48 #ifndef __NEURAL_NETWORK_TRAINING_MODEL_H__ 49 #define __NEURAL_NETWORK_TRAINING_MODEL_H__ 51 #include "services/daal_defines.h" 52 #include "data_management/data/tensor.h" 53 #include "data_management/data/numeric_table.h" 54 #include "services/daal_memory.h" 55 #include "algorithms/neural_networks/layers/layer.h" 56 #include "algorithms/neural_networks/layers/layer_types.h" 57 #include "algorithms/neural_networks/layers/loss/loss_layer_forward.h" 58 #include "algorithms/neural_networks/layers/split/split_layer_forward.h" 59 #include "algorithms/neural_networks/neural_networks_prediction_model.h" 60 #include "algorithms/neural_networks/neural_networks_training_topology.h" 62 #include "algorithms/optimization_solver/iterative_solver/iterative_solver_batch.h" 71 namespace neural_networks
85 class Parameter :
public daal::algorithms::Parameter
93 Parameter(
const services::SharedPtr<optimization_solver::iterative_solver::Batch > &optimizationSolver_ = services::SharedPtr<optimization_solver::iterative_solver::Batch>(),
94 engines::EnginePtr engine_ = engines::mt19937::Batch<DAAL_ALGORITHM_FP_TYPE>::create()) :
95 optimizationSolver(optimizationSolver_),
98 services::SharedPtr<optimization_solver::iterative_solver::Batch> optimizationSolver;
99 engines::EnginePtr engine;
106 class DAAL_EXPORT Model :
public neural_networks::ModelImpl
109 DECLARE_SERIALIZABLE_CAST(Model);
111 using neural_networks::ModelImpl::getWeightsAndBiases;
112 using neural_networks::ModelImpl::setWeightsAndBiases;
117 static services::SharedPtr<Model> create(services::Status *stat = NULL);
120 Model(
const Model &model) :
122 _backwardLayers(model.getBackwardLayers()),
123 _storeWeightDerivativesInTable(model._storeWeightDerivativesInTable)
137 template<
typename modelFPType>
138 services::Status initialize(
const services::Collection<size_t> &sampleSize,
const Topology &topology,
139 const Parameter ¶meter = Parameter())
141 using namespace layers;
142 using namespace services;
144 size_t nLayers = topology.size();
146 _backwardNextLayers = SharedPtr<Collection<NextLayers> >(
new Collection<NextLayers>(nLayers));
147 if (!_backwardNextLayers)
149 st.add(services::ErrorMemoryAllocationFailed);
153 for(
size_t i = 0; i < nLayers; i++)
155 insertLayer(topology[i]);
158 for(
int i = (
int)nLayers - 1; i >= 0; i--)
160 size_t layerId = topology[i].index();
161 const NextLayers &next = topology[i].nextLayers();
162 for (
size_t j = 0; j < next.size(); j++)
164 (*_backwardNextLayers)[next[j]].push_back(layerId);
168 for(
int i = (
int)nLayers - 1; i >= 0; i--)
170 layers::forward::LayerIfacePtr layer = getForwardLayer(i);
171 SharedPtr<split::forward::Batch<float> > splitLayerFloat = dynamicPointerCast<split::forward::Batch<float>, forward::LayerIface>(layer);
172 SharedPtr<split::forward::Batch<double> > splitLayerDouble = dynamicPointerCast<split::forward::Batch<double>, forward::LayerIface>(layer);
173 if(splitLayerFloat.get() || splitLayerDouble.get())
175 const NextLayers &next = topology[i].nextLayers();
176 for (
size_t j = 0; j < next.size(); j++)
178 layers::forward::LayerIfacePtr nextLayer = getForwardLayer(next[j]);
179 nextLayer->getLayerParameter()->allowInplaceComputation =
false;
184 allocate<modelFPType>(sampleSize, parameter);
186 for(
size_t i = 0; i < nLayers; i++)
188 getForwardLayer(i)->enableResetOnCompute(
false);
189 getBackwardLayer(i)->enableResetOnCompute(
false);
198 const ForwardLayersPtr getForwardLayers()
const 200 return _forwardLayers;
208 const layers::forward::LayerIfacePtr getForwardLayer(
size_t index)
const 210 return _forwardLayers->get(index);
217 const BackwardLayersPtr getBackwardLayers()
const 219 return _backwardLayers;
227 const layers::backward::LayerIfacePtr getBackwardLayer(
size_t index)
const 229 return _backwardLayers->get(index);
236 template<
typename modelFPType>
237 const prediction::ModelPtr getPredictionModel()
239 using namespace services;
240 using namespace data_management;
241 using namespace layers;
243 size_t nLayers = _forwardLayers->size();
246 ForwardLayersPtr _predictionForwardLayers(
new ForwardLayers(nLayers));
247 SharedPtr<Collection<NextLayers> > _predictionNextLayers(
new Collection<NextLayers>(nLayers));
248 for (
size_t i = 0; i < nLayers; i++)
250 (*_predictionNextLayers)[i] = _nextLayers->get(i);
251 (*_predictionForwardLayers)[i] = ((*_forwardLayers)[i])->getLayerForPrediction();
252 (*_predictionForwardLayers)[i]->getLayerParameter()->predictionStage =
true;
255 bool storeWeightsInTable =
true;
256 prediction::ModelPtr predictionModel(
new prediction::Model(
257 _predictionForwardLayers, _predictionNextLayers, (modelFPType)0.0, storeWeightsInTable));
259 predictionModel->setWeightsAndBiases(getWeightsAndBiases());
260 return predictionModel;
268 bool getWeightsAndBiasesStorageStatus()
const 270 return _storeWeightsInTable;
280 services::Status setWeightsAndBiases(
size_t idx,
const data_management::NumericTablePtr &table);
287 data_management::NumericTablePtr getWeightsAndBiases(
size_t idx)
const;
293 data_management::NumericTablePtr getWeightsAndBiasesDerivatives()
const;
300 data_management::NumericTablePtr getWeightsAndBiasesDerivatives(
size_t idx)
const;
309 DAAL_DEPRECATED services::Status setErrors(services::ErrorCollection &errors)
311 return services::Status();
319 DAAL_DEPRECATED
const services::ErrorCollection &getErrors()
const {
return _errors; }
328 template<
typename modelFPType>
329 services::Status allocate(
const services::Collection<size_t> &sampleSize,
const Parameter ¶meter = Parameter())
331 using namespace services;
332 using namespace data_management;
333 using namespace layers;
337 if (_sampleSize.size() > 0) { _sampleSize.clear(); }
338 _sampleSize = sampleSize;
340 _forwardLayers->get(0)->getLayerInput()->set(forward::data,
341 TensorPtr(
new HomogenTensor<modelFPType>(_sampleSize, Tensor::doAllocate)));
343 size_t nLayers = _forwardLayers->size();
345 for (
size_t i = 0; i < nLayers; i++)
347 layers::Parameter *lParameter = _forwardLayers->get(i)->getLayerParameter();
348 initializers::Parameter *wParameter = lParameter->weightsInitializer->getParameter();
349 initializers::Parameter *bParameter = lParameter->biasesInitializer->getParameter();
351 s |= connectForwardLayers(i);
353 if(!wParameter->engine)
355 wParameter->engine = parameter.engine;
357 if(!bParameter->engine)
359 bParameter->engine = parameter.engine;
363 bool checkWeightsAndBiasesAlloc =
true;
364 s |= createWeightsAndBiases<modelFPType>(checkWeightsAndBiasesAlloc);
365 s |= enableConditionalGradientPropagation();
368 for (
size_t i = 0; i < nLayers; i++)
370 forward::LayerIfacePtr forwardLayer = _forwardLayers->get(i);
371 forward::Input *forwardInput = forwardLayer->getLayerInput();
373 forwardLayer->getLayerResult()->setResultForBackward(forwardInput);
377 s |= checkWeightsAndBiasesDerivativesAllocation();
379 for (
int i = (
int)nLayers - 1; i >= 0; i--)
381 s |= connectBackwardLayers(i);
384 s |= createWeightsAndBiasesDerivatives<modelFPType>();
385 if(_solverOptionalArgumentCollection.size() == 0)
387 if(_storeWeightsInTable) _solverOptionalArgumentCollection = DataCollection(1);
388 else _solverOptionalArgumentCollection = DataCollection(nLayers);
395 Model(services::Status &st);
398 template<
typename Archive,
bool onDeserialize>
399 services::Status serialImpl(Archive *arch)
401 return services::Status();
404 void insertLayer(
const layers::LayerDescriptor &layerDescriptor)
406 _forwardLayers->insert(layerDescriptor.index(), layerDescriptor.layer()->forwardLayer->clone());
407 _backwardLayers->insert(layerDescriptor.index(), layerDescriptor.layer()->backwardLayer->clone());
408 _nextLayers->insert(layerDescriptor.index(), layerDescriptor.nextLayers());
411 services::Status enableConditionalGradientPropagation()
413 using namespace services;
414 using namespace layers;
418 size_t nLayers = _forwardLayers->size();
421 bool *flags = (
bool *)daal_malloc(nLayers *
sizeof(
bool));
425 s |= disableGradientPropagationInStartingLayers(nLayers, flags);
429 s |= enableGradientPropagation(nLayers, flags);
435 services::Status disableGradientPropagationInStartingLayers(
size_t nLayers,
bool *visited)
437 using namespace services;
438 using namespace layers;
440 for (
size_t i = 0; i < nLayers; i++)
445 Collection<size_t> stack;
447 while (stack.size() > 0)
449 size_t layerId = stack[stack.size() - 1];
450 stack.erase(stack.size() - 1);
451 if (!visited[layerId])
453 visited[layerId] =
true;
455 forward::LayerIfacePtr forwardLayer = _forwardLayers->get(layerId);
456 forward::Input *forwardInput = forwardLayer->getLayerInput();
457 layers::Parameter *forwardParameter = forwardLayer->getLayerParameter();
458 layers::Parameter *backwardParameter = _backwardLayers->get(layerId)->getLayerParameter();
460 backwardParameter->propagateGradient =
false;
462 if (forwardInput->getWeightsSizes(forwardParameter).size() +
463 forwardInput->getBiasesSizes(forwardParameter) .size() == 0)
466 const NextLayers &next = _nextLayers->get(layerId);
467 for (
size_t i = 0; i < next.size(); i++)
469 stack.push_back(next[i]);
474 return services::Status();
477 services::Status enableGradientPropagationInSubsequentLayers(
size_t startLayerId,
size_t nLayers,
bool *enabledPropagation)
479 using namespace services;
480 using namespace layers;
481 Collection<size_t> stack;
482 const NextLayers &next = _nextLayers->get(startLayerId);
483 for (
size_t i = 0; i < next.size(); i++)
485 stack.push_back(next[i]);
487 while (stack.size() > 0)
489 size_t layerId = stack[stack.size() - 1];
490 stack.erase(stack.size() - 1);
491 if (!enabledPropagation[layerId])
493 enabledPropagation[layerId] =
true;
494 backward::LayerIfacePtr backwardLayer = _backwardLayers->get(layerId);
495 backwardLayer->getLayerParameter()->propagateGradient =
true;
496 const NextLayers &next = _nextLayers->get(layerId);
497 for (
size_t i = 0; i < next.size(); i++)
499 stack.push_back(next[i]);
503 return services::Status();
506 services::Status enableGradientPropagation(
size_t nLayers,
bool *enabledPropagation)
508 using namespace services;
509 using namespace layers;
510 Collection<size_t> stack;
513 for (
size_t i = 0; i < nLayers; i++)
515 enabledPropagation[i] =
false;
518 while (stack.size() > 0)
520 size_t layerId = stack[stack.size() - 1];
521 stack.erase(stack.size() - 1);
522 if (!enabledPropagation[layerId])
524 forward::LayerIfacePtr forwardLayer = _forwardLayers->get(layerId);
525 forward::Input *forwardInput = forwardLayer->getLayerInput();
526 layers::Parameter *forwardParameter = forwardLayer->getLayerParameter();
527 layers::Parameter *backwardParameter = _backwardLayers->get(layerId)->getLayerParameter();
529 if (backwardParameter->propagateGradient ==
false &&
530 (forwardInput->getWeightsSizes(forwardParameter).size() +
531 forwardInput->getBiasesSizes(forwardParameter) .size()) > 0)
533 enableGradientPropagationInSubsequentLayers(layerId, nLayers, enabledPropagation);
537 const NextLayers &next = _nextLayers->get(layerId);
538 for (
size_t i = 0; i < next.size(); i++)
540 stack.push_back(next[i]);
545 return services::Status();
548 services::Status checkWeightsAndBiasesDerivativesAllocation()
550 using namespace services;
551 using namespace layers;
553 _storeWeightDerivativesInTable =
true;
554 size_t nLayers = _backwardLayers->size();
555 for (
size_t i = 0; i < nLayers; i++)
557 backward::LayerIfacePtr &backwardLayer = _backwardLayers->get(i);
558 if (!backwardLayer) {
continue; }
559 backward::ResultPtr backwardResult = backwardLayer->getLayerResult();
561 if (backwardResult->get(backward::weightDerivatives) || backwardResult->get(backward::biasDerivatives))
563 _storeWeightDerivativesInTable =
false;
567 return services::Status();
570 services::Status connectBackwardLayers(
size_t layerId)
572 using namespace services;
573 using namespace data_management;
574 using namespace layers;
576 forward::LayerIfacePtr &forwardLayer = _forwardLayers->get(layerId);
577 backward::LayerIfacePtr &backwardLayer = _backwardLayers->get(layerId);
579 if (!forwardLayer || !backwardLayer) {
return services::Status(); }
581 backward::Input *backwardInput = backwardLayer->getLayerInput();
582 forward::ResultPtr forwardResult = forwardLayer->getLayerResult();
584 backwardInput->setInputFromForward(forwardResult);
585 backwardLayer->allocateResult();
589 if (!backwardLayer->getLayerParameter()->propagateGradient) {
return services::Status(); }
591 backward::ResultPtr backwardResult = backwardLayer->getLayerResult();
593 const NextLayers &next = _backwardNextLayers->get(layerId);
594 const size_t nextLayersSize = next.size();
595 for(
size_t j = 0; j < nextLayersSize; j++)
597 size_t inputIndex = nextLayersSize - j - 1;
598 _backwardLayers->get(next[j])->addInput(backwardResult, inputIndex, 0 );
600 return services::Status();
603 template<
typename modelFPType>
604 DAAL_EXPORT services::Status createWeightsAndBiasesDerivatives();
612 algorithms::OptionalArgumentPtr getSolverOptionalArgument(
size_t index)
614 return services::dynamicPointerCast<algorithms::OptionalArgument, data_management::SerializationIface>(_solverOptionalArgumentCollection[index]);
624 services::Status setSolverOptionalArgument(
const algorithms::OptionalArgumentPtr& solverOptionalArgument,
size_t index)
626 _solverOptionalArgumentCollection[index] = solverOptionalArgument;
627 return services::Status();
634 data_management::DataCollection getSolverOptionalArgumentCollection()
636 return _solverOptionalArgumentCollection;
645 services::Status setSolverOptionalArgumentCollection(
const data_management::DataCollection &solverOptionalArgumentCollection)
647 _solverOptionalArgumentCollection = solverOptionalArgumentCollection;
648 return services::Status();
652 data_management::DataCollection _solverOptionalArgumentCollection;
653 services::Collection<size_t> _sampleSize;
654 BackwardLayersPtr _backwardLayers;
655 services::SharedPtr<services::Collection<layers::NextLayers> > _backwardNextLayers;
656 mutable services::ErrorCollection _errors;
658 bool _storeWeightDerivativesInTable;
659 LearnableParametersIfacePtr _weightsAndBiasesDerivatives;
662 typedef services::SharedPtr<Model> ModelPtr;
666 using interface1::Parameter;
667 using interface1::Model;
668 using interface1::ModelPtr;
daal::algorithms::neural_networks::layers::interface1::LayerDescriptor
Class defining descriptor for layer on both forward and backward stages and its parameters.
Definition: layer_descriptor.h:71
daal::services::interface1::Collection::get
T & get(size_t index)
Definition: collection.h:177
daal::algorithms::neural_networks::training::interface1::Model::setSolverOptionalArgumentCollection
services::Status setSolverOptionalArgumentCollection(const data_management::DataCollection &solverOptionalArgumentCollection)
Definition: neural_networks_training_model.h:645
daal::algorithms::neural_networks::training::interface1::Model
Class representing the model of neural network.
Definition: neural_networks_training_model.h:106
daal::algorithms::neural_networks::training::interface1::Model::setSolverOptionalArgument
services::Status setSolverOptionalArgument(const algorithms::OptionalArgumentPtr &solverOptionalArgument, size_t index)
Definition: neural_networks_training_model.h:624
daal::algorithms::neural_networks::layers::interface1::LayerDescriptor::layer
const layers::LayerIfacePtr & layer() const
Definition: layer_descriptor.h:119
daal::algorithms::neural_networks::training::interface1::Topology
Class defining a neural network topology - a set of layers and connection between them - on the train...
Definition: neural_networks_training_topology.h:66
daal::algorithms::neural_networks::training::interface1::Topology::push_back
size_t push_back(const layers::LayerIfacePtr &layer)
Definition: neural_networks_training_topology.h:95
daal::services::interface1::Status
Class that holds the results of API calls. In case of API routine failure it contains the list of err...
Definition: error_handling.h:491
daal
Definition: algorithm_base_common.h:57
daal::algorithms::neural_networks::interface1::ModelImpl::setWeightsAndBiases
services::Status setWeightsAndBiases(const data_management::NumericTablePtr &weightsAndBiases)
Definition: neural_networks_learnable_parameters.h:109
daal::algorithms::neural_networks::training::interface1::Model::getWeightsAndBiasesStorageStatus
bool getWeightsAndBiasesStorageStatus() const
Definition: neural_networks_training_model.h:268
daal::algorithms::neural_networks::training::interface1::Model::initialize
services::Status initialize(const services::Collection< size_t > &sampleSize, const Topology &topology, const Parameter ¶meter=Parameter())
Definition: neural_networks_training_model.h:138
daal::data_management::interface1::DataCollection
Class that provides functionality of Collection container for objects derived from SerializationIface...
Definition: data_collection.h:71
daal::algorithms::neural_networks::layers::interface1::Parameter
Definition: layer_types.h:110
daal::algorithms::neural_networks::training::interface1::Model::allocate
services::Status allocate(const services::Collection< size_t > &sampleSize, const Parameter ¶meter=Parameter())
Definition: neural_networks_training_model.h:329
daal::algorithms::neural_networks::layers::interface1::LayerDescriptor::nextLayers
const NextLayers & nextLayers() const
Definition: layer_descriptor.h:125
daal::services::ErrorMemoryAllocationFailed
Definition: error_indexes.h:172
daal::services::interface1::Status::clear
void clear()
daal::algorithms::neural_networks::training::interface1::Parameter::optimizationSolver
services::SharedPtr< optimization_solver::iterative_solver::Batch > optimizationSolver
Definition: neural_networks_training_model.h:98
daal::algorithms::neural_networks::training::interface1::Topology::size
size_t size() const
Definition: neural_networks_training_topology.h:88
daal::services::interface1::ErrorCollection
Class that represents an error collection.
Definition: error_handling.h:338
daal::algorithms::neural_networks::initializers::interface1::Parameter::engine
engines::EnginePtr engine
Definition: initializer_types.h:114
daal::algorithms::neural_networks::interface1::ModelImpl::getWeightsAndBiases
const data_management::NumericTablePtr getWeightsAndBiases() const
Definition: neural_networks_learnable_parameters.h:118
daal::algorithms::neural_networks::training::interface1::Parameter::engine
engines::EnginePtr engine
Definition: neural_networks_training_model.h:99
daal::algorithms::interface1::Parameter
Base class to represent computation parameters. Algorithm-specific parameters are represented as deri...
Definition: algorithm_types.h:86
daal::services::interface1::SharedPtr
Shared pointer that retains shared ownership of an object through a pointer. Several SharedPtr object...
Definition: daal_shared_ptr.h:187
daal::algorithms::neural_networks::training::interface1::Model::getSolverOptionalArgument
algorithms::OptionalArgumentPtr getSolverOptionalArgument(size_t index)
Definition: neural_networks_training_model.h:612
daal::algorithms::neural_networks::training::interface1::Parameter
Class representing the parameters of neural network.
Definition: neural_networks_training_model.h:85
daal::algorithms::neural_networks::layers::interface1::Parameter::weightsInitializer
initializers::InitializerIfacePtr weightsInitializer
Definition: layer_types.h:117
daal::services::daal_malloc
DAAL_EXPORT void * daal_malloc(size_t size, size_t alignment=DAAL_MALLOC_DEFAULT_ALIGNMENT)
daal::algorithms::association_rules::data
Definition: apriori_types.h:107
daal::algorithms::neural_networks::training::interface1::Parameter::Parameter
Parameter(const services::SharedPtr< optimization_solver::iterative_solver::Batch > &optimizationSolver_=services::SharedPtr< optimization_solver::iterative_solver::Batch >(), engines::EnginePtr engine_=engines::mt19937::Batch< DAAL_ALGORITHM_FP_TYPE >::create())
Definition: neural_networks_training_model.h:93
daal::algorithms::neural_networks::training::interface1::Model::getForwardLayers
const ForwardLayersPtr getForwardLayers() const
Definition: neural_networks_training_model.h:198
daal::algorithms::neural_networks::training::interface1::Model::~Model
virtual ~Model()
Destructor.
Definition: neural_networks_training_model.h:127
daal::services::daal_free
DAAL_EXPORT void daal_free(void *ptr)
daal::algorithms::neural_networks::training::interface1::Model::getSolverOptionalArgumentCollection
data_management::DataCollection getSolverOptionalArgumentCollection()
Definition: neural_networks_training_model.h:634
daal::algorithms::neural_networks::layers::interface1::LayerDescriptor::index
size_t index() const
Definition: layer_descriptor.h:103
daal::algorithms::neural_networks::training::interface1::Model::Model
Model(const Model &model)
Copy constructor.
Definition: neural_networks_training_model.h:120
daal::algorithms::neural_networks::prediction::interface1::Model
Class Model object for the prediction stage of neural network algorithm.
Definition: neural_networks_prediction_model.h:108
daal::algorithms::neural_networks::layers::interface1::Parameter::biasesInitializer
initializers::InitializerIfacePtr biasesInitializer
Definition: layer_types.h:119
daal::algorithms::engines::mt19937::interface1::Batch
Provides methods for mt19937 engine computations in the batch processing mode.
Definition: mt19937.h:113
daal::algorithms::neural_networks::training::interface1::Model::getForwardLayer
const layers::forward::LayerIfacePtr getForwardLayer(size_t index) const
Definition: neural_networks_training_model.h:208
daal::algorithms::neural_networks::training::interface1::Model::getBackwardLayer
const layers::backward::LayerIfacePtr getBackwardLayer(size_t index) const
Definition: neural_networks_training_model.h:227
daal::algorithms::neural_networks::layers::interface1::Parameter::propagateGradient
bool propagateGradient
Definition: layer_types.h:123
daal::algorithms::neural_networks::training::interface1::Model::getBackwardLayers
const BackwardLayersPtr getBackwardLayers() const
Definition: neural_networks_training_model.h:217
daal::algorithms::neural_networks::training::model
Definition: neural_networks_training_result.h:78
daal::algorithms::neural_networks::interface1::ModelImpl
Class Model object for the prediction stage of neural network algorithm.
Definition: neural_networks_learnable_parameters.h:89
daal::algorithms::neural_networks::initializers::interface1::Parameter
Definition: initializer_types.h:107
daal::services::interface1::Collection
Class that implements functionality of the Collection container.
Definition: collection.h:69