dnn.hpp 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725
  1. /*M///////////////////////////////////////////////////////////////////////////////////////
  2. //
  3. // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  4. //
  5. // By downloading, copying, installing or using the software you agree to this license.
  6. // If you do not agree to this license, do not download, install,
  7. // copy or use the software.
  8. //
  9. //
  10. // License Agreement
  11. // For Open Source Computer Vision Library
  12. //
  13. // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14. // Third party copyrights are property of their respective owners.
  15. //
  16. // Redistribution and use in source and binary forms, with or without modification,
  17. // are permitted provided that the following conditions are met:
  18. //
  19. // * Redistribution's of source code must retain the above copyright notice,
  20. // this list of conditions and the following disclaimer.
  21. //
  22. // * Redistribution's in binary form must reproduce the above copyright notice,
  23. // this list of conditions and the following disclaimer in the documentation
  24. // and/or other materials provided with the distribution.
  25. //
  26. // * The name of the copyright holders may not be used to endorse or promote products
  27. // derived from this software without specific prior written permission.
  28. //
  29. // This software is provided by the copyright holders and contributors "as is" and
  30. // any express or implied warranties, including, but not limited to, the implied
  31. // warranties of merchantability and fitness for a particular purpose are disclaimed.
  32. // In no event shall the Intel Corporation or contributors be liable for any direct,
  33. // indirect, incidental, special, exemplary, or consequential damages
  34. // (including, but not limited to, procurement of substitute goods or services;
  35. // loss of use, data, or profits; or business interruption) however caused
  36. // and on any theory of liability, whether in contract, strict liability,
  37. // or tort (including negligence or otherwise) arising in any way out of
  38. // the use of this software, even if advised of the possibility of such damage.
  39. //
  40. //M*/
  41. #ifndef OPENCV_DNN_DNN_HPP
  42. #define OPENCV_DNN_DNN_HPP
  43. #include <vector>
  44. #include <opencv2/core.hpp>
  45. #include "opencv2/core/async.hpp"
  46. #include "../dnn/version.hpp"
  47. #include <opencv2/dnn/dict.hpp>
  48. namespace cv {
  49. namespace dnn {
  50. CV__DNN_INLINE_NS_BEGIN
  51. //! @addtogroup dnn
  52. //! @{
  53. typedef std::vector<int> MatShape;
  54. /**
  55. * @brief Enum of computation backends supported by layers.
  56. * @see Net::setPreferableBackend
  57. */
  58. enum Backend
  59. {
  60. //! DNN_BACKEND_DEFAULT equals to DNN_BACKEND_INFERENCE_ENGINE if
  61. //! OpenCV is built with Intel's Inference Engine library or
  62. //! DNN_BACKEND_OPENCV otherwise.
  63. DNN_BACKEND_DEFAULT = 0,
  64. DNN_BACKEND_HALIDE,
  65. DNN_BACKEND_INFERENCE_ENGINE, //!< Intel's Inference Engine computational backend
  66. //!< @sa setInferenceEngineBackendType
  67. DNN_BACKEND_OPENCV,
  68. DNN_BACKEND_VKCOM,
  69. DNN_BACKEND_CUDA,
  70. DNN_BACKEND_WEBNN,
  71. #ifdef __OPENCV_BUILD
  72. DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
  73. DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
  74. #endif
  75. };
  76. /**
  77. * @brief Enum of target devices for computations.
  78. * @see Net::setPreferableTarget
  79. */
  80. enum Target
  81. {
  82. DNN_TARGET_CPU = 0,
  83. DNN_TARGET_OPENCL,
  84. DNN_TARGET_OPENCL_FP16,
  85. DNN_TARGET_MYRIAD,
  86. DNN_TARGET_VULKAN,
  87. DNN_TARGET_FPGA, //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin.
  88. DNN_TARGET_CUDA,
  89. DNN_TARGET_CUDA_FP16,
  90. DNN_TARGET_HDDL
  91. };
  92. CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends();
  93. CV_EXPORTS_W std::vector<Target> getAvailableTargets(dnn::Backend be);
  94. /**
  95. * @brief Enables detailed logging of the DNN model loading with CV DNN API.
  96. * @param[in] isDiagnosticsMode Indicates whether diagnostic mode should be set.
  97. *
  98. * Diagnostic mode provides detailed logging of the model loading stage to explore
  99. * potential problems (ex.: not implemented layer type).
  100. *
  101. * @note In diagnostic mode series of assertions will be skipped, it can lead to the
  102. * expected application crash.
  103. */
  104. CV_EXPORTS void enableModelDiagnostics(bool isDiagnosticsMode);
  105. /** @brief This class provides all data needed to initialize layer.
  106. *
  107. * It includes dictionary with scalar params (which can be read by using Dict interface),
  108. * blob params #blobs and optional meta information: #name and #type of layer instance.
  109. */
  110. class CV_EXPORTS LayerParams : public Dict
  111. {
  112. public:
  113. //TODO: Add ability to name blob params
  114. std::vector<Mat> blobs; //!< List of learned parameters stored as blobs.
  115. String name; //!< Name of the layer instance (optional, can be used internal purposes).
  116. String type; //!< Type name which was used for creating layer by layer factory (optional).
  117. };
  118. /**
  119. * @brief Derivatives of this class encapsulates functions of certain backends.
  120. */
  121. class BackendNode
  122. {
  123. public:
  124. explicit BackendNode(int backendId);
  125. virtual ~BackendNode(); //!< Virtual destructor to make polymorphism.
  126. int backendId; //!< Backend identifier.
  127. };
  128. /**
  129. * @brief Derivatives of this class wraps cv::Mat for different backends and targets.
  130. */
  131. class BackendWrapper
  132. {
  133. public:
  134. BackendWrapper(int backendId, int targetId);
  135. /**
  136. * @brief Wrap cv::Mat for specific backend and target.
  137. * @param[in] targetId Target identifier.
  138. * @param[in] m cv::Mat for wrapping.
  139. *
  140. * Make CPU->GPU data transfer if it's require for the target.
  141. */
  142. BackendWrapper(int targetId, const cv::Mat& m);
  143. /**
  144. * @brief Make wrapper for reused cv::Mat.
  145. * @param[in] base Wrapper of cv::Mat that will be reused.
  146. * @param[in] shape Specific shape.
  147. *
  148. * Initialize wrapper from another one. It'll wrap the same host CPU
  149. * memory and mustn't allocate memory on device(i.e. GPU). It might
  150. * has different shape. Use in case of CPU memory reusing for reuse
  151. * associated memory on device too.
  152. */
  153. BackendWrapper(const Ptr<BackendWrapper>& base, const MatShape& shape);
  154. virtual ~BackendWrapper(); //!< Virtual destructor to make polymorphism.
  155. /**
  156. * @brief Transfer data to CPU host memory.
  157. */
  158. virtual void copyToHost() = 0;
  159. /**
  160. * @brief Indicate that an actual data is on CPU.
  161. */
  162. virtual void setHostDirty() = 0;
  163. int backendId; //!< Backend identifier.
  164. int targetId; //!< Target identifier.
  165. };
  166. class CV_EXPORTS ActivationLayer;
  167. /** @brief This interface class allows to build new Layers - are building blocks of networks.
  168. *
  169. * Each class, derived from Layer, must implement allocate() methods to declare own outputs and forward() to compute outputs.
  170. * Also before using the new layer into networks you must register your layer by using one of @ref dnnLayerFactory "LayerFactory" macros.
  171. */
  172. class CV_EXPORTS_W Layer : public Algorithm
  173. {
  174. public:
  175. //! List of learned parameters must be stored here to allow read them by using Net::getParam().
  176. CV_PROP_RW std::vector<Mat> blobs;
  177. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  178. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  179. * @param[in] input vector of already allocated input blobs
  180. * @param[out] output vector of already allocated output blobs
  181. *
  182. * If this method is called after network has allocated all memory for input and output blobs
  183. * and before inferencing.
  184. */
  185. CV_DEPRECATED_EXTERNAL
  186. virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output);
  187. /** @brief Computes and sets internal parameters according to inputs, outputs and blobs.
  188. * @param[in] inputs vector of already allocated input blobs
  189. * @param[out] outputs vector of already allocated output blobs
  190. *
  191. * If this method is called after network has allocated all memory for input and output blobs
  192. * and before inferencing.
  193. */
  194. CV_WRAP virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs);
  195. /** @brief Given the @p input blobs, computes the output @p blobs.
  196. * @deprecated Use Layer::forward(InputArrayOfArrays, OutputArrayOfArrays, OutputArrayOfArrays) instead
  197. * @param[in] input the input blobs.
  198. * @param[out] output allocated output blobs, which will store results of the computation.
  199. * @param[out] internals allocated internal blobs
  200. */
  201. CV_DEPRECATED_EXTERNAL
  202. virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals);
  203. /** @brief Given the @p input blobs, computes the output @p blobs.
  204. * @param[in] inputs the input blobs.
  205. * @param[out] outputs allocated output blobs, which will store results of the computation.
  206. * @param[out] internals allocated internal blobs
  207. */
  208. virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
  209. /** @brief Tries to quantize the given layer and compute the quantization parameters required for fixed point implementation.
  210. * @param[in] scales input and output scales.
  211. * @param[in] zeropoints input and output zeropoints.
  212. * @param[out] params Quantized parameters required for fixed point implementation of that layer.
  213. * @returns True if layer can be quantized.
  214. */
  215. virtual bool tryQuantize(const std::vector<std::vector<float> > &scales,
  216. const std::vector<std::vector<int> > &zeropoints, LayerParams& params);
  217. /** @brief Given the @p input blobs, computes the output @p blobs.
  218. * @param[in] inputs the input blobs.
  219. * @param[out] outputs allocated output blobs, which will store results of the computation.
  220. * @param[out] internals allocated internal blobs
  221. */
  222. void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals);
  223. /** @brief
  224. * @overload
  225. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  226. */
  227. CV_DEPRECATED_EXTERNAL
  228. void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs);
  229. /** @brief
  230. * @overload
  231. * @deprecated Use Layer::finalize(InputArrayOfArrays, OutputArrayOfArrays) instead
  232. */
  233. CV_DEPRECATED std::vector<Mat> finalize(const std::vector<Mat> &inputs);
  234. /** @brief Allocates layer and computes output.
  235. * @deprecated This method will be removed in the future release.
  236. */
  237. CV_DEPRECATED CV_WRAP void run(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs,
  238. CV_IN_OUT std::vector<Mat> &internals);
  239. /** @brief Returns index of input blob into the input array.
  240. * @param inputName label of input blob
  241. *
  242. * Each layer input and output can be labeled to easily identify them using "%<layer_name%>[.output_name]" notation.
  243. * This method maps label of input blob to its index into input vector.
  244. */
  245. virtual int inputNameToIndex(String inputName); // FIXIT const
  246. /** @brief Returns index of output blob in output array.
  247. * @see inputNameToIndex()
  248. */
  249. CV_WRAP virtual int outputNameToIndex(const String& outputName); // FIXIT const
  250. /**
  251. * @brief Ask layer if it support specific backend for doing computations.
  252. * @param[in] backendId computation backend identifier.
  253. * @see Backend
  254. */
  255. virtual bool supportBackend(int backendId); // FIXIT const
  256. /**
  257. * @brief Returns Halide backend node.
  258. * @param[in] inputs Input Halide buffers.
  259. * @see BackendNode, BackendWrapper
  260. *
  261. * Input buffers should be exactly the same that will be used in forward invocations.
  262. * Despite we can use Halide::ImageParam based on input shape only,
  263. * it helps prevent some memory management issues (if something wrong,
  264. * Halide tests will be failed).
  265. */
  266. virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs);
  267. virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs);
  268. virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
  269. virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs);
  270. virtual Ptr<BackendNode> initWebnn(const std::vector<Ptr<BackendWrapper> > &inputs, const std::vector<Ptr<BackendNode> >& nodes);
  271. /**
  272. * @brief Returns a CUDA backend node
  273. *
  274. * @param context void pointer to CSLContext object
  275. * @param inputs layer inputs
  276. * @param outputs layer outputs
  277. */
  278. virtual Ptr<BackendNode> initCUDA(
  279. void *context,
  280. const std::vector<Ptr<BackendWrapper>>& inputs,
  281. const std::vector<Ptr<BackendWrapper>>& outputs
  282. );
  283. /**
  284. * @brief Automatic Halide scheduling based on layer hyper-parameters.
  285. * @param[in] node Backend node with Halide functions.
  286. * @param[in] inputs Blobs that will be used in forward invocations.
  287. * @param[in] outputs Blobs that will be used in forward invocations.
  288. * @param[in] targetId Target identifier
  289. * @see BackendNode, Target
  290. *
  291. * Layer don't use own Halide::Func members because we can have applied
  292. * layers fusing. In this way the fused function should be scheduled.
  293. */
  294. virtual void applyHalideScheduler(Ptr<BackendNode>& node,
  295. const std::vector<Mat*> &inputs,
  296. const std::vector<Mat> &outputs,
  297. int targetId) const;
  298. /**
  299. * @brief Implement layers fusing.
  300. * @param[in] node Backend node of bottom layer.
  301. * @see BackendNode
  302. *
  303. * Actual for graph-based backends. If layer attached successfully,
  304. * returns non-empty cv::Ptr to node of the same backend.
  305. * Fuse only over the last function.
  306. */
  307. virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node);
  308. /**
  309. * @brief Tries to attach to the layer the subsequent activation layer, i.e. do the layer fusion in a partial case.
  310. * @param[in] layer The subsequent activation layer.
  311. *
  312. * Returns true if the activation layer has been attached successfully.
  313. */
  314. virtual bool setActivation(const Ptr<ActivationLayer>& layer);
  315. /**
  316. * @brief Try to fuse current layer with a next one
  317. * @param[in] top Next layer to be fused.
  318. * @returns True if fusion was performed.
  319. */
  320. virtual bool tryFuse(Ptr<Layer>& top);
  321. /**
  322. * @brief Returns parameters of layers with channel-wise multiplication and addition.
  323. * @param[out] scale Channel-wise multipliers. Total number of values should
  324. * be equal to number of channels.
  325. * @param[out] shift Channel-wise offsets. Total number of values should
  326. * be equal to number of channels.
  327. *
  328. * Some layers can fuse their transformations with further layers.
  329. * In example, convolution + batch normalization. This way base layer
  330. * use weights from layer after it. Fused layer is skipped.
  331. * By default, @p scale and @p shift are empty that means layer has no
  332. * element-wise multiplications or additions.
  333. */
  334. virtual void getScaleShift(Mat& scale, Mat& shift) const;
  335. /**
  336. * @brief Returns scale and zeropoint of layers
  337. * @param[out] scale Output scale
  338. * @param[out] zeropoint Output zeropoint
  339. *
  340. * By default, @p scale is 1 and @p zeropoint is 0.
  341. */
  342. virtual void getScaleZeropoint(float& scale, int& zeropoint) const;
  343. /**
  344. * @brief "Deattaches" all the layers, attached to particular layer.
  345. */
  346. virtual void unsetAttached();
  347. virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
  348. const int requiredOutputs,
  349. std::vector<MatShape> &outputs,
  350. std::vector<MatShape> &internals) const;
  351. virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
  352. const std::vector<MatShape> &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;}
  353. virtual bool updateMemoryShapes(const std::vector<MatShape> &inputs);
  354. CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes.
  355. CV_PROP String type; //!< Type name which was used for creating layer by layer factory.
  356. CV_PROP int preferableTarget; //!< prefer target for layer forwarding
  357. Layer();
  358. explicit Layer(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  359. void setParamsFrom(const LayerParams &params); //!< Initializes only #name, #type and #blobs fields.
  360. virtual ~Layer();
  361. };
  362. /** @brief This class allows to create and manipulate comprehensive artificial neural networks.
  363. *
  364. * Neural network is presented as directed acyclic graph (DAG), where vertices are Layer instances,
  365. * and edges specify relationships between layers inputs and outputs.
  366. *
  367. * Each network layer has unique integer id and unique string name inside its network.
  368. * LayerId can store either layer name or layer id.
  369. *
  370. * This class supports reference counting of its instances, i. e. copies point to the same instance.
  371. */
  372. class CV_EXPORTS_W_SIMPLE Net
  373. {
  374. public:
  375. CV_WRAP Net(); //!< Default constructor.
  376. CV_WRAP ~Net(); //!< Destructor frees the net only if there aren't references to the net anymore.
  377. /** @brief Create a network from Intel's Model Optimizer intermediate representation (IR).
  378. * @param[in] xml XML configuration file with network's topology.
  379. * @param[in] bin Binary file with trained weights.
  380. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  381. * backend.
  382. */
  383. CV_WRAP static Net readFromModelOptimizer(const String& xml, const String& bin);
  384. /** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
  385. * @param[in] bufferModelConfig buffer with model's configuration.
  386. * @param[in] bufferWeights buffer with model's trained weights.
  387. * @returns Net object.
  388. */
  389. CV_WRAP static
  390. Net readFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
  391. /** @brief Create a network from Intel's Model Optimizer in-memory buffers with intermediate representation (IR).
  392. * @param[in] bufferModelConfigPtr buffer pointer of model's configuration.
  393. * @param[in] bufferModelConfigSize buffer size of model's configuration.
  394. * @param[in] bufferWeightsPtr buffer pointer of model's trained weights.
  395. * @param[in] bufferWeightsSize buffer size of model's trained weights.
  396. * @returns Net object.
  397. */
  398. static
  399. Net readFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
  400. const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
  401. /** Returns true if there are no layers in the network. */
  402. CV_WRAP bool empty() const;
  403. /** @brief Dump net to String
  404. * @returns String with structure, hyperparameters, backend, target and fusion
  405. * Call method after setInput(). To see correct backend, target and fusion run after forward().
  406. */
  407. CV_WRAP String dump();
  408. /** @brief Dump net structure, hyperparameters, backend, target and fusion to dot file
  409. * @param path path to output file with .dot extension
  410. * @see dump()
  411. */
  412. CV_WRAP void dumpToFile(const String& path);
  413. /** @brief Adds new layer to the net.
  414. * @param name unique name of the adding layer.
  415. * @param type typename of the adding layer (type must be registered in LayerRegister).
  416. * @param dtype datatype of output blobs.
  417. * @param params parameters which will be used to initialize the creating layer.
  418. * @returns unique identifier of created layer, or -1 if a failure will happen.
  419. */
  420. int addLayer(const String &name, const String &type, const int &dtype, LayerParams &params);
  421. /** @overload Datatype of output blobs set to default CV_32F */
  422. int addLayer(const String &name, const String &type, LayerParams &params);
  423. /** @brief Adds new layer and connects its first input to the first output of previously added layer.
  424. * @see addLayer()
  425. */
  426. int addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams &params);
  427. /** @overload */
  428. int addLayerToPrev(const String &name, const String &type, LayerParams &params);
  429. /** @brief Converts string name of the layer to the integer identifier.
  430. * @returns id of the layer, or -1 if the layer wasn't found.
  431. */
  432. CV_WRAP int getLayerId(const String &layer) const;
  433. CV_WRAP std::vector<String> getLayerNames() const;
  434. /** @brief Container for strings and integers.
  435. *
  436. * @deprecated Use getLayerId() with int result.
  437. */
  438. typedef DictValue LayerId;
  439. /** @brief Returns pointer to layer with specified id or name which the network use. */
  440. CV_WRAP Ptr<Layer> getLayer(int layerId) const;
  441. /** @overload
  442. * @deprecated Use int getLayerId(const String &layer)
  443. */
  444. CV_WRAP inline Ptr<Layer> getLayer(const String& layerName) const { return getLayer(getLayerId(layerName)); }
  445. /** @overload
  446. * @deprecated to be removed
  447. */
  448. CV_WRAP Ptr<Layer> getLayer(const LayerId& layerId) const;
  449. /** @brief Returns pointers to input layers of specific layer. */
  450. std::vector<Ptr<Layer> > getLayerInputs(int layerId) const; // FIXIT: CV_WRAP
  451. /** @brief Connects output of the first layer to input of the second layer.
  452. * @param outPin descriptor of the first layer output.
  453. * @param inpPin descriptor of the second layer input.
  454. *
  455. * Descriptors have the following template <DFN>&lt;layer_name&gt;[.input_number]</DFN>:
  456. * - the first part of the template <DFN>layer_name</DFN> is string name of the added layer.
  457. * If this part is empty then the network input pseudo layer will be used;
  458. * - the second optional part of the template <DFN>input_number</DFN>
  459. * is either number of the layer input, either label one.
  460. * If this part is omitted then the first layer input will be used.
  461. *
  462. * @see setNetInputs(), Layer::inputNameToIndex(), Layer::outputNameToIndex()
  463. */
  464. CV_WRAP void connect(String outPin, String inpPin);
  465. /** @brief Connects #@p outNum output of the first layer to #@p inNum input of the second layer.
  466. * @param outLayerId identifier of the first layer
  467. * @param outNum number of the first layer output
  468. * @param inpLayerId identifier of the second layer
  469. * @param inpNum number of the second layer input
  470. */
  471. void connect(int outLayerId, int outNum, int inpLayerId, int inpNum);
  472. /** @brief Registers network output with name
  473. *
  474. * Function may create additional 'Identity' layer.
  475. *
  476. * @param outputName identifier of the output
  477. * @param layerId identifier of the second layer
  478. * @param outputPort number of the second layer input
  479. *
  480. * @returns index of bound layer (the same as layerId or newly created)
  481. */
  482. int registerOutput(const std::string& outputName, int layerId, int outputPort);
  483. /** @brief Sets outputs names of the network input pseudo layer.
  484. *
  485. * Each net always has special own the network input pseudo layer with id=0.
  486. * This layer stores the user blobs only and don't make any computations.
  487. * In fact, this layer provides the only way to pass user data into the network.
  488. * As any other layer, this layer can label its outputs and this function provides an easy way to do this.
  489. */
  490. CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames);
  491. /** @brief Specify shape of network input.
  492. */
  493. CV_WRAP void setInputShape(const String &inputName, const MatShape& shape);
  494. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  495. * @param outputName name for layer which output is needed to get
  496. * @return blob for first output of specified layer.
  497. * @details By default runs forward pass for the whole network.
  498. */
  499. CV_WRAP Mat forward(const String& outputName = String());
  500. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  501. * @param outputName name for layer which output is needed to get
  502. * @details By default runs forward pass for the whole network.
  503. *
  504. * This is an asynchronous version of forward(const String&).
  505. * dnn::DNN_BACKEND_INFERENCE_ENGINE backend is required.
  506. */
  507. CV_WRAP AsyncArray forwardAsync(const String& outputName = String());
  508. /** @brief Runs forward pass to compute output of layer with name @p outputName.
  509. * @param outputBlobs contains all output blobs for specified layer.
  510. * @param outputName name for layer which output is needed to get
  511. * @details If @p outputName is empty, runs forward pass for the whole network.
  512. */
  513. CV_WRAP void forward(OutputArrayOfArrays outputBlobs, const String& outputName = String());
  514. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  515. * @param outputBlobs contains blobs for first outputs of specified layers.
  516. * @param outBlobNames names for layers which outputs are needed to get
  517. */
  518. CV_WRAP void forward(OutputArrayOfArrays outputBlobs,
  519. const std::vector<String>& outBlobNames);
  520. /** @brief Runs forward pass to compute outputs of layers listed in @p outBlobNames.
  521. * @param outputBlobs contains all output blobs for each layer specified in @p outBlobNames.
  522. * @param outBlobNames names for layers which outputs are needed to get
  523. */
  524. CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector<std::vector<Mat> >& outputBlobs,
  525. const std::vector<String>& outBlobNames);
  526. /** @brief Returns a quantized Net from a floating-point Net.
  527. * @param calibData Calibration data to compute the quantization parameters.
  528. * @param inputsDtype Datatype of quantized net's inputs. Can be CV_32F or CV_8S.
  529. * @param outputsDtype Datatype of quantized net's outputs. Can be CV_32F or CV_8S.
  530. */
  531. CV_WRAP Net quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype);
  532. /** @brief Returns input scale and zeropoint for a quantized Net.
  533. * @param scales output parameter for returning input scales.
  534. * @param zeropoints output parameter for returning input zeropoints.
  535. */
  536. CV_WRAP void getInputDetails(CV_OUT std::vector<float>& scales, CV_OUT std::vector<int>& zeropoints) const;
  537. /** @brief Returns output scale and zeropoint for a quantized Net.
  538. * @param scales output parameter for returning output scales.
  539. * @param zeropoints output parameter for returning output zeropoints.
  540. */
  541. CV_WRAP void getOutputDetails(CV_OUT std::vector<float>& scales, CV_OUT std::vector<int>& zeropoints) const;
  542. /**
  543. * @brief Compile Halide layers.
  544. * @param[in] scheduler Path to YAML file with scheduling directives.
  545. * @see setPreferableBackend
  546. *
  547. * Schedule layers that support Halide backend. Then compile them for
  548. * specific target. For layers that not represented in scheduling file
  549. * or if no manual scheduling used at all, automatic scheduling will be applied.
  550. */
  551. CV_WRAP void setHalideScheduler(const String& scheduler);
  552. /**
  553. * @brief Ask network to use specific computation backend where it supported.
  554. * @param[in] backendId backend identifier.
  555. * @see Backend
  556. *
  557. * If OpenCV is compiled with Intel's Inference Engine library, DNN_BACKEND_DEFAULT
  558. * means DNN_BACKEND_INFERENCE_ENGINE. Otherwise it equals to DNN_BACKEND_OPENCV.
  559. */
  560. CV_WRAP void setPreferableBackend(int backendId);
  561. /**
  562. * @brief Ask network to make computations on specific target device.
  563. * @param[in] targetId target identifier.
  564. * @see Target
  565. *
  566. * List of supported combinations backend / target:
  567. * | | DNN_BACKEND_OPENCV | DNN_BACKEND_INFERENCE_ENGINE | DNN_BACKEND_HALIDE | DNN_BACKEND_CUDA |
  568. * |------------------------|--------------------|------------------------------|--------------------|-------------------|
  569. * | DNN_TARGET_CPU | + | + | + | |
  570. * | DNN_TARGET_OPENCL | + | + | + | |
  571. * | DNN_TARGET_OPENCL_FP16 | + | + | | |
  572. * | DNN_TARGET_MYRIAD | | + | | |
  573. * | DNN_TARGET_FPGA | | + | | |
  574. * | DNN_TARGET_CUDA | | | | + |
  575. * | DNN_TARGET_CUDA_FP16 | | | | + |
  576. * | DNN_TARGET_HDDL | | + | | |
  577. */
  578. CV_WRAP void setPreferableTarget(int targetId);
  579. /** @brief Sets the new input value for the network
  580. * @param blob A new blob. Should have CV_32F or CV_8U depth.
  581. * @param name A name of input layer.
  582. * @param scalefactor An optional normalization scale.
  583. * @param mean An optional mean subtraction values.
  584. * @see connect(String, String) to know format of the descriptor.
  585. *
  586. * If scale or mean values are specified, a final input blob is computed
  587. * as:
  588. * \f[input(n,c,h,w) = scalefactor \times (blob(n,c,h,w) - mean_c)\f]
  589. */
  590. CV_WRAP void setInput(InputArray blob, const String& name = "",
  591. double scalefactor = 1.0, const Scalar& mean = Scalar());
  592. /** @brief Sets the new value for the learned param of the layer.
  593. * @param layer name or id of the layer.
  594. * @param numParam index of the layer parameter in the Layer::blobs array.
  595. * @param blob the new value.
  596. * @see Layer::blobs
  597. * @note If shape of the new blob differs from the previous shape,
  598. * then the following forward pass may fail.
  599. */
  600. CV_WRAP void setParam(int layer, int numParam, const Mat &blob);
  601. CV_WRAP inline void setParam(const String& layerName, int numParam, const Mat &blob) { return setParam(getLayerId(layerName), numParam, blob); }
  602. /** @brief Returns parameter blob of the layer.
  603. * @param layer name or id of the layer.
  604. * @param numParam index of the layer parameter in the Layer::blobs array.
  605. * @see Layer::blobs
  606. */
  607. CV_WRAP Mat getParam(int layer, int numParam = 0) const;
  608. CV_WRAP inline Mat getParam(const String& layerName, int numParam = 0) const { return getParam(getLayerId(layerName), numParam); }
  609. /** @brief Returns indexes of layers with unconnected outputs.
  610. *
  611. * FIXIT: Rework API to registerOutput() approach, deprecate this call
  612. */
  613. CV_WRAP std::vector<int> getUnconnectedOutLayers() const;
  614. /** @brief Returns names of layers with unconnected outputs.
  615. *
  616. * FIXIT: Rework API to registerOutput() approach, deprecate this call
  617. */
  618. CV_WRAP std::vector<String> getUnconnectedOutLayersNames() const;
  619. /** @brief Returns input and output shapes for all layers in loaded model;
  620. * preliminary inferencing isn't necessary.
  621. * @param netInputShapes shapes for all input blobs in net input layer.
  622. * @param layersIds output parameter for layer IDs.
  623. * @param inLayersShapes output parameter for input layers shapes;
  624. * order is the same as in layersIds
  625. * @param outLayersShapes output parameter for output layers shapes;
  626. * order is the same as in layersIds
  627. */
  628. CV_WRAP void getLayersShapes(const std::vector<MatShape>& netInputShapes,
  629. CV_OUT std::vector<int>& layersIds,
  630. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  631. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  632. /** @overload */
  633. CV_WRAP void getLayersShapes(const MatShape& netInputShape,
  634. CV_OUT std::vector<int>& layersIds,
  635. CV_OUT std::vector<std::vector<MatShape> >& inLayersShapes,
  636. CV_OUT std::vector<std::vector<MatShape> >& outLayersShapes) const;
  637. /** @brief Returns input and output shapes for layer with specified
  638. * id in loaded model; preliminary inferencing isn't necessary.
  639. * @param netInputShape shape input blob in net input layer.
  640. * @param layerId id for layer.
  641. * @param inLayerShapes output parameter for input layers shapes;
  642. * order is the same as in layersIds
  643. * @param outLayerShapes output parameter for output layers shapes;
  644. * order is the same as in layersIds
  645. */
  646. void getLayerShapes(const MatShape& netInputShape,
  647. const int layerId,
  648. CV_OUT std::vector<MatShape>& inLayerShapes,
  649. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  650. /** @overload */
  651. void getLayerShapes(const std::vector<MatShape>& netInputShapes,
  652. const int layerId,
  653. CV_OUT std::vector<MatShape>& inLayerShapes,
  654. CV_OUT std::vector<MatShape>& outLayerShapes) const; // FIXIT: CV_WRAP
  655. /** @brief Computes FLOP for whole loaded model with specified input shapes.
  656. * @param netInputShapes vector of shapes for all net inputs.
  657. * @returns computed FLOP.
  658. */
  659. CV_WRAP int64 getFLOPS(const std::vector<MatShape>& netInputShapes) const;
  660. /** @overload */
  661. CV_WRAP int64 getFLOPS(const MatShape& netInputShape) const;
  662. /** @overload */
  663. CV_WRAP int64 getFLOPS(const int layerId,
  664. const std::vector<MatShape>& netInputShapes) const;
  665. /** @overload */
  666. CV_WRAP int64 getFLOPS(const int layerId,
  667. const MatShape& netInputShape) const;
  668. /** @brief Returns list of types for layer used in model.
  669. * @param layersTypes output parameter for returning types.
  670. */
  671. CV_WRAP void getLayerTypes(CV_OUT std::vector<String>& layersTypes) const;
  672. /** @brief Returns count of layers of specified type.
  673. * @param layerType type.
  674. * @returns count of layers
  675. */
  676. CV_WRAP int getLayersCount(const String& layerType) const;
  677. /** @brief Computes bytes number which are required to store
  678. * all weights and intermediate blobs for model.
  679. * @param netInputShapes vector of shapes for all net inputs.
  680. * @param weights output parameter to store resulting bytes for weights.
  681. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  682. */
  683. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  684. CV_OUT size_t& weights, CV_OUT size_t& blobs) const; // FIXIT: CV_WRAP
  685. /** @overload */
  686. CV_WRAP void getMemoryConsumption(const MatShape& netInputShape,
  687. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  688. /** @overload */
  689. CV_WRAP void getMemoryConsumption(const int layerId,
  690. const std::vector<MatShape>& netInputShapes,
  691. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  692. /** @overload */
  693. CV_WRAP void getMemoryConsumption(const int layerId,
  694. const MatShape& netInputShape,
  695. CV_OUT size_t& weights, CV_OUT size_t& blobs) const;
  696. /** @brief Computes bytes number which are required to store
  697. * all weights and intermediate blobs for each layer.
  698. * @param netInputShapes vector of shapes for all net inputs.
  699. * @param layerIds output vector to save layer IDs.
  700. * @param weights output parameter to store resulting bytes for weights.
  701. * @param blobs output parameter to store resulting bytes for intermediate blobs.
  702. */
  703. void getMemoryConsumption(const std::vector<MatShape>& netInputShapes,
  704. CV_OUT std::vector<int>& layerIds,
  705. CV_OUT std::vector<size_t>& weights,
  706. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  707. /** @overload */
  708. void getMemoryConsumption(const MatShape& netInputShape,
  709. CV_OUT std::vector<int>& layerIds,
  710. CV_OUT std::vector<size_t>& weights,
  711. CV_OUT std::vector<size_t>& blobs) const; // FIXIT: CV_WRAP
  712. /** @brief Enables or disables layer fusion in the network.
  713. * @param fusion true to enable the fusion, false to disable. The fusion is enabled by default.
  714. */
  715. CV_WRAP void enableFusion(bool fusion);
  716. /** @brief Returns overall time for inference and timings (in ticks) for layers.
  717. *
  718. * Indexes in returned vector correspond to layers ids. Some layers can be fused with others,
  719. * in this case zero ticks count will be return for that skipped layers. Supported by DNN_BACKEND_OPENCV on DNN_TARGET_CPU only.
  720. *
  721. * @param[out] timings vector for tick timings for all layers.
  722. * @return overall ticks for model inference.
  723. */
  724. CV_WRAP int64 getPerfProfile(CV_OUT std::vector<double>& timings);
  725. private:
  726. struct Impl;
  727. Ptr<Impl> impl;
  728. };
  729. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  730. * @param cfgFile path to the .cfg file with text description of the network architecture.
  731. * @param darknetModel path to the .weights file with learned network.
  732. * @returns Network object that ready to do forward, throw an exception in failure cases.
  733. * @returns Net object.
  734. */
  735. CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String());
  736. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  737. * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
  738. * @param bufferModel A buffer contains a content of .weights file with learned network.
  739. * @returns Net object.
  740. */
  741. CV_EXPORTS_W Net readNetFromDarknet(const std::vector<uchar>& bufferCfg,
  742. const std::vector<uchar>& bufferModel = std::vector<uchar>());
  743. /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
  744. * @param bufferCfg A buffer contains a content of .cfg file with text description of the network architecture.
  745. * @param lenCfg Number of bytes to read from bufferCfg
  746. * @param bufferModel A buffer contains a content of .weights file with learned network.
  747. * @param lenModel Number of bytes to read from bufferModel
  748. * @returns Net object.
  749. */
  750. CV_EXPORTS Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg,
  751. const char *bufferModel = NULL, size_t lenModel = 0);
  752. /** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format.
  753. * @param prototxt path to the .prototxt file with text description of the network architecture.
  754. * @param caffeModel path to the .caffemodel file with learned network.
  755. * @returns Net object.
  756. */
  757. CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String());
  758. /** @brief Reads a network model stored in Caffe model in memory.
  759. * @param bufferProto buffer containing the content of the .prototxt file
  760. * @param bufferModel buffer containing the content of the .caffemodel file
  761. * @returns Net object.
  762. */
  763. CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
  764. const std::vector<uchar>& bufferModel = std::vector<uchar>());
  765. /** @brief Reads a network model stored in Caffe model in memory.
  766. * @details This is an overloaded member function, provided for convenience.
  767. * It differs from the above function only in what argument(s) it accepts.
  768. * @param bufferProto buffer containing the content of the .prototxt file
  769. * @param lenProto length of bufferProto
  770. * @param bufferModel buffer containing the content of the .caffemodel file
  771. * @param lenModel length of bufferModel
  772. * @returns Net object.
  773. */
  774. CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
  775. const char *bufferModel = NULL, size_t lenModel = 0);
  776. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  777. * @param model path to the .pb file with binary protobuf description of the network architecture
  778. * @param config path to the .pbtxt file that contains text graph definition in protobuf format.
  779. * Resulting Net object is built by text graph using weights from a binary one that
  780. * let us make it more flexible.
  781. * @returns Net object.
  782. */
  783. CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String());
  784. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  785. * @param bufferModel buffer containing the content of the pb file
  786. * @param bufferConfig buffer containing the content of the pbtxt file
  787. * @returns Net object.
  788. */
  789. CV_EXPORTS_W Net readNetFromTensorflow(const std::vector<uchar>& bufferModel,
  790. const std::vector<uchar>& bufferConfig = std::vector<uchar>());
  791. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
  792. * @details This is an overloaded member function, provided for convenience.
  793. * It differs from the above function only in what argument(s) it accepts.
  794. * @param bufferModel buffer containing the content of the pb file
  795. * @param lenModel length of bufferModel
  796. * @param bufferConfig buffer containing the content of the pbtxt file
  797. * @param lenConfig length of bufferConfig
  798. */
  799. CV_EXPORTS Net readNetFromTensorflow(const char *bufferModel, size_t lenModel,
  800. const char *bufferConfig = NULL, size_t lenConfig = 0);
  801. /**
  802. * @brief Reads a network model stored in <a href="http://torch.ch">Torch7</a> framework's format.
  803. * @param model path to the file, dumped from Torch by using torch.save() function.
  804. * @param isBinary specifies whether the network was serialized in ascii mode or binary.
  805. * @param evaluate specifies testing phase of network. If true, it's similar to evaluate() method in Torch.
  806. * @returns Net object.
  807. *
  808. * @note Ascii mode of Torch serializer is more preferable, because binary mode extensively use `long` type of C language,
  809. * which has various bit-length on different systems.
  810. *
  811. * The loading file must contain serialized <a href="https://github.com/torch/nn/blob/master/doc/module.md">nn.Module</a> object
  812. * with importing network. Try to eliminate a custom objects from serialazing data to avoid importing errors.
  813. *
  814. * List of supported layers (i.e. object instances derived from Torch nn.Module class):
  815. * - nn.Sequential
  816. * - nn.Parallel
  817. * - nn.Concat
  818. * - nn.Linear
  819. * - nn.SpatialConvolution
  820. * - nn.SpatialMaxPooling, nn.SpatialAveragePooling
  821. * - nn.ReLU, nn.TanH, nn.Sigmoid
  822. * - nn.Reshape
  823. * - nn.SoftMax, nn.LogSoftMax
  824. *
  825. * Also some equivalents of these classes from cunn, cudnn, and fbcunn may be successfully imported.
  826. */
  827. CV_EXPORTS_W Net readNetFromTorch(const String &model, bool isBinary = true, bool evaluate = true);
  828. /**
  829. * @brief Read deep learning network represented in one of the supported formats.
  830. * @param[in] model Binary file contains trained weights. The following file
  831. * extensions are expected for models from different frameworks:
  832. * * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/)
  833. * * `*.pb` (TensorFlow, https://www.tensorflow.org/)
  834. * * `*.t7` | `*.net` (Torch, http://torch.ch/)
  835. * * `*.weights` (Darknet, https://pjreddie.com/darknet/)
  836. * * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit)
  837. * * `*.onnx` (ONNX, https://onnx.ai/)
  838. * @param[in] config Text file contains network configuration. It could be a
  839. * file with the following extensions:
  840. * * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/)
  841. * * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/)
  842. * * `*.cfg` (Darknet, https://pjreddie.com/darknet/)
  843. * * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit)
  844. * @param[in] framework Explicit framework name tag to determine a format.
  845. * @returns Net object.
  846. *
  847. * This function automatically detects an origin framework of trained model
  848. * and calls an appropriate function such @ref readNetFromCaffe, @ref readNetFromTensorflow,
  849. * @ref readNetFromTorch or @ref readNetFromDarknet. An order of @p model and @p config
  850. * arguments does not matter.
  851. */
  852. CV_EXPORTS_W Net readNet(const String& model, const String& config = "", const String& framework = "");
  853. /**
  854. * @brief Read deep learning network represented in one of the supported formats.
  855. * @details This is an overloaded member function, provided for convenience.
  856. * It differs from the above function only in what argument(s) it accepts.
  857. * @param[in] framework Name of origin framework.
  858. * @param[in] bufferModel A buffer with a content of binary file with weights
  859. * @param[in] bufferConfig A buffer with a content of text file contains network configuration.
  860. * @returns Net object.
  861. */
  862. CV_EXPORTS_W Net readNet(const String& framework, const std::vector<uchar>& bufferModel,
  863. const std::vector<uchar>& bufferConfig = std::vector<uchar>());
  864. /** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework.
  865. * @warning This function has the same limitations as readNetFromTorch().
  866. */
  867. CV_EXPORTS_W Mat readTorchBlob(const String &filename, bool isBinary = true);
  868. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  869. * @param[in] xml XML configuration file with network's topology.
  870. * @param[in] bin Binary file with trained weights.
  871. * @returns Net object.
  872. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  873. * backend.
  874. */
  875. CV_EXPORTS_W
  876. Net readNetFromModelOptimizer(const String &xml, const String &bin);
  877. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  878. * @param[in] bufferModelConfig Buffer contains XML configuration with network's topology.
  879. * @param[in] bufferWeights Buffer contains binary data with trained weights.
  880. * @returns Net object.
  881. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  882. * backend.
  883. */
  884. CV_EXPORTS_W
  885. Net readNetFromModelOptimizer(const std::vector<uchar>& bufferModelConfig, const std::vector<uchar>& bufferWeights);
  886. /** @brief Load a network from Intel's Model Optimizer intermediate representation.
  887. * @param[in] bufferModelConfigPtr Pointer to buffer which contains XML configuration with network's topology.
  888. * @param[in] bufferModelConfigSize Binary size of XML configuration data.
  889. * @param[in] bufferWeightsPtr Pointer to buffer which contains binary data with trained weights.
  890. * @param[in] bufferWeightsSize Binary size of trained weights data.
  891. * @returns Net object.
  892. * Networks imported from Intel's Model Optimizer are launched in Intel's Inference Engine
  893. * backend.
  894. */
  895. CV_EXPORTS
  896. Net readNetFromModelOptimizer(const uchar* bufferModelConfigPtr, size_t bufferModelConfigSize,
  897. const uchar* bufferWeightsPtr, size_t bufferWeightsSize);
  898. /** @brief Reads a network model <a href="https://onnx.ai/">ONNX</a>.
  899. * @param onnxFile path to the .onnx file with text description of the network architecture.
  900. * @returns Network object that ready to do forward, throw an exception in failure cases.
  901. */
  902. CV_EXPORTS_W Net readNetFromONNX(const String &onnxFile);
  903. /** @brief Reads a network model from <a href="https://onnx.ai/">ONNX</a>
  904. * in-memory buffer.
  905. * @param buffer memory address of the first byte of the buffer.
  906. * @param sizeBuffer size of the buffer.
  907. * @returns Network object that ready to do forward, throw an exception
  908. * in failure cases.
  909. */
  910. CV_EXPORTS Net readNetFromONNX(const char* buffer, size_t sizeBuffer);
  911. /** @brief Reads a network model from <a href="https://onnx.ai/">ONNX</a>
  912. * in-memory buffer.
  913. * @param buffer in-memory buffer that stores the ONNX model bytes.
  914. * @returns Network object that ready to do forward, throw an exception
  915. * in failure cases.
  916. */
  917. CV_EXPORTS_W Net readNetFromONNX(const std::vector<uchar>& buffer);
  918. /** @brief Creates blob from .pb file.
  919. * @param path to the .pb file with input tensor.
  920. * @returns Mat.
  921. */
  922. CV_EXPORTS_W Mat readTensorFromONNX(const String& path);
  923. /** @brief Creates 4-dimensional blob from image. Optionally resizes and crops @p image from center,
  924. * subtract @p mean values, scales values by @p scalefactor, swap Blue and Red channels.
  925. * @param image input image (with 1-, 3- or 4-channels).
  926. * @param size spatial size for output image
  927. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  928. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  929. * @param scalefactor multiplier for @p image values.
  930. * @param swapRB flag which indicates that swap first and last channels
  931. * in 3-channel image is necessary.
  932. * @param crop flag which indicates whether image will be cropped after resize or not
  933. * @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
  934. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
  935. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  936. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  937. * @returns 4-dimensional Mat with NCHW dimensions order.
  938. */
  939. CV_EXPORTS_W Mat blobFromImage(InputArray image, double scalefactor=1.0, const Size& size = Size(),
  940. const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  941. int ddepth=CV_32F);
  942. /** @brief Creates 4-dimensional blob from image.
  943. * @details This is an overloaded member function, provided for convenience.
  944. * It differs from the above function only in what argument(s) it accepts.
  945. */
  946. CV_EXPORTS void blobFromImage(InputArray image, OutputArray blob, double scalefactor=1.0,
  947. const Size& size = Size(), const Scalar& mean = Scalar(),
  948. bool swapRB=false, bool crop=false, int ddepth=CV_32F);
  949. /** @brief Creates 4-dimensional blob from series of images. Optionally resizes and
  950. * crops @p images from center, subtract @p mean values, scales values by @p scalefactor,
  951. * swap Blue and Red channels.
  952. * @param images input images (all with 1-, 3- or 4-channels).
  953. * @param size spatial size for output image
  954. * @param mean scalar with mean values which are subtracted from channels. Values are intended
  955. * to be in (mean-R, mean-G, mean-B) order if @p image has BGR ordering and @p swapRB is true.
  956. * @param scalefactor multiplier for @p images values.
  957. * @param swapRB flag which indicates that swap first and last channels
  958. * in 3-channel image is necessary.
  959. * @param crop flag which indicates whether image will be cropped after resize or not
  960. * @param ddepth Depth of output blob. Choose CV_32F or CV_8U.
  961. * @details if @p crop is true, input image is resized so one side after resize is equal to corresponding
  962. * dimension in @p size and another one is equal or larger. Then, crop from the center is performed.
  963. * If @p crop is false, direct resize without cropping and preserving aspect ratio is performed.
  964. * @returns 4-dimensional Mat with NCHW dimensions order.
  965. */
  966. CV_EXPORTS_W Mat blobFromImages(InputArrayOfArrays images, double scalefactor=1.0,
  967. Size size = Size(), const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  968. int ddepth=CV_32F);
  969. /** @brief Creates 4-dimensional blob from series of images.
  970. * @details This is an overloaded member function, provided for convenience.
  971. * It differs from the above function only in what argument(s) it accepts.
  972. */
  973. CV_EXPORTS void blobFromImages(InputArrayOfArrays images, OutputArray blob,
  974. double scalefactor=1.0, Size size = Size(),
  975. const Scalar& mean = Scalar(), bool swapRB=false, bool crop=false,
  976. int ddepth=CV_32F);
  977. /** @brief Parse a 4D blob and output the images it contains as 2D arrays through a simpler data structure
  978. * (std::vector<cv::Mat>).
  979. * @param[in] blob_ 4 dimensional array (images, channels, height, width) in floating point precision (CV_32F) from
  980. * which you would like to extract the images.
  981. * @param[out] images_ array of 2D Mat containing the images extracted from the blob in floating point precision
  982. * (CV_32F). They are non normalized neither mean added. The number of returned images equals the first dimension
  983. * of the blob (batch size). Every image has a number of channels equals to the second dimension of the blob (depth).
  984. */
  985. CV_EXPORTS_W void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_);
  986. /** @brief Convert all weights of Caffe network to half precision floating point.
  987. * @param src Path to origin model from Caffe framework contains single
  988. * precision floating point weights (usually has `.caffemodel` extension).
  989. * @param dst Path to destination model with updated weights.
  990. * @param layersTypes Set of layers types which parameters will be converted.
  991. * By default, converts only Convolutional and Fully-Connected layers'
  992. * weights.
  993. *
  994. * @note Shrinked model has no origin float32 weights so it can't be used
  995. * in origin Caffe framework anymore. However the structure of data
  996. * is taken from NVidia's Caffe fork: https://github.com/NVIDIA/caffe.
  997. * So the resulting model may be used there.
  998. */
  999. CV_EXPORTS_W void shrinkCaffeModel(const String& src, const String& dst,
  1000. const std::vector<String>& layersTypes = std::vector<String>());
  1001. /** @brief Create a text representation for a binary network stored in protocol buffer format.
  1002. * @param[in] model A path to binary network.
  1003. * @param[in] output A path to output text file to be created.
  1004. *
  1005. * @note To reduce output file size, trained weights are not included.
  1006. */
  1007. CV_EXPORTS_W void writeTextGraph(const String& model, const String& output);
  1008. /** @brief Performs non maximum suppression given boxes and corresponding scores.
  1009. * @param bboxes a set of bounding boxes to apply NMS.
  1010. * @param scores a set of corresponding confidences.
  1011. * @param score_threshold a threshold used to filter boxes by score.
  1012. * @param nms_threshold a threshold used in non maximum suppression.
  1013. * @param indices the kept indices of bboxes after NMS.
  1014. * @param eta a coefficient in adaptive threshold formula: \f$nms\_threshold_{i+1}=eta\cdot nms\_threshold_i\f$.
  1015. * @param top_k if `>0`, keep at most @p top_k picked indices.
  1016. */
  1017. CV_EXPORTS void NMSBoxes(const std::vector<Rect>& bboxes, const std::vector<float>& scores,
  1018. const float score_threshold, const float nms_threshold,
  1019. CV_OUT std::vector<int>& indices,
  1020. const float eta = 1.f, const int top_k = 0);
  1021. CV_EXPORTS_W void NMSBoxes(const std::vector<Rect2d>& bboxes, const std::vector<float>& scores,
  1022. const float score_threshold, const float nms_threshold,
  1023. CV_OUT std::vector<int>& indices,
  1024. const float eta = 1.f, const int top_k = 0);
  1025. CV_EXPORTS_AS(NMSBoxesRotated) void NMSBoxes(const std::vector<RotatedRect>& bboxes, const std::vector<float>& scores,
  1026. const float score_threshold, const float nms_threshold,
  1027. CV_OUT std::vector<int>& indices,
  1028. const float eta = 1.f, const int top_k = 0);
  1029. /**
  1030. * @brief Enum of Soft NMS methods.
  1031. * @see softNMSBoxes
  1032. */
  1033. enum class SoftNMSMethod
  1034. {
  1035. SOFTNMS_LINEAR = 1,
  1036. SOFTNMS_GAUSSIAN = 2
  1037. };
  1038. /** @brief Performs soft non maximum suppression given boxes and corresponding scores.
  1039. * Reference: https://arxiv.org/abs/1704.04503
  1040. * @param bboxes a set of bounding boxes to apply Soft NMS.
  1041. * @param scores a set of corresponding confidences.
  1042. * @param updated_scores a set of corresponding updated confidences.
  1043. * @param score_threshold a threshold used to filter boxes by score.
  1044. * @param nms_threshold a threshold used in non maximum suppression.
  1045. * @param indices the kept indices of bboxes after NMS.
  1046. * @param top_k keep at most @p top_k picked indices.
  1047. * @param sigma parameter of Gaussian weighting.
  1048. * @param method Gaussian or linear.
  1049. * @see SoftNMSMethod
  1050. */
  1051. CV_EXPORTS_W void softNMSBoxes(const std::vector<Rect>& bboxes,
  1052. const std::vector<float>& scores,
  1053. CV_OUT std::vector<float>& updated_scores,
  1054. const float score_threshold,
  1055. const float nms_threshold,
  1056. CV_OUT std::vector<int>& indices,
  1057. size_t top_k = 0,
  1058. const float sigma = 0.5,
  1059. SoftNMSMethod method = SoftNMSMethod::SOFTNMS_GAUSSIAN);
  1060. /** @brief This class is presented high-level API for neural networks.
  1061. *
  1062. * Model allows to set params for preprocessing input image.
  1063. * Model creates net from file with trained weights and config,
  1064. * sets preprocessing input and runs forward pass.
  1065. */
  1066. class CV_EXPORTS_W_SIMPLE Model
  1067. {
  1068. public:
  1069. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1070. Model();
  1071. Model(const Model&) = default;
  1072. Model(Model&&) = default;
  1073. Model& operator=(const Model&) = default;
  1074. Model& operator=(Model&&) = default;
  1075. /**
  1076. * @brief Create model from deep learning network represented in one of the supported formats.
  1077. * An order of @p model and @p config arguments does not matter.
  1078. * @param[in] model Binary file contains trained weights.
  1079. * @param[in] config Text file contains network configuration.
  1080. */
  1081. CV_WRAP Model(const String& model, const String& config = "");
  1082. /**
  1083. * @brief Create model from deep learning network.
  1084. * @param[in] network Net object.
  1085. */
  1086. CV_WRAP Model(const Net& network);
  1087. /** @brief Set input size for frame.
  1088. * @param[in] size New input size.
  1089. * @note If shape of the new blob less than 0, then frame size not change.
  1090. */
  1091. CV_WRAP Model& setInputSize(const Size& size);
  1092. /** @overload
  1093. * @param[in] width New input width.
  1094. * @param[in] height New input height.
  1095. */
  1096. CV_WRAP inline
  1097. Model& setInputSize(int width, int height) { return setInputSize(Size(width, height)); }
  1098. /** @brief Set mean value for frame.
  1099. * @param[in] mean Scalar with mean values which are subtracted from channels.
  1100. */
  1101. CV_WRAP Model& setInputMean(const Scalar& mean);
  1102. /** @brief Set scalefactor value for frame.
  1103. * @param[in] scale Multiplier for frame values.
  1104. */
  1105. CV_WRAP Model& setInputScale(double scale);
  1106. /** @brief Set flag crop for frame.
  1107. * @param[in] crop Flag which indicates whether image will be cropped after resize or not.
  1108. */
  1109. CV_WRAP Model& setInputCrop(bool crop);
  1110. /** @brief Set flag swapRB for frame.
  1111. * @param[in] swapRB Flag which indicates that swap first and last channels.
  1112. */
  1113. CV_WRAP Model& setInputSwapRB(bool swapRB);
  1114. /** @brief Set preprocessing parameters for frame.
  1115. * @param[in] size New input size.
  1116. * @param[in] mean Scalar with mean values which are subtracted from channels.
  1117. * @param[in] scale Multiplier for frame values.
  1118. * @param[in] swapRB Flag which indicates that swap first and last channels.
  1119. * @param[in] crop Flag which indicates whether image will be cropped after resize or not.
  1120. * blob(n, c, y, x) = scale * resize( frame(y, x, c) ) - mean(c) )
  1121. */
  1122. CV_WRAP void setInputParams(double scale = 1.0, const Size& size = Size(),
  1123. const Scalar& mean = Scalar(), bool swapRB = false, bool crop = false);
  1124. /** @brief Given the @p input frame, create input blob, run net and return the output @p blobs.
  1125. * @param[in] frame The input image.
  1126. * @param[out] outs Allocated output blobs, which will store results of the computation.
  1127. */
  1128. CV_WRAP void predict(InputArray frame, OutputArrayOfArrays outs) const;
  1129. // ============================== Net proxy methods ==============================
  1130. // Never expose methods with network implementation details, like:
  1131. // - addLayer, addLayerToPrev, connect, setInputsNames, setInputShape, setParam, getParam
  1132. // - getLayer*, getUnconnectedOutLayers, getUnconnectedOutLayersNames, getLayersShapes
  1133. // - forward* methods, setInput
  1134. /// @sa Net::setPreferableBackend
  1135. CV_WRAP Model& setPreferableBackend(dnn::Backend backendId);
  1136. /// @sa Net::setPreferableTarget
  1137. CV_WRAP Model& setPreferableTarget(dnn::Target targetId);
  1138. CV_DEPRECATED_EXTERNAL
  1139. operator Net&() const { return getNetwork_(); }
  1140. //protected: - internal/tests usage only
  1141. Net& getNetwork_() const;
  1142. inline Net& getNetwork_() { return const_cast<const Model*>(this)->getNetwork_(); }
  1143. struct Impl;
  1144. inline Impl* getImpl() const { return impl.get(); }
  1145. inline Impl& getImplRef() const { CV_DbgAssert(impl); return *impl.get(); }
  1146. protected:
  1147. Ptr<Impl> impl;
  1148. };
  1149. /** @brief This class represents high-level API for classification models.
  1150. *
  1151. * ClassificationModel allows to set params for preprocessing input image.
  1152. * ClassificationModel creates net from file with trained weights and config,
  1153. * sets preprocessing input, runs forward pass and return top-1 prediction.
  1154. */
  1155. class CV_EXPORTS_W_SIMPLE ClassificationModel : public Model
  1156. {
  1157. public:
  1158. /**
  1159. * @brief Create classification model from network represented in one of the supported formats.
  1160. * An order of @p model and @p config arguments does not matter.
  1161. * @param[in] model Binary file contains trained weights.
  1162. * @param[in] config Text file contains network configuration.
  1163. */
  1164. CV_WRAP ClassificationModel(const String& model, const String& config = "");
  1165. /**
  1166. * @brief Create model from deep learning network.
  1167. * @param[in] network Net object.
  1168. */
  1169. CV_WRAP ClassificationModel(const Net& network);
  1170. /** @brief Given the @p input frame, create input blob, run net and return top-1 prediction.
  1171. * @param[in] frame The input image.
  1172. */
  1173. std::pair<int, float> classify(InputArray frame);
  1174. /** @overload */
  1175. CV_WRAP void classify(InputArray frame, CV_OUT int& classId, CV_OUT float& conf);
  1176. };
  1177. /** @brief This class represents high-level API for keypoints models
  1178. *
  1179. * KeypointsModel allows to set params for preprocessing input image.
  1180. * KeypointsModel creates net from file with trained weights and config,
  1181. * sets preprocessing input, runs forward pass and returns the x and y coordinates of each detected keypoint
  1182. */
  1183. class CV_EXPORTS_W_SIMPLE KeypointsModel: public Model
  1184. {
  1185. public:
  1186. /**
  1187. * @brief Create keypoints model from network represented in one of the supported formats.
  1188. * An order of @p model and @p config arguments does not matter.
  1189. * @param[in] model Binary file contains trained weights.
  1190. * @param[in] config Text file contains network configuration.
  1191. */
  1192. CV_WRAP KeypointsModel(const String& model, const String& config = "");
  1193. /**
  1194. * @brief Create model from deep learning network.
  1195. * @param[in] network Net object.
  1196. */
  1197. CV_WRAP KeypointsModel(const Net& network);
  1198. /** @brief Given the @p input frame, create input blob, run net
  1199. * @param[in] frame The input image.
  1200. * @param thresh minimum confidence threshold to select a keypoint
  1201. * @returns a vector holding the x and y coordinates of each detected keypoint
  1202. *
  1203. */
  1204. CV_WRAP std::vector<Point2f> estimate(InputArray frame, float thresh=0.5);
  1205. };
  1206. /** @brief This class represents high-level API for segmentation models
  1207. *
  1208. * SegmentationModel allows to set params for preprocessing input image.
  1209. * SegmentationModel creates net from file with trained weights and config,
  1210. * sets preprocessing input, runs forward pass and returns the class prediction for each pixel.
  1211. */
  1212. class CV_EXPORTS_W_SIMPLE SegmentationModel: public Model
  1213. {
  1214. public:
  1215. /**
  1216. * @brief Create segmentation model from network represented in one of the supported formats.
  1217. * An order of @p model and @p config arguments does not matter.
  1218. * @param[in] model Binary file contains trained weights.
  1219. * @param[in] config Text file contains network configuration.
  1220. */
  1221. CV_WRAP SegmentationModel(const String& model, const String& config = "");
  1222. /**
  1223. * @brief Create model from deep learning network.
  1224. * @param[in] network Net object.
  1225. */
  1226. CV_WRAP SegmentationModel(const Net& network);
  1227. /** @brief Given the @p input frame, create input blob, run net
  1228. * @param[in] frame The input image.
  1229. * @param[out] mask Allocated class prediction for each pixel
  1230. */
  1231. CV_WRAP void segment(InputArray frame, OutputArray mask);
  1232. };
  1233. /** @brief This class represents high-level API for object detection networks.
  1234. *
  1235. * DetectionModel allows to set params for preprocessing input image.
  1236. * DetectionModel creates net from file with trained weights and config,
  1237. * sets preprocessing input, runs forward pass and return result detections.
  1238. * For DetectionModel SSD, Faster R-CNN, YOLO topologies are supported.
  1239. */
  1240. class CV_EXPORTS_W_SIMPLE DetectionModel : public Model
  1241. {
  1242. public:
  1243. /**
  1244. * @brief Create detection model from network represented in one of the supported formats.
  1245. * An order of @p model and @p config arguments does not matter.
  1246. * @param[in] model Binary file contains trained weights.
  1247. * @param[in] config Text file contains network configuration.
  1248. */
  1249. CV_WRAP DetectionModel(const String& model, const String& config = "");
  1250. /**
  1251. * @brief Create model from deep learning network.
  1252. * @param[in] network Net object.
  1253. */
  1254. CV_WRAP DetectionModel(const Net& network);
  1255. CV_DEPRECATED_EXTERNAL // avoid using in C++ code (need to fix bindings first)
  1256. DetectionModel();
  1257. /**
  1258. * @brief nmsAcrossClasses defaults to false,
  1259. * such that when non max suppression is used during the detect() function, it will do so per-class.
  1260. * This function allows you to toggle this behaviour.
  1261. * @param[in] value The new value for nmsAcrossClasses
  1262. */
  1263. CV_WRAP DetectionModel& setNmsAcrossClasses(bool value);
  1264. /**
  1265. * @brief Getter for nmsAcrossClasses. This variable defaults to false,
  1266. * such that when non max suppression is used during the detect() function, it will do so only per-class
  1267. */
  1268. CV_WRAP bool getNmsAcrossClasses();
  1269. /** @brief Given the @p input frame, create input blob, run net and return result detections.
  1270. * @param[in] frame The input image.
  1271. * @param[out] classIds Class indexes in result detection.
  1272. * @param[out] confidences A set of corresponding confidences.
  1273. * @param[out] boxes A set of bounding boxes.
  1274. * @param[in] confThreshold A threshold used to filter boxes by confidences.
  1275. * @param[in] nmsThreshold A threshold used in non maximum suppression.
  1276. */
  1277. CV_WRAP void detect(InputArray frame, CV_OUT std::vector<int>& classIds,
  1278. CV_OUT std::vector<float>& confidences, CV_OUT std::vector<Rect>& boxes,
  1279. float confThreshold = 0.5f, float nmsThreshold = 0.0f);
  1280. };
  1281. /** @brief This class represents high-level API for text recognition networks.
  1282. *
  1283. * TextRecognitionModel allows to set params for preprocessing input image.
  1284. * TextRecognitionModel creates net from file with trained weights and config,
  1285. * sets preprocessing input, runs forward pass and return recognition result.
  1286. * For TextRecognitionModel, CRNN-CTC is supported.
  1287. */
  1288. class CV_EXPORTS_W_SIMPLE TextRecognitionModel : public Model
  1289. {
  1290. public:
  1291. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1292. TextRecognitionModel();
  1293. /**
  1294. * @brief Create Text Recognition model from deep learning network
  1295. * Call setDecodeType() and setVocabulary() after constructor to initialize the decoding method
  1296. * @param[in] network Net object
  1297. */
  1298. CV_WRAP TextRecognitionModel(const Net& network);
  1299. /**
  1300. * @brief Create text recognition model from network represented in one of the supported formats
  1301. * Call setDecodeType() and setVocabulary() after constructor to initialize the decoding method
  1302. * @param[in] model Binary file contains trained weights
  1303. * @param[in] config Text file contains network configuration
  1304. */
  1305. CV_WRAP inline
  1306. TextRecognitionModel(const std::string& model, const std::string& config = "")
  1307. : TextRecognitionModel(readNet(model, config)) { /* nothing */ }
  1308. /**
  1309. * @brief Set the decoding method of translating the network output into string
  1310. * @param[in] decodeType The decoding method of translating the network output into string, currently supported type:
  1311. * - `"CTC-greedy"` greedy decoding for the output of CTC-based methods
  1312. * - `"CTC-prefix-beam-search"` Prefix beam search decoding for the output of CTC-based methods
  1313. */
  1314. CV_WRAP
  1315. TextRecognitionModel& setDecodeType(const std::string& decodeType);
  1316. /**
  1317. * @brief Get the decoding method
  1318. * @return the decoding method
  1319. */
  1320. CV_WRAP
  1321. const std::string& getDecodeType() const;
  1322. /**
  1323. * @brief Set the decoding method options for `"CTC-prefix-beam-search"` decode usage
  1324. * @param[in] beamSize Beam size for search
  1325. * @param[in] vocPruneSize Parameter to optimize big vocabulary search,
  1326. * only take top @p vocPruneSize tokens in each search step, @p vocPruneSize <= 0 stands for disable this prune.
  1327. */
  1328. CV_WRAP
  1329. TextRecognitionModel& setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize = 0);
  1330. /**
  1331. * @brief Set the vocabulary for recognition.
  1332. * @param[in] vocabulary the associated vocabulary of the network.
  1333. */
  1334. CV_WRAP
  1335. TextRecognitionModel& setVocabulary(const std::vector<std::string>& vocabulary);
  1336. /**
  1337. * @brief Get the vocabulary for recognition.
  1338. * @return vocabulary the associated vocabulary
  1339. */
  1340. CV_WRAP
  1341. const std::vector<std::string>& getVocabulary() const;
  1342. /**
  1343. * @brief Given the @p input frame, create input blob, run net and return recognition result
  1344. * @param[in] frame The input image
  1345. * @return The text recognition result
  1346. */
  1347. CV_WRAP
  1348. std::string recognize(InputArray frame) const;
  1349. /**
  1350. * @brief Given the @p input frame, create input blob, run net and return recognition result
  1351. * @param[in] frame The input image
  1352. * @param[in] roiRects List of text detection regions of interest (cv::Rect, CV_32SC4). ROIs is be cropped as the network inputs
  1353. * @param[out] results A set of text recognition results.
  1354. */
  1355. CV_WRAP
  1356. void recognize(InputArray frame, InputArrayOfArrays roiRects, CV_OUT std::vector<std::string>& results) const;
  1357. };
  1358. /** @brief Base class for text detection networks
  1359. */
  1360. class CV_EXPORTS_W_SIMPLE TextDetectionModel : public Model
  1361. {
  1362. protected:
  1363. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1364. TextDetectionModel();
  1365. public:
  1366. /** @brief Performs detection
  1367. *
  1368. * Given the input @p frame, prepare network input, run network inference, post-process network output and return result detections.
  1369. *
  1370. * Each result is quadrangle's 4 points in this order:
  1371. * - bottom-left
  1372. * - top-left
  1373. * - top-right
  1374. * - bottom-right
  1375. *
  1376. * Use cv::getPerspectiveTransform function to retrive image region without perspective transformations.
  1377. *
  1378. * @note If DL model doesn't support that kind of output then result may be derived from detectTextRectangles() output.
  1379. *
  1380. * @param[in] frame The input image
  1381. * @param[out] detections array with detections' quadrangles (4 points per result)
  1382. * @param[out] confidences array with detection confidences
  1383. */
  1384. CV_WRAP
  1385. void detect(
  1386. InputArray frame,
  1387. CV_OUT std::vector< std::vector<Point> >& detections,
  1388. CV_OUT std::vector<float>& confidences
  1389. ) const;
  1390. /** @overload */
  1391. CV_WRAP
  1392. void detect(
  1393. InputArray frame,
  1394. CV_OUT std::vector< std::vector<Point> >& detections
  1395. ) const;
  1396. /** @brief Performs detection
  1397. *
  1398. * Given the input @p frame, prepare network input, run network inference, post-process network output and return result detections.
  1399. *
  1400. * Each result is rotated rectangle.
  1401. *
  1402. * @note Result may be inaccurate in case of strong perspective transformations.
  1403. *
  1404. * @param[in] frame the input image
  1405. * @param[out] detections array with detections' RotationRect results
  1406. * @param[out] confidences array with detection confidences
  1407. */
  1408. CV_WRAP
  1409. void detectTextRectangles(
  1410. InputArray frame,
  1411. CV_OUT std::vector<cv::RotatedRect>& detections,
  1412. CV_OUT std::vector<float>& confidences
  1413. ) const;
  1414. /** @overload */
  1415. CV_WRAP
  1416. void detectTextRectangles(
  1417. InputArray frame,
  1418. CV_OUT std::vector<cv::RotatedRect>& detections
  1419. ) const;
  1420. };
  1421. /** @brief This class represents high-level API for text detection DL networks compatible with EAST model.
  1422. *
  1423. * Configurable parameters:
  1424. * - (float) confThreshold - used to filter boxes by confidences, default: 0.5f
  1425. * - (float) nmsThreshold - used in non maximum suppression, default: 0.0f
  1426. */
  1427. class CV_EXPORTS_W_SIMPLE TextDetectionModel_EAST : public TextDetectionModel
  1428. {
  1429. public:
  1430. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1431. TextDetectionModel_EAST();
  1432. /**
  1433. * @brief Create text detection algorithm from deep learning network
  1434. * @param[in] network Net object
  1435. */
  1436. CV_WRAP TextDetectionModel_EAST(const Net& network);
  1437. /**
  1438. * @brief Create text detection model from network represented in one of the supported formats.
  1439. * An order of @p model and @p config arguments does not matter.
  1440. * @param[in] model Binary file contains trained weights.
  1441. * @param[in] config Text file contains network configuration.
  1442. */
  1443. CV_WRAP inline
  1444. TextDetectionModel_EAST(const std::string& model, const std::string& config = "")
  1445. : TextDetectionModel_EAST(readNet(model, config)) { /* nothing */ }
  1446. /**
  1447. * @brief Set the detection confidence threshold
  1448. * @param[in] confThreshold A threshold used to filter boxes by confidences
  1449. */
  1450. CV_WRAP
  1451. TextDetectionModel_EAST& setConfidenceThreshold(float confThreshold);
  1452. /**
  1453. * @brief Get the detection confidence threshold
  1454. */
  1455. CV_WRAP
  1456. float getConfidenceThreshold() const;
  1457. /**
  1458. * @brief Set the detection NMS filter threshold
  1459. * @param[in] nmsThreshold A threshold used in non maximum suppression
  1460. */
  1461. CV_WRAP
  1462. TextDetectionModel_EAST& setNMSThreshold(float nmsThreshold);
  1463. /**
  1464. * @brief Get the detection confidence threshold
  1465. */
  1466. CV_WRAP
  1467. float getNMSThreshold() const;
  1468. };
  1469. /** @brief This class represents high-level API for text detection DL networks compatible with DB model.
  1470. *
  1471. * Related publications: @cite liao2020real
  1472. * Paper: https://arxiv.org/abs/1911.08947
  1473. * For more information about the hyper-parameters setting, please refer to https://github.com/MhLiao/DB
  1474. *
  1475. * Configurable parameters:
  1476. * - (float) binaryThreshold - The threshold of the binary map. It is usually set to 0.3.
  1477. * - (float) polygonThreshold - The threshold of text polygons. It is usually set to 0.5, 0.6, and 0.7. Default is 0.5f
  1478. * - (double) unclipRatio - The unclip ratio of the detected text region, which determines the output size. It is usually set to 2.0.
  1479. * - (int) maxCandidates - The max number of the output results.
  1480. */
  1481. class CV_EXPORTS_W_SIMPLE TextDetectionModel_DB : public TextDetectionModel
  1482. {
  1483. public:
  1484. CV_DEPRECATED_EXTERNAL // avoid using in C++ code, will be moved to "protected" (need to fix bindings first)
  1485. TextDetectionModel_DB();
  1486. /**
  1487. * @brief Create text detection algorithm from deep learning network.
  1488. * @param[in] network Net object.
  1489. */
  1490. CV_WRAP TextDetectionModel_DB(const Net& network);
  1491. /**
  1492. * @brief Create text detection model from network represented in one of the supported formats.
  1493. * An order of @p model and @p config arguments does not matter.
  1494. * @param[in] model Binary file contains trained weights.
  1495. * @param[in] config Text file contains network configuration.
  1496. */
  1497. CV_WRAP inline
  1498. TextDetectionModel_DB(const std::string& model, const std::string& config = "")
  1499. : TextDetectionModel_DB(readNet(model, config)) { /* nothing */ }
  1500. CV_WRAP TextDetectionModel_DB& setBinaryThreshold(float binaryThreshold);
  1501. CV_WRAP float getBinaryThreshold() const;
  1502. CV_WRAP TextDetectionModel_DB& setPolygonThreshold(float polygonThreshold);
  1503. CV_WRAP float getPolygonThreshold() const;
  1504. CV_WRAP TextDetectionModel_DB& setUnclipRatio(double unclipRatio);
  1505. CV_WRAP double getUnclipRatio() const;
  1506. CV_WRAP TextDetectionModel_DB& setMaxCandidates(int maxCandidates);
  1507. CV_WRAP int getMaxCandidates() const;
  1508. };
  1509. //! @}
  1510. CV__DNN_INLINE_NS_END
  1511. }
  1512. }
  1513. #include <opencv2/dnn/layer.hpp>
  1514. #include <opencv2/dnn/dnn.inl.hpp>
  1515. /// @deprecated Include this header directly from application. Automatic inclusion will be removed
  1516. #include <opencv2/dnn/utils/inference_engine.hpp>
  1517. #endif /* OPENCV_DNN_DNN_HPP */