JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Network.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <jevois/Types/Enum.H>
22 
23 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
24 
25 #include <opencv2/core/core.hpp>
26 #include <vector>
27 
28 namespace jevois
29 {
30  namespace dnn
31  {
32  namespace network
33  {
34  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
35  // will use different subsets of all available parameters:
36  static jevois::ParameterCategory const ParamCateg("DNN Network Options");
37 
38  //! Parameter \relates jevois::Network
39  JEVOIS_DECLARE_PARAMETER(comment, std::string, "Optional comment about the network",
40  "", ParamCateg);
41 
42  //! Parameter \relates jevois::Network
43  JEVOIS_DECLARE_PARAMETER(url, std::string, "Optional URL for the network",
44  "", ParamCateg);
45 
46  //! Parameter \relates jevois::Network
47  JEVOIS_DECLARE_PARAMETER(dataroot, std::string, "Root directory to use when config or model parameters "
48  "are relative paths.",
49  JEVOIS_SHARE_PATH, ParamCateg);
50 
51  //! Parameter \relates jevois::Network
52  JEVOIS_DECLARE_PARAMETER(config, std::string, "Path to a text file that contains network configuration. "
53  "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet). "
54  "If path is relative, it will be prefixed by dataroot.",
55  "", ParamCateg);
56 
57  //! Parameter \relates jevois::Network
58  JEVOIS_DECLARE_PARAMETER(model, std::string, "Path to a binary file of model contains trained weights. "
59  "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), "
60  ".tflite (TensorFlow Lite), or .weights (Darknet). If path is relative, it will be "
61  "prefixed by dataroot.",
62  "", ParamCateg);
63 
64 #ifdef JEVOIS_PRO
65  //! Enum \relates jevois::Network
66  JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) (OpenCL) (OpenCL_FP16) (Myriad) (NPU) );
67 #else
68  //! Enum \relates jevois::Network
69  JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) );
70 #endif
71 
72  //! Parameter \relates jevois::Network
73  JEVOIS_DECLARE_PARAMETER(target, Target, "OpenCV compute target to use. Changes will take effect "
74  "next time you load a different model.",
75  Target::CPU, Target_Values, ParamCateg);
76 #ifdef JEVOIS_PRO
77  //! Enum \relates jevois::Network
78  JEVOIS_DEFINE_ENUM_CLASS(Backend, (OpenCV) (InferenceEngine) (TimVX) );
79 #define JEVOIS_BACKEND_DEFAULT Backend::OpenCV
80 #else
81  //! Enum \relates jevois::Network
82  JEVOIS_DEFINE_ENUM_CLASS(Backend, (Default) );
83 #define JEVOIS_BACKEND_DEFAULT Backend::Default
84 #endif
85 
86  //! Parameter \relates jevois::Network
87  JEVOIS_DECLARE_PARAMETER(backend, Backend, "OpenCV compute backend to use. Default will use the inference "
88  "engine if available, otherwise OpenCV (note that inference engine only works on Intel "
89  "processors or MyriadX hardware, thus you should normally select OpenCV when running "
90  "on JeVois-Pro Platform, unless you want to use an optional MyriadX accelerator). "
91  "Changes will take effect next time you load a model.",
92  JEVOIS_BACKEND_DEFAULT, Backend_Values, ParamCateg);
93 
94  //! Parameter \relates jevois::Network
95  JEVOIS_DECLARE_PARAMETER(intensors, std::string, "Specification of input tensors",
96  "", ParamCateg);
97 
98  //! Parameter \relates jevois::Network
99  JEVOIS_DECLARE_PARAMETER(outtensors, std::string, "Specification of output tensors",
100  "", ParamCateg);
101 
102  //! Parameter \relates jevois::Network
103  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(outreshape, std::string, "Specification of reshaped output tensors; "
104  "sometimes useful to re-interpret tensors to what a post-processor expects; for "
105  "example, TPU YoloV4-Int-VOC outputs 5D tensors 32F:1x52x52x3x85, 32F:1x26x26x3x85, "
106  "32F:1x13x13x3x85 but the YOLO post-processor expects 4D, which would be specified here "
107  "as 32F:1x52x52x255, 32F:1x26x26x255, 32F:1x13x13x255. Note that this only changes "
108  "the description of dimensions, but does not move any pixel data around (e.g., cannot "
109  "convert from NCHW to NHWC, convert data types, etc). Use sparingly and with caution.",
110  "", ParamCateg);
111 
112  //! Parameter \relates jevois::Network
113  JEVOIS_DECLARE_PARAMETER(dequant, bool, "Dequantize output tensors to float32 from their native quantized type",
114  true, ParamCateg);
115 
116  //! Parameter \relates jevois::NetworkPython
117  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pynet, std::string, "Full path of the python network processor file. "
118  "Name of class defined in the file must match the file name without "
119  "the trailing '.py'",
120  "", ParamCateg);
121 #ifdef JEVOIS_PRO
122  //! Parameter \relates jevois::Network
123  JEVOIS_DECLARE_PARAMETER(tpunum, size_t, "Coral EdgeTPU number to use to run this model, typically 0, or can be "
124  "1 when using a dual-TPU add-on board, or more when using additional TPUs connected "
125  "to USB ports",
126  0, ParamCateg);
127 
128  //! Parameter \relates jevois::Network
129  JEVOIS_DECLARE_PARAMETER(spunum, size_t, "Hailo8 device number to use to run this model, typically 0 unless "
130  "several Hailo8 accelerators are connected to the system",
131  0, ParamCateg);
132 
133  //! Parameter \relates jevois::NetworkNPU
134  JEVOIS_DECLARE_PARAMETER(verifygraph, bool, "Verify NPU graph after loading it",
135  true, ParamCateg);
136 
137  //! Parameter \relates jevois::NetworkNPU
138  JEVOIS_DECLARE_PARAMETER(ovxver, std::string, "ovxlib version to use with NPU network, or leave blank "
139  "to use latest version",
140  "", boost::regex("^$|^[0-9]+\\.[0-9]+\\.[0-9]+$"), ParamCateg);
141 
142  //! Parameter \relates jevois::NetworkHailo
143  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(turbo, bool, "Turbo mode. Has no significant effect on small or "
144  "fast networks. Use with caution as it may lead to overheating, not "
145  "recommended for production",
146  false, ParamCateg);
147 #endif
148  }
149 
150  //! Abstract class to represent a neural network
151  /*! Derived classes provide implementation via OpenCV (on CPU, OpenCL, or OpenVino/Myriad-X), Amlogic/Vivante NPU,
152  Hailo-8, Python, or Google Coral TPU. \ingroup dnn */
153  class Network : public Component,
154  public Parameter<network::comment, network::url, network::outreshape>
155  {
156  public:
157  //! Inherited constructor ok
159 
160  //! Destructor
161  /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
162  virtual ~Network();
163 
164  //! If network is currently loading, wait until that is done before destroying
165  /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
166  void waitBeforeDestroy();
167 
168  //! Returns true when network is ready to run (loaded and initialized)
169  bool ready();
170 
171  //! Get shapes of all input tensors
172  virtual std::vector<vsi_nn_tensor_attr_t> inputShapes() = 0;
173 
174  //! Get shapes of all output tensors
175  virtual std::vector<vsi_nn_tensor_attr_t> outputShapes() = 0;
176 
177  //! Process input blobs and obtain output blobs
178  /*! Network implementations may push information data into the info string, which will be displayed to
179  user. Convention is: if an info line starts with '* ', it is a header, and if it starts with '- ' it is a
180  bullet. Info should always be organized into headers at the top level. */
181  std::vector<cv::Mat> process(std::vector<cv::Mat> const & blobs, std::vector<std::string> & info);
182 
183  //! Freeze/unfreeze parameters that users should not change while running
184  virtual void freeze(bool doit) = 0;
185 
186  protected:
187  //! Load from disk
188  virtual void load() = 0;
189 
190  //! Process input blobs and obtain output blobs
191  virtual std::vector<cv::Mat> doprocess(std::vector<cv::Mat> const & blobs,
192  std::vector<std::string> & info) = 0;
193 
194  void onParamChange(network::outreshape const & param, std::string const & val) override;
195 
196  private:
197  std::atomic<bool> itsLoading = false;
198  std::atomic<bool> itsLoaded = false;
199  std::future<void> itsLoadFut;
200  std::vector<vsi_nn_tensor_attr_t> itsReshape;
201  };
202 
203  } // namespace dnn
204 } // namespace jevois
jevois::ParameterRegistry::Component
friend class Component
Allow Component and DynamicParameter to access our registry data, everyone else is locked out.
Definition: ParameterRegistry.H:51
jevois::dnn::Network::ready
bool ready()
Returns true when network is ready to run (loaded and initialized)
Definition: Network.C:50
jevois::dnn::Network::~Network
virtual ~Network()
Destructor.
Definition: Network.C:23
jevois::dnn::Network::onParamChange
void onParamChange(network::outreshape const &param, std::string const &val) override
Definition: Network.C:27
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
jevois::dnn::Network::process
std::vector< cv::Mat > process(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)
Process input blobs and obtain output blobs.
Definition: Network.C:75
jevois::Component
A component of a model hierarchy.
Definition: Component.H:181
jevois::dnn::Network::freeze
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
JEVOIS_BACKEND_DEFAULT
#define JEVOIS_BACKEND_DEFAULT
Definition: Network.H:79
jevois::dnn::Network::inputShapes
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes()=0
Get shapes of all input tensors.
jevois::dnn::Network::waitBeforeDestroy
void waitBeforeDestroy()
If network is currently loading, wait until that is done before destroying.
Definition: Network.C:37
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::dnn::Network::load
virtual void load()=0
Load from disk.
jevois
Definition: Concepts.dox:1
Component.H
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290)(os08a10))
Enum for different sensor models.
jevois::dnn::Network::outputShapes
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes()=0
Get shapes of all output tensors.
jevois::dnn::Network::doprocess
virtual std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)=0
Process input blobs and obtain output blobs.
jevois::dnn::Network
Abstract class to represent a neural network.
Definition: Network.H:153
Enum.H