JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Network.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <jevois/Types/Enum.H>
22 
23 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
24 
25 #include <opencv2/core/core.hpp>
26 #include <vector>
27 
28 namespace jevois
29 {
30  namespace dnn
31  {
32  namespace network
33  {
34  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
35  // will use different subsets of all available parameters:
36  static jevois::ParameterCategory const ParamCateg("DNN Network Options");
37 
38  //! Parameter \relates jevois::Network
39  JEVOIS_DECLARE_PARAMETER(dataroot, std::string, "Root directory to use when config or model parameters "
40  "are relative paths.",
41  JEVOIS_SHARE_PATH, ParamCateg);
42 
43  //! Parameter \relates jevois::Network
44  JEVOIS_DECLARE_PARAMETER(config, std::string, "Path to a text file that contains network configuration. "
45  "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet). "
46  "If path is relative, it will be prefixed by dataroot.",
47  "", ParamCateg);
48 
49  //! Parameter \relates jevois::Network
50  JEVOIS_DECLARE_PARAMETER(model, std::string, "Path to a binary file of model contains trained weights. "
51  "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), "
52  ".tflite (TensorFlow Lite), or .weights (Darknet). If path is relative, it will be "
53  "prefixed by dataroot.",
54  "", ParamCateg);
55 
56 #ifdef JEVOIS_PRO
57  //! Enum \relates jevois::Network
58  JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) (OpenCL) (OpenCL_FP16) (Myriad) );
59 #else
60  //! Enum \relates jevois::Network
61  JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) );
62 #endif
63 
64  //! Parameter \relates jevois::Network
65  JEVOIS_DECLARE_PARAMETER(target, Target, "OpenCV compute target to use. Changes will take effect "
66  "next time you load a different model.",
67  Target::CPU, Target_Values, ParamCateg);
68 #ifdef JEVOIS_PRO
69  //! Enum \relates jevois::Network
70  JEVOIS_DEFINE_ENUM_CLASS(Backend, (Default) (OpenCV) (InferenceEngine) );
71 
72 #ifdef JEVOIS_PLATFORM
73 #define JEVOIS_BACKEND_DEFAULT Backend::OpenCV
74 #else
75 #define JEVOIS_BACKEND_DEFAULT Backend::Default
76 #endif
77 #else
78  //! Enum \relates jevois::Network
79  JEVOIS_DEFINE_ENUM_CLASS(Backend, (Default) );
80 #define JEVOIS_BACKEND_DEFAULT Backend::Default
81 #endif
82 
83  //! Parameter \relates jevois::Network
84  JEVOIS_DECLARE_PARAMETER(backend, Backend, "OpenCV compute backend to use. Default will use the inference "
85  "engine if available, otherwise OpenCV (note that inference engine only works on Intel "
86  "processors or MyriadX hardware, thus you should normally select OpenCV when running "
87  "on JeVois-Pro Platform, unless you want to use an optional MyriadX accelerator). "
88  "Changes will take effect next time you load a model.",
89  JEVOIS_BACKEND_DEFAULT, Backend_Values, ParamCateg);
90 
91 #ifdef JEVOIS_PRO
92  //! Parameter \relates jevois::Network
93  JEVOIS_DECLARE_PARAMETER(tpunum, size_t, "Coral EdgeTPU number to use to run this model, typically 0, or can be "
94  "1 when using a dual-TPU add-on board, or more when using additional TPUs connected "
95  "to USB ports",
96  0, ParamCateg);
97 #endif
98 
99  //! Parameter \relates jevois::Network
100  JEVOIS_DECLARE_PARAMETER(intensors, std::string, "Specification of input tensors",
101  "", ParamCateg);
102 
103  //! Parameter \relates jevois::Network
104  JEVOIS_DECLARE_PARAMETER(outtensors, std::string, "Specification of output tensors",
105  "", ParamCateg);
106 
107  //! Parameter \relates jevois::Network
108  JEVOIS_DECLARE_PARAMETER(dequant, bool, "Dequantize output tensors",
109  true, ParamCateg);
110 
111  //! Parameter \relates jevois::Network
112  JEVOIS_DECLARE_PARAMETER(flattenoutputs, bool, "Dequant, flatten and concatenate all NPU outputs into a "
113  "single 1D vector",
114  false, ParamCateg);
115  }
116 
117  //! Abstract class to represent a neural network
118  /*! Derived classes provide implementation via OpenCV (on CPU, OpenCL, or OpenVino), Amlogic/Vivante NPU, or Google
119  Coral TPU. \ingroup dnn */
120  class Network : public jevois::Component
121  {
122  public:
123  //! Inherited constructor ok
125 
126  //! Destructor
127  /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
128  virtual ~Network();
129 
130  //! If network is currently loading, wait until that is done before destroying
131  /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
132  void waitBeforeDestroy();
133 
134  //! Returns true when network is ready to run (loaded and initialized)
135  bool ready();
136 
137  //! Get shapes of all input tensors
138  virtual std::vector<vsi_nn_tensor_attr_t> inputShapes() = 0;
139 
140  //! Get shapes of all output tensors
141  virtual std::vector<vsi_nn_tensor_attr_t> outputShapes() = 0;
142 
143  //! Process input blobs and obtain output blobs
144  /*! Network implementations may push information data into the info string, which will be displayed to
145  user. Convention is: if an info line starts with '* ', it is a header, and if it starts with '- ' it is a
146  bullet. Info should always be organized into headers at the top level. */
147  std::vector<cv::Mat> process(std::vector<cv::Mat> const & blobs, std::vector<std::string> & info);
148 
149  //! Freeze/unfreeze parameters that users should not change while running
150  virtual void freeze(bool doit) = 0;
151 
152  protected:
153  //! Load from disk
154  virtual void load() = 0;
155 
156  //! Process input blobs and obtain output blobs
157  virtual std::vector<cv::Mat> doprocess(std::vector<cv::Mat> const & blobs,
158  std::vector<std::string> & info) = 0;
159 
160  private:
161  std::atomic<bool> itsLoading = false;
162  std::atomic<bool> itsLoaded = false;
163  std::future<void> itsLoadFut;
164  };
165 
166  } // namespace dnn
167 } // namespace jevois
jevois::ParameterRegistry::Component
friend class Component
Allow Component to access our registry data, everyone else is locked out.
Definition: ParameterRegistry.H:51
jevois::dnn::Network::ready
bool ready()
Returns true when network is ready to run (loaded and initialized)
Definition: Network.C:40
jevois::dnn::Network::~Network
virtual ~Network()
Destructor.
Definition: Network.C:23
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290))
Enum for different sensor models.
jevois::dnn::Network::process
std::vector< cv::Mat > process(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)
Process input blobs and obtain output blobs.
Definition: Network.C:65
jevois::Component
A component of a model hierarchy.
Definition: Component.H:180
jevois::dnn::Network::freeze
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
JEVOIS_BACKEND_DEFAULT
#define JEVOIS_BACKEND_DEFAULT
Definition: Network.H:75
jevois::dnn::Network::inputShapes
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes()=0
Get shapes of all input tensors.
jevois::dnn::Network::waitBeforeDestroy
void waitBeforeDestroy()
If network is currently loading, wait until that is done before destroying.
Definition: Network.C:27
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::dnn::Network::load
virtual void load()=0
Load from disk.
jevois
Definition: Concepts.dox:1
Component.H
jevois::dnn::Network::outputShapes
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes()=0
Get shapes of all output tensors.
jevois::dnn::Network::doprocess
virtual std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)=0
Process input blobs and obtain output blobs.
jevois::dnn::Network
Abstract class to represent a neural network.
Definition: Network.H:120
Enum.H