JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Network.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <jevois/Types/Enum.H>
22 
23 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
24 
25 #include <opencv2/core/core.hpp>
26 #include <vector>
27 
28 namespace jevois
29 {
30  namespace dnn
31  {
32  namespace network
33  {
34  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
35  // will use different subsets of all available parameters:
36  static jevois::ParameterCategory const ParamCateg("DNN Network Options");
37 
38  //! Parameter \relates jevois::Network
39  JEVOIS_DECLARE_PARAMETER(dataroot, std::string, "Root directory to use when config or model parameters "
40  "are relative paths.",
41  JEVOIS_SHARE_PATH, ParamCateg);
42 
43  //! Parameter \relates jevois::Network
44  JEVOIS_DECLARE_PARAMETER(config, std::string, "Path to a text file that contains network configuration. "
45  "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet). "
46  "If path is relative, it will be prefixed by dataroot.",
47  "", ParamCateg);
48 
49  //! Parameter \relates jevois::Network
50  JEVOIS_DECLARE_PARAMETER(model, std::string, "Path to a binary file of model contains trained weights. "
51  "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), "
52  ".tflite (TensorFlow Lite), or .weights (Darknet). If path is relative, it will be "
53  "prefixed by dataroot.",
54  "", ParamCateg);
55 
56 #ifdef JEVOIS_PRO
57  //! Enum \relates jevois::Network
58  JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) (OpenCL) (OpenCL_FP16) (Myriad) );
59 #else
60  //! Enum \relates jevois::Network
61  JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) );
62 #endif
63 
64  //! Parameter \relates jevois::Network
65  JEVOIS_DECLARE_PARAMETER(target, Target, "OpenCV compute target to use. Changes will take effect "
66  "next time you load a different model.",
67  Target::CPU, Target_Values, ParamCateg);
68 #ifdef JEVOIS_PRO
69  //! Enum \relates jevois::Network
70  JEVOIS_DEFINE_ENUM_CLASS(Backend, (OpenCV) (InferenceEngine) );
71 #define JEVOIS_BACKEND_DEFAULT Backend::OpenCV
72 #else
73  //! Enum \relates jevois::Network
74  JEVOIS_DEFINE_ENUM_CLASS(Backend, (Default) );
75 #define JEVOIS_BACKEND_DEFAULT Backend::Default
76 #endif
77 
78  //! Parameter \relates jevois::Network
79  JEVOIS_DECLARE_PARAMETER(backend, Backend, "OpenCV compute backend to use. Default will use the inference "
80  "engine if available, otherwise OpenCV (note that inference engine only works on Intel "
81  "processors or MyriadX hardware, thus you should normally select OpenCV when running "
82  "on JeVois-Pro Platform, unless you want to use an optional MyriadX accelerator). "
83  "Changes will take effect next time you load a model.",
84  JEVOIS_BACKEND_DEFAULT, Backend_Values, ParamCateg);
85 
86 #ifdef JEVOIS_PRO
87  //! Parameter \relates jevois::Network
88  JEVOIS_DECLARE_PARAMETER(tpunum, size_t, "Coral EdgeTPU number to use to run this model, typically 0, or can be "
89  "1 when using a dual-TPU add-on board, or more when using additional TPUs connected "
90  "to USB ports",
91  0, ParamCateg);
92 #endif
93 
94  //! Parameter \relates jevois::Network
95  JEVOIS_DECLARE_PARAMETER(intensors, std::string, "Specification of input tensors",
96  "", ParamCateg);
97 
98  //! Parameter \relates jevois::Network
99  JEVOIS_DECLARE_PARAMETER(outtensors, std::string, "Specification of output tensors",
100  "", ParamCateg);
101 
102  //! Parameter \relates jevois::Network
103  JEVOIS_DECLARE_PARAMETER(dequant, bool, "Dequantize output tensors",
104  true, ParamCateg);
105 
106  //! Parameter \relates jevois::Network
107  JEVOIS_DECLARE_PARAMETER(flattenoutputs, bool, "Dequant, flatten and concatenate all NPU outputs into a "
108  "single 1D vector",
109  false, ParamCateg);
110  }
111 
112  //! Abstract class to represent a neural network
113  /*! Derived classes provide implementation via OpenCV (on CPU, OpenCL, or OpenVino), Amlogic/Vivante NPU, or Google
114  Coral TPU. \ingroup dnn */
115  class Network : public jevois::Component
116  {
117  public:
118  //! Inherited constructor ok
120 
121  //! Destructor
122  /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
123  virtual ~Network();
124 
125  //! If network is currently loading, wait until that is done before destroying
126  /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
127  void waitBeforeDestroy();
128 
129  //! Returns true when network is ready to run (loaded and initialized)
130  bool ready();
131 
132  //! Get shapes of all input tensors
133  virtual std::vector<vsi_nn_tensor_attr_t> inputShapes() = 0;
134 
135  //! Get shapes of all output tensors
136  virtual std::vector<vsi_nn_tensor_attr_t> outputShapes() = 0;
137 
138  //! Process input blobs and obtain output blobs
139  /*! Network implementations may push information data into the info string, which will be displayed to
140  user. Convention is: if an info line starts with '* ', it is a header, and if it starts with '- ' it is a
141  bullet. Info should always be organized into headers at the top level. */
142  std::vector<cv::Mat> process(std::vector<cv::Mat> const & blobs, std::vector<std::string> & info);
143 
144  //! Freeze/unfreeze parameters that users should not change while running
145  virtual void freeze(bool doit) = 0;
146 
147  protected:
148  //! Load from disk
149  virtual void load() = 0;
150 
151  //! Process input blobs and obtain output blobs
152  virtual std::vector<cv::Mat> doprocess(std::vector<cv::Mat> const & blobs,
153  std::vector<std::string> & info) = 0;
154 
155  private:
156  std::atomic<bool> itsLoading = false;
157  std::atomic<bool> itsLoaded = false;
158  std::future<void> itsLoadFut;
159  };
160 
161  } // namespace dnn
162 } // namespace jevois
jevois::ParameterRegistry::Component
friend class Component
Allow Component to access our registry data, everyone else is locked out.
Definition: ParameterRegistry.H:51
jevois::dnn::Network::ready
bool ready()
Returns true when network is ready to run (loaded and initialized)
Definition: Network.C:40
jevois::dnn::Network::~Network
virtual ~Network()
Destructor.
Definition: Network.C:23
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290))
Enum for different sensor models.
jevois::dnn::Network::process
std::vector< cv::Mat > process(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)
Process input blobs and obtain output blobs.
Definition: Network.C:65
jevois::Component
A component of a model hierarchy.
Definition: Component.H:180
jevois::dnn::Network::freeze
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
JEVOIS_BACKEND_DEFAULT
#define JEVOIS_BACKEND_DEFAULT
Definition: Network.H:71
jevois::dnn::Network::inputShapes
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes()=0
Get shapes of all input tensors.
jevois::dnn::Network::waitBeforeDestroy
void waitBeforeDestroy()
If network is currently loading, wait until that is done before destroying.
Definition: Network.C:27
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::dnn::Network::load
virtual void load()=0
Load from disk.
jevois
Definition: Concepts.dox:1
Component.H
jevois::dnn::Network::outputShapes
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes()=0
Get shapes of all output tensors.
jevois::dnn::Network::doprocess
virtual std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)=0
Process input blobs and obtain output blobs.
jevois::dnn::Network
Abstract class to represent a neural network.
Definition: Network.H:115
Enum.H