JeVois  1.21
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Loading...
Searching...
No Matches
Network.H
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
18#pragma once
19
21#include <jevois/Types/Enum.H>
22
23#include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
24
25#include <opencv2/core/core.hpp>
26#include <vector>
27
28namespace jevois
29{
30 namespace dnn
31 {
32 namespace network
33 {
34 // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
35 // will use different subsets of all available parameters:
36 static jevois::ParameterCategory const ParamCateg("DNN Network Options");
37
38 //! Parameter \relates jevois::dnn::Network
39 JEVOIS_DECLARE_PARAMETER(comment, std::string, "Optional comment about the network",
40 "", ParamCateg);
41
42 //! Parameter \relates jevois::dnn::Network
43 JEVOIS_DECLARE_PARAMETER(url, std::string, "Optional URL for the network",
44 "", ParamCateg);
45
46 //! Parameter \relates jevois::dnn::Network
47 JEVOIS_DECLARE_PARAMETER(dataroot, std::string, "Root directory to use when config or model parameters "
48 "are relative paths.",
49 JEVOIS_SHARE_PATH, ParamCateg);
50
51 //! Parameter \relates jevois::dnn::Network
52 JEVOIS_DECLARE_PARAMETER(config, std::string, "Path to a text file that contains network configuration. "
53 "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet). "
54 "If path is relative, it will be prefixed by dataroot.",
55 "", ParamCateg);
56
57 //! Parameter \relates jevois::dnn::Network
58 JEVOIS_DECLARE_PARAMETER(model, std::string, "Path to a binary file of model contains trained weights. "
59 "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), "
60 ".tflite (TensorFlow Lite), or .weights (Darknet). If path is relative, it will be "
61 "prefixed by dataroot.",
62 "", ParamCateg);
63
64#ifdef JEVOIS_PRO
65 //! Enum \relates jevois::dnn::Network
66 JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) (OpenCL) (OpenCL_FP16) (Myriad) (NPU) );
67#else
68 //! Enum \relates jevois::dnn::Network
69 JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) );
70#endif
71
72 //! Parameter \relates jevois::dnn::Network
73 JEVOIS_DECLARE_PARAMETER(target, Target, "OpenCV compute target to use. Changes will take effect "
74 "next time you load a different model.",
75 Target::CPU, Target_Values, ParamCateg);
76#ifdef JEVOIS_PRO
77 //! Enum \relates jevois::dnn::Network
78 JEVOIS_DEFINE_ENUM_CLASS(Backend, (OpenCV) (InferenceEngine) (TimVX) );
79#define JEVOIS_BACKEND_DEFAULT Backend::OpenCV
80#else
81 //! Enum \relates jevois::dnn::Network
82 JEVOIS_DEFINE_ENUM_CLASS(Backend, (Default) );
83#define JEVOIS_BACKEND_DEFAULT Backend::Default
84#endif
85
86 //! Parameter \relates jevois::dnn::Network
87 JEVOIS_DECLARE_PARAMETER(backend, Backend, "OpenCV compute backend to use. Default will use the inference "
88 "engine if available, otherwise OpenCV (note that inference engine only works on Intel "
89 "processors or MyriadX hardware, thus you should normally select OpenCV when running "
90 "on JeVois-Pro Platform, unless you want to use an optional MyriadX accelerator). "
91 "Changes will take effect next time you load a model.",
92 JEVOIS_BACKEND_DEFAULT, Backend_Values, ParamCateg);
93
94 //! Parameter \relates jevois::dnn::Network
95 JEVOIS_DECLARE_PARAMETER(intensors, std::string, "Specification of input tensors",
96 "", ParamCateg);
97
98 //! Parameter \relates jevois::dnn::Network
99 JEVOIS_DECLARE_PARAMETER(extraintensors, std::string, "Specification of extra fixed input tensors that will be "
100 "added after the regular intensors. Format is: "
101 "<type>:<shape>:val1 val2 ... valN, <type>:<shape>:val1 ... valN. For example, for "
102 "URetinex-Net: 32F:1x1x1:3.0",
103 "", ParamCateg);
104
105 //! Parameter \relates jevois::dnn::Network
106 JEVOIS_DECLARE_PARAMETER(outtensors, std::string, "Specification of output tensors",
107 "", ParamCateg);
108
109 //! Parameter \relates jevois::dnn::Network
110 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(outreshape, std::string, "Specification of reshaped output tensors; "
111 "sometimes useful to re-interpret tensors to what a post-processor expects; for "
112 "example, TPU YoloV4-Int-VOC outputs 5D tensors 32F:1x52x52x3x85, 32F:1x26x26x3x85, "
113 "32F:1x13x13x3x85 but the YOLO post-processor expects 4D, which would be specified here "
114 "as 32F:1x52x52x255, 32F:1x26x26x255, 32F:1x13x13x255. Note that this only changes "
115 "the description of dimensions, but does not move any pixel data around (e.g., cannot "
116 "convert from NCHW to NHWC, convert data types, etc). Use sparingly and with caution.",
117 "", ParamCateg);
118
119 //! Parameter \relates jevois::dnn::Network
120 JEVOIS_DECLARE_PARAMETER(dequant, bool, "Dequantize output tensors to float32 from their native quantized type",
121 true, ParamCateg);
122
123 //! Parameter \relates jevois::dnn::NetworkPython
124 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pynet, std::string, "Full path of the python network processor file. "
125 "Name of class defined in the file must match the file name without "
126 "the trailing '.py'",
127 "", ParamCateg);
128#ifdef JEVOIS_PRO
129 //! Parameter \relates jevois::dnn::Network
130 JEVOIS_DECLARE_PARAMETER(tpunum, size_t, "Coral EdgeTPU number to use to run this model, typically 0, or can be "
131 "1 when using a dual-TPU add-on board, or more when using additional TPUs connected "
132 "to USB ports",
133 0, ParamCateg);
134
135 //! Parameter \relates jevois::dnn::Network
136 JEVOIS_DECLARE_PARAMETER(spunum, size_t, "Hailo8 device number to use to run this model, typically 0 unless "
137 "several Hailo8 accelerators are connected to the system",
138 0, ParamCateg);
139
140 //! Parameter \relates jevois::dnn::NetworkNPU
141 JEVOIS_DECLARE_PARAMETER(verifygraph, bool, "Verify NPU graph after loading it",
142 true, ParamCateg);
143
144 //! Parameter \relates jevois::dnn::NetworkNPU
145 JEVOIS_DECLARE_PARAMETER(ovxver, std::string, "ovxlib version to use with NPU network, or leave blank "
146 "to use latest version",
147 "", boost::regex("^$|^[0-9]+\\.[0-9]+\\.[0-9]+$"), ParamCateg);
148
149 //! Parameter \relates jevois::dnn::NetworkHailo
150 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(turbo, bool, "Turbo mode. Has no significant effect on small or "
151 "fast networks. Use with caution as it may lead to overheating, not "
152 "recommended for production",
153 false, ParamCateg);
154#endif
155 }
156
157 //! Abstract class to represent a neural network
158 /*! Derived classes provide implementation via OpenCV (on CPU, OpenCL, or OpenVino/Myriad-X), Amlogic/Vivante NPU,
159 Hailo-8, Python, or Google Coral TPU. \ingroup dnn */
160 class Network : public Component,
161 public Parameter<network::comment, network::url, network::outreshape, network::extraintensors>
162 {
163 public:
164 //! Inherited constructor ok
166
167 //! Destructor
168 /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
169 virtual ~Network();
170
171 //! If network is currently loading, wait until that is done before destroying
172 /*! CAUTION: derived classes must call waitBeforeDestroy() in their destructor */
173 void waitBeforeDestroy();
174
175 //! Returns true when network is ready to run (loaded and initialized)
176 bool ready();
177
178 //! Get shapes of all input tensors
179 virtual std::vector<vsi_nn_tensor_attr_t> inputShapes() = 0;
180
181 //! Get shapes of all output tensors
182 virtual std::vector<vsi_nn_tensor_attr_t> outputShapes() = 0;
183
184 //! Process input blobs and obtain output blobs
185 /*! Network implementations may push information data into the info string, which will be displayed to
186 user. Convention is: if an info line starts with '* ', it is a header, and if it starts with '- ' it is a
187 bullet. Info should always be organized into headers at the top level. */
188 std::vector<cv::Mat> process(std::vector<cv::Mat> const & blobs, std::vector<std::string> & info);
189
190 //! Freeze/unfreeze parameters that users should not change while running
191 /*! Note: derived classes can freeze their own params by overriding this function, and should remember to still
192 call the base class jevois::dnn::Network::freeze(doit) */
193 virtual void freeze(bool doit);
194
195 protected:
196 //! Load from disk
197 virtual void load() = 0;
198
199 //! Process input blobs and obtain output blobs
200 virtual std::vector<cv::Mat> doprocess(std::vector<cv::Mat> const & blobs,
201 std::vector<std::string> & info) = 0;
202
203 void onParamChange(network::outreshape const & param, std::string const & val) override;
204
205 private:
206 std::atomic<bool> itsLoading = false;
207 std::atomic<bool> itsLoaded = false;
208 std::future<void> itsLoadFut;
209 std::vector<vsi_nn_tensor_attr_t> itsReshape;
210 };
211
212 } // namespace dnn
213} // namespace jevois
#define JEVOIS_SHARE_PATH
Base path for shared files (e.g., neural network weights, etc)
Definition Config.H:82
#define JEVOIS_BACKEND_DEFAULT
Definition Network.H:79
A component of a model hierarchy.
Definition Component.H:182
friend class Component
Allow Component and DynamicParameter to access our registry data, everyone else is locked out.
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(turbo, bool, "Turbo mode. Has no significant effect on small or " "fast networks. Use with caution as it may lead to overheating, not " "recommended for production", false, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(verifygraph, bool, "Verify NPU graph after loading it", true, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(ovxver, std::string, "ovxlib version to use with NPU network, or leave blank " "to use latest version", "", boost::regex("^$|^[0-9]+\\.[0-9]+\\.[0-9]+$"), ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pynet, std::string, "Full path of the python network processor file. " "Name of class defined in the file must match the file name without " "the trailing '.py'", "", ParamCateg)
Parameter.
Abstract class to represent a neural network.
Definition Network.H:162
JEVOIS_DECLARE_PARAMETER(url, std::string, "Optional URL for the network", "", ParamCateg)
Parameter.
virtual void load()=0
Load from disk.
bool ready()
Returns true when network is ready to run (loaded and initialized)
Definition Network.C:57
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(outreshape, std::string, "Specification of reshaped output tensors " "sometimes useful to re-interpret tensors to what a post-processor expects for " "example, TPU YoloV4-Int-VOC outputs 5D tensors 32F:1x52x52x3x85, 32F:1x26x26x3x85, " "32F:1x13x13x3x85 but the YOLO post-processor expects 4D, which would be specified here " "as 32F:1x52x52x255, 32F:1x26x26x255, 32F:1x13x13x255. Note that this only changes " "the description of dimensions, but does not move any pixel data around (e.g., cannot " "convert from NCHW to NHWC, convert data types, etc). Use sparingly and with caution.", "", ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(dequant, bool, "Dequantize output tensors to float32 from their native quantized type", true, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(spunum, size_t, "Hailo8 device number to use to run this model, typically 0 unless " "several Hailo8 accelerators are connected to the system", 0, ParamCateg)
Parameter.
std::vector< cv::Mat > process(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)
Process input blobs and obtain output blobs.
Definition Network.C:82
JEVOIS_DECLARE_PARAMETER(config, std::string, "Path to a text file that contains network configuration. " "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet). " "If path is relative, it will be prefixed by dataroot.", "", ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(intensors, std::string, "Specification of input tensors", "", ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(target, Target, "OpenCV compute target to use. Changes will take effect " "next time you load a different model.", Target::CPU, Target_Values, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(model, std::string, "Path to a binary file of model contains trained weights. " "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), " ".tflite (TensorFlow Lite), or .weights (Darknet). If path is relative, it will be " "prefixed by dataroot.", "", ParamCateg)
Parameter.
void waitBeforeDestroy()
If network is currently loading, wait until that is done before destroying.
Definition Network.C:44
JEVOIS_DECLARE_PARAMETER(backend, Backend, "OpenCV compute backend to use. Default will use the inference " "engine if available, otherwise OpenCV (note that inference engine only works on Intel " "processors or MyriadX hardware, thus you should normally select OpenCV when running " "on JeVois-Pro Platform, unless you want to use an optional MyriadX accelerator). " "Changes will take effect next time you load a model.", JEVOIS_BACKEND_DEFAULT, Backend_Values, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(outtensors, std::string, "Specification of output tensors", "", ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(tpunum, size_t, "Coral EdgeTPU number to use to run this model, typically 0, or can be " "1 when using a dual-TPU add-on board, or more when using additional TPUs connected " "to USB ports", 0, ParamCateg)
Parameter.
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes()=0
Get shapes of all output tensors.
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes()=0
Get shapes of all input tensors.
JEVOIS_DECLARE_PARAMETER(extraintensors, std::string, "Specification of extra fixed input tensors that will be " "added after the regular intensors. Format is: " "<type>:<shape>:val1 val2 ... valN, <type>:<shape>:val1 ... valN. For example, for " "URetinex-Net: 32F:1x1x1:3.0", "", ParamCateg)
Parameter.
virtual std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info)=0
Process input blobs and obtain output blobs.
JEVOIS_DECLARE_PARAMETER(comment, std::string, "Optional comment about the network", "", ParamCateg)
Parameter.
virtual ~Network()
Destructor.
Definition Network.C:23
JEVOIS_DEFINE_ENUM_CLASS(Target,(CPU)(OpenCL)(OpenCL_FP16)(Myriad)(NPU))
Enum.
void onParamChange(network::outreshape const &param, std::string const &val) override
Definition Network.C:35
JEVOIS_DEFINE_ENUM_CLASS(Backend,(OpenCV)(InferenceEngine)(TimVX))
Enum.
virtual void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition Network.C:27
JEVOIS_DECLARE_PARAMETER(dataroot, std::string, "Root directory to use when config or model parameters " "are relative paths.", JEVOIS_SHARE_PATH, ParamCateg)
Parameter.
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(imx290)(os08a10)(ar0234))
Enum for different sensor models.
Main namespace for all JeVois classes and functions.
Definition Concepts.dox:2
A category to which multiple ParameterDef definitions can belong.