JeVois  1.20
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
NetworkNPU.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #ifdef JEVOIS_PLATFORM_PRO
19 
20 #include <jevois/DNN/NetworkNPU.H>
21 #include <jevois/Util/Utils.H>
22 #include <jevois/DNN/Utils.H>
23 #include <vsi_nn_version.h>
24 
25 #define VNN_APP_DEBUG (FALSE)
26 
27 /*-------------------------------------------
28  Macros
29  -------------------------------------------*/
30 
31 #define NEW_VXNODE(_node, _type, _in, _out, _uid) do { \
32  _node = vsi_nn_AddNode(itsGraph, _type, _in, _out, NULL); \
33  _node->uid = (uint32_t)_uid; \
34  if (NULL == _node) LFATAL("NEW_VXNODE failed"); \
35  } while(0)
36 
37 #define NEW_VIRTUAL_TENSOR(_id, _attr, _dtype) do { \
38  memset(_attr.size, 0, VSI_NN_MAX_DIM_NUM * sizeof(vsi_size_t)); \
39  _attr.dim_num = VSI_NN_DIM_AUTO; \
40  _attr.vtl = !VNN_APP_DEBUG; \
41  _attr.is_const = FALSE; \
42  _attr.dtype.vx_type = _dtype; \
43  _id = vsi_nn_AddTensor(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, NULL); \
44  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_VIRTUAL_TENSOR failed"); \
45  } while(0)
46 
47 // Set const tensor dims out of this macro.
48 #define NEW_CONST_TENSOR(_id, _attr, _dtype, _ofst, _size) do { \
49  data = load_data(fp, _ofst, _size); \
50  _attr.vtl = FALSE; \
51  _attr.is_const = TRUE; \
52  _attr.dtype.vx_type = _dtype; \
53  _id = vsi_nn_AddTensor(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, data); \
54  free(data); \
55  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_CONST_TENSOR failed"); \
56  } while(0)
57 
58 // Set generic tensor dims out of this macro.
59 #define NEW_NORM_TENSOR(_id, _attr, _dtype) do { \
60  _attr.vtl = FALSE; \
61  _attr.is_const = FALSE; \
62  _attr.dtype.vx_type = _dtype; \
63  _id = vsi_nn_AddTensor(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, NULL); \
64  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_NORM_TENSOR failed"); \
65  } while(0)
66 
67 // Set generic tensor dims out of this macro.
68 #define NEW_NORM_TENSOR_FROM_HANDLE(_id, _attr, _dtype) do { \
69  _attr.vtl = FALSE; \
70  _attr.is_const = FALSE; \
71  _attr.dtype.vx_type = _dtype; \
72  _id = vsi_nn_AddTensorFromHandle(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, NULL); \
73  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_NORM_TENSOR_FROM_HANDLE failed"); \
74  } while(0)
75 
76 // ####################################################################################################
77 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::NetworkNPU::inputShapes()
78 {
80 }
81 
82 // ####################################################################################################
83 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::NetworkNPU::outputShapes()
84 {
86 }
87 
88 // ####################################################################################################
89 void jevois::dnn::NetworkNPU::create_tensors(std::vector<vsi_nn_tensor_attr_t> & attrs, vsi_nn_node_t * node, bool isin)
90 {
91  if (attrs.empty()) LFATAL("Invalid empty " << (isin ? "in" : "out") << "tensors specification");
92 
93  for (int tnum = 0; vsi_nn_tensor_attr_t & attr : attrs)
94  {
95  // Allocate the tensor:
96  vsi_nn_tensor_id_t id;
97  NEW_NORM_TENSOR(id, attr, attr.dtype.vx_type);
98 
99  // Connect the tensor:
100  if (isin)
101  {
102  node->input.tensors[tnum] = id;
103  itsGraph->input.tensors[tnum] = id;
104  LINFO("Input tensor " << tnum << ": " << jevois::dnn::attrstr(attr));
105  }
106  else
107  {
108  node->output.tensors[tnum] = id;
109  itsGraph->output.tensors[tnum] = id;
110  LINFO("Output tensor " << tnum << ": " << jevois::dnn::attrstr(attr));
111  }
112  ++tnum;
113  }
114 }
115 
116 // ####################################################################################################
118 {
119  waitBeforeDestroy();
120  if (itsGraph) vsi_nn_ReleaseGraph(&itsGraph);
121  if (itsCtx) vsi_nn_ReleaseContext(&itsCtx);
122 }
123 
124 // ####################################################################################################
126 {
127  dataroot::freeze(doit);
128  model::freeze(doit);
129  intensors::freeze(doit);
130  outtensors::freeze(doit);
131  ovxver::freeze(doit);
132  jevois::dnn::Network::freeze(doit); // base class parameters
133 }
134 
135 // ####################################################################################################
137 {
138  // Need to nuke the network first if it exists or we could run out of RAM:
139  if (itsGraph) { vsi_nn_ReleaseGraph(&itsGraph); itsGraph = nullptr; }
140 
141  // Create context if needed:
142  if (itsCtx == 0) itsCtx = vsi_nn_CreateContext();
143 
144  // Parse input and output tensor specs:
145  std::vector<vsi_nn_tensor_attr_t> iattrs = jevois::dnn::parseTensorSpecs(intensors::get());
146  std::vector<vsi_nn_tensor_attr_t> oattrs = jevois::dnn::parseTensorSpecs(outtensors::get());
147  size_t const numin = iattrs.size();
148  size_t const numout = oattrs.size();
149 
150  // Create graph:
151  itsGraph = vsi_nn_CreateGraph(itsCtx, numin + numout * 2, 1);
152  if (itsGraph == NULL) LFATAL("Graph creation failed");
153 
154  if (ovxver::get().empty() == false)
155  {
156  std::vector<std::string> tok = jevois::split(ovxver::get(), "\\.");
157  if (tok.size() != 3) LFATAL("Malformed ovxver version [" << ovxver::get() <<"] -- should be x.y.z");
158  vsi_nn_SetGraphVersion(itsGraph, std::stoi(tok[0]), std::stoi(tok[1]), std::stoi(tok[2]));
159  }
160  else
161  vsi_nn_SetGraphVersion(itsGraph, VSI_NN_VERSION_MAJOR, VSI_NN_VERSION_MINOR, VSI_NN_VERSION_PATCH);
162 
163  vsi_nn_SetGraphInputs(itsGraph, NULL, numin);
164  vsi_nn_SetGraphOutputs(itsGraph, NULL, numout);
165 
166  LINFO("Created graph with " << numin << " inputs and " << numout << " outputs");
167 
168  // Get NBG file name:
169  std::string const m = jevois::absolutePath(dataroot::get(), model::get());
170 
171  // Initialize node:
172  vsi_nn_node_t * node[1];
173  NEW_VXNODE(node[0], VSI_NN_OP_NBG, numin, numout, 0);
174  node[0]->nn_param.nbg.type = VSI_NN_NBG_FILE;
175  node[0]->nn_param.nbg.url = m.c_str();
176 
177  // Create input and output tensors and attach them to the node and graph:
178  create_tensors(oattrs, node[0], false);
179  create_tensors(iattrs, node[0], true);
180 
181  // Setup the graph:
182  auto status = vsi_nn_SetupGraph(itsGraph, FALSE);
183  if (status != VSI_SUCCESS)
184  LFATAL("Failed to setup graph -- Possible causes:\n"
185  "- Your source network uses unsupported layer types?\n"
186  "- Wrong NPU model? Check --optimize VIPNANOQI_PID0X88\n"
187  "- Wrong NPU SDK version? Running ovxlib " <<
188  VSI_NN_VERSION_MAJOR << '.' << VSI_NN_VERSION_MINOR << '.' << VSI_NN_VERSION_PATCH);
189  LINFO("Graph ready.");
190 
191  if (verifygraph::get())
192  {
193  status = vsi_nn_VerifyGraph(itsGraph);
194  if (status != VSI_SUCCESS) LFATAL("Graph verification failed");
195  else LINFO("Graph verification ok");
196  }
197 }
198 
199 // ####################################################################################################
200 std::vector<cv::Mat> jevois::dnn::NetworkNPU::doprocess(std::vector<cv::Mat> const & blobs,
201  std::vector<std::string> & info)
202 {
203  if (blobs.size() != itsGraph->input.num)
204  LFATAL("Received " << blobs.size() << " blobs, but network has " << itsGraph->input.num << " inputs");
205 
206  for (size_t b = 0; b < blobs.size(); ++b)
207  {
208  cv::Mat const & blob = blobs[b];
209 
210  // Get the input tensor:
211  vsi_nn_tensor_t * tensor = vsi_nn_GetTensor(itsGraph, itsGraph->input.tensors[b]);
212  if (tensor == nullptr) LFATAL("Network does not have input tensor " << b);
213  auto const & iattr = tensor->attr;
214 
215  // Check that blob and tensor are a complete match:
216  if (jevois::dnn::attrmatch(iattr, blob) == false)
217  LFATAL("Input " << b << ": received " << jevois::dnn::shapestr(blob) <<
218  " but want: " << jevois::dnn::shapestr(iattr));
219 
220  // Copy blob data to tensor:
221  auto status = vsi_nn_CopyDataToTensor(itsGraph, tensor, (uint8_t *)blob.data);
222  if (status != VSI_SUCCESS) LFATAL("Error setting input tensor: " << status);
223 
224  info.emplace_back("- In " + std::to_string(b) + ": " + jevois::dnn::attrstr(iattr));
225  }
226 
227  // Ok, let's run the network:
228  auto status = vsi_nn_RunGraph(itsGraph);
229  if (status != VSI_SUCCESS) LFATAL("Error running graph: " << status);
230  info.emplace_back("- Network forward pass ok");
231 
232  // Collect the outputs:
233  std::vector<cv::Mat> outs;
234  for (uint32_t i = 0; i < itsGraph->output.num; ++i)
235  {
236  vsi_nn_tensor_t * ot = vsi_nn_GetTensor(itsGraph, itsGraph->output.tensors[i]);
237  vsi_nn_tensor_attr_t const & oattr = ot->attr;
238  uint8_t * tensor_data = (uint8_t *)vsi_nn_ConvertTensorToData(itsGraph, ot);
239 
240  try
241  {
242  cv::Mat rawout = jevois::dnn::attrmat(oattr, tensor_data);
243 
244  if (dequant::get())
245  {
246  outs.emplace_back(jevois::dnn::dequantize(rawout, oattr));
247  info.emplace_back("- Out " + std::to_string(i) + ": " + jevois::dnn::attrstr(oattr) + " -> 32F");
248  }
249  else
250  {
251  outs.emplace_back(rawout.clone());
252  info.emplace_back("- Out " + std::to_string(i) + ": " + jevois::dnn::attrstr(oattr));
253  }
254  } catch (...) { vsi_nn_Free(tensor_data); jevois::warnAndRethrowException(); }
255 
256  vsi_nn_Free(tensor_data);
257  }
258 
259  return outs;
260 }
261 
262 #endif // JEVOIS_PLATFORM_PRO
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::dnn::NetworkNPU::~NetworkNPU
virtual ~NetworkNPU()
Destructor.
Definition: NetworkNPU.C:117
NEW_NORM_TENSOR
#define NEW_NORM_TENSOR(_id, _attr, _dtype)
Definition: NetworkNPU.C:59
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:270
jevois::warnAndRethrowException
void warnAndRethrowException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition: Log.C:203
Utils.H
jevois::dnn::dequantize
cv::Mat dequantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Dequantize an output to float32 according to the quantization spec in attr.
Definition: Utils.C:792
jevois::dnn::Network::freeze
virtual void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition: Network.C:27
jevois::dnn::NetworkNPU::inputShapes
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes() override
Get shapes of all input tensors.
Definition: NetworkNPU.C:77
jevois::dnn::parseTensorSpecs
std::vector< vsi_nn_tensor_attr_t > parseTensorSpecs(std::string const &specs)
Parse tensor specification.
Definition: Utils.C:386
jevois::dnn::attrmatch
bool attrmatch(vsi_nn_tensor_attr_t const &attr, cv::Mat const &blob)
Check that a cv::Mat blob matches exactly the spec of an attr.
Definition: Utils.C:703
jevois::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &root, std::filesystem::path const &path)
Compute an absolute path from two paths.
Definition: Utils.C:385
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois::dnn::NetworkNPU::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: NetworkNPU.C:125
NEW_VXNODE
#define NEW_VXNODE(_node, _type, _in, _out, _uid)
Definition: NetworkNPU.C:31
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:511
jevois::dnn::attrmat
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition: Utils.C:470
Utils.H
jevois::dnn::NetworkNPU::outputShapes
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes() override
Get shapes of all output tensors.
Definition: NetworkNPU.C:83
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
jevois::dnn::NetworkNPU::doprocess
std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info) override
Process input blobs and obtain output blobs.
Definition: NetworkNPU.C:200
jevois::dnn::NetworkNPU::load
void load() override
Load from disk.
Definition: NetworkNPU.C:136
NetworkNPU.H