JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
NetworkNPU.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #ifdef JEVOIS_PLATFORM_PRO
19 
20 #include <jevois/DNN/NetworkNPU.H>
21 #include <jevois/Util/Utils.H>
22 #include <jevois/DNN/Utils.H>
23 #include <vsi_nn_version.h>
24 
25 #define VNN_APP_DEBUG (FALSE)
26 
27 /*-------------------------------------------
28  Macros
29  -------------------------------------------*/
30 
31 #define NEW_VXNODE(_node, _type, _in, _out, _uid) do { \
32  _node = vsi_nn_AddNode(itsGraph, _type, _in, _out, NULL); \
33  _node->uid = (uint32_t)_uid; \
34  if (NULL == _node) LFATAL("NEW_VXNODE failed"); \
35  } while(0)
36 
37 #define NEW_VIRTUAL_TENSOR(_id, _attr, _dtype) do { \
38  memset(_attr.size, 0, VSI_NN_MAX_DIM_NUM * sizeof(vsi_size_t)); \
39  _attr.dim_num = VSI_NN_DIM_AUTO; \
40  _attr.vtl = !VNN_APP_DEBUG; \
41  _attr.is_const = FALSE; \
42  _attr.dtype.vx_type = _dtype; \
43  _id = vsi_nn_AddTensor(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, NULL); \
44  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_VIRTUAL_TENSOR failed"); \
45  } while(0)
46 
47 // Set const tensor dims out of this macro.
48 #define NEW_CONST_TENSOR(_id, _attr, _dtype, _ofst, _size) do { \
49  data = load_data(fp, _ofst, _size); \
50  _attr.vtl = FALSE; \
51  _attr.is_const = TRUE; \
52  _attr.dtype.vx_type = _dtype; \
53  _id = vsi_nn_AddTensor(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, data); \
54  free(data); \
55  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_CONST_TENSOR failed"); \
56  } while(0)
57 
58 // Set generic tensor dims out of this macro.
59 #define NEW_NORM_TENSOR(_id, _attr, _dtype) do { \
60  _attr.vtl = FALSE; \
61  _attr.is_const = FALSE; \
62  _attr.dtype.vx_type = _dtype; \
63  _id = vsi_nn_AddTensor(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, NULL); \
64  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_NORM_TENSOR failed"); \
65  } while(0)
66 
67 // Set generic tensor dims out of this macro.
68 #define NEW_NORM_TENSOR_FROM_HANDLE(_id, _attr, _dtype) do { \
69  _attr.vtl = FALSE; \
70  _attr.is_const = FALSE; \
71  _attr.dtype.vx_type = _dtype; \
72  _id = vsi_nn_AddTensorFromHandle(itsGraph, VSI_NN_TENSOR_ID_AUTO, & _attr, NULL); \
73  if (VSI_NN_TENSOR_ID_NA == _id) LFATAL("NEW_NORM_TENSOR_FROM_HANDLE failed"); \
74  } while(0)
75 
76 // ####################################################################################################
77 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::NetworkNPU::inputShapes()
78 {
80 }
81 
82 // ####################################################################################################
83 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::NetworkNPU::outputShapes()
84 {
86 }
87 
88 // ####################################################################################################
89 void jevois::dnn::NetworkNPU::create_tensors(std::vector<vsi_nn_tensor_attr_t> & attrs, vsi_nn_node_t * node, bool isin)
90 {
91  if (attrs.empty()) LFATAL("Invalid empty " << (isin ? "in" : "out") << "tensors specification");
92 
93  for (int tnum = 0; vsi_nn_tensor_attr_t & attr : attrs)
94  {
95  // Allocate the tensor:
96  vsi_nn_tensor_id_t id;
97  NEW_NORM_TENSOR(id, attr, attr.dtype.vx_type);
98 
99  // Connect the tensor:
100  if (isin)
101  {
102  node->input.tensors[tnum] = id;
103  itsGraph->input.tensors[tnum] = id;
104  LINFO("Input tensor " << tnum << ": " << jevois::dnn::attrstr(attr));
105  }
106  else
107  {
108  node->output.tensors[tnum] = id;
109  itsGraph->output.tensors[tnum] = id;
110  LINFO("Output tensor " << tnum << ": " << jevois::dnn::attrstr(attr));
111  }
112  ++tnum;
113  }
114 }
115 
116 // ####################################################################################################
118 {
119  waitBeforeDestroy();
120  if (itsGraph) vsi_nn_ReleaseGraph(&itsGraph);
121  if (itsCtx) vsi_nn_ReleaseContext(&itsCtx);
122 }
123 
124 // ####################################################################################################
126 {
127  dataroot::freeze(doit);
128  model::freeze(doit);
129  intensors::freeze(doit);
130  outtensors::freeze(doit);
131  ovxver::freeze(doit);
132 }
133 
134 // ####################################################################################################
136 {
137  // Need to nuke the network first if it exists or we could run out of RAM:
138  if (itsGraph) { vsi_nn_ReleaseGraph(&itsGraph); itsGraph = nullptr; }
139 
140  // Create context if needed:
141  if (itsCtx == 0) itsCtx = vsi_nn_CreateContext();
142 
143  // Parse input and output tensor specs:
144  std::vector<vsi_nn_tensor_attr_t> iattrs = jevois::dnn::parseTensorSpecs(intensors::get());
145  std::vector<vsi_nn_tensor_attr_t> oattrs = jevois::dnn::parseTensorSpecs(outtensors::get());
146  size_t const numin = iattrs.size();
147  size_t const numout = oattrs.size();
148 
149  // Create graph:
150  itsGraph = vsi_nn_CreateGraph(itsCtx, numin + numout * 2, 1);
151  if (itsGraph == NULL) LFATAL("Graph creation failed");
152 
153  if (ovxver::get().empty() == false)
154  {
155  std::vector<std::string> tok = jevois::split(ovxver::get(), "\\.");
156  if (tok.size() != 3) LFATAL("Malformed ovxver version [" << ovxver::get() <<"] -- should be x.y.z");
157  vsi_nn_SetGraphVersion(itsGraph, std::stoi(tok[0]), std::stoi(tok[1]), std::stoi(tok[2]));
158  }
159  else
160  vsi_nn_SetGraphVersion(itsGraph, VSI_NN_VERSION_MAJOR, VSI_NN_VERSION_MINOR, VSI_NN_VERSION_PATCH);
161 
162  vsi_nn_SetGraphInputs(itsGraph, NULL, numin);
163  vsi_nn_SetGraphOutputs(itsGraph, NULL, numout);
164 
165  LINFO("Created graph with " << numin << " inputs and " << numout << " outputs");
166 
167  // Get NBG file name:
168  std::string const m = jevois::absolutePath(dataroot::get(), model::get());
169 
170  // Initialize node:
171  vsi_nn_node_t * node[1];
172  NEW_VXNODE(node[0], VSI_NN_OP_NBG, numin, numout, 0);
173  node[0]->nn_param.nbg.type = VSI_NN_NBG_FILE;
174  node[0]->nn_param.nbg.url = m.c_str();
175 
176  // Create input and output tensors and attach them to the node and graph:
177  create_tensors(oattrs, node[0], false);
178  create_tensors(iattrs, node[0], true);
179 
180  // Setup the graph:
181  auto status = vsi_nn_SetupGraph(itsGraph, FALSE);
182  if (status != VSI_SUCCESS)
183  LFATAL("Failed to setup graph -- Possible causes:\n"
184  "- Your source network uses unsupported layer types?\n"
185  "- Wrong NPU model? Check --optimize VIPNANOQI_PID0X88\n"
186  "- Wrong NPU SDK version? Running ovxlib " <<
187  VSI_NN_VERSION_MAJOR << '.' << VSI_NN_VERSION_MINOR << '.' << VSI_NN_VERSION_PATCH);
188  LINFO("Graph ready.");
189 
190  if (verifygraph::get())
191  {
192  status = vsi_nn_VerifyGraph(itsGraph);
193  if (status != VSI_SUCCESS) LFATAL("Graph verification failed");
194  else LINFO("Graph verification ok");
195  }
196 }
197 
198 // ####################################################################################################
199 std::vector<cv::Mat> jevois::dnn::NetworkNPU::doprocess(std::vector<cv::Mat> const & blobs,
200  std::vector<std::string> & info)
201 {
202  if (blobs.size() != itsGraph->input.num)
203  LFATAL("Received " << blobs.size() << " blobs, but network has " << itsGraph->input.num << " inputs");
204 
205  for (size_t b = 0; b < blobs.size(); ++b)
206  {
207  cv::Mat const & blob = blobs[b];
208 
209  // Get the input tensor:
210  vsi_nn_tensor_t * tensor = vsi_nn_GetTensor(itsGraph, itsGraph->input.tensors[b]);
211  if (tensor == nullptr) LFATAL("Network does not have input tensor " << b);
212  auto const & iattr = tensor->attr;
213 
214  // Check that blob and tensor are a complete match:
215  if (jevois::dnn::attrmatch(iattr, blob) == false)
216  LFATAL("Input " << b << ": received " << jevois::dnn::shapestr(blob) <<
217  " but want: " << jevois::dnn::shapestr(iattr));
218 
219  // Copy blob data to tensor:
220  auto status = vsi_nn_CopyDataToTensor(itsGraph, tensor, (uint8_t *)blob.data);
221  if (status != VSI_SUCCESS) LFATAL("Error setting input tensor: " << status);
222 
223  info.emplace_back("- In " + std::to_string(b) + ": " + jevois::dnn::attrstr(iattr));
224  }
225 
226  // Ok, let's run the network:
227  auto status = vsi_nn_RunGraph(itsGraph);
228  if (status != VSI_SUCCESS) LFATAL("Error running graph: " << status);
229  info.emplace_back("- Network forward pass ok");
230 
231  // Collect the outputs:
232  std::vector<cv::Mat> outs;
233  for (uint32_t i = 0; i < itsGraph->output.num; ++i)
234  {
235  vsi_nn_tensor_t * ot = vsi_nn_GetTensor(itsGraph, itsGraph->output.tensors[i]);
236  vsi_nn_tensor_attr_t const & oattr = ot->attr;
237  uint8_t * tensor_data = (uint8_t *)vsi_nn_ConvertTensorToData(itsGraph, ot);
238 
239  try
240  {
241  cv::Mat rawout = jevois::dnn::attrmat(oattr, tensor_data);
242 
243  if (dequant::get())
244  {
245  outs.emplace_back(jevois::dnn::dequantize(rawout, oattr));
246  info.emplace_back("- Out " + std::to_string(i) + ": " + jevois::dnn::attrstr(oattr) + " -> 32F");
247  }
248  else
249  {
250  outs.emplace_back(rawout.clone());
251  info.emplace_back("- Out " + std::to_string(i) + ": " + jevois::dnn::attrstr(oattr));
252  }
253  } catch (...) { vsi_nn_Free(tensor_data); jevois::warnAndRethrowException(); }
254 
255  vsi_nn_Free(tensor_data);
256  }
257 
258  return outs;
259 }
260 
261 #endif // JEVOIS_PLATFORM_PRO
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::dnn::NetworkNPU::~NetworkNPU
virtual ~NetworkNPU()
Destructor.
Definition: NetworkNPU.C:117
NEW_NORM_TENSOR
#define NEW_NORM_TENSOR(_id, _attr, _dtype)
Definition: NetworkNPU.C:59
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:258
jevois::warnAndRethrowException
void warnAndRethrowException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition: Log.C:203
Utils.H
jevois::dnn::dequantize
cv::Mat dequantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Dequantize an output to float32 according to the quantization spec in attr.
Definition: Utils.C:714
jevois::dnn::NetworkNPU::inputShapes
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes() override
Get shapes of all input tensors.
Definition: NetworkNPU.C:77
jevois::dnn::parseTensorSpecs
std::vector< vsi_nn_tensor_attr_t > parseTensorSpecs(std::string const &specs)
Parse tensor specification.
Definition: Utils.C:308
jevois::dnn::attrmatch
bool attrmatch(vsi_nn_tensor_attr_t const &attr, cv::Mat const &blob)
Check that a cv::Mat blob matches exactly the spec of an attr.
Definition: Utils.C:625
jevois::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &root, std::filesystem::path const &path)
Compute an absolute path from two paths.
Definition: Utils.C:365
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois::dnn::NetworkNPU::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: NetworkNPU.C:125
NEW_VXNODE
#define NEW_VXNODE(_node, _type, _in, _out, _uid)
Definition: NetworkNPU.C:31
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:433
jevois::dnn::attrmat
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition: Utils.C:392
Utils.H
jevois::dnn::NetworkNPU::outputShapes
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes() override
Get shapes of all output tensors.
Definition: NetworkNPU.C:83
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
jevois::dnn::NetworkNPU::doprocess
std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info) override
Process input blobs and obtain output blobs.
Definition: NetworkNPU.C:199
jevois::dnn::NetworkNPU::load
void load() override
Load from disk.
Definition: NetworkNPU.C:135
NetworkNPU.H