JeVois  1.20
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Utils.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2020 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/DNN/Utils.H>
19 #include <jevois/Util/Utils.H>
20 #include <jevois/Debug/Log.H>
21 #include <fstream>
22 
23 // ##############################################################################################################
24 std::map<int, std::string> jevois::dnn::readLabelsFile(std::string const & fname)
25 {
26  std::ifstream ifs(fname);
27  if (ifs.is_open() == false) LFATAL("Failed to open file " << fname);
28 
29  size_t linenum = 1; std::map<int, std::string> ret; int id = 0;
30  for (std::string line; std::getline(ifs, line); ++linenum)
31  {
32  size_t idx1 = line.find_first_not_of(" \t"); if (idx1 == line.npos) continue;
33  size_t idx2 = line.find_last_not_of(" \t\r\n"); if (idx2 == line.npos) continue;
34  if (line[idx1] == '#') continue;
35 
36  try { id = std::stoi(line, &idx1); idx1 = line.find_first_not_of("0123456789 \t,:", idx1); } catch (...) { }
37 
38  std::string classname;
39  if (idx1 >= idx2)
40  {
41  LERROR(fname << ':' << linenum << ": empty class name -- REPLACING BY 'unspecified'");
42  classname = "unspecified";
43  }
44  else classname = line.substr(idx1, idx2 + 1 - idx1);
45 
46  // Possibly replace two double quotes by one:
47  jevois::replaceStringAll(classname, "\"\"", "\"");
48 
49  // Possibly remove enclosing double quotes:
50  size_t len = classname.length();
51  if (len > 1 && classname[0] == '"' && classname[len-1] == '"') classname = classname.substr(1, len-2);
52 
53  ret[id] = classname;
54 
55  // Increment id in case no ID number is given in the file:
56  ++id;
57  }
58 
59  ifs.close();
60 
61  LINFO("Loaded " << ret.size() << " class names from " << fname);
62 
63  return ret;
64 }
65 
66 // ##############################################################################################################
67 std::string jevois::dnn::getLabel(std::map<int, std::string> const & labels, int id)
68 {
69  auto itr = labels.find(id);
70  if (itr == labels.end()) return std::to_string(id);
71  return itr->second;
72 }
73 
74 // ##############################################################################################################
75 int jevois::dnn::stringToRGBA(std::string const & label, unsigned char alpha)
76 {
77  int col = 0x80808080;
78  for (char const c : label) col = c + ((col << 5) - col);
79  col = (col & 0xffffff) | (alpha << 24);
80  return col;
81 }
82 
83 // ##############################################################################################################
84 void jevois::dnn::topK(float const * pfProb, float * pfMaxProb, uint32_t * pMaxClass, uint32_t outputCount,
85  uint32_t topNum)
86 {
87  memset(pfMaxProb, 0xfe, sizeof(float) * topNum);
88  memset(pMaxClass, 0xff, sizeof(float) * topNum);
89 
90  for (uint32_t j = 0; j < topNum; ++j)
91  {
92  for (uint32_t i = 0; i < outputCount; ++i)
93  {
94  uint32_t k;
95  for (k = 0; k < topNum; ++k) if (i == pMaxClass[k]) break;
96  if (k != topNum) continue;
97 
98  if (pfProb[i] > pfMaxProb[j]) { pfMaxProb[j] = pfProb[i]; pMaxClass[j] = i; }
99  }
100  }
101 }
102 
103 // ##############################################################################################################
104 std::string jevois::dnn::shapestr(cv::Mat const & m)
105 {
106  cv::MatSize const & ms = m.size; int const nd = ms.dims();
107  std::string ret = std::to_string(nd) + "D ";
108  for (int i = 0; i < nd; ++i) ret += std::to_string(ms[i]) + (i < nd-1 ? "x" : "");
109  ret += ' ' + jevois::cvtypestr(m.type());
110  return ret;
111 }
112 
113 // ##############################################################################################################
114 std::string jevois::dnn::shapestr(TfLiteTensor const * t)
115 {
116 
117  TfLiteIntArray const & dims = *t->dims;
118  std::string ret = std::to_string(dims.size) + "D ";
119  for (int i = 0; i < dims.size; ++i) ret += std::to_string(dims.data[i]) + (i < dims.size-1 ? "x" : "");
120 
121  // Do not use TfLiteTypeGetName() as it returns different names...
122  switch (t->type)
123  {
124  case kTfLiteNoType: ret += " NoType"; break;
125  case kTfLiteFloat32: ret += " 32F"; break;
126  case kTfLiteInt32: ret += " 32S"; break;
127  case kTfLiteUInt8: ret += " 8U"; break;
128  case kTfLiteInt64: ret += " 64S"; break;
129  case kTfLiteString: ret += " String"; break;
130  case kTfLiteBool: ret += " 8B"; break;
131  case kTfLiteInt16: ret += " 16S"; break;
132  case kTfLiteComplex64: ret += " 64C"; break;
133  case kTfLiteInt8: ret += " 8I"; break;
134  case kTfLiteFloat16: ret += " 16F"; break;
135  case kTfLiteFloat64: ret += " 64F"; break;
136  case kTfLiteComplex128: ret += " 128C"; break;
137  default: ret += " UnknownType"; break;
138  }
139  return ret;
140 }
141 
142 // ##############################################################################################################
143 std::string jevois::dnn::shapestr(vsi_nn_tensor_attr_t const & attr)
144 {
145  std::string ret = std::to_string(attr.dim_num) + "D ";
146  for (uint32_t i = 0; i < attr.dim_num; ++i)
147  ret += std::to_string(attr.size[attr.dim_num-1-i]) + (i < attr.dim_num-1 ? "x" : "");
148 
149  // Value type:
150  switch (attr.dtype.vx_type)
151  {
152  case VSI_NN_TYPE_UINT8: ret += " 8U"; break;
153  case VSI_NN_TYPE_INT8: ret += " 8S"; break;
154  case VSI_NN_TYPE_BOOL8: ret += " 8B"; break;
155  case VSI_NN_TYPE_UINT16: ret += " 16U"; break;
156  case VSI_NN_TYPE_INT16: ret += " 16S"; break;
157  case VSI_NN_TYPE_FLOAT16: ret += " 16F"; break;
158  case VSI_NN_TYPE_BFLOAT16: ret += " 16B"; break;
159  case VSI_NN_TYPE_UINT32: ret += " 32U"; break;
160  case VSI_NN_TYPE_INT32: ret += " 32S"; break;
161  case VSI_NN_TYPE_FLOAT32: ret += " 32F"; break;
162  case VSI_NN_TYPE_UINT64: ret += " 64U"; break;
163  case VSI_NN_TYPE_INT64: ret += " 64S"; break;
164  case VSI_NN_TYPE_FLOAT64: ret += " 64F"; break;
165  default: throw std::range_error("shapestr: Unsupported tensor type " + std::to_string(attr.dtype.vx_type));
166  }
167 
168  return ret;
169 }
170 
171 #ifdef JEVOIS_PRO
172 // ##############################################################################################################
173 std::string jevois::dnn::shapestr(Ort::ConstTensorTypeAndShapeInfo const & ti)
174 {
175  std::ostringstream os;
176  std::vector<int64_t> input_node_dims = ti.GetShape();
177  os << input_node_dims.size() << "D ";
178  for (int64_t d : input_node_dims) os << d << 'x';
179  os.seekp(-1, os.cur); // will overwrite last 'x'
180 
181  switch (ti.GetElementType())
182  {
183  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED: os << " UNDEFINED"; break;
184  case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: os << " 32F"; break;
185  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: os << " 8U"; break;
186  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: os << " 8S"; break;
187  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: os << " 16U"; break;
188  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: os << " 16S"; break;
189  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: os << " 32S"; break;
190  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: os << " 64S"; break;
191  case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: os << " STR"; break;
192  case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: os << " BOOL"; break;
193  case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: os << " 16F"; break;
194  case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: os << " 64F"; break;
195  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: os << " 32U"; break;
196  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: os << " 64U"; break;
197  case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: os << " 64CPLX"; break;
198  case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: os << " 128CPLX"; break;
199  case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: os << " 16B"; break;
200  default: throw std::range_error("shapestr: Unsupported tensor type " + std::to_string(ti.GetElementType()));
201  }
202 
203  return os.str();
204 }
205 
206 // ##############################################################################################################
207 vsi_nn_type_e jevois::dnn::onnx2vsi(ONNXTensorElementDataType t)
208 {
209  switch (t)
210  {
211  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED: return VSI_NN_TYPE_NONE;
212  case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return VSI_NN_TYPE_FLOAT32;
213  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return VSI_NN_TYPE_UINT8;
214  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return VSI_NN_TYPE_INT8;
215  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return VSI_NN_TYPE_UINT16;
216  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return VSI_NN_TYPE_INT16;
217  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return VSI_NN_TYPE_INT32;
218  case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return VSI_NN_TYPE_INT64;
219  case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return VSI_NN_TYPE_BOOL8;
220  case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return VSI_NN_TYPE_FLOAT16;
221  case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return VSI_NN_TYPE_FLOAT64;
222  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return VSI_NN_TYPE_UINT32;
223  case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return VSI_NN_TYPE_UINT64;
224  case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: return VSI_NN_TYPE_BFLOAT16;
225  //case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: // unsupported by VSI
226  //case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64:
227  //case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128:
228  default: throw std::range_error("onnx2vsi: Unsupported tensor type " + std::to_string(t));
229  }
230 }
231 
232 // ##############################################################################################################
233 vsi_nn_tensor_attr_t jevois::dnn::tensorattr(Ort::ConstTensorTypeAndShapeInfo const & ti)
234 {
235  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
236  attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO;
237  attr.dtype.vx_type = jevois::dnn::onnx2vsi(ti.GetElementType());
238 
239  std::vector<int64_t> const dims = ti.GetShape();
240  size_t const ds = dims.size();
241  attr.dim_num = ds;
242  for (size_t i = 0; i < ds; ++i) attr.size[ds - 1 - i] = dims[i];
243 
244  // FIXME: quantization not yet supported
245  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
246 
247  return attr;
248 }
249 
250 // ##############################################################################################################
251 std::string jevois::dnn::shapestr(hailo_vstream_info_t const & vi)
252 {
253  // FIXME: should optimize but beware that vi.shape may have different interpretations depending on NCHW vs NWCH etc
254  return shapestr(tensorattr(vi));
255 
256  /*
257  std::string ret = "3D ";
258  switch (+ std::to_string(vi.shape[0]) + 'x' +
259  std::to_string(vi.shape[1]) + 'x' + std::to_string(vi.shape[2]);
260 
261  switch (vi.format.type)
262  {
263  case HAILO_FORMAT_TYPE_AUTO: ret += " AUTO"; break;
264  case HAILO_FORMAT_TYPE_UINT8: ret += " 8U"; break;
265  case HAILO_FORMAT_TYPE_UINT16: ret += " 16U"; break;
266  case HAILO_FORMAT_TYPE_FLOAT32: ret += " 32F"; break;
267  default: throw std::range_error("shapestr: Unsupported tensor type " + std::to_string(vi.format));
268  }
269 
270  return ret;
271  */
272 }
273 #endif
274 
275 // ##############################################################################################################
276 std::vector<size_t> jevois::dnn::strshape(std::string const & str)
277 {
278  std::vector<size_t> ret;
279  auto tok = jevois::split(str, "x");
280  for (std::string const & t : tok) ret.emplace_back(std::stoi(t));
281  return ret;
282 }
283 
284 // ##############################################################################################################
285 int jevois::dnn::tf2cv(TfLiteType t)
286 {
287  switch (t)
288  {
289  case kTfLiteFloat32: return CV_32F;
290  case kTfLiteInt32: return CV_32S;
291  case kTfLiteUInt8: return CV_8U;
292  case kTfLiteInt16: return CV_16S;
293  case kTfLiteInt8: return CV_8S;
294  case kTfLiteFloat16: return CV_16F;
295  case kTfLiteFloat64: return CV_64F;
296  //case kTfLiteComplex128:
297  //case kTfLiteComplex64:
298  //case kTfLiteBool:
299  //case kTfLiteString:
300  //case kTfLiteInt64:
301  //case kTfLiteNoType:
302  default: throw std::range_error(std::string("tf2cv: Unsupported type ") + TfLiteTypeGetName(t));
303  }
304 }
305 
306 // ##############################################################################################################
307 int jevois::dnn::vsi2cv(vsi_nn_type_e t)
308 {
309  switch (t)
310  {
311  case VSI_NN_TYPE_UINT8: return CV_8U;
312  case VSI_NN_TYPE_INT8: return CV_8S;
313  case VSI_NN_TYPE_BOOL8: return CV_8U;
314  case VSI_NN_TYPE_UINT16: return CV_16U;
315  case VSI_NN_TYPE_INT16: return CV_16S;
316  case VSI_NN_TYPE_FLOAT16: return CV_16F;
317  //case VSI_NN_TYPE_BFLOAT16: return CV_16F; // check
318  //case VSI_NN_TYPE_UINT32: return CV_32U; // unsupported by opencv
319  case VSI_NN_TYPE_INT32: return CV_32S;
320  case VSI_NN_TYPE_FLOAT32: return CV_32F;
321  //case VSI_NN_TYPE_UINT64: return CV_64U; // unsupported by opencv
322  //case VSI_NN_TYPE_INT64: return CV_64S; // unsupported by opencv
323  case VSI_NN_TYPE_FLOAT64: return CV_64F;
324  default: throw std::range_error("vsi2cv: Unsupported tensor type " + std::to_string(t));
325  }
326 }
327 
328 // ##############################################################################################################
329 vsi_nn_type_e jevois::dnn::tf2vsi(TfLiteType t)
330 {
331  switch (t)
332  {
333  case kTfLiteFloat32: return VSI_NN_TYPE_FLOAT32;
334  case kTfLiteInt32: return VSI_NN_TYPE_INT32;
335  case kTfLiteUInt8: return VSI_NN_TYPE_UINT8;
336  case kTfLiteInt16: return VSI_NN_TYPE_INT16;
337  case kTfLiteInt8: return VSI_NN_TYPE_INT8;
338  case kTfLiteFloat16: return VSI_NN_TYPE_FLOAT16;
339  case kTfLiteFloat64: return VSI_NN_TYPE_FLOAT64;
340  case kTfLiteInt64: return VSI_NN_TYPE_INT64;
341  case kTfLiteBool: return VSI_NN_TYPE_BOOL8; // fixme: need to check
342  case kTfLiteNoType: return VSI_NN_TYPE_NONE;
343  //case kTfLiteComplex128:
344  //case kTfLiteComplex64:
345  //case kTfLiteString:
346  default: throw std::range_error(std::string("tf2vsi: Unsupported type ") + TfLiteTypeGetName(t));
347  }
348 }
349 
350 // ##############################################################################################################
351 #ifdef JEVOIS_PRO
352 vsi_nn_type_e jevois::dnn::hailo2vsi(hailo_format_type_t t)
353 {
354  switch (t)
355  {
356  case HAILO_FORMAT_TYPE_AUTO: return VSI_NN_TYPE_NONE; break; // or throw?
357  case HAILO_FORMAT_TYPE_UINT8: return VSI_NN_TYPE_UINT8; break;
358  case HAILO_FORMAT_TYPE_UINT16: return VSI_NN_TYPE_UINT16; break;
359  case HAILO_FORMAT_TYPE_FLOAT32: return VSI_NN_TYPE_FLOAT32; break;
360  default: throw std::range_error("hailo2vsi: Unsupported tensor type " + std::to_string(t));
361  }
362 }
363 #endif
364 
365 // ##############################################################################################################
366 void jevois::dnn::clamp(cv::Rect & r, int width, int height)
367 {
368  int tx = std::min(width - 1, std::max(0, r.x));
369  int ty = std::min(height - 1, std::max(0, r.y));
370  int bx = std::min(width - 1, std::max(0, r.x + r.width));
371  int by = std::min(height - 1, std::max(0, r.y + r.height));
372  r.x = tx; r.y = ty; r.width = bx - tx; r.height = by - ty;
373 }
374 
375 // ##############################################################################################################
376 void jevois::dnn::clamp(cv::Rect2f & r, float width, float height)
377 {
378  float tx = std::min(width - 1.0F, std::max(0.0F, r.x));
379  float ty = std::min(height - 1.0F, std::max(0.0F, r.y));
380  float bx = std::min(width - 1.0F, std::max(0.0F, r.x + r.width));
381  float by = std::min(height - 1.0F, std::max(0.0F, r.y + r.height));
382  r.x = tx; r.y = ty; r.width = bx - tx; r.height = by - ty;
383 }
384 
385 // ##############################################################################################################
386 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::parseTensorSpecs(std::string const & specs)
387 {
388  char const * const specdef = "[NCHW:|NHWC:|NA:|AUTO:]Type:[NxCxHxW|NxHxWxC|...][:QNT[:fl|:scale:zero]]";
389  std::vector<std::string> spectok = jevois::split(specs, ",\\s*");
390  std::vector<vsi_nn_tensor_attr_t> ret;
391 
392  for (std::string const & spec : spectok)
393  {
394  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
395 
396  // NCHW:Type:NxCxHxW:QNT:scale:mean
397  std::vector<std::string> tok = jevois::split(spec, ":");
398  if (tok.size() < 2) throw std::runtime_error("parseTensorSpecs: Malformed tensor spec ["+spec+"] not "+specdef);
399 
400  // Decode optional shape:
401  size_t n = 0; // next tok to parse
402  if (tok[0] == "NCHW") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_NCHW; } // planar RGB
403  else if (tok[0] == "NHWC") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC; } // packed RGB
404  else if (tok[0] == "NA") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_NA; }
405  else if (tok[0] == "AUTO") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO; }
406  else attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO; // use AUTO if it was not given
407 
408  // We need at least type and dims:
409  if (tok.size() < n+2) throw std::runtime_error("parseTensorSpecs: Malformed tensor spec ["+spec+"] not "+specdef);
410 
411  // Decode type and convert to vsi:
412  if (tok[n] == "8U") attr.dtype.vx_type = VSI_NN_TYPE_UINT8;
413  else if (tok[n] == "8S") attr.dtype.vx_type = VSI_NN_TYPE_INT8;
414  else if (tok[n] == "8B") attr.dtype.vx_type = VSI_NN_TYPE_BOOL8;
415  else if (tok[n] == "16U") attr.dtype.vx_type = VSI_NN_TYPE_UINT16;
416  else if (tok[n] == "16S") attr.dtype.vx_type = VSI_NN_TYPE_INT16;
417  else if (tok[n] == "16F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT16;
418  else if (tok[n] == "16B") attr.dtype.vx_type = VSI_NN_TYPE_BFLOAT16;
419  else if (tok[n] == "32U") attr.dtype.vx_type = VSI_NN_TYPE_UINT32;
420  else if (tok[n] == "32S") attr.dtype.vx_type = VSI_NN_TYPE_INT32;
421  else if (tok[n] == "32F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT32;
422  else if (tok[n] == "64U") attr.dtype.vx_type = VSI_NN_TYPE_UINT64;
423  else if (tok[n] == "64S") attr.dtype.vx_type = VSI_NN_TYPE_INT64;
424  else if (tok[n] == "64F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT64;
425  else throw std::range_error("parseTensorSpecs: Invalid tensor type [" + tok[n] + "] in " + spec);
426  ++n; // next token
427 
428  // Decode the dims:
429  std::vector<size_t> dims = jevois::dnn::strshape(tok[n]);
430  attr.dim_num = dims.size();
431  for (size_t i = 0; i < attr.dim_num; ++i) attr.size[attr.dim_num - 1 - i] = dims[i];
432  ++n; // next token
433 
434  // Decode optional quantization type and its possible extra parameters:
435  if (n == tok.size() || tok[n] == "NONE")
436  {
437  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
438  }
439  else if (tok[n] == "DFP")
440  {
441  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_DFP;
442  if (tok.size() != n+2)
443  throw std::range_error("parseTensorSpecs: In "+spec+", DFP quantization needs :fl (" + specdef + ')');
444  attr.dtype.fl = std::stoi(tok[n+1]);
445  }
446 
447  else if (tok[n] == "AA" || tok[n] == "AS") // affine asymmetric and symmetric same, see ovxlib/vsi_nn_tensor.h
448  {
449  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
450  if (tok.size() != n+3)
451  throw std::range_error("parseTensorSpecs: In "+spec+", AA/AS quantization needs :scale:zero ("+specdef+')');
452  attr.dtype.scale = std::stof(tok[n+1]);
453  attr.dtype.zero_point = std::stoi(tok[n+2]);
454  }
455  else if (tok[n] == "APS")
456  {
457  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC;
458  throw std::range_error("parseTensorSpecs: In " + spec + ", AFFINE_PERCHANNEL_SYMMETRIC quant not yet supported");
459  }
460  else throw std::range_error("parseTensorSpecs: Invalid quantization type in " + spec);
461 
462  // Done with this tensor:
463  ret.emplace_back(attr);
464  }
465 
466  return ret;
467 }
468 
469 // ##############################################################################################################
470 cv::Mat jevois::dnn::attrmat(vsi_nn_tensor_attr_t const & attr, void * dataptr)
471 {
472  if (dataptr) return cv::Mat(jevois::dnn::attrdims(attr), jevois::dnn::vsi2cv(attr.dtype.vx_type), dataptr);
473  else return cv::Mat(jevois::dnn::attrdims(attr), jevois::dnn::vsi2cv(attr.dtype.vx_type));
474 }
475 
476 // ##############################################################################################################
477 std::vector<int> jevois::dnn::attrdims(vsi_nn_tensor_attr_t const & attr)
478 {
479  size_t const ndim = attr.dim_num;
480  std::vector<int> cvdims(ndim);
481  for (size_t i = 0; i < ndim; ++i) cvdims[ndim - 1 - i] = attr.size[i];
482  return cvdims;
483 }
484 
485 // ##############################################################################################################
486 cv::Size jevois::dnn::attrsize(vsi_nn_tensor_attr_t const & attr)
487 {
488  switch (attr.dtype.fmt)
489  {
490  case VSI_NN_DIM_FMT_NHWC:
491  if (attr.dim_num < 3) throw std::range_error("attrsize: need at least 3D, got " + jevois::dnn::attrstr(attr));
492  return cv::Size(attr.size[1], attr.size[2]);
493 
494  case VSI_NN_DIM_FMT_NCHW:
495  if (attr.dim_num < 2) throw std::range_error("attrsize: need at least 2D, got " + jevois::dnn::attrstr(attr));
496  return cv::Size(attr.size[0], attr.size[1]);
497 
498  case VSI_NN_DIM_FMT_AUTO:
499  if (attr.dim_num < 2) return cv::Size(attr.size[0], 1);
500  if (attr.dim_num < 3) return cv::Size(attr.size[0], attr.size[1]);
501  // ok, size[] starts with either CWH (when dim index goes 0..2) or WHC, assume C<H
502  if (attr.size[0] > attr.size[2]) return cv::Size(attr.size[0], attr.size[1]); // WHCN
503  else return cv::Size(attr.size[1], attr.size[2]); // CWHN
504 
505  default:
506  throw std::range_error("attrsize: cannot extract width and height, got " + jevois::dnn::attrstr(attr));
507  }
508 }
509 
510 // ##############################################################################################################
511 std::string jevois::dnn::attrstr(vsi_nn_tensor_attr_t const & attr)
512 {
513  std::string ret;
514 
515  // Dimension ordering:
516  switch (attr.dtype.fmt)
517  {
518  case VSI_NN_DIM_FMT_NCHW: ret += "NCHW:"; break;
519  case VSI_NN_DIM_FMT_NHWC: ret += "NHWC:"; break;
520  default: break;
521  }
522 
523  // Value type:
524  switch (attr.dtype.vx_type)
525  {
526  case VSI_NN_TYPE_UINT8: ret += "8U:"; break;
527  case VSI_NN_TYPE_INT8: ret += "8S:"; break;
528  case VSI_NN_TYPE_BOOL8: ret += "8B:"; break;
529  case VSI_NN_TYPE_UINT16: ret += "16U:"; break;
530  case VSI_NN_TYPE_INT16: ret += "16S:"; break;
531  case VSI_NN_TYPE_FLOAT16: ret += "16F:"; break;
532  case VSI_NN_TYPE_BFLOAT16: ret += "16B:"; break;
533  case VSI_NN_TYPE_UINT32: ret += "32U:"; break;
534  case VSI_NN_TYPE_INT32: ret += "32S:"; break;
535  case VSI_NN_TYPE_FLOAT32: ret += "32F:"; break;
536  case VSI_NN_TYPE_UINT64: ret += "64U:"; break;
537  case VSI_NN_TYPE_INT64: ret += "64S:"; break;
538  case VSI_NN_TYPE_FLOAT64: ret += "64F:"; break;
539  default: ret += "TYPE_UNKNOWN:";
540  }
541 
542  // Dims:
543  for (uint32_t i = 0; i < attr.dim_num; ++i)
544  ret += std::to_string(attr.size[attr.dim_num - 1 - i]) + ((i<attr.dim_num-1) ? 'x' : ':');
545 
546  // Quantization:
547  switch (attr.dtype.qnt_type)
548  {
549  case VSI_NN_QNT_TYPE_NONE: ret += "NONE"; break;
550  case VSI_NN_QNT_TYPE_DFP: ret += "DFP:" + std::to_string(attr.dtype.fl); break;
551  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: // same value as VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC:
552  ret += "AA:" + std::to_string(attr.dtype.scale) + ':' + std::to_string(attr.dtype.zero_point);
553  break;
554  case VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC: ret += "APS:unsupported"; break;
555  default: ret += "QUANT_UNKNOWN";
556  }
557 
558  return ret;
559 }
560 
561 // ##############################################################################################################
562 vsi_nn_tensor_attr_t jevois::dnn::tensorattr(TfLiteTensor const * t)
563 {
564  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
565  attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO;
566  attr.dtype.vx_type = jevois::dnn::tf2vsi(t->type);
567 
568  switch (t->quantization.type)
569  {
570  case kTfLiteNoQuantization:
571  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
572  break;
573 
574  case kTfLiteAffineQuantization:
575  {
576  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
577  attr.dtype.scale = t->params.scale;
578  attr.dtype.zero_point = t->params.zero_point;
579  }
580  break;
581 
582  default: LFATAL("unsupported quantization " << t->quantization.type);
583  }
584 
585  TfLiteIntArray const & dims = *t->dims;
586  attr.dim_num = dims.size;
587  for (int i = 0; i < dims.size; ++i) attr.size[dims.size - 1 - i] = dims.data[i];
588 
589  // Set the fmt to NCHW or NHWC if possible:
590  if (attr.dim_num == 4)
591  {
592  if (attr.size[0] > attr.size[2]) attr.dtype.fmt = VSI_NN_DIM_FMT_NCHW; // assume H>C
593  else attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
594  }
595 
596  return attr;
597 }
598 
599 // ##############################################################################################################
600 #ifdef JEVOIS_PRO
601 vsi_nn_tensor_attr_t jevois::dnn::tensorattr(hailo_vstream_info_t const & vi)
602 {
603  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
604 
605  attr.dtype.vx_type = hailo2vsi(vi.format.type);
606 
607  switch (vi.format.order)
608  {
609  case HAILO_FORMAT_ORDER_HAILO_NMS:
610  attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO;
611  attr.dim_num = 2;
612  attr.size[0] = vi.nms_shape.number_of_classes;
613  attr.size[1] = vi.nms_shape.max_bboxes_per_class * 5; // Each box has: xmin, ymin, xmax, ymax, score
614  break;
615 
616  case HAILO_FORMAT_ORDER_NHWC:
617  case HAILO_FORMAT_ORDER_FCR:
618  case HAILO_FORMAT_ORDER_F8CR:
619  attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
620  attr.dim_num = 4;
621  attr.size[0] = vi.shape.features;
622  attr.size[1] = vi.shape.width;
623  attr.size[2] = vi.shape.height;
624  attr.size[3] = 1;
625  break;
626 
627  case HAILO_FORMAT_ORDER_NHW:
628  attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
629  attr.dim_num = 4;
630  attr.size[0] = 1;
631  attr.size[1] = vi.shape.width;
632  attr.size[2] = vi.shape.height;
633  attr.size[3] = 1;
634  break;
635 
636  case HAILO_FORMAT_ORDER_NC:
637  attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
638  attr.dim_num = 4;
639  attr.size[0] = 1;
640  attr.size[1] = 1;
641  attr.size[2] = vi.shape.features;
642  attr.size[3] = 1;
643  break;
644 
645  case HAILO_FORMAT_ORDER_NCHW:
646  attr.dtype.fmt = VSI_NN_DIM_FMT_NCHW;
647  attr.dim_num = 4;
648  attr.size[0] = vi.shape.features;
649  attr.size[1] = vi.shape.width;
650  attr.size[2] = vi.shape.height;
651  attr.size[3] = 1;
652  break;
653 
654  default: throw std::range_error("tensorattr: Unsupported Hailo order " +std::to_string(vi.format.order));
655  }
656 
657  // Hailo only supports one quantization type:
658  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
659  attr.dtype.scale = vi.quant_info.qp_scale;
660  attr.dtype.zero_point = int32_t(vi.quant_info.qp_zp);
661 
662  return attr;
663 }
664 #endif
665 
666 // ##############################################################################################################
667 size_t jevois::dnn::softmax(float const * input, size_t const n, size_t const stride, float const fac, float * output,
668  bool maxonly)
669 {
670  if (stride == 0) LFATAL("Cannot work with stride = 0");
671 
672  float sum = 0.0F;
673  float largest = -FLT_MAX; size_t largest_idx = 0;
674  size_t const ns = n * stride;
675 
676  for (size_t i = 0; i < ns; i += stride) if (input[i] > largest) { largest = input[i]; largest_idx = i; }
677 
678  if (fac == 1.0F)
679  for (size_t i = 0; i < ns; i += stride)
680  {
681  float const e = expf(input[i] - largest);
682  sum += e;
683  output[i] = e;
684  }
685  else
686  for (size_t i = 0; i < ns; i += stride)
687  {
688  float const e = expf(input[i]/fac - largest/fac);
689  sum += e;
690  output[i] = e;
691  }
692 
693  if (sum)
694  {
695  if (maxonly) output[largest_idx] /= sum;
696  else for (size_t i = 0; i < ns; i += stride) output[i] /= sum;
697  }
698 
699  return largest_idx;
700 }
701 
702 // ##############################################################################################################
703 bool jevois::dnn::attrmatch(vsi_nn_tensor_attr_t const & attr, cv::Mat const & blob)
704 {
705  // Check that blob and tensor are a complete match:
706  if (blob.channels() != 1) return false;
707  if (blob.depth() != jevois::dnn::vsi2cv(attr.dtype.vx_type)) return false;
708  if (uint32_t(blob.size.dims()) != attr.dim_num) return false;
709 
710  for (size_t i = 0; i < attr.dim_num; ++i)
711  if (int(attr.size[attr.dim_num - 1 - i]) != blob.size[i]) return false;
712 
713  return true;
714 }
715 
716 // ##############################################################################################################
717 cv::Mat jevois::dnn::quantize(cv::Mat const & m, vsi_nn_tensor_attr_t const & attr)
718 {
719  if (m.depth() != CV_32F) LFATAL("Tensor to quantize must be 32F");
720 
721  // Do a sloppy match for total size only since m may still be 2D RGB packed vs 4D attr...
722  std::vector<int> adims = jevois::dnn::attrdims(attr);
723  size_t tot = 1; for (int d : adims) tot *= d;
724 
725  if (tot != m.total() * m.channels())
726  LFATAL("Mismatched tensor: " << jevois::dnn::shapestr(m) << " vs attr: " << jevois::dnn::shapestr(attr));
727 
728  unsigned int const tt = jevois::dnn::vsi2cv(attr.dtype.vx_type);
729 
730  switch (attr.dtype.qnt_type)
731  {
732  case VSI_NN_QNT_TYPE_NONE:
733  {
734  cv::Mat ret;
735  m.convertTo(ret, tt);
736  return ret;
737  }
738 
739  case VSI_NN_QNT_TYPE_DFP:
740  {
741  switch (tt)
742  {
743  case CV_8S:
744  {
745  if (attr.dtype.fl > 7) LFATAL("Invalid DFP fl value " << attr.dtype.fl << ": must be in [0..7]");
746  cv::Mat ret;
747  m.convertTo(ret, tt, 1 << attr.dtype.fl, 0.0);
748  return ret;
749  }
750  case CV_16S:
751  {
752  if (attr.dtype.fl > 15) LFATAL("Invalid DFP fl value " << attr.dtype.fl << ": must be in [0..15]");
753  cv::Mat ret;
754  m.convertTo(ret, tt, 1 << attr.dtype.fl, 0.0);
755  return ret;
756  }
757  default:
758  break;
759  }
760  break;
761  }
762 
763  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: // same value as VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC:
764  {
765  switch (tt)
766  {
767  case CV_8U:
768  {
769  cv::Mat ret;
770  if (attr.dtype.scale == 0.0) LFATAL("Quantization scale must not be zero in " << jevois::dnn::shapestr(attr));
771  m.convertTo(ret, tt, 1.0 / attr.dtype.scale, attr.dtype.zero_point);
772  return ret;
773  }
774 
775  default:
776  break;
777  }
778  break;
779  }
780 
781  case VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC:
782  LFATAL("Affine per-channel symmetric not supported yet");
783 
784  default:
785  break;
786  }
787 
788  LFATAL("Quantization to " << jevois::dnn::shapestr(attr) << " not yet supported");
789 }
790 
791 // ##############################################################################################################
792 cv::Mat jevois::dnn::dequantize(cv::Mat const & m, vsi_nn_tensor_attr_t const & attr)
793 {
794  if (! jevois::dnn::attrmatch(attr, m))
795  LFATAL("Mismatched tensor: " << jevois::dnn::shapestr(m) << " vs attr: " << jevois::dnn::shapestr(attr));
796 
797  switch (attr.dtype.qnt_type)
798  {
799  case VSI_NN_QNT_TYPE_NONE:
800  {
801  cv::Mat ret;
802  m.convertTo(ret, CV_32F);
803  return ret;
804  }
805 
806  case VSI_NN_QNT_TYPE_DFP:
807  {
808  cv::Mat ret;
809  m.convertTo(ret, CV_32F, 1.0 / (1 << attr.dtype.fl), 0.0);
810  return ret;
811  }
812 
813  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: // same value as VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC:
814  {
815  cv::Mat ret;
816  m.convertTo(ret, CV_32F);
817  if (attr.dtype.zero_point) ret -= attr.dtype.zero_point;
818  if (attr.dtype.scale != 1.0F) ret *= attr.dtype.scale;
819  return ret;
820  }
821 
822  case VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC:
823  LFATAL("Affine per-channel symmetric not supported yet");
824 
825  default:
826  LFATAL("Unknown quantization type " << int(attr.dtype.qnt_type));
827  }
828 }
829 
830 // ##############################################################################################################
831 size_t jevois::dnn::effectiveDims(cv::Mat const & m)
832 {
833  cv::MatSize const & rs = m.size;
834  size_t const ndims = rs.dims();
835  size_t ret = ndims;
836  for (size_t i = 0; i < ndims; ++i) if (rs[i] == 1) --ret; else break;
837  return ret;
838 }
839 
jevois::dnn::clamp
void clamp(cv::Rect &r, int width, int height)
Clamp a rectangle to within given image width and height.
Definition: Utils.C:366
jevois::dnn::tf2cv
int tf2cv(TfLiteType t)
Convert from TensorFlow data type to OpenCV.
Definition: Utils.C:285
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:270
jevois::cvtypestr
std::string cvtypestr(unsigned int cvtype)
Convert cv::Mat::type() code to to a string (e.g., CV_8UC1, CV_32SC3, etc)
Definition: Utils.C:58
Utils.H
jevois::dnn::softmax
size_t softmax(float const *input, size_t const n, size_t const stride, float const fac, float *output, bool maxonly)
Apply softmax to a float vector.
Definition: Utils.C:667
jevois::dnn::dequantize
cv::Mat dequantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Dequantize an output to float32 according to the quantization spec in attr.
Definition: Utils.C:792
jevois::dnn::vsi2cv
int vsi2cv(vsi_nn_type_e t)
Convert from NPU data type to OpenCV.
Definition: Utils.C:307
jevois::dnn::tensorattr
vsi_nn_tensor_attr_t tensorattr(TfLiteTensor const *t)
Get tensor shape and type attributes for a TensorFlow Lite tensor.
Definition: Utils.C:562
jevois::dnn::stringToRGBA
int stringToRGBA(std::string const &label, unsigned char alpha=128)
Compute a color from a label name.
Definition: Utils.C:75
jevois::dnn::readLabelsFile
std::map< int, std::string > readLabelsFile(std::string const &fname)
Read a label file.
Definition: Utils.C:24
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:211
jevois::dnn::effectiveDims
size_t effectiveDims(cv::Mat const &m)
Returns the number of non-unit dims in a cv::Mat.
Definition: Utils.C:831
jevois::dnn::parseTensorSpecs
std::vector< vsi_nn_tensor_attr_t > parseTensorSpecs(std::string const &specs)
Parse tensor specification.
Definition: Utils.C:386
jevois::dnn::attrmatch
bool attrmatch(vsi_nn_tensor_attr_t const &attr, cv::Mat const &blob)
Check that a cv::Mat blob matches exactly the spec of an attr.
Definition: Utils.C:703
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
F
float F
Definition: GUIhelper.C:2373
jevois::dnn::getLabel
std::string getLabel(std::map< int, std::string > const &labels, int id)
Get a label from an id.
Definition: Utils.C:67
Log.H
jevois::dnn::onnx2vsi
vsi_nn_type_e onnx2vsi(ONNXTensorElementDataType t)
Convert from ONNX-Runtime data type to vsi_nn.
Definition: Utils.C:207
jevois::replaceStringAll
size_t replaceStringAll(std::string &str, std::string const &from, std::string const &to)
Replace all instances of 'from' with 'to'.
Definition: Utils.C:344
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::dnn::attrdims
std::vector< int > attrdims(vsi_nn_tensor_attr_t const &attr)
Get a tensor dims as a vector of int, useful to construct a matching cv::Mat.
Definition: Utils.C:477
jevois::dnn::strshape
std::vector< size_t > strshape(std::string const &str)
Get a vector of size_t from a string containing AxBxC...
Definition: Utils.C:276
jevois::dnn::hailo2vsi
vsi_nn_type_e hailo2vsi(hailo_format_type_t t)
Convert from Hailo data type to vsi_nn.
Definition: Utils.C:352
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's (width, height) size in cv::Size format, skipping over other dimensions.
Definition: Utils.C:486
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:511
jevois::dnn::attrmat
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition: Utils.C:470
Utils.H
jevois::dnn::tf2vsi
vsi_nn_type_e tf2vsi(TfLiteType t)
Convert from TensorFlow data type to vsi_nn.
Definition: Utils.C:329
jevois::dnn::quantize
cv::Mat quantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Quantize from float32 to fixed-point according to the quantization spec in attr.
Definition: Utils.C:717
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
jevois::dnn::topK
void topK(float const *pfProb, float *pfMaxProb, uint32_t *pMaxClass, uint32_t outputCount, uint32_t topNum)
Get top-k entries and their indices.
Definition: Utils.C:84