JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Utils.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2020 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/DNN/Utils.H>
19 #include <jevois/Util/Utils.H>
20 #include <jevois/Debug/Log.H>
21 #include <fstream>
22 
23 // ##############################################################################################################
24 std::map<int, std::string> jevois::dnn::readLabelsFile(std::string const & fname)
25 {
26  std::ifstream ifs(fname);
27  if (ifs.is_open() == false) LFATAL("Failed to open file " << fname);
28 
29  size_t linenum = 1; std::map<int, std::string> ret; int id = 0;
30  for (std::string line; std::getline(ifs, line); ++linenum)
31  {
32  size_t idx1 = line.find_first_not_of(" \t"); if (idx1 == line.npos) continue;
33  size_t idx2 = line.find_last_not_of(" \t\r\n"); if (idx2 == line.npos) continue;
34  if (line[idx1] == '#') continue;
35 
36  try { id = std::stoi(line, &idx1); idx1 = line.find_first_not_of("0123456789 \t,:", idx1); } catch (...) { }
37 
38  std::string classname;
39  if (idx1 >= idx2)
40  {
41  LERROR(fname << ':' << linenum << ": empty class name -- REPLACING BY 'unspecified'");
42  classname = "unspecified";
43  }
44  else classname = line.substr(idx1, idx2 + 1 - idx1);
45 
46  // Possibly replace two double quotes by one:
47  jevois::replaceStringAll(classname, "\"\"", "\"");
48 
49  // Possibly remove enclosing double quotes:
50  size_t len = classname.length();
51  if (len > 1 && classname[0] == '"' && classname[len-1] == '"') classname = classname.substr(1, len-2);
52 
53  ret[id] = classname;
54 
55  // Increment id in case no ID number is given in the file:
56  ++id;
57  }
58 
59  ifs.close();
60 
61  LINFO("Loaded " << ret.size() << " class names from " << fname);
62 
63  return ret;
64 }
65 
66 // ##############################################################################################################
67 std::string jevois::dnn::getLabel(std::map<int, std::string> const & labels, int id)
68 {
69  auto itr = labels.find(id);
70  if (itr == labels.end()) return std::to_string(id);
71  return itr->second;
72 }
73 
74 // ##############################################################################################################
75 int jevois::dnn::stringToRGBA(std::string const & label, unsigned char alpha)
76 {
77  int col = 0x80808080;
78  for (char const c : label) col = c + ((col << 5) - col);
79  col = (col & 0xffffff) | (alpha << 24);
80  return col;
81 }
82 
83 // ##############################################################################################################
84 void jevois::dnn::topK(float const * pfProb, float * pfMaxProb, uint32_t * pMaxClass, uint32_t outputCount,
85  uint32_t topNum)
86 {
87  memset(pfMaxProb, 0xfe, sizeof(float) * topNum);
88  memset(pMaxClass, 0xff, sizeof(float) * topNum);
89 
90  for (uint32_t j = 0; j < topNum; ++j)
91  {
92  for (uint32_t i = 0; i < outputCount; ++i)
93  {
94  uint32_t k;
95  for (k = 0; k < topNum; ++k) if (i == pMaxClass[k]) break;
96  if (k != topNum) continue;
97 
98  if (pfProb[i] > pfMaxProb[j]) { pfMaxProb[j] = pfProb[i]; pMaxClass[j] = i; }
99  }
100  }
101 }
102 
103 // ##############################################################################################################
104 std::string jevois::dnn::shapestr(cv::Mat const & m)
105 {
106  cv::MatSize const & ms = m.size; int const nd = ms.dims();
107  std::string ret = std::to_string(nd) + "D ";
108  for (int i = 0; i < nd; ++i) ret += std::to_string(ms[i]) + (i < nd-1 ? "x" : "");
109  ret += ' ' + jevois::cvtypestr(m.type());
110  return ret;
111 }
112 
113 // ##############################################################################################################
114 std::string jevois::dnn::shapestr(TfLiteTensor const * t)
115 {
116 
117  TfLiteIntArray const & dims = *t->dims;
118  std::string ret = std::to_string(dims.size) + "D ";
119  for (int i = 0; i < dims.size; ++i) ret += std::to_string(dims.data[i]) + (i < dims.size-1 ? "x" : "");
120 
121  // Do not use TfLiteTypeGetName() as it returns different names...
122  switch (t->type)
123  {
124  case kTfLiteNoType: ret += " NoType"; break;
125  case kTfLiteFloat32: ret += " 32F"; break;
126  case kTfLiteInt32: ret += " 32S"; break;
127  case kTfLiteUInt8: ret += " 8U"; break;
128  case kTfLiteInt64: ret += " 64S"; break;
129  case kTfLiteString: ret += " String"; break;
130  case kTfLiteBool: ret += " 8B"; break;
131  case kTfLiteInt16: ret += " 16S"; break;
132  case kTfLiteComplex64: ret += " 64C"; break;
133  case kTfLiteInt8: ret += " 8I"; break;
134  case kTfLiteFloat16: ret += " 16F"; break;
135  case kTfLiteFloat64: ret += " 64F"; break;
136  case kTfLiteComplex128: ret += " 128C"; break;
137  default: ret += " UnknownType"; break;
138  }
139  return ret;
140 }
141 
142 // ##############################################################################################################
143 std::string jevois::dnn::shapestr(vsi_nn_tensor_attr_t const & attr)
144 {
145  std::string ret = std::to_string(attr.dim_num) + "D ";
146  for (uint32_t i = 0; i < attr.dim_num; ++i)
147  ret += std::to_string(attr.size[attr.dim_num-1-i]) + (i < attr.dim_num-1 ? "x" : "");
148 
149  // Value type:
150  switch (attr.dtype.vx_type)
151  {
152  case VSI_NN_TYPE_UINT8: ret += " 8U"; break;
153  case VSI_NN_TYPE_INT8: ret += " 8S"; break;
154  case VSI_NN_TYPE_BOOL8: ret += " 8B"; break;
155  case VSI_NN_TYPE_UINT16: ret += " 16U"; break;
156  case VSI_NN_TYPE_INT16: ret += " 16S"; break;
157  case VSI_NN_TYPE_FLOAT16: ret += " 16F"; break;
158  case VSI_NN_TYPE_BFLOAT16: ret += " 16B"; break;
159  case VSI_NN_TYPE_UINT32: ret += " 32U"; break;
160  case VSI_NN_TYPE_INT32: ret += " 32S"; break;
161  case VSI_NN_TYPE_FLOAT32: ret += " 32F"; break;
162  case VSI_NN_TYPE_UINT64: ret += " 64U"; break;
163  case VSI_NN_TYPE_INT64: ret += " 64S"; break;
164  case VSI_NN_TYPE_FLOAT64: ret += " 64F"; break;
165  default: throw std::range_error("shapestr: Unsupported tensor type " + std::to_string(attr.dtype.vx_type));
166  }
167 
168  return ret;
169 }
170 
171 // ##############################################################################################################
172 #ifdef JEVOIS_PRO
173 std::string jevois::dnn::shapestr(hailo_vstream_info_t const & vi)
174 {
175  // FIXME: should optimize but beware that vi.shape may have different interpretations depending on NCHW vs NWCH etc
176  return shapestr(tensorattr(vi));
177 
178  /*
179  std::string ret = "3D ";
180  switch (+ std::to_string(vi.shape[0]) + 'x' +
181  std::to_string(vi.shape[1]) + 'x' + std::to_string(vi.shape[2]);
182 
183  switch (vi.format.type)
184  {
185  case HAILO_FORMAT_TYPE_AUTO: ret += " AUTO"; break;
186  case HAILO_FORMAT_TYPE_UINT8: ret += " 8U"; break;
187  case HAILO_FORMAT_TYPE_UINT16: ret += " 16U"; break;
188  case HAILO_FORMAT_TYPE_FLOAT32: ret += " 32F"; break;
189  default: throw std::range_error("shapestr: Unsupported tensor type " + std::to_string(vi.format));
190  }
191 
192  return ret;
193  */
194 }
195 #endif
196 
197 // ##############################################################################################################
198 std::vector<size_t> jevois::dnn::strshape(std::string const & str)
199 {
200  std::vector<size_t> ret;
201  auto tok = jevois::split(str, "x");
202  for (std::string const & t : tok) ret.emplace_back(std::stoi(t));
203  return ret;
204 }
205 
206 // ##############################################################################################################
207 int jevois::dnn::tf2cv(TfLiteType t)
208 {
209  switch (t)
210  {
211  case kTfLiteFloat32: return CV_32F;
212  case kTfLiteInt32: return CV_32S;
213  case kTfLiteUInt8: return CV_8U;
214  case kTfLiteInt16: return CV_16S;
215  case kTfLiteInt8: return CV_8S;
216  case kTfLiteFloat16: return CV_16S;
217  case kTfLiteFloat64: return CV_64F;
218  //case kTfLiteComplex128:
219  //case kTfLiteComplex64:
220  //case kTfLiteBool:
221  //case kTfLiteString:
222  //case kTfLiteInt64:
223  //case kTfLiteNoType:
224  default: throw std::range_error(std::string("tf2cv: Unsupported type ") + TfLiteTypeGetName(t));
225  }
226 }
227 
228 // ##############################################################################################################
229 int jevois::dnn::vsi2cv(vsi_nn_type_e t)
230 {
231  switch (t)
232  {
233  case VSI_NN_TYPE_UINT8: return CV_8U;
234  case VSI_NN_TYPE_INT8: return CV_8S;
235  case VSI_NN_TYPE_BOOL8: return CV_8U;
236  case VSI_NN_TYPE_UINT16: return CV_16U;
237  case VSI_NN_TYPE_INT16: return CV_16S;
238  case VSI_NN_TYPE_FLOAT16: return CV_16F;
239  case VSI_NN_TYPE_BFLOAT16: return CV_16F; // check
240  //case VSI_NN_TYPE_UINT32: return CV_32U; // unsupported by opencv
241  case VSI_NN_TYPE_INT32: return CV_32S;
242  case VSI_NN_TYPE_FLOAT32: return CV_32F;
243  //case VSI_NN_TYPE_UINT64: return CV_64U; // unsupported by opencv
244  //case VSI_NN_TYPE_INT64: return CV_64S; // unsupported by opencv
245  case VSI_NN_TYPE_FLOAT64: return CV_64F;
246  default: throw std::range_error("vsi2cv: Unsupported tensor type " + std::to_string(t));
247  }
248 }
249 
250 // ##############################################################################################################
251 vsi_nn_type_e jevois::dnn::tf2vsi(TfLiteType t)
252 {
253  switch (t)
254  {
255  case kTfLiteFloat32: return VSI_NN_TYPE_FLOAT32;
256  case kTfLiteInt32: return VSI_NN_TYPE_INT32;
257  case kTfLiteUInt8: return VSI_NN_TYPE_UINT8;
258  case kTfLiteInt16: return VSI_NN_TYPE_INT16;
259  case kTfLiteInt8: return VSI_NN_TYPE_INT8;
260  case kTfLiteFloat16: return VSI_NN_TYPE_FLOAT16;
261  case kTfLiteFloat64: return VSI_NN_TYPE_FLOAT64;
262  case kTfLiteInt64: return VSI_NN_TYPE_INT64;
263  case kTfLiteBool: return VSI_NN_TYPE_BOOL8; // fixme: need to check
264  case kTfLiteNoType: return VSI_NN_TYPE_NONE;
265  //case kTfLiteComplex128:
266  //case kTfLiteComplex64:
267  //case kTfLiteString:
268  default: throw std::range_error(std::string("tf2vsi: Unsupported type ") + TfLiteTypeGetName(t));
269  }
270 }
271 
272 // ##############################################################################################################
273 #ifdef JEVOIS_PRO
274 vsi_nn_type_e jevois::dnn::hailo2vsi(hailo_format_type_t t)
275 {
276  switch (t)
277  {
278  case HAILO_FORMAT_TYPE_AUTO: return VSI_NN_TYPE_NONE; break; // or throw?
279  case HAILO_FORMAT_TYPE_UINT8: return VSI_NN_TYPE_UINT8; break;
280  case HAILO_FORMAT_TYPE_UINT16: return VSI_NN_TYPE_UINT16; break;
281  case HAILO_FORMAT_TYPE_FLOAT32: return VSI_NN_TYPE_FLOAT32; break;
282  default: throw std::range_error("hailo2vsi: Unsupported tensor type " + std::to_string(t));
283  }
284 }
285 #endif
286 
287 // ##############################################################################################################
288 void jevois::dnn::clamp(cv::Rect & r, int width, int height)
289 {
290  int tx = std::min(width - 1, std::max(0, r.x));
291  int ty = std::min(height - 1, std::max(0, r.y));
292  int bx = std::min(width - 1, std::max(0, r.x + r.width));
293  int by = std::min(height - 1, std::max(0, r.y + r.height));
294  r.x = tx; r.y = ty; r.width = bx - tx; r.height = by - ty;
295 }
296 
297 // ##############################################################################################################
298 void jevois::dnn::clamp(cv::Rect2f & r, float width, float height)
299 {
300  float tx = std::min(width - 1.0F, std::max(0.0F, r.x));
301  float ty = std::min(height - 1.0F, std::max(0.0F, r.y));
302  float bx = std::min(width - 1.0F, std::max(0.0F, r.x + r.width));
303  float by = std::min(height - 1.0F, std::max(0.0F, r.y + r.height));
304  r.x = tx; r.y = ty; r.width = bx - tx; r.height = by - ty;
305 }
306 
307 // ##############################################################################################################
308 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::parseTensorSpecs(std::string const & specs)
309 {
310  char const * const specdef = "[NCHW:|NHWC:|NA:|AUTO:]Type:[NxCxHxW|NxHxWxC|...][:QNT[:fl|:scale:zero]]";
311  std::vector<std::string> spectok = jevois::split(specs, ",\\s*");
312  std::vector<vsi_nn_tensor_attr_t> ret;
313 
314  for (std::string const & spec : spectok)
315  {
316  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
317 
318  // NCHW:Type:NxCxHxW:QNT:scale:mean
319  std::vector<std::string> tok = jevois::split(spec, ":");
320  if (tok.size() < 2) throw std::runtime_error("parseTensorSpecs: Malformed tensor spec ["+spec+"] not "+specdef);
321 
322  // Decode optional shape:
323  size_t n = 0; // next tok to parse
324  if (tok[0] == "NCHW") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_NCHW; } // planar RGB
325  else if (tok[0] == "NHWC") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC; } // packed RGB
326  else if (tok[0] == "NA") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_NA; }
327  else if (tok[0] == "AUTO") { ++n; attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO; }
328  else attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO; // use AUTO if it was not given
329 
330  // We need at least type and dims:
331  if (tok.size() < n+2) throw std::runtime_error("parseTensorSpecs: Malformed tensor spec ["+spec+"] not "+specdef);
332 
333  // Decode type and convert to vsi:
334  if (tok[n] == "8U") attr.dtype.vx_type = VSI_NN_TYPE_UINT8;
335  else if (tok[n] == "8S") attr.dtype.vx_type = VSI_NN_TYPE_INT8;
336  else if (tok[n] == "8B") attr.dtype.vx_type = VSI_NN_TYPE_BOOL8;
337  else if (tok[n] == "16U") attr.dtype.vx_type = VSI_NN_TYPE_UINT16;
338  else if (tok[n] == "16S") attr.dtype.vx_type = VSI_NN_TYPE_INT16;
339  else if (tok[n] == "16F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT16;
340  else if (tok[n] == "16B") attr.dtype.vx_type = VSI_NN_TYPE_BFLOAT16;
341  else if (tok[n] == "32U") attr.dtype.vx_type = VSI_NN_TYPE_UINT32;
342  else if (tok[n] == "32S") attr.dtype.vx_type = VSI_NN_TYPE_INT32;
343  else if (tok[n] == "32F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT32;
344  else if (tok[n] == "64U") attr.dtype.vx_type = VSI_NN_TYPE_UINT64;
345  else if (tok[n] == "64S") attr.dtype.vx_type = VSI_NN_TYPE_INT64;
346  else if (tok[n] == "64F") attr.dtype.vx_type = VSI_NN_TYPE_FLOAT64;
347  else throw std::range_error("parseTensorSpecs: Invalid tensor type [" + tok[n] + "] in " + spec);
348  ++n; // next token
349 
350  // Decode the dims:
351  std::vector<size_t> dims = jevois::dnn::strshape(tok[n]);
352  attr.dim_num = dims.size();
353  for (size_t i = 0; i < attr.dim_num; ++i) attr.size[attr.dim_num - 1 - i] = dims[i];
354  ++n; // next token
355 
356  // Decode optional quantization type and its possible extra parameters:
357  if (n == tok.size() || tok[n] == "NONE")
358  {
359  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
360  }
361  else if (tok[n] == "DFP")
362  {
363  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_DFP;
364  if (tok.size() != n+2)
365  throw std::range_error("parseTensorSpecs: In "+spec+", DFP quantization needs :fl (" + specdef + ')');
366  attr.dtype.fl = std::stoi(tok[n+1]);
367  }
368 
369  else if (tok[n] == "AA" || tok[n] == "AS") // affine asymmetric and symmetric same, see ovxlib/vsi_nn_tensor.h
370  {
371  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
372  if (tok.size() != n+3)
373  throw std::range_error("parseTensorSpecs: In "+spec+", AA/AS quantization needs :scale:zero ("+specdef+')');
374  attr.dtype.scale = std::stof(tok[n+1]);
375  attr.dtype.zero_point = std::stoi(tok[n+2]);
376  }
377  else if (tok[n] == "APS")
378  {
379  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC;
380  throw std::range_error("parseTensorSpecs: In " + spec + ", AFFINE_PERCHANNEL_SYMMETRIC quant not yet supported");
381  }
382  else throw std::range_error("parseTensorSpecs: Invalid quantization type in " + spec);
383 
384  // Done with this tensor:
385  ret.emplace_back(attr);
386  }
387 
388  return ret;
389 }
390 
391 // ##############################################################################################################
392 cv::Mat jevois::dnn::attrmat(vsi_nn_tensor_attr_t const & attr, void * dataptr)
393 {
394  if (dataptr) return cv::Mat(jevois::dnn::attrdims(attr), jevois::dnn::vsi2cv(attr.dtype.vx_type), dataptr);
395  else return cv::Mat(jevois::dnn::attrdims(attr), jevois::dnn::vsi2cv(attr.dtype.vx_type));
396 }
397 
398 // ##############################################################################################################
399 std::vector<int> jevois::dnn::attrdims(vsi_nn_tensor_attr_t const & attr)
400 {
401  size_t const ndim = attr.dim_num;
402  std::vector<int> cvdims(ndim);
403  for (size_t i = 0; i < ndim; ++i) cvdims[ndim - 1 - i] = attr.size[i];
404  return cvdims;
405 }
406 
407 // ##############################################################################################################
408 cv::Size jevois::dnn::attrsize(vsi_nn_tensor_attr_t const & attr)
409 {
410  switch (attr.dtype.fmt)
411  {
412  case VSI_NN_DIM_FMT_NHWC:
413  if (attr.dim_num < 3) throw std::range_error("attrsize: need at least 3D, got " + jevois::dnn::attrstr(attr));
414  return cv::Size(attr.size[1], attr.size[2]);
415 
416  case VSI_NN_DIM_FMT_NCHW:
417  if (attr.dim_num < 2) throw std::range_error("attrsize: need at least 2D, got " + jevois::dnn::attrstr(attr));
418  return cv::Size(attr.size[0], attr.size[1]);
419 
420  case VSI_NN_DIM_FMT_AUTO:
421  if (attr.dim_num < 2) throw std::range_error("attrsize: need at least 2D, got " + jevois::dnn::attrstr(attr));
422  if (attr.dim_num < 3) return cv::Size(attr.size[0], attr.size[1]);
423  // ok, size[] starts with either CWH (when dim index goes 0..2) or WHC, assume C<H
424  if (attr.size[0] > attr.size[2]) return cv::Size(attr.size[0], attr.size[1]); // WHCN
425  else return cv::Size(attr.size[1], attr.size[2]); // CWHN
426 
427  default:
428  throw std::range_error("attrsize: cannot extract width and height, got " + jevois::dnn::attrstr(attr));
429  }
430 }
431 
432 // ##############################################################################################################
433 std::string jevois::dnn::attrstr(vsi_nn_tensor_attr_t const & attr)
434 {
435  std::string ret;
436 
437  // Dimension ordering:
438  switch (attr.dtype.fmt)
439  {
440  case VSI_NN_DIM_FMT_NCHW: ret += "NCHW:"; break;
441  case VSI_NN_DIM_FMT_NHWC: ret += "NHWC:"; break;
442  default: break;
443  }
444 
445  // Value type:
446  switch (attr.dtype.vx_type)
447  {
448  case VSI_NN_TYPE_UINT8: ret += "8U:"; break;
449  case VSI_NN_TYPE_INT8: ret += "8S:"; break;
450  case VSI_NN_TYPE_BOOL8: ret += "8B:"; break;
451  case VSI_NN_TYPE_UINT16: ret += "16U:"; break;
452  case VSI_NN_TYPE_INT16: ret += "16S:"; break;
453  case VSI_NN_TYPE_FLOAT16: ret += "16F:"; break;
454  case VSI_NN_TYPE_BFLOAT16: ret += "16B:"; break;
455  case VSI_NN_TYPE_UINT32: ret += "32U:"; break;
456  case VSI_NN_TYPE_INT32: ret += "32S:"; break;
457  case VSI_NN_TYPE_FLOAT32: ret += "32F:"; break;
458  case VSI_NN_TYPE_UINT64: ret += "64U:"; break;
459  case VSI_NN_TYPE_INT64: ret += "64S:"; break;
460  case VSI_NN_TYPE_FLOAT64: ret += "64F:"; break;
461  default: ret += "TYPE_UNKNOWN";
462  }
463 
464  // Dims:
465  for (uint32_t i = 0; i < attr.dim_num; ++i)
466  ret += std::to_string(attr.size[attr.dim_num - 1 - i]) + ((i<attr.dim_num-1) ? 'x' : ':');
467 
468  // Quantization:
469  switch (attr.dtype.qnt_type)
470  {
471  case VSI_NN_QNT_TYPE_NONE: ret += "NONE"; break;
472  case VSI_NN_QNT_TYPE_DFP: ret += "DFP:" + std::to_string(attr.dtype.fl); break;
473  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: // same value as VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC:
474  ret += "AA:" + std::to_string(attr.dtype.scale) + ':' + std::to_string(attr.dtype.zero_point);
475  break;
476  case VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC: ret += "APS:unsupported"; break;
477  default: ret += "QUANT_UNKNOWN";
478  }
479 
480  return ret;
481 }
482 
483 // ##############################################################################################################
484 vsi_nn_tensor_attr_t jevois::dnn::tensorattr(TfLiteTensor const * t)
485 {
486  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
487  attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO;
488  attr.dtype.vx_type = jevois::dnn::tf2vsi(t->type);
489 
490  switch (t->quantization.type)
491  {
492  case kTfLiteNoQuantization:
493  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_NONE;
494  break;
495 
496  case kTfLiteAffineQuantization:
497  {
498  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
499  attr.dtype.scale = t->params.scale;
500  attr.dtype.zero_point = t->params.zero_point;
501  }
502  break;
503 
504  default: LFATAL("unsupported quantization " << t->quantization.type);
505  }
506 
507  TfLiteIntArray const & dims = *t->dims;
508  attr.dim_num = dims.size;
509  for (int i = 0; i < dims.size; ++i) attr.size[dims.size - 1 - i] = dims.data[i];
510 
511  // Set the fmt to NCHW or NHWC if possible:
512  if (attr.dim_num == 4)
513  {
514  if (attr.size[0] > attr.size[2]) attr.dtype.fmt = VSI_NN_DIM_FMT_NCHW; // assume H>C
515  else attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
516  }
517 
518  return attr;
519 }
520 
521 // ##############################################################################################################
522 #ifdef JEVOIS_PRO
523 vsi_nn_tensor_attr_t jevois::dnn::tensorattr(hailo_vstream_info_t const & vi)
524 {
525  vsi_nn_tensor_attr_t attr; memset(&attr, 0, sizeof(attr));
526 
527  attr.dtype.vx_type = hailo2vsi(vi.format.type);
528 
529  switch (vi.format.order)
530  {
531  case HAILO_FORMAT_ORDER_HAILO_NMS:
532  attr.dtype.fmt = VSI_NN_DIM_FMT_AUTO;
533  attr.dim_num = 2;
534  attr.size[0] = vi.nms_shape.number_of_classes;
535  attr.size[1] = vi.nms_shape.max_bboxes_per_class * 5; // Each box has: xmin, ymin, xmax, ymax, score
536  break;
537 
538  case HAILO_FORMAT_ORDER_NHWC:
539  case HAILO_FORMAT_ORDER_FCR:
540  case HAILO_FORMAT_ORDER_F8CR:
541  attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
542  attr.dim_num = 4;
543  attr.size[0] = vi.shape.features;
544  attr.size[1] = vi.shape.width;
545  attr.size[2] = vi.shape.height;
546  attr.size[3] = 1;
547  break;
548 
549  case HAILO_FORMAT_ORDER_NHW:
550  attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
551  attr.dim_num = 4;
552  attr.size[0] = 1;
553  attr.size[1] = vi.shape.width;
554  attr.size[2] = vi.shape.height;
555  attr.size[3] = 1;
556  break;
557 
558  case HAILO_FORMAT_ORDER_NC:
559  attr.dtype.fmt = VSI_NN_DIM_FMT_NHWC;
560  attr.dim_num = 4;
561  attr.size[0] = 1;
562  attr.size[1] = 1;
563  attr.size[2] = vi.shape.features;
564  attr.size[3] = 1;
565  break;
566 
567  case HAILO_FORMAT_ORDER_NCHW:
568  attr.dtype.fmt = VSI_NN_DIM_FMT_NCHW;
569  attr.dim_num = 4;
570  attr.size[0] = vi.shape.features;
571  attr.size[1] = vi.shape.width;
572  attr.size[2] = vi.shape.height;
573  attr.size[3] = 1;
574  break;
575 
576  default: throw std::range_error("tensorattr: Unsupported Hailo order " +std::to_string(vi.format.order));
577  }
578 
579  // Hailo only supports one quantization type:
580  attr.dtype.qnt_type = VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC;
581  attr.dtype.scale = vi.quant_info.qp_scale;
582  attr.dtype.zero_point = int32_t(vi.quant_info.qp_zp);
583 
584  return attr;
585 }
586 #endif
587 
588 // ##############################################################################################################
589 size_t jevois::dnn::softmax(float const * input, size_t const n, size_t const stride, float const fac, float * output,
590  bool maxonly)
591 {
592  if (stride == 0) LFATAL("Cannot work with stride = 0");
593 
594  float sum = 0.0F;
595  float largest = -FLT_MAX; size_t largest_idx = 0;
596  size_t const ns = n * stride;
597 
598  for (size_t i = 0; i < ns; i += stride) if (input[i] > largest) { largest = input[i]; largest_idx = i; }
599 
600  if (fac == 1.0F)
601  for (size_t i = 0; i < ns; i += stride)
602  {
603  float const e = expf(input[i] - largest);
604  sum += e;
605  output[i] = e;
606  }
607  else
608  for (size_t i = 0; i < ns; i += stride)
609  {
610  float const e = expf(input[i]/fac - largest/fac);
611  sum += e;
612  output[i] = e;
613  }
614 
615  if (sum)
616  {
617  if (maxonly) output[largest_idx] /= sum;
618  else for (size_t i = 0; i < ns; i += stride) output[i] /= sum;
619  }
620 
621  return largest_idx;
622 }
623 
624 // ##############################################################################################################
625 bool jevois::dnn::attrmatch(vsi_nn_tensor_attr_t const & attr, cv::Mat const & blob)
626 {
627  // Check that blob and tensor are a complete match:
628  if (blob.channels() != 1) return false;
629  if (blob.depth() != jevois::dnn::vsi2cv(attr.dtype.vx_type)) return false;
630  if (uint32_t(blob.size.dims()) != attr.dim_num) return false;
631 
632  for (size_t i = 0; i < attr.dim_num; ++i)
633  if (int(attr.size[attr.dim_num - 1 - i]) != blob.size[i]) return false;
634 
635  return true;
636 }
637 
638 // ##############################################################################################################
639 cv::Mat jevois::dnn::quantize(cv::Mat const & m, vsi_nn_tensor_attr_t const & attr)
640 {
641  if (m.depth() != CV_32F) LFATAL("Tensor to quantize must be 32F");
642 
643  // Do a sloppy match for total size only since m may still be 2D RGB packed vs 4D attr...
644  std::vector<int> adims = jevois::dnn::attrdims(attr);
645  size_t tot = 1; for (int d : adims) tot *= d;
646 
647  if (tot != m.total() * m.channels())
648  LFATAL("Mismatched tensor: " << jevois::dnn::shapestr(m) << " vs attr: " << jevois::dnn::shapestr(attr));
649 
650  unsigned int const tt = jevois::dnn::vsi2cv(attr.dtype.vx_type);
651 
652  switch (attr.dtype.qnt_type)
653  {
654  case VSI_NN_QNT_TYPE_NONE:
655  {
656  cv::Mat ret;
657  m.convertTo(ret, tt);
658  return ret;
659  }
660 
661  case VSI_NN_QNT_TYPE_DFP:
662  {
663  switch (tt)
664  {
665  case CV_8S:
666  {
667  if (attr.dtype.fl > 7) LFATAL("Invalid DFP fl value " << attr.dtype.fl << ": must be in [0..7]");
668  cv::Mat ret;
669  m.convertTo(ret, tt, 1 << attr.dtype.fl, 0.0);
670  return ret;
671  }
672  case CV_16S:
673  {
674  if (attr.dtype.fl > 15) LFATAL("Invalid DFP fl value " << attr.dtype.fl << ": must be in [0..15]");
675  cv::Mat ret;
676  m.convertTo(ret, tt, 1 << attr.dtype.fl, 0.0);
677  return ret;
678  }
679  default:
680  break;
681  }
682  break;
683  }
684 
685  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: // same value as VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC:
686  {
687  switch (tt)
688  {
689  case CV_8U:
690  {
691  cv::Mat ret;
692  if (attr.dtype.scale == 0.0) LFATAL("Quantization scale must not be zero in " << jevois::dnn::shapestr(attr));
693  m.convertTo(ret, tt, 1.0 / attr.dtype.scale, attr.dtype.zero_point);
694  return ret;
695  }
696 
697  default:
698  break;
699  }
700  break;
701  }
702 
703  case VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC:
704  LFATAL("Affine per-channel symmetric not supported yet");
705 
706  default:
707  break;
708  }
709 
710  LFATAL("Quantization to " << jevois::dnn::shapestr(attr) << " not yet supported");
711 }
712 
713 // ##############################################################################################################
714 cv::Mat jevois::dnn::dequantize(cv::Mat const & m, vsi_nn_tensor_attr_t const & attr)
715 {
716  if (! jevois::dnn::attrmatch(attr, m))
717  LFATAL("Mismatched tensor: " << jevois::dnn::shapestr(m) << " vs attr: " << jevois::dnn::shapestr(attr));
718 
719  switch (attr.dtype.qnt_type)
720  {
721  case VSI_NN_QNT_TYPE_NONE:
722  {
723  cv::Mat ret;
724  m.convertTo(ret, CV_32F);
725  return ret;
726  }
727 
728  case VSI_NN_QNT_TYPE_DFP:
729  {
730  cv::Mat ret;
731  m.convertTo(ret, CV_32F, 1.0 / (1 << attr.dtype.fl), 0.0);
732  return ret;
733  }
734 
735  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: // same value as VSI_NN_QNT_TYPE_AFFINE_SYMMETRIC:
736  {
737  cv::Mat ret;
738  m.convertTo(ret, CV_32F);
739  if (attr.dtype.zero_point) ret -= attr.dtype.zero_point;
740  if (attr.dtype.scale != 1.0F) ret *= attr.dtype.scale;
741  return ret;
742  }
743 
744  case VSI_NN_QNT_TYPE_AFFINE_PERCHANNEL_SYMMETRIC:
745  LFATAL("Affine per-channel symmetric not supported yet");
746 
747  default:
748  LFATAL("Unknown quantization type " << int(attr.dtype.qnt_type));
749  }
750 }
751 
752 // ##############################################################################################################
753 size_t jevois::dnn::effectiveDims(cv::Mat const & m)
754 {
755  cv::MatSize const & rs = m.size;
756  size_t const ndims = rs.dims();
757  size_t ret = ndims;
758  for (size_t i = 0; i < ndims; ++i) if (rs[i] == 1) --ret; else break;
759  return ret;
760 }
761 
jevois::dnn::clamp
void clamp(cv::Rect &r, int width, int height)
Clamp a rectangle to within given image width and height.
Definition: Utils.C:288
jevois::dnn::tf2cv
int tf2cv(TfLiteType t)
Convert from TensorFlow data type to OpenCV.
Definition: Utils.C:207
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:258
jevois::cvtypestr
std::string cvtypestr(unsigned int cvtype)
Convert cv::Mat::type() code to to a string (e.g., CV_8UC1, CV_32SC3, etc)
Definition: Utils.C:58
Utils.H
jevois::dnn::softmax
size_t softmax(float const *input, size_t const n, size_t const stride, float const fac, float *output, bool maxonly)
Apply softmax to a float vector.
Definition: Utils.C:589
jevois::dnn::dequantize
cv::Mat dequantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Dequantize an output to float32 according to the quantization spec in attr.
Definition: Utils.C:714
jevois::dnn::vsi2cv
int vsi2cv(vsi_nn_type_e t)
Convert from NPU data type to OpenCV.
Definition: Utils.C:229
jevois::dnn::tensorattr
vsi_nn_tensor_attr_t tensorattr(TfLiteTensor const *t)
Get tensor shape and type attributes for a TensorFlow Lite tensor.
Definition: Utils.C:484
jevois::dnn::stringToRGBA
int stringToRGBA(std::string const &label, unsigned char alpha=128)
Compute a color from a label name.
Definition: Utils.C:75
jevois::dnn::readLabelsFile
std::map< int, std::string > readLabelsFile(std::string const &fname)
Read a label file.
Definition: Utils.C:24
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:211
jevois::dnn::effectiveDims
size_t effectiveDims(cv::Mat const &m)
Returns the number of non-unit dims in a cv::Mat.
Definition: Utils.C:753
jevois::dnn::parseTensorSpecs
std::vector< vsi_nn_tensor_attr_t > parseTensorSpecs(std::string const &specs)
Parse tensor specification.
Definition: Utils.C:308
jevois::dnn::attrmatch
bool attrmatch(vsi_nn_tensor_attr_t const &attr, cv::Mat const &blob)
Check that a cv::Mat blob matches exactly the spec of an attr.
Definition: Utils.C:625
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
F
float F
Definition: GUIhelper.C:1968
jevois::dnn::getLabel
std::string getLabel(std::map< int, std::string > const &labels, int id)
Get a label from an id.
Definition: Utils.C:67
Log.H
jevois::replaceStringAll
size_t replaceStringAll(std::string &str, std::string const &from, std::string const &to)
Replace all instances of 'from' with 'to'.
Definition: Utils.C:332
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::dnn::attrdims
std::vector< int > attrdims(vsi_nn_tensor_attr_t const &attr)
Get a tensor dims as a vector of int, useful to construct a matching cv::Mat.
Definition: Utils.C:399
jevois::dnn::strshape
std::vector< size_t > strshape(std::string const &str)
Get a vector of size_t from a string containing AxBxC...
Definition: Utils.C:198
jevois::dnn::hailo2vsi
vsi_nn_type_e hailo2vsi(hailo_format_type_t t)
Convert from Hailo data type to vsi_nn.
Definition: Utils.C:274
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's (width, height) size in cv::Size format, skipping over other dimensions.
Definition: Utils.C:408
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:433
jevois::dnn::attrmat
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition: Utils.C:392
Utils.H
jevois::dnn::tf2vsi
vsi_nn_type_e tf2vsi(TfLiteType t)
Convert from TensorFlow data type to vsi_nn.
Definition: Utils.C:251
jevois::dnn::quantize
cv::Mat quantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Quantize from float32 to fixed-point according to the quantization spec in attr.
Definition: Utils.C:639
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
jevois::dnn::topK
void topK(float const *pfProb, float *pfMaxProb, uint32_t *pMaxClass, uint32_t outputCount, uint32_t topNum)
Get top-k entries and their indices.
Definition: Utils.C:84