JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Utils.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2020 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
20 #include <map>
21 #include <string>
22 #include <opencv2/core/core.hpp>
23 #include <tensorflow/lite/c/common.h> // for TfLiteType
24 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
25 
26 #ifdef JEVOIS_PRO
27 #include <hailo/hailort.h>
28 #endif
29 
30 namespace jevois
31 {
32  namespace dnn
33  {
34  /*! \defgroup dnn Tensor/Neural Processing networks
35 
36  Classes and utilities to provide abstraction to deep neural networks. Provides interfacing to OpenCV backends
37  (CPU, OpenCL), tensor processing units (TPU) such as Coral Edge TPU and neural processing units (NPU) such as
38  Amlogic A311D NPU. */
39 
40  /*! @{ */ // **********************************************************************
41 
42  //! Read a label file
43  /*! Two formats are allowed: one class name per line, or one class number followed by one class name per file.*/
44  std::map<int, std::string> readLabelsFile(std::string const & fname);
45 
46  //! Get a label from an id
47  /*! If no entry is found in the map, return the id as a string. */
48  std::string getLabel(std::map<int, std::string> const & labels, int id);
49 
50  //! Compute a color from a label name
51  int stringToRGBA(std::string const & label, unsigned char alpha = 128);
52 
53  //! Get top-k entries and their indices
54  void topK(float const * pfProb, float * pfMaxProb, uint32_t * pMaxClass, uint32_t outputCount, uint32_t topNum);
55 
56  //! Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE
57  std::string shapestr(cv::Mat const & m);
58 
59  //! Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional TfLiteTensor with data type TYPE
60  std::string shapestr(TfLiteTensor const * t);
61 
62  //! Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional NPU tensor with data type TYPE
63  std::string shapestr(vsi_nn_tensor_attr_t const & attr);
64 
65  //! Get a vector of size_t from a string containing AxBxC...
66  std::vector<size_t> strshape(std::string const & str);
67 
68  //! Convert from TensorFlow data type to OpenCV
69  int tf2cv(TfLiteType t);
70 
71  //! Convert from TensorFlow data type to vsi_nn
72  vsi_nn_type_e tf2vsi(TfLiteType t);
73 
74  //! Convert from NPU data type to OpenCV
75  int vsi2cv(vsi_nn_type_e t);
76 
77  //! Clamp a rectangle to within given image width and height
78  void clamp(cv::Rect & r, int width, int height);
79 
80  //! Clamp a rectangle to within given image width and height
81  void clamp(cv::Rect2f & r, float width, float height);
82 
83  //! Parse tensor specification
84  /*! If the specification is empty, an empty vector is returned. Throws std::range_error on any parsing error. */
85  std::vector<vsi_nn_tensor_attr_t> parseTensorSpecs(std::string const & specs);
86 
87  //! Construct a cv::Mat from attr and possibly data pointer
88  /*! If dataptr is nullptr, new memory will be allocated for the cv::Mat. Caller must ensure data outlives the
89  cv::Mat, and is responsible for eventually de-allocating the data. Usually, with non-null dataptr, this is only
90  to be used as a temporary re-casting, e.g., to recast a received tensor into a Mat before dequantizing it, then
91  forgetting about that Mat. */
92  cv::Mat attrmat(vsi_nn_tensor_attr_t const & attr, void * dataptr = nullptr);
93 
94  //! Get a tensor dims as a vector of int, useful to construct a matching cv::Mat
95  std::vector<int> attrdims(vsi_nn_tensor_attr_t const & attr);
96 
97  //! Get a tensor's (width, height) size in cv::Size format, skipping over other dimensions
98  cv::Size attrsize(vsi_nn_tensor_attr_t const & attr);
99 
100  //! Get a string describing the specs of a tensor, including quantification specs (not provided by shapestr())
101  std::string attrstr(vsi_nn_tensor_attr_t const & attr);
102 
103  //! Check that a cv::Mat blob matches exactly the spec of an attr
104  bool attrmatch(vsi_nn_tensor_attr_t const & attr, cv::Mat const & blob);
105 
106  //! Get tensor shape and type attributes for a TensorFlow Lite tensor
107  vsi_nn_tensor_attr_t tensorattr(TfLiteTensor const * t);
108 
109  //! Apply softmax to a float vector
110  /*! n is the number of elements to process, stride is the increment in the arrays from one element to the next. So
111  the arrays should have size n * stride. Returns the index in [0..n*stride[ of the highest scoring element. If
112  maxonly is true, only output[returned index] is valid. */
113  size_t softmax(float const * input, size_t const n, size_t const stride, float const fac, float * output,
114  bool maxonly);
115 
116  //! Quantize from float32 to fixed-point according to the quantization spec in attr
117  /*! m should be float32, typically normalized to [0..1[ or [-1..1[ already. attr is the desired quantized type and
118  method (DFP, AA, etc) */
119  cv::Mat quantize(cv::Mat const & m, vsi_nn_tensor_attr_t const & attr);
120 
121  //! Dequantize an output to float32 according to the quantization spec in attr
122  /*! attr should have the type and quantization details of m, returned tensor is float32 */
123  cv::Mat dequantize(cv::Mat const & m, vsi_nn_tensor_attr_t const & attr);
124 
125  //! Returns the number of non-unit dims in a cv::Mat
126  /*! For example, returns 2 for a 4D Mat with size 1x1x224x224, since it effectively is a 224x224 2D array */
127  size_t effectiveDims(cv::Mat const & m);
128 
129 #ifdef JEVOIS_PRO
130  //! Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional Hailo tensor with data type TYPE
131  std::string shapestr(hailo_vstream_info_t const & vi);
132 
133  //! Get tensor shape and type attributes for a Hailo tensor
134  vsi_nn_tensor_attr_t tensorattr(hailo_vstream_info_t const & vi);
135 
136  //! Convert from Hailo data type to vsi_nn
137  vsi_nn_type_e hailo2vsi(hailo_format_type_t t);
138 #endif
139 
140  /*! @} */ // **********************************************************************
141 
142 
143  /*! \defgroup pydnn DNN-related processors written in python
144 
145  In addition to writing DNN pre/net/post processors in C++, JeVois supports writing them in Python.
146 
147  \ingroup dnn */
148 
149 
150  } // namespace dnn
151 } // namespace jevois
jevois::dnn::clamp
void clamp(cv::Rect &r, int width, int height)
Clamp a rectangle to within given image width and height.
Definition: Utils.C:288
jevois::dnn::tf2cv
int tf2cv(TfLiteType t)
Convert from TensorFlow data type to OpenCV.
Definition: Utils.C:207
jevois::dnn::softmax
size_t softmax(float const *input, size_t const n, size_t const stride, float const fac, float *output, bool maxonly)
Apply softmax to a float vector.
Definition: Utils.C:589
jevois::dnn::dequantize
cv::Mat dequantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Dequantize an output to float32 according to the quantization spec in attr.
Definition: Utils.C:714
jevois::dnn::vsi2cv
int vsi2cv(vsi_nn_type_e t)
Convert from NPU data type to OpenCV.
Definition: Utils.C:229
jevois::dnn::tensorattr
vsi_nn_tensor_attr_t tensorattr(TfLiteTensor const *t)
Get tensor shape and type attributes for a TensorFlow Lite tensor.
Definition: Utils.C:484
jevois::dnn::stringToRGBA
int stringToRGBA(std::string const &label, unsigned char alpha=128)
Compute a color from a label name.
Definition: Utils.C:75
jevois::dnn::readLabelsFile
std::map< int, std::string > readLabelsFile(std::string const &fname)
Read a label file.
Definition: Utils.C:24
jevois::dnn::effectiveDims
size_t effectiveDims(cv::Mat const &m)
Returns the number of non-unit dims in a cv::Mat.
Definition: Utils.C:753
jevois::dnn::parseTensorSpecs
std::vector< vsi_nn_tensor_attr_t > parseTensorSpecs(std::string const &specs)
Parse tensor specification.
Definition: Utils.C:308
jevois::dnn::attrmatch
bool attrmatch(vsi_nn_tensor_attr_t const &attr, cv::Mat const &blob)
Check that a cv::Mat blob matches exactly the spec of an attr.
Definition: Utils.C:625
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois
Definition: Concepts.dox:1
jevois::dnn::getLabel
std::string getLabel(std::map< int, std::string > const &labels, int id)
Get a label from an id.
Definition: Utils.C:67
jevois::dnn::attrdims
std::vector< int > attrdims(vsi_nn_tensor_attr_t const &attr)
Get a tensor dims as a vector of int, useful to construct a matching cv::Mat.
Definition: Utils.C:399
jevois::dnn::strshape
std::vector< size_t > strshape(std::string const &str)
Get a vector of size_t from a string containing AxBxC...
Definition: Utils.C:198
jevois::dnn::hailo2vsi
vsi_nn_type_e hailo2vsi(hailo_format_type_t t)
Convert from Hailo data type to vsi_nn.
Definition: Utils.C:274
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's (width, height) size in cv::Size format, skipping over other dimensions.
Definition: Utils.C:408
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:433
jevois::dnn::attrmat
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition: Utils.C:392
jevois::dnn::tf2vsi
vsi_nn_type_e tf2vsi(TfLiteType t)
Convert from TensorFlow data type to vsi_nn.
Definition: Utils.C:251
jevois::dnn::quantize
cv::Mat quantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Quantize from float32 to fixed-point according to the quantization spec in attr.
Definition: Utils.C:639
jevois::dnn::topK
void topK(float const *pfProb, float *pfMaxProb, uint32_t *pMaxClass, uint32_t outputCount, uint32_t topNum)
Get top-k entries and their indices.
Definition: Utils.C:84