JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PreProcessorBlob.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/DNN/Utils.H>
21 
22 #include <opencv2/dnn.hpp>
23 #include <opencv2/imgproc/imgproc.hpp>
24 
25 // ####################################################################################################
27 { }
28 
29 // ####################################################################################################
30 void jevois::dnn::PreProcessorBlob::freeze(bool JEVOIS_UNUSED_PARAM(doit))
31 { }
32 
33 // ####################################################################################################
34 std::vector<cv::Mat> jevois::dnn::PreProcessorBlob::process(cv::Mat const & img, bool swaprb,
35  std::vector<vsi_nn_tensor_attr_t> const & attrs,
36  std::vector<cv::Rect> & crops)
37 {
38  float sc = scale::get(); bool docrop = letterbox::get();
39 
40  // Get the blobs:
41  std::vector<cv::Mat> blobs;
42  for (vsi_nn_tensor_attr_t const & attr : attrs)
43  {
44  cv::Mat blob;
45  cv::Size bsiz = jevois::dnn::attrsize(attr);
46  cv::Rect crop;
47 
48  // Start with an unscaled crop:
49  unsigned int bw = bsiz.width, bh = bsiz.height;
50  if (bw == 1 || bw == 3 || bh == 1 || bh == 3)
51  LFATAL("Incorrect input tensor " << jevois::dnn::shapestr(attr) <<"; did you swap NHWC vs NCHW?");
52 
53  if (docrop)
54  {
55  jevois::applyLetterBox(bw, bh, img.cols, img.rows, false);
56 
57  cv::Rect roi;
58  roi.x = (img.cols - bw) / 2;
59  roi.y = (img.rows - bh) / 2;
60  roi.width = bw;
61  roi.height = bh;
62  blob = img(roi);
63 
64  crop.x = (img.cols - bw) / 2;
65  crop.y = (img.rows - bh) / 2;
66  crop.width = bw;
67  crop.height = bh;
68  }
69  else
70  {
71  blob = img;
72 
73  crop.x = 0;
74  crop.y = 0;
75  crop.width = img.cols;
76  crop.height = img.rows;
77  }
78 
79  // Resize to desired network input dims:
80  cv::resize(blob, blob, bsiz);
81 
82  // Swap red/blue byte order if we have color and will not do planar; would be better below except that cvtColor
83  // always outputs 8U pixels so we have to do this here before possible conversion to 8S:
84  bool swapped = false;
85  if (swaprb && attr.dtype.fmt == VSI_NN_DIM_FMT_NHWC)
86  {
87  switch (blob.channels())
88  {
89  case 3: cv::cvtColor(blob, blob, cv::COLOR_RGB2BGR); swapped = true; break;
90  case 4: cv::cvtColor(blob, blob, cv::COLOR_RGBA2BGRA); swapped = true; break;
91  default: break; // Ignore swaprb value if not 3 or 4 channels
92  }
93  }
94 
95  // Convert if needed:
96  unsigned int tt = jevois::dnn::vsi2cv(attr.dtype.vx_type);
97  unsigned int bt = blob.depth();
98  if (bt == tt)
99  {
100  // Ready to go, no conversion
101  }
102  else if (bt == CV_8U && tt == CV_8S)
103  {
104  // Convert from 8U to 8S: need DFP quantization:
105  if (attr.dtype.qnt_type != VSI_NN_QNT_TYPE_DFP) LFATAL("Need DFP quantization for 8S intensors");
106  cv::Mat newblob(bsiz, CV_MAKETYPE(tt, blob.channels()));
107 
108  uint8_t const * bdata = (uint8_t const *)blob.data;
109  uint32_t const sz = blob.total() * blob.channels();
110  int8_t * data = (int8_t *)newblob.data;
111  int const shift = 8 - attr.dtype.fl;
112  for (uint32_t i = 0; i < sz; i++) data[i] = bdata[i] >> shift;
113  blob = newblob;
114  }
115  else
116  {
117  // This is the slowest path... you should add optimizations above for some specific cases:
118  blob.convertTo(blob, CV_32F);
119 
120  // Apply mean and scale:
121  cv::Scalar m = mean::get();
122  if (swaprb && swapped == false) std::swap(m[0], m[2]);
123  if (m != cv::Scalar()) blob -= m;
124  if (sc != 1.0F) blob *= sc;
125 
126  if (tt != CV_32F) blob.convertTo(blob, tt);
127  /*
128  if (attr.dtype.vx_type != VSI_NN_TYPE_FLOAT32)
129  {
130  // Convert to tensor type, applying whatever quantization is specified in attr:
131  cv::Mat newblob(bsiz, blob.type());
132  float const * fdata = (float const *)blob.data;
133  uint32_t const sz = blob.total() * blob.channels();
134  uint32_t const stride = vsi_nn_TypeGetBytes(attr.dtype.vx_type);
135  uint8_t * data = (uint8_t *)newblob.data;
136  for (uint32_t i = 0; i < sz; i++) vsi_nn_Float32ToDtype(fdata[i], &data[stride * i], &attr.dtype);
137  blob = newblob;
138  }
139  */
140  }
141 
142  // Ok, blob has desired width, height, and type, but is still packed RGB. Now deal with making a 4D shape, and R/G
143  // swapping if we have channels:
144  int const nch = blob.channels();
145  switch (nch)
146  {
147  case 1:
148  break; // Nothing to do
149 
150  case 3:
151  case 4:
152  {
153  switch (attr.dtype.fmt)
154  {
155  case VSI_NN_DIM_FMT_NCHW:
156  {
157  // Convert from packed to planar:
158  int dims[] = { 1, nch, blob.rows, blob.cols };
159  cv::Mat newblob(4, dims, tt);
160 
161  // Create some pointers in newblob for each channel:
162  cv::Mat nbc[nch];
163  for (int i = 0; i < nch; ++i) nbc[i] = cv::Mat(blob.rows, blob.cols, tt, newblob.ptr(0, i));
164  if (swaprb) std::swap(nbc[0], nbc[2]);
165 
166  // Split:
167  cv::split(blob, nbc);
168 
169  // This our final 4D blob:
170  blob = newblob;
171  }
172  break;
173 
174  case VSI_NN_DIM_FMT_NHWC:
175  {
176  // red/blue byte swap was handled above...
177 
178  // Finally convert to a 4D blob:
179  blob = blob.reshape(1, { 1, bsiz.height, bsiz.width, 3 });
180  }
181  break;
182 
183  default: LFATAL("Can only handle NCHW or NHWC intensors shapes");
184  }
185  }
186  break;
187 
188  default: LFATAL("Can only handle input images with 1, 3, or 4 channels");
189  }
190 
191  // Done with this blob:
192  blobs.emplace_back(blob);
193  crops.emplace_back(crop);
194  }
195  return blobs;
196 }
197 
198 // ####################################################################################################
200  jevois::RawImage * JEVOIS_UNUSED_PARAM(outimg),
201  jevois::OptGUIhelper * JEVOIS_UNUSED_PARAM(helper),
202  bool JEVOIS_UNUSED_PARAM(overlay),
203  bool JEVOIS_UNUSED_PARAM(idle))
204 {
205 
206 }
207 
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
RawImageOps.H
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:257
jevois::dnn::PreProcessorBlob::process
std::vector< cv::Mat > process(cv::Mat const &img, bool swaprb, std::vector< vsi_nn_tensor_attr_t > const &attrs, std::vector< cv::Rect > &crops) override
Extract blobs from input image.
Definition: PreProcessorBlob.C:34
jevois::dnn::PreProcessorBlob::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PreProcessorBlob.C:199
Utils.H
jevois::dnn::PreProcessorBlob::~PreProcessorBlob
virtual ~PreProcessorBlob()
Destructor.
Definition: PreProcessorBlob.C:26
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::dnn::vsi2cv
int vsi2cv(vsi_nn_type_e t)
Convert from NPU data type to OpenCV.
Definition: Utils.C:205
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:105
F
float F
Definition: GUIhelper.C:2150
jevois::applyLetterBox
void applyLetterBox(unsigned int &imw, unsigned int &imh, unsigned int const winw, unsigned int const winh, bool noalias)
Apply a letterbox resizing to fit an image into a window.
Definition: Utils.C:209
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's size in cv::Size format.
Definition: Utils.C:344
PreProcessorBlob.H
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
jevois::dnn::PreProcessorBlob::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PreProcessorBlob.C:30