JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PreProcessorBlob.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/DNN/Utils.H>
21 
22 #include <opencv2/dnn.hpp>
23 #include <opencv2/imgproc/imgproc.hpp>
24 
25 #define DETAILS(fmt, ...) \
26  do { if (detail) itsInfo.emplace_back(prefix + jevois::sformat(fmt, ## __VA_ARGS__)); } while(0)
27 
28 #define DETAILS2(fmt, ...) \
29  do { itsInfo.emplace_back(prefix + jevois::sformat(fmt, ## __VA_ARGS__)); } while(0)
30 
31 // ####################################################################################################
33 { }
34 
35 // ####################################################################################################
36 void jevois::dnn::PreProcessorBlob::freeze(bool JEVOIS_UNUSED_PARAM(doit))
37 { }
38 
39 // ####################################################################################################
40 std::vector<cv::Mat> jevois::dnn::PreProcessorBlob::process(cv::Mat const & img, bool swaprb,
41  std::vector<vsi_nn_tensor_attr_t> const & attrs,
42  std::vector<cv::Rect> & crops)
43 {
44  bool const detail = details::get();
45  itsInfo.clear();
46  cv::Scalar m = mean::get();
47  cv::Scalar sd = stdev::get();
48  if (sd[0] == 0.0 || sd[1] == 0.0 || sd[2] == 0.0) LFATAL("stdev cannot be zero");
49  float sc = scale::get();
50  if (sc == 0.0F) LFATAL("Scale cannot be zero");
51 
52  std::vector<cv::Mat> blobs; size_t bnum = 0;
53  for (vsi_nn_tensor_attr_t const & attr : attrs)
54  {
55  // --------------------------------------------------------------------------------
56  // Get the blob:
57  cv::Mat blob;
58  cv::Size bsiz = jevois::dnn::attrsize(attr);
59  cv::Rect crop;
60  std::string prefix; if (detail) prefix = "Blob " + std::to_string(bnum) + ": ";
61 
62  // Start with an unscaled crop:
63  unsigned int bw = bsiz.width, bh = bsiz.height;
64  if (bw == 1 || bw == 3 || bh == 1 || bh == 3)
65  LFATAL("Incorrect input tensor " << jevois::dnn::shapestr(attr) <<
66  "; did you swap NHWC vs NCHW in your intensors specification?");
67 
68  // --------------------------------------------------------------------------------
69  // Compute crop rectangle:
70  if (letterbox::get())
71  {
72  jevois::applyLetterBox(bw, bh, img.cols, img.rows, false);
73 
74  cv::Rect roi;
75  roi.x = (img.cols - bw) / 2;
76  roi.y = (img.rows - bh) / 2;
77  roi.width = bw;
78  roi.height = bh;
79  blob = img(roi);
80 
81  crop.x = (img.cols - bw) / 2;
82  crop.y = (img.rows - bh) / 2;
83  crop.width = bw;
84  crop.height = bh;
85  DETAILS("Letterbox %dx%d @ %d,%d", bw, bh, crop.x, crop.y);
86  }
87  else
88  {
89  blob = img;
90 
91  crop.x = 0;
92  crop.y = 0;
93  crop.width = img.cols;
94  crop.height = img.rows;
95  }
96 
97  // --------------------------------------------------------------------------------
98  // Crop and resize to desired network input dims:
99  cv::InterpolationFlags interpflags;
100  switch (interp::get())
101  {
102  case jevois::dnn::preprocessor::InterpMode::Linear: interpflags = cv::INTER_LINEAR; break;
103  case jevois::dnn::preprocessor::InterpMode::Cubic: interpflags = cv::INTER_CUBIC; break;
104  case jevois::dnn::preprocessor::InterpMode::Area: interpflags = cv::INTER_AREA; break;
105  case jevois::dnn::preprocessor::InterpMode::Lanczos4: interpflags = cv::INTER_LANCZOS4; break;
106  default: interpflags = cv::INTER_NEAREST;
107  }
108 
109  cv::resize(blob, blob, bsiz, 0.0, 0.0, interpflags);
110  DETAILS("Resize to %dx%d%s", blob.cols, blob.rows, letterbox::get() ? "" : " (stretch)");
111 
112  // --------------------------------------------------------------------------------
113  // Swap red/blue byte order if we have color and will not do planar; would be better below except that cvtColor
114  // always outputs 8U pixels so we have to do this here before possible conversion to 8S or others:
115  bool swapped = false;
116  if (swaprb && attr.dtype.fmt == VSI_NN_DIM_FMT_NHWC)
117  {
118  switch (blob.channels())
119  {
120  case 3: cv::cvtColor(blob, blob, cv::COLOR_RGB2BGR); swapped = true; break;
121  case 4: cv::cvtColor(blob, blob, cv::COLOR_RGBA2BGRA); swapped = true; break;
122  default: break; // Ignore swaprb value if not 3 or 4 channels
123  }
124  DETAILS("Swap Red <-> Blue");
125  }
126 
127  // If we need to swap but will do it later, swap mean and std red/blue now:
128  if (swaprb && swapped == false) { std::swap(m[0], m[2]); std::swap(sd[0], sd[2]); }
129 
130  // --------------------------------------------------------------------------------
131  // Convert and quantize if needed: First try some fast paths:
132  unsigned int const tt = jevois::dnn::vsi2cv(attr.dtype.vx_type);
133  unsigned int const bt = blob.depth();
134  bool const uniformsd = (sd[0] == sd[1] && sd[1] == sd[2]);
135  bool const uniformmean = (m[0] == m[1] && m[1] == m[2]);
136  bool const unitsd = (uniformsd && sd[0] > 0.99 && sd[0] < 1.01);
137  bool notdone = true;
138 
139  if (bt == CV_8U && tt == CV_8U && attr.dtype.qnt_type == VSI_NN_QNT_TYPE_NONE)
140  {
141  DETAILS("8U to 8U direct no quantization");
142  DETAILS("(ignoring mean, scale, stdev)");
143  notdone = false;
144  }
145 
146  else if (unitsd && attr.dtype.qnt_type == VSI_NN_QNT_TYPE_DFP)
147  {
148  if (bt == CV_8U && tt == CV_8S)
149  {
150  // --------------------
151  // Convert from 8U to 8S with DFP quantization:
152  cv::Mat newblob(bsiz, CV_MAKETYPE(tt, blob.channels()));
153 
154  uint8_t const * bdata = (uint8_t const *)blob.data;
155  uint32_t const sz = blob.total() * blob.channels();
156  int8_t * data = (int8_t *)newblob.data;
157  if (attr.dtype.fl > 7) LFATAL("Invalid DFP fl value " << attr.dtype.fl << ": must be in [0..7]");
158  int const shift = 8 - attr.dtype.fl;
159  for (uint32_t i = 0; i < sz; ++i) *data++ = *bdata++ >> shift;
160 
161  DETAILS("8U to 8S DFP:%d: bit-shift >> %d", attr.dtype.fl, shift);
162  blob = newblob;
163 
164  if (m[0] > 1.0 || m[1] > 1.0 || m[2] > 1.0)
165  {
166  blob -= m;
167  DETAILS("Subtract mean [%.2f %.2f %.2f]", m[0], m[1], m[2]);
168  }
169  notdone = false;
170  }
171  else if (bt == CV_8U && tt == CV_16S)
172  {
173  // --------------------
174  // Convert from 8U to 16S with DFP quantization:
175  int const fl = attr.dtype.fl;
176  uint8_t const * bdata = (uint8_t const *)blob.data;
177  uint32_t const sz = blob.total() * blob.channels();
178  if (fl > 15) LFATAL("Invalid DFP fl value " << fl << ": must be in [0..15]");
179  if (fl > 8)
180  {
181  cv::Mat newblob(bsiz, CV_MAKETYPE(tt, blob.channels()));
182  int16_t * data = (int16_t *)newblob.data;
183  int const shift = fl - 8;
184  for (uint32_t i = 0; i < sz; ++i) *data++ = int16_t(*bdata++) << shift;
185  blob = newblob;
186  DETAILS("8U to 16S DFP:%d: bit-shift << %d", fl, shift);
187  }
188  else if (fl < 8)
189  {
190  cv::Mat newblob(bsiz, CV_MAKETYPE(tt, blob.channels()));
191  int16_t * data = (int16_t *)newblob.data;
192  int const shift = 8 - fl;
193  for (uint32_t i = 0; i < sz; ++i) *data++ = int16_t(*bdata++) >> shift;
194  blob = newblob;
195  DETAILS("8U to 16S DFP:%d: bit-shift >> %d", fl, shift);
196  }
197  else
198  {
199  blob.convertTo(blob, tt);
200  DETAILS("8U to 16S DFP:%d: direct conversion", fl);
201  }
202 
203  if (m[0] > 1.0 || m[1] > 1.0 || m[2] > 1.0)
204  {
205  blob -= m;
206  DETAILS("Subtract mean [%.2f %.2f %.2f]", m[0], m[1], m[2]);
207  }
208  notdone = false;
209  }
210  // We only handle DFP: 8U->8S and 8U->16S with unit stdev here, more general code below for other cases.
211  }
212 
213  if (notdone && uniformsd && uniformmean)
214  {
215  double qs, zp;
216  switch (attr.dtype.qnt_type)
217  {
218  case VSI_NN_QNT_TYPE_AFFINE_ASYMMETRIC: qs = attr.dtype.scale; zp = attr.dtype.zero_point; notdone = false; break;
219  case VSI_NN_QNT_TYPE_DFP: qs = 1.0 / (1 << attr.dtype.fl); zp = 0.0; notdone = false; break;
220  default: break;
221  }
222 
223  if (notdone == false)
224  {
225  if (qs == 0.0) LFATAL("Quantizer scale must not be zero");
226  double alpha = sc / (sd[0] * qs);
227  double beta = zp - m[0] * alpha;
228  if (alpha > 0.99 && alpha < 1.01) alpha = 1.0; // will run faster
229  if (beta > -0.51 && beta < 0.51) beta = 0.0; // will run faster
230 
231  if (alpha == 1.0 && beta == 0.0 && bt == tt)
232  DETAILS("No conversion needed");
233  else
234  {
235  cv::Mat newblob;
236  blob.convertTo(newblob, tt, alpha, beta);
237  blob = newblob;
238  if (detail)
239  {
240  DETAILS2("%s to %s fast path", jevois::cvtypestr(bt).c_str(), jevois::cvtypestr(tt).c_str());
241  if (m[0]) DETAILS2("Subtract mean [%.2f %.2f %.2f]", m[0], m[1], m[2]);
242  if (sd[0] != 1.0) DETAILS2("Divide by stdev [%f %f %f]", sd[0], sd[1], sd[2]);
243  if (sc != 1.0F) DETAILS2("Multiply by scale %f (=1/%.2f)", sc, 1.0/sc);
244  if (qs != 1.0F) DETAILS2("Divide by quantizer scale %f (=1/%.2f)", qs, 1.0/qs);
245  if (zp) DETAILS2("Add quantizer zero-point %.2f", zp);
246  if (alpha == 1.0 && beta == 0.0) DETAILS2("Summary: out = in");
247  else if (alpha == 1.0) DETAILS2("Summary: out = in%+f", beta);
248  else if (beta == 0.0) DETAILS2("Summary: out = in*%f", alpha);
249  else DETAILS2("Summary: out = in*%f%+f", alpha, beta);
250  }
251  }
252  }
253  }
254 
255  if (notdone)
256  {
257  // This is the slowest path... you should add optimizations above for some specific cases:
258  blob.convertTo(blob, CV_32F);
259  DETAILS("Convert to 32F");
260 
261  // Apply mean and scale:
262  if (m != cv::Scalar())
263  {
264  blob -= m;
265  DETAILS("Subtract mean [%.2f %.2f %.2f]", m[0], m[1], m[2]);
266  }
267 
268  if (sd != cv::Scalar(1.0F, 1.0F, 1.0F))
269  {
270  if (sd[0] == 0.0F || sd[1] == 0.0F || sd[2] == 0.0F) LFATAL("Parameter stdev cannot contain any zero");
271  if (sc != 1.0F && sc != 0.0F)
272  {
273  sd *= 1.0F / sc;
274  DETAILS("Divide stdev by scale %f (=1/%.2f)", sc, 1.0/sc);
275  }
276  blob /= sd;
277  DETAILS("Divide by stdev [%f %f %f]", sd[0], sd[1], sd[2]);
278  }
279  else if (sc != 1.0F)
280  {
281  blob *= sc;
282  DETAILS("Multiply by scale %f (=1/%.2f)", sc, 1.0/sc);
283  }
284 
285  if (tt == CV_16F || tt == CV_64F)
286  {
287  blob.convertTo(blob, tt);
288  DETAILS("Convert to %s", jevois::dnn::attrstr(attr).c_str());
289  }
290  else if (tt != CV_32F)
291  {
292  blob = jevois::dnn::quantize(blob, attr);
293  DETAILS("Quantize to %s", jevois::dnn::attrstr(attr).c_str());
294  }
295  }
296 
297  // --------------------------------------------------------------------------------
298  // Ok, blob has desired width, height, and type, but is still packed RGB. Now deal with making a 4D shape, and R/G
299  // swapping if we have channels:
300  int const nch = blob.channels();
301  switch (nch)
302  {
303  case 1:
304  break; // Nothing to do
305 
306  case 3:
307  case 4:
308  {
309  switch (attr.dtype.fmt)
310  {
311  case VSI_NN_DIM_FMT_NCHW:
312  {
313  // Convert from packed to planar:
314  int dims[] = { 1, nch, blob.rows, blob.cols };
315  cv::Mat newblob(4, dims, tt);
316 
317  // Create some pointers in newblob for each channel:
318  cv::Mat nbc[nch];
319  for (int i = 0; i < nch; ++i) nbc[i] = cv::Mat(blob.rows, blob.cols, tt, newblob.ptr(0, i));
320  if (swaprb)
321  {
322  std::swap(nbc[0], nbc[2]);
323  DETAILS("Swap Red <-> Blue");
324  }
325 
326  // Split:
327  cv::split(blob, nbc);
328  DETAILS("Split channels (NHWC->NCHW)");
329 
330  // This our final 4D blob:
331  blob = newblob;
332  }
333  break;
334 
335  case VSI_NN_DIM_FMT_NHWC:
336  {
337  // red/blue byte swap was handled above... Just convert to a 4D blob:
338  blob = blob.reshape(1, { 1, bsiz.height, bsiz.width, 3 });
339  }
340  break;
341 
342  default: LFATAL("Can only handle NCHW or NHWC intensors shapes");
343  }
344  }
345  break;
346 
347  default: LFATAL("Can only handle input images with 1, 3, or 4 channels");
348  }
349 
350  // --------------------------------------------------------------------------------
351  // Done with this blob:
352  DETAILS("%s", jevois::dnn::attrstr(attr).c_str());
353  blobs.emplace_back(blob);
354  crops.emplace_back(crop);
355  ++bnum;
356  }
357  return blobs;
358 }
359 
360 // ####################################################################################################
362  jevois::RawImage * JEVOIS_UNUSED_PARAM(outimg),
363  jevois::OptGUIhelper * helper, bool JEVOIS_UNUSED_PARAM(overlay), bool idle)
364 {
365 #ifdef JEVOIS_PRO
366  if (helper && idle == false)
367  for (std::string const & s : itsInfo) ImGui::BulletText(s.c_str());
368 #else
369  (void)helper; (void)idle;
370 #endif
371 }
372 
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
RawImageOps.H
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:258
jevois::dnn::PreProcessorBlob::process
std::vector< cv::Mat > process(cv::Mat const &img, bool swaprb, std::vector< vsi_nn_tensor_attr_t > const &attrs, std::vector< cv::Rect > &crops) override
Extract blobs from input image.
Definition: PreProcessorBlob.C:40
jevois::cvtypestr
std::string cvtypestr(unsigned int cvtype)
Convert cv::Mat::type() code to to a string (e.g., CV_8UC1, CV_32SC3, etc)
Definition: Utils.C:58
jevois::dnn::PreProcessorBlob::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PreProcessorBlob.C:361
Utils.H
jevois::dnn::PreProcessorBlob::~PreProcessorBlob
virtual ~PreProcessorBlob()
Destructor.
Definition: PreProcessorBlob.C:32
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::dnn::vsi2cv
int vsi2cv(vsi_nn_type_e t)
Convert from NPU data type to OpenCV.
Definition: Utils.C:229
DETAILS2
#define DETAILS2(fmt,...)
Definition: PreProcessorBlob.C:28
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
F
float F
Definition: GUIhelper.C:1968
jevois::applyLetterBox
void applyLetterBox(unsigned int &imw, unsigned int &imh, unsigned int const winw, unsigned int const winh, bool noalias)
Apply a letterbox resizing to fit an image into a window.
Definition: Utils.C:210
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's (width, height) size in cv::Size format, skipping over other dimensions.
Definition: Utils.C:408
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:433
PreProcessorBlob.H
DETAILS
#define DETAILS(fmt,...)
Definition: PreProcessorBlob.C:25
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
jevois::dnn::quantize
cv::Mat quantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Quantize from float32 to fixed-point according to the quantization spec in attr.
Definition: Utils.C:639
jevois::dnn::PreProcessorBlob::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PreProcessorBlob.C:36