JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PreProcessor.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/DNN/Utils.H>
21 #include <jevois/Util/Utils.H>
22 #include <jevois/Core/Engine.H>
23 
24 // ####################################################################################################
26 { }
27 
28 // ####################################################################################################
29 std::vector<cv::Mat> const & jevois::dnn::PreProcessor::blobs() const
30 { return itsBlobs; }
31 
32 // ####################################################################################################
33 cv::Size const & jevois::dnn::PreProcessor::imagesize() const
34 { return itsImageSize; }
35 
36 // ####################################################################################################
37 cv::Size jevois::dnn::PreProcessor::blobsize(size_t num) const
38 {
39  if (num >= itsAttrs.size()) LFATAL("Invalid blob number " << num << ", only have " << itsAttrs.size() << " blobs");
40  return jevois::dnn::attrsize(itsAttrs[num]);
41 }
42 
43 // ####################################################################################################
44 void jevois::dnn::PreProcessor::b2i(float & x, float & y, size_t blobnum)
45 {
46  if (blobnum >= itsCrops.size())
47  LFATAL("Invalid blob number " << blobnum << ", only have " << itsCrops.size() << " crops");
48 
49  cv::Rect const & r = itsCrops[blobnum];
50  b2i(x, y, blobsize(blobnum), (r.x != 0 || r.y != 0));
51 }
52 
53 // ####################################################################################################
54 void jevois::dnn::PreProcessor::b2i(float & x, float & y, cv::Size const & bsiz, bool letterboxed)
55 {
56  if (bsiz.width == 0 || bsiz.height == 0) LFATAL("Cannot handle zero blob width or height");
57 
58  if (letterboxed)
59  {
60  // We did letterbox and crop, so we need to apply scale and offset:
61  float const fac = std::min(itsImageSize.width / float(bsiz.width), itsImageSize.height / float(bsiz.height));
62  float const cropw = fac * bsiz.width + 0.4999F;
63  float const croph = fac * bsiz.height + 0.4999F;
64  x = (itsImageSize.width - cropw) * 0.5F + x * fac;
65  y = (itsImageSize.height - croph) * 0.5F + y * fac;
66  }
67  else
68  {
69  x *= itsImageSize.width / float(bsiz.width);
70  y *= itsImageSize.height / float(bsiz.height);
71  }
72 }
73 
74 // ####################################################################################################
76 {
77  if (num >= itsCrops.size()) LFATAL("Invalid blob number " << num << ", only have " << itsCrops.size() << " blobs");
78  return itsCrops[num];
79 }
80 
81 // ####################################################################################################
82 void jevois::dnn::PreProcessor::getUnscaledCropRect(size_t num, int & tlx, int & tly, int & brx, int & bry)
83 {
84  cv::Rect const & r = getUnscaledCropRect(num);
85  tlx = r.x; tly = r.y; brx = r.x + r.width; bry = r.y + r.height;
86 }
87 
88 // ####################################################################################################
89 std::vector<cv::Mat> jevois::dnn::PreProcessor::process(jevois::RawImage const & img,
90  std::vector<vsi_nn_tensor_attr_t> const & attrs)
91 {
92  // Store input image size and format for future use:
93  itsImageSize.width = img.width; itsImageSize.height = img.height; itsImageFmt = img.fmt;
94  itsCrops.clear(); itsBlobs.clear();
95 
96  if (itsAttrs.empty()) itsAttrs = attrs;
97  if (itsAttrs.empty()) LFATAL("Cannot work with no input tensors");
98 
99  // Do the pre-processing:
100  if (img.fmt == V4L2_PIX_FMT_RGB24)
101  itsBlobs = process(jevois::rawimage::cvImage(img), rgb::get(), itsAttrs, itsCrops);
102  else if (img.fmt == V4L2_PIX_FMT_BGR24)
103  itsBlobs = process(jevois::rawimage::cvImage(img), ! rgb::get(), itsAttrs, itsCrops);
104  else if (rgb::get())
105  itsBlobs = process(jevois::rawimage::convertToCvRGB(img), false, itsAttrs, itsCrops);
106  else
107  itsBlobs = process(jevois::rawimage::convertToCvBGR(img), false, itsAttrs, itsCrops);
108 
109  return itsBlobs;
110 }
111 
112 // ####################################################################################################
114  jevois::OptGUIhelper * helper, bool overlay, bool idle)
115 {
116  // First some info about the input:
117 #ifdef JEVOIS_PRO
118  if (helper && idle == false && ImGui::CollapsingHeader("Pre-Processing", ImGuiTreeNodeFlags_DefaultOpen))
119  {
120  ImGui::BulletText("Input image: %dx%d %s", itsImageSize.width, itsImageSize.height,
121  jevois::fccstr(itsImageFmt).c_str());
122 
123  if (itsImageFmt != V4L2_PIX_FMT_RGB24 && itsImageFmt != V4L2_PIX_FMT_BGR24)
124  {
125  if (rgb::get()) ImGui::BulletText("Convert to RGB");
126  else ImGui::BulletText("Convert to BGR");
127  }
128  }
129 #endif
130 
131  // Now a report from the derived class:
132  report(mod, outimg, helper, overlay, idle);
133 
134  // If desired, draw a rectangle around the network input:
135  if (showin::get())
136  {
137 #ifdef JEVOIS_PRO
138  if (helper)
139  for (cv::Rect const & r : itsCrops)
140  ImGui::GetBackgroundDrawList()->AddRect(helper->i2d(r.x, r.y),
141  helper->i2d(r.x + r.width, r.y + r.height), 0x80808080, 0, 0, 5);
142 #else
143  (void)helper; // keep compiler happy
144 #endif
145 
146  if (outimg)
147  for (cv::Rect const & r : itsCrops)
148  jevois::rawimage::drawRect(*outimg, r.x, r.y, r.width, r.height, 3, jevois::yuyv::MedGrey);
149  }
150 
151  // Finally some info about the blobs:
152 #ifdef JEVOIS_PRO
153  if (helper && idle == false)
154  {
155  int idx = 0;
156  for (cv::Mat const & blob : itsBlobs)
157  {
158  cv::Rect const & r = itsCrops[idx];
159  bool const stretch = (r.x == 0 && r.y == 0);
160 
161  ImGui::BulletText("Crop %d: %dx%d @ %d,%d %s", idx, r.width, r.height, r.x, r.y,
162  stretch ? "" : "(letterbox)");
163  ImGui::BulletText("Blob %d: %s %s", idx, jevois::dnn::shapestr(blob).c_str(),
164  stretch ? "(stretch)" : "(uniform)");
165 
166  ++idx;
167  }
168  }
169 #endif
170 }
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::dnn::PreProcessor::blobsize
cv::Size blobsize(size_t num) const
Access the width and height of a given blob, accounting for NCHW or NHWC.
Definition: PreProcessor.C:37
jevois::dnn::PreProcessor::getUnscaledCropRect
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
Definition: PreProcessor.C:75
RawImageOps.H
Utils.H
jevois::rawimage::convertToCvRGB
cv::Mat convertToCvRGB(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV RGB byte.
Definition: RawImageOps.C:307
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::GUIhelper::i2d
ImVec2 i2d(ImVec2 p, char const *name=nullptr)
Convert coordinates of a point from within a rendered image to on-screen.
Definition: GUIhelper.C:343
jevois::dnn::PreProcessor::blobs
const std::vector< cv::Mat > & blobs() const
Access the last computed blobs (or empty if process() has not yet been called)
Definition: PreProcessor.C:29
jevois::rawimage::convertToCvBGR
cv::Mat convertToCvBGR(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV BGR byte.
Definition: RawImageOps.C:282
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois::RawImage::width
unsigned int width
Image width in pixels.
Definition: RawImage.H:145
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:105
F
float F
Definition: GUIhelper.C:2157
Engine.H
jevois::fccstr
std::string fccstr(unsigned int fcc)
Convert a V4L2 four-cc code (V4L2_PIX_FMT_...) to a 4-char string.
Definition: Utils.C:44
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
jevois::dnn::PreProcessor::sendreport
virtual void sendreport(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)
Report what happened in last process() to console/output video/GUI.
Definition: PreProcessor.C:113
jevois::RawImage::height
unsigned int height
Image height in pixels.
Definition: RawImage.H:146
jevois::rawimage::cvImage
cv::Mat cvImage(RawImage const &src)
Create an OpenCV image from the existing RawImage data, sharing the pixel memory rather than copying ...
Definition: RawImageOps.C:31
PreProcessor.H
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's size in cv::Size format.
Definition: Utils.C:344
jevois::rawimage::drawRect
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
Draw a rectangle in a YUYV image.
Definition: RawImageOps.C:607
jevois::RawImage::fmt
unsigned int fmt
Pixel format as a V4L2_PIX_FMT_XXX.
Definition: RawImage.H:147
Utils.H
jevois::dnn::PreProcessor::imagesize
const cv::Size & imagesize() const
Access the last processed image size.
Definition: PreProcessor.C:33
jevois::dnn::PreProcessor::b2i
void b2i(float &x, float &y, size_t blobnum=0)
Convert coordinates from blob back to original image.
Definition: PreProcessor.C:44
jevois::dnn::PreProcessor::process
std::vector< cv::Mat > process(jevois::RawImage const &img, std::vector< vsi_nn_tensor_attr_t > const &attrs)
Extract blobs from input image.
Definition: PreProcessor.C:89
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
jevois::dnn::PreProcessor::~PreProcessor
virtual ~PreProcessor()
Destructor.
Definition: PreProcessor.C:25