JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PreProcessor.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/DNN/Utils.H>
21 #include <jevois/Util/Utils.H>
22 #include <jevois/Core/Engine.H>
24 
25 // ####################################################################################################
26 jevois::dnn::PreProcessor::PreProcessor(std::string const & instance) :
27  jevois::Component(instance), itsPP(new jevois::dnn::PreProcessorForPython(this))
28 { }
29 
30 // ####################################################################################################
32 { }
33 
34 // ####################################################################################################
35 std::vector<cv::Mat> const & jevois::dnn::PreProcessor::blobs() const
36 { return itsBlobs; }
37 
38 // ####################################################################################################
39 cv::Size const & jevois::dnn::PreProcessor::imagesize() const
40 { return itsImageSize; }
41 
42 // ####################################################################################################
43 cv::Size jevois::dnn::PreProcessor::blobsize(size_t num) const
44 {
45  if (num >= itsAttrs.size()) LFATAL("Invalid blob number " << num << ", only have " << itsAttrs.size() << " blobs");
46  return jevois::dnn::attrsize(itsAttrs[num]);
47 }
48 
49 // ####################################################################################################
50 void jevois::dnn::PreProcessor::b2i(float & x, float & y, size_t blobnum)
51 {
52  if (blobnum >= itsCrops.size())
53  LFATAL("Invalid blob number " << blobnum << ", only have " << itsCrops.size() << " crops");
54 
55  cv::Rect const & r = itsCrops[blobnum];
56  b2i(x, y, blobsize(blobnum), (r.x != 0 || r.y != 0));
57 }
58 
59 // ####################################################################################################
60 void jevois::dnn::PreProcessor::b2i(float & x, float & y, cv::Size const & bsiz, bool letterboxed)
61 {
62  if (bsiz.width == 0 || bsiz.height == 0) LFATAL("Cannot handle zero blob width or height");
63 
64  if (letterboxed)
65  {
66  // We did letterbox and crop, so we need to apply scale and offset:
67  float const fac = std::min(itsImageSize.width / float(bsiz.width), itsImageSize.height / float(bsiz.height));
68  float const cropw = fac * bsiz.width + 0.4999F;
69  float const croph = fac * bsiz.height + 0.4999F;
70  x = (itsImageSize.width - cropw) * 0.5F + x * fac;
71  y = (itsImageSize.height - croph) * 0.5F + y * fac;
72  }
73  else
74  {
75  x *= itsImageSize.width / float(bsiz.width);
76  y *= itsImageSize.height / float(bsiz.height);
77  }
78 }
79 
80 // ####################################################################################################
82 {
83  if (num >= itsCrops.size()) LFATAL("Invalid blob number " << num << ", only have " << itsCrops.size() << " blobs");
84  return itsCrops[num];
85 }
86 
87 // ####################################################################################################
88 void jevois::dnn::PreProcessor::getUnscaledCropRect(size_t num, int & tlx, int & tly, int & brx, int & bry)
89 {
90  cv::Rect const & r = getUnscaledCropRect(num);
91  tlx = r.x; tly = r.y; brx = r.x + r.width; bry = r.y + r.height;
92 }
93 
94 // ####################################################################################################
95 std::shared_ptr<jevois::dnn::PreProcessorForPython> jevois::dnn::PreProcessor::getPreProcForPy() const
96 { return itsPP; }
97 
98 // ####################################################################################################
99 std::vector<cv::Mat> jevois::dnn::PreProcessor::process(jevois::RawImage const & img,
100  std::vector<vsi_nn_tensor_attr_t> const & attrs)
101 {
102  // Store input image size and format for future use:
103  itsImageSize.width = img.width; itsImageSize.height = img.height; itsImageFmt = img.fmt;
104  itsCrops.clear(); itsBlobs.clear();
105 
106  if (itsAttrs.empty()) itsAttrs = attrs;
107  if (itsAttrs.empty()) LFATAL("Cannot work with no input tensors");
108 
109  // Do the pre-processing:
110  if (img.fmt == V4L2_PIX_FMT_RGB24)
111  itsBlobs = process(jevois::rawimage::cvImage(img), ! rgb::get(), itsAttrs, itsCrops);
112  else if (img.fmt == V4L2_PIX_FMT_BGR24)
113  itsBlobs = process(jevois::rawimage::cvImage(img), rgb::get(), itsAttrs, itsCrops);
114  else if (rgb::get())
115  itsBlobs = process(jevois::rawimage::convertToCvRGB(img), false, itsAttrs, itsCrops);
116  else
117  itsBlobs = process(jevois::rawimage::convertToCvBGR(img), false, itsAttrs, itsCrops);
118 
119  return itsBlobs;
120 }
121 
122 // ####################################################################################################
124  jevois::OptGUIhelper * helper, bool overlay, bool idle)
125 {
126  // First some info about the input:
127 #ifdef JEVOIS_PRO
128  if (helper && idle == false && ImGui::CollapsingHeader("Pre-Processing", ImGuiTreeNodeFlags_DefaultOpen))
129  {
130  ImGui::BulletText("Input image: %dx%d %s", itsImageSize.width, itsImageSize.height,
131  jevois::fccstr(itsImageFmt).c_str());
132 
133  if (itsImageFmt != V4L2_PIX_FMT_RGB24 && itsImageFmt != V4L2_PIX_FMT_BGR24)
134  {
135  if (rgb::get()) ImGui::BulletText("Convert to RGB");
136  else ImGui::BulletText("Convert to BGR");
137  }
138 
139  // Now a report from the derived class:
140  report(mod, outimg, helper, overlay, idle);
141  }
142 #endif
143 
144  // Now a report from the derived class, when not using a GUI helper:
145  if (helper == nullptr) report(mod, outimg, helper, overlay, idle);
146 
147  // If desired, draw a rectangle around the network input:
148  if (showin::get())
149  {
150 #ifdef JEVOIS_PRO
151  if (helper)
152  for (cv::Rect const & r : itsCrops)
153  ImGui::GetBackgroundDrawList()->AddRect(helper->i2d(r.x, r.y),
154  helper->i2d(r.x + r.width, r.y + r.height), 0x80808080, 0, 0, 5);
155 #endif
156 
157  if (outimg)
158  for (cv::Rect const & r : itsCrops)
159  jevois::rawimage::drawRect(*outimg, r.x, r.y, r.width, r.height, 3, jevois::yuyv::MedGrey);
160  }
161 
162  // Finally some info about the blobs, if detailed info was not requested (detailed info is provided by derived class):
163 #ifdef JEVOIS_PRO
164  if (helper && idle == false && details::get() == false)
165  {
166  int idx = 0;
167  for (cv::Mat const & blob : itsBlobs)
168  {
169  cv::Rect const & r = itsCrops[idx];
170  bool const stretch = (r.x == 0 && r.y == 0);
171 
172  ImGui::BulletText("Crop %d: %dx%d @ %d,%d %s", idx, r.width, r.height, r.x, r.y,
173  stretch ? "" : "(letterbox)");
174  ImGui::BulletText("Blob %d: %s %s", idx, jevois::dnn::shapestr(blob).c_str(),
175  stretch ? "(stretch)" : "(uniform)");
176 
177  ++idx;
178  }
179  }
180 #endif
181 }
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::dnn::PreProcessor::blobsize
cv::Size blobsize(size_t num) const
Access the width and height of a given blob, accounting for NCHW or NHWC.
Definition: PreProcessor.C:43
jevois::dnn::PreProcessor::getUnscaledCropRect
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
Definition: PreProcessor.C:81
RawImageOps.H
jevois::Component
A component of a model hierarchy.
Definition: Component.H:181
Utils.H
jevois::rawimage::convertToCvRGB
cv::Mat convertToCvRGB(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV RGB byte.
Definition: RawImageOps.C:307
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::GUIhelper::i2d
ImVec2 i2d(ImVec2 p, char const *name=nullptr)
Convert coordinates of a point from within a rendered image to on-screen.
Definition: GUIhelper.C:374
jevois::dnn::PreProcessor::blobs
const std::vector< cv::Mat > & blobs() const
Access the last computed blobs (or empty if process() has not yet been called)
Definition: PreProcessor.C:35
jevois::rawimage::convertToCvBGR
cv::Mat convertToCvBGR(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV BGR byte.
Definition: RawImageOps.C:282
jevois::dnn::PreProcessorForPython
Pre-Processor interface expose to the python side.
Definition: PythonModule.H:380
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois::RawImage::width
unsigned int width
Image width in pixels.
Definition: RawImage.H:145
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois
Definition: Concepts.dox:1
F
float F
Definition: GUIhelper.C:1968
Engine.H
jevois::fccstr
std::string fccstr(unsigned int fcc)
Convert a V4L2 four-cc code (V4L2_PIX_FMT_...) to a 4-char string.
Definition: Utils.C:45
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::dnn::PreProcessor::sendreport
virtual void sendreport(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)
Report what happened in last process() to console/output video/GUI.
Definition: PreProcessor.C:123
PythonModule.H
jevois::RawImage::height
unsigned int height
Image height in pixels.
Definition: RawImage.H:146
jevois::rawimage::cvImage
cv::Mat cvImage(RawImage const &src)
Create an OpenCV image from the existing RawImage data, sharing the pixel memory rather than copying ...
Definition: RawImageOps.C:31
PreProcessor.H
jevois::dnn::attrsize
cv::Size attrsize(vsi_nn_tensor_attr_t const &attr)
Get a tensor's (width, height) size in cv::Size format, skipping over other dimensions.
Definition: Utils.C:408
jevois::rawimage::drawRect
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
Draw a rectangle in a YUYV image.
Definition: RawImageOps.C:607
jevois::RawImage::fmt
unsigned int fmt
Pixel format as a V4L2_PIX_FMT_XXX.
Definition: RawImage.H:147
Utils.H
jevois::dnn::PreProcessor::imagesize
const cv::Size & imagesize() const
Access the last processed image size.
Definition: PreProcessor.C:39
jevois::dnn::PreProcessor::b2i
void b2i(float &x, float &y, size_t blobnum=0)
Convert coordinates from blob back to original image.
Definition: PreProcessor.C:50
jevois::dnn::PreProcessor::process
std::vector< cv::Mat > process(jevois::RawImage const &img, std::vector< vsi_nn_tensor_attr_t > const &attrs)
Extract blobs from input image.
Definition: PreProcessor.C:99
jevois::dnn::PreProcessor::getPreProcForPy
std::shared_ptr< PreProcessorForPython > getPreProcForPy() const
Get a pointer to our python-friendly interface.
Definition: PreProcessor.C:95
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
jevois::dnn::PreProcessor::~PreProcessor
virtual ~PreProcessor()
Destructor.
Definition: PreProcessor.C:31
jevois::dnn::PreProcessor::PreProcessor
PreProcessor(std::string const &instance)
Constructor.
Definition: PreProcessor.C:26