JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PreProcessor.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <opencv2/core/core.hpp>
22 #include <jevois/Image/RawImage.H>
23 #include <jevois/GPU/GUIhelper.H>
24 #include <jevois/Core/Module.H>
25 
26 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
27 
28 namespace jevois
29 {
30  namespace dnn
31  {
32  class PreProcessorForPython;
33 
34  namespace preprocessor
35  {
36  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
37  // will use different subsets of all available parameters:
38  static jevois::ParameterCategory const ParamCateg("DNN Pre-Processing Options");
39 
40  //! Parameter \relates jevois::PreProcessor
41  JEVOIS_DECLARE_PARAMETER(rgb, bool, "When true, model works with RGB input images instead BGR ones",
42  true, ParamCateg);
43 
44  //! Parameter \relates jevois::PreProcessor
45  JEVOIS_DECLARE_PARAMETER(scale, float, "Value scaling factor applied to input pixels after mean subtraction, "
46  "or 0.0 to extract an unscaled UINT8 blob, typically for use with quantized networks",
47  2.0F / 255.0F, ParamCateg);
48 
49  //! Parameter \relates jevois::PreProcessor
50  JEVOIS_DECLARE_PARAMETER(mean, cv::Scalar, "Mean values subtracted from input image, in the same RGB/BGR "
51  "order as the network's input",
52  cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg);
53 
54  //! Parameter \relates jevois::PreProcessor
55  JEVOIS_DECLARE_PARAMETER(stdev, cv::Scalar, "Input image is divided by stdev after mean subtraction and scale "
56  "factor are applied. This is rarely used. Same RGB/BGR order as the network's input",
57  cv::Scalar(1.0F, 1.0F, 1.0F), ParamCateg);
58 
59  //! Parameter \relates jevois::PreProcessor
60  JEVOIS_DECLARE_PARAMETER(letterbox, bool, "When true, extract the largest possible box from the input image "
61  "with same aspect ratio as the network's input tensor, and then rescale it to that "
62  "tensor's width and height (hence with cropping but no distortion). Otherwise, use "
63  "the whole image and rescale it to the network's input width and height with some "
64  "possible stretching.",
65  false, ParamCateg);
66 
67  //! Parameter \relates jevois::PreProcessor
68  JEVOIS_DECLARE_PARAMETER(showin, bool, "Show outline of cropped image fed to network",
69  true, ParamCateg);
70 
71  //! Parameter \relates jevois::PreProcessorPython
72  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pypre, std::string, "Full path of the python pre-processor file. Name of "
73  "class defined in the file must match the file name without the "
74  "trailing '.py'",
75  "", ParamCateg);
76 
77  //! Parameter \relates jevois::PreProcessor
78  JEVOIS_DECLARE_PARAMETER(details, bool, "Show more details about the pre-processing steps",
79  false, ParamCateg);
80 
81  //! Enum for image resizing modes \relates jevois::PreProcessor
82  JEVOIS_DEFINE_ENUM_CLASS(InterpMode, (Nearest) (Linear) (Cubic) (Area) (Lanczos4));
83 
84  //! Parameter \relates jevois::PreProcessor
85  JEVOIS_DECLARE_PARAMETER(interp, InterpMode, "Image interpolation to use when resizing the input image "
86  "from camera to network input dims",
87  InterpMode::Nearest, InterpMode_Values, ParamCateg);
88  }
89 
90  //! Pre-Processor for neural network pipeline
91  /*! This is the first step in a deep neural network processing Pipeline.
92 
93  Derived classes must implement the pure virtual methods:
94  - process(): process an input image and generate some tensors (blobs)
95  - report(): describe what process() did to humans
96  - freeze(): freeze/unfreeze parameters that users should not change at runtime
97 
98  They should keep some internal state about what to report, since report() is always called on
99  every frame, but process() may be called less often if the network is slow.
100 
101  \ingroup dnn */
103  public jevois::Parameter<preprocessor::rgb, preprocessor::showin, preprocessor::details>
104  {
105  public:
106 
107  //! Constructor
108  PreProcessor(std::string const & instance);
109 
110  //! Destructor
111  virtual ~PreProcessor();
112 
113  //! Freeze/unfreeze parameters that users should not change while running
114  virtual void freeze(bool doit) = 0;
115 
116  //! Extract blobs from input image
117  std::vector<cv::Mat> process(jevois::RawImage const & img, std::vector<vsi_nn_tensor_attr_t> const & attrs);
118 
119  //! Report what happened in last process() to console/output video/GUI
120  virtual void sendreport(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
121  jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false);
122 
123  //! Access the last processed image size
124  cv::Size const & imagesize() const;
125 
126  //! Access the last computed blobs (or empty if process() has not yet been called)
127  std::vector<cv::Mat> const & blobs() const;
128 
129  //! Access the width and height of a given blob, accounting for NCHW or NHWC
130  cv::Size blobsize(size_t num) const;
131 
132  //! Convert coordinates from blob back to original image
133  /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the blob's width and height. This is useful to
134  convert detected boxes back into original input coordinates. */
135  void b2i(float & x, float & y, size_t blobnum = 0);
136 
137  //! Convert coordinates from blob back to original image, given a known blob size
138  /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the blob's width and height. This is useful to
139  convert detected boxes back into original input coordinates. */
140  void b2i(float & x, float & y, cv::Size const & bsiz, bool letterboxed);
141 
142  //! Get unscaled crop rectangle in image coordinates
143  /*! This is useful to display an image overlay on top of the input image. */
144  cv::Rect getUnscaledCropRect(size_t blobnum = 0);
145 
146  //! Get unscaled crop rectangle in image coordinates
147  /*! This is useful to display an image overlay on top of the input image. */
148  void getUnscaledCropRect(size_t blobnum, int & tlx, int & tly, int & brx, int & bry);
149 
150  //! Get a pointer to our python-friendly interface
151  std::shared_ptr<PreProcessorForPython> getPreProcForPy() const;
152 
153  protected:
154  //! Extract blobs from input image
155  /*! isrgb should be true if the given img has RGB color order, or false for BGR. Only 3-channel byte images are
156  supported as input. */
157  virtual std::vector<cv::Mat> process(cv::Mat const & img, bool isrgb,
158  std::vector<vsi_nn_tensor_attr_t> const & attrs,
159  std::vector<cv::Rect> & crops) = 0;
160 
161  //! Report what happened in last process() to console/output video/GUI
162  virtual void report(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
163  jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false) = 0;
164 
165  private:
166  std::vector<vsi_nn_tensor_attr_t> itsAttrs;
167  std::vector<cv::Mat> itsBlobs;
168  std::vector<cv::Rect> itsCrops; // Unscaled crops, one per blob, used for rescaling from blob to image
169 
170  cv::Size itsImageSize;
171  unsigned int itsImageFmt;
172 
173  // Helper class exposed to python
174  std::shared_ptr<PreProcessorForPython> itsPP;
175  };
176 
177  } // namespace dnn
178 } // namespace jevois
jevois::dnn::PreProcessor::blobsize
cv::Size blobsize(size_t num) const
Access the width and height of a given blob, accounting for NCHW or NHWC.
Definition: PreProcessor.C:43
Module.H
jevois::dnn::PreProcessor::getUnscaledCropRect
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
Definition: PreProcessor.C:81
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
RawImage.H
jevois::Component
A component of a model hierarchy.
Definition: Component.H:181
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::dnn::PreProcessor::blobs
const std::vector< cv::Mat > & blobs() const
Access the last computed blobs (or empty if process() has not yet been called)
Definition: PreProcessor.C:35
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois
Definition: Concepts.dox:1
F
float F
Definition: GUIhelper.C:1968
Component.H
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:102
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290)(os08a10))
Enum for different sensor models.
jevois::dnn::PreProcessor::freeze
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
jevois::dnn::PreProcessor::sendreport
virtual void sendreport(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)
Report what happened in last process() to console/output video/GUI.
Definition: PreProcessor.C:123
jevois::dnn::PreProcessor::report
virtual void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)=0
Report what happened in last process() to console/output video/GUI.
jevois::dnn::PreProcessor::imagesize
const cv::Size & imagesize() const
Access the last processed image size.
Definition: PreProcessor.C:39
jevois::dnn::PreProcessor::b2i
void b2i(float &x, float &y, size_t blobnum=0)
Convert coordinates from blob back to original image.
Definition: PreProcessor.C:50
jevois::dnn::PreProcessor::process
std::vector< cv::Mat > process(jevois::RawImage const &img, std::vector< vsi_nn_tensor_attr_t > const &attrs)
Extract blobs from input image.
Definition: PreProcessor.C:99
jevois::dnn::PreProcessor::getPreProcForPy
std::shared_ptr< PreProcessorForPython > getPreProcForPy() const
Get a pointer to our python-friendly interface.
Definition: PreProcessor.C:95
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
jevois::dnn::PreProcessor::~PreProcessor
virtual ~PreProcessor()
Destructor.
Definition: PreProcessor.C:31
GUIhelper.H
jevois::dnn::PreProcessor::PreProcessor
PreProcessor(std::string const &instance)
Constructor.
Definition: PreProcessor.C:26