JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PreProcessor.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <opencv2/core/core.hpp>
22 #include <jevois/Image/RawImage.H>
23 #include <jevois/GPU/GUIhelper.H>
24 #include <jevois/Core/Module.H>
25 
26 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
27 
28 namespace jevois
29 {
30  namespace dnn
31  {
32  namespace preprocessor
33  {
34  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
35  // will use different subsets of all available parameters:
36  static jevois::ParameterCategory const ParamCateg("DNN Pre-Processing Options");
37 
38  //! Parameter \relates jevois::PreProcessor
39  JEVOIS_DECLARE_PARAMETER(rgb, bool, "When true, model works with RGB input images instead BGR ones",
40  true, ParamCateg);
41 
42  //! Parameter \relates jevois::PreProcessor
43  JEVOIS_DECLARE_PARAMETER(scale, float, "Value scaling factor applied to input pixels, or 0.0 to extract a "
44  "UINT8 blob, typically for use with quantized networks",
45  2.0F / 255.0F, ParamCateg);
46 
47  //! Parameter \relates jevois::PreProcessor
48  JEVOIS_DECLARE_PARAMETER(mean, cv::Scalar, "Mean BGR value subtracted from input image",
49  cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg);
50 
51  //! Parameter \relates jevois::PreProcessor
52  JEVOIS_DECLARE_PARAMETER(letterbox, bool, "When true, extract the largest possible box from the input image "
53  "with same aspect ratio as the network's input tensor, and then rescale it to that "
54  "tensor's width and height (hence with cropping but no distortion). Otherwise, use "
55  "the whole image and rescale it to the network's input width and height with some "
56  "possible stretching.",
57  false, ParamCateg);
58 
59  //! Parameter \relates jevois::PreProcessor
60  JEVOIS_DECLARE_PARAMETER(showin, bool, "Show outline of cropped image fed to network",
61  true, ParamCateg);
62  }
63 
64  //! Pre-Processor for neural network pipeline
65  /*! This is the first step in a deep neural network processing Pipeline.
66 
67  Derived classes must implement the pure virtual methods:
68  - process(cv::Mat const & img, bool isrgb): process and input image and generate some tensors (blobs)
69  - sendreport(): describe what process() did to humans
70  - freeze(): freeze/unfreeze parameters that users should not change at runtime
71 
72  They should keep some internal state about what to report, since sendreport() is always called on
73  every frame, but process() may be called less often if the network is slow.
74 
75  \ingroup dnn */
77  public jevois::Parameter<preprocessor::rgb, preprocessor::showin>
78  {
79  public:
80 
81  //! Inherited constructor ok
83 
84  //! Destructor
85  virtual ~PreProcessor();
86 
87  //! Freeze/unfreeze parameters that users should not change while running
88  virtual void freeze(bool doit) = 0;
89 
90  //! Extract blobs from input image
91  std::vector<cv::Mat> process(jevois::RawImage const & img, std::vector<vsi_nn_tensor_attr_t> const & attrs);
92 
93  //! Report what happened in last process() to console/output video/GUI
94  virtual void sendreport(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
95  jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false);
96 
97  //! Access the last processed image size
98  cv::Size const & imagesize() const;
99 
100  //! Access the last computed blobs (or empty if process() has not yet been called)
101  std::vector<cv::Mat> const & blobs() const;
102 
103  //! Access the width and height of a given blob, accounting for NCHW or NHWC
104  cv::Size blobsize(size_t num) const;
105 
106  //! Convert coordinates from blob back to original image
107  /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the blob's width and height. This is useful to
108  convert detected boxes back into original input coordinates. */
109  void b2i(float & x, float & y, size_t blobnum = 0);
110 
111  //! Convert coordinates from blob back to original image, given a known blob size
112  /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the blob's width and height. This is useful to
113  convert detected boxes back into original input coordinates. */
114  void b2i(float & x, float & y, cv::Size const & bsiz, bool letterboxed);
115 
116  //! Get unscaled crop rectangle in image coordinates
117  /*! This is useful to display an image overlay on top of the input image. */
118  cv::Rect getUnscaledCropRect(size_t blobnum = 0);
119 
120  //! Get unscaled crop rectangle in image coordinates
121  /*! This is useful to display an image overlay on top of the input image. */
122  void getUnscaledCropRect(size_t blobnum, int & tlx, int & tly, int & brx, int & bry);
123 
124  protected:
125  //! Extract blobs from input image
126  /*! isrgb should be true if the given img has RGB color order, or false for BGR. Only 3-channel byte images are
127  supported as input. */
128  virtual std::vector<cv::Mat> process(cv::Mat const & img, bool isrgb,
129  std::vector<vsi_nn_tensor_attr_t> const & attrs,
130  std::vector<cv::Rect> & crops) = 0;
131 
132  //! Report what happened in last process() to console/output video/GUI
133  virtual void report(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
134  jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false) = 0;
135 
136  private:
137  std::vector<vsi_nn_tensor_attr_t> itsAttrs;
138  std::vector<cv::Mat> itsBlobs;
139  std::vector<cv::Rect> itsCrops; // Unscaled crops, one per blob, used for rescaling from blob to image
140 
141  cv::Size itsImageSize;
142  unsigned int itsImageFmt;
143  };
144 
145  } // namespace dnn
146 } // namespace jevois
jevois::dnn::PreProcessor::blobsize
cv::Size blobsize(size_t num) const
Access the width and height of a given blob, accounting for NCHW or NHWC.
Definition: PreProcessor.C:37
jevois::ParameterRegistry::Component
friend class Component
Allow Component to access our registry data, everyone else is locked out.
Definition: ParameterRegistry.H:51
Module.H
jevois::dnn::PreProcessor::getUnscaledCropRect
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
Definition: PreProcessor.C:75
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
RawImage.H
jevois::Component
A component of a model hierarchy.
Definition: Component.H:180
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::dnn::PreProcessor::blobs
const std::vector< cv::Mat > & blobs() const
Access the last computed blobs (or empty if process() has not yet been called)
Definition: PreProcessor.C:29
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois
Definition: Concepts.dox:1
F
float F
Definition: GUIhelper.C:2157
Component.H
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:76
jevois::dnn::PreProcessor::freeze
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
jevois::dnn::PreProcessor::sendreport
virtual void sendreport(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)
Report what happened in last process() to console/output video/GUI.
Definition: PreProcessor.C:113
jevois::dnn::PreProcessor::report
virtual void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)=0
Report what happened in last process() to console/output video/GUI.
jevois::dnn::PreProcessor::imagesize
const cv::Size & imagesize() const
Access the last processed image size.
Definition: PreProcessor.C:33
jevois::dnn::PreProcessor::b2i
void b2i(float &x, float &y, size_t blobnum=0)
Convert coordinates from blob back to original image.
Definition: PreProcessor.C:44
jevois::dnn::PreProcessor::process
std::vector< cv::Mat > process(jevois::RawImage const &img, std::vector< vsi_nn_tensor_attr_t > const &attrs)
Extract blobs from input image.
Definition: PreProcessor.C:89
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
jevois::dnn::PreProcessor::~PreProcessor
virtual ~PreProcessor()
Destructor.
Definition: PreProcessor.C:25
GUIhelper.H