JeVois  1.21
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Loading...
Searching...
No Matches
PreProcessor.H
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
18#pragma once
19
21#include <opencv2/core/core.hpp>
24#include <jevois/Core/Module.H>
25
26#include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
27
28namespace jevois
29{
30 namespace dnn
31 {
32 class PreProcessorForPython;
33
34 namespace preprocessor
35 {
36 // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
37 // will use different subsets of all available parameters:
38 static jevois::ParameterCategory const ParamCateg("DNN Pre-Processing Options");
39
40 //! Parameter \relates jevois::dnn::PreProcessor
41 JEVOIS_DECLARE_PARAMETER(rgb, bool, "When true, model works with RGB input images instead BGR ones",
42 true, ParamCateg);
43
44 //! Parameter \relates jevois::dnn::PreProcessor
45 JEVOIS_DECLARE_PARAMETER(scale, float, "Value scaling factor applied to input pixels after mean subtraction, "
46 "or 0.0 to extract an unscaled UINT8 blob, typically for use with quantized networks",
47 2.0F / 255.0F, ParamCateg);
48
49 //! Parameter \relates jevois::dnn::PreProcessor
50 JEVOIS_DECLARE_PARAMETER(mean, cv::Scalar, "Mean values subtracted from input image, in the same RGB/BGR "
51 "order as the network's input",
52 cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg);
53
54 //! Parameter \relates jevois::dnn::PreProcessor
55 JEVOIS_DECLARE_PARAMETER(stdev, cv::Scalar, "Input image is divided by stdev after mean subtraction and scale "
56 "factor are applied. This is rarely used. Same RGB/BGR order as the network's input",
57 cv::Scalar(1.0F, 1.0F, 1.0F), ParamCateg);
58
59 //! Parameter \relates jevois::dnn::PreProcessor
60 JEVOIS_DECLARE_PARAMETER(letterbox, bool, "When true, extract the largest possible box from the input image "
61 "with same aspect ratio as the network's input tensor, and then rescale it to that "
62 "tensor's width and height (hence with cropping but no distortion). Otherwise, use "
63 "the whole image and rescale it to the network's input width and height with some "
64 "possible stretching.",
65 false, ParamCateg);
66
67 //! Parameter \relates jevois::dnn::PreProcessor
68 JEVOIS_DECLARE_PARAMETER(showin, bool, "Show outline of cropped image fed to network",
69 true, ParamCateg);
70
71 //! Parameter \relates jevois::dnn::PreProcessorPython
72 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pypre, std::string, "Full path of the python pre-processor file. Name of "
73 "class defined in the file must match the file name without the "
74 "trailing '.py'",
75 "", ParamCateg);
76
77 //! Parameter \relates jevois::dnn::PreProcessor
78 JEVOIS_DECLARE_PARAMETER(details, bool, "Show more details about the pre-processing steps",
79 false, ParamCateg);
80
81 //! Enum for image resizing modes \relates jevois::dnn::PreProcessor
82 JEVOIS_DEFINE_ENUM_CLASS(InterpMode, (Nearest) (Linear) (Cubic) (Area) (Lanczos4) );
83
84 //! Parameter \relates jevois::dnn::PreProcessor
85 JEVOIS_DECLARE_PARAMETER(interp, InterpMode, "Image interpolation to use when resizing the input image "
86 "from camera to network input dims",
87 InterpMode::Nearest, InterpMode_Values, ParamCateg);
88
89 //! Parameter \relates jevois::dnn::PreProcessorBlob
90 JEVOIS_DECLARE_PARAMETER(numin, size_t, "Number of input blobs to generate from the received video image. "
91 "Any additional inputs required by the network would have to be specified using "
92 "Network parameter extraintensors",
93 1, ParamCateg);
94 }
95
96 //! Pre-Processor for neural network pipeline
97 /*! This is the first step in a deep neural network processing Pipeline.
98
99 Derived classes must implement the pure virtual methods:
100 - process(): process an input image and generate some tensors (blobs)
101 - report(): describe what process() did to humans
102 - freeze(): freeze/unfreeze parameters that users should not change at runtime
103
104 They should keep some internal state about what to report, since report() is always called on
105 every frame, but process() may be called less often if the network is slow.
106
107 \ingroup dnn */
109 public jevois::Parameter<preprocessor::rgb, preprocessor::showin, preprocessor::details>
110 {
111 public:
112
113 //! Constructor
114 PreProcessor(std::string const & instance);
115
116 //! Destructor
117 virtual ~PreProcessor();
118
119 //! Freeze/unfreeze parameters that users should not change while running
120 virtual void freeze(bool doit) = 0;
121
122 //! Extract blobs from input image
123 std::vector<cv::Mat> process(jevois::RawImage const & img, std::vector<vsi_nn_tensor_attr_t> const & attrs);
124
125 //! Report what happened in last process() to console/output video/GUI
126 virtual void sendreport(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
127 jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false);
128
129 //! Access the last processed image size
130 cv::Size const & imagesize() const;
131
132 //! Access the last computed blobs (or empty if process() has not yet been called)
133 std::vector<cv::Mat> const & blobs() const;
134
135 //! Access the width and height of a given blob, accounting for NCHW or NHWC
136 cv::Size blobsize(size_t num) const;
137
138 //! Convert coordinates from blob back to original image
139 /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the blob's width and height. This is useful to
140 convert detected boxes back into original input coordinates. */
141 void b2i(float & x, float & y, size_t blobnum = 0);
142
143 //! Convert coordinates from blob back to original image, given a known blob size
144 /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the blob's width and height. This is useful to
145 convert detected boxes back into original input coordinates. */
146 void b2i(float & x, float & y, cv::Size const & bsiz, bool letterboxed);
147
148 //! Get unscaled crop rectangle in image coordinates
149 /*! This is useful to display an image overlay on top of the input image. */
150 cv::Rect getUnscaledCropRect(size_t blobnum = 0);
151
152 //! Get unscaled crop rectangle in image coordinates
153 /*! This is useful to display an image overlay on top of the input image. */
154 void getUnscaledCropRect(size_t blobnum, int & tlx, int & tly, int & brx, int & bry);
155
156 //! Convert coordinates from image to blob
157 /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the image's width and height. This is useful
158 to convert mouse coordinates (after they have been converted from screen to image coords) to locations
159 within an input blob. */
160 void i2b(float & x, float & y, size_t blobnum = 0);
161
162 //! Convert coordinates from image to blob
163 /*! Given coords x,y should be in [0..w-1]x[0..h-1] where w,h are the image's width and height. This is useful
164 to convert mouse coordinates (after they have been converted from screen to image coords) to locations
165 within an input blob. */
166 void i2b(float & x, float & y, cv::Size const & bsiz, bool letterboxed);
167
168 //! Get a pointer to our python-friendly interface
169 std::shared_ptr<PreProcessorForPython> getPreProcForPy() const;
170
171 protected:
172 //! Extract blobs from input image
173 /*! isrgb should be true if the given img has RGB color order, or false for BGR. Only 3-channel byte images are
174 supported as input. */
175 virtual std::vector<cv::Mat> process(cv::Mat const & img, bool isrgb,
176 std::vector<vsi_nn_tensor_attr_t> const & attrs,
177 std::vector<cv::Rect> & crops) = 0;
178
179 //! Report what happened in last process() to console/output video/GUI
180 virtual void report(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
181 jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false) = 0;
182
183 private:
184 std::vector<vsi_nn_tensor_attr_t> itsAttrs;
185 std::vector<cv::Mat> itsBlobs;
186 std::vector<cv::Rect> itsCrops; // Unscaled crops, one per blob, used for rescaling from blob to image
187
188 cv::Size itsImageSize;
189 unsigned int itsImageFmt;
190
191 // Helper class exposed to python
192 std::shared_ptr<PreProcessorForPython> itsPP;
193 };
194
195 } // namespace dnn
196} // namespace jevois
A component of a model hierarchy.
Definition Component.H:182
Helper class to assist modules in creating graphical and GUI elements.
Definition GUIhelper.H:133
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition RawImage.H:111
Base class for a module that supports standardized serial messages.
Definition Module.H:234
JEVOIS_DECLARE_PARAMETER(numin, size_t, "Number of input blobs to generate from the received video image. " "Any additional inputs required by the network would have to be specified using " "Network parameter extraintensors", 1, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pypre, std::string, "Full path of the python pre-processor file. Name of " "class defined in the file must match the file name without the " "trailing '.py'", "", ParamCateg)
Parameter.
Pre-Processor for neural network pipeline.
JEVOIS_DECLARE_PARAMETER(rgb, bool, "When true, model works with RGB input images instead BGR ones", true, ParamCateg)
Parameter.
std::shared_ptr< PreProcessorForPython > getPreProcForPy() const
Get a pointer to our python-friendly interface.
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
void i2b(float &x, float &y, size_t blobnum=0)
Convert coordinates from image to blob.
JEVOIS_DECLARE_PARAMETER(showin, bool, "Show outline of cropped image fed to network", true, ParamCateg)
Parameter.
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
virtual void sendreport(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)
Report what happened in last process() to console/output video/GUI.
JEVOIS_DECLARE_PARAMETER(scale, float, "Value scaling factor applied to input pixels after mean subtraction, " "or 0.0 to extract an unscaled UINT8 blob, typically for use with quantized networks", 2.0F/255.0F, ParamCateg)
Parameter.
JEVOIS_DEFINE_ENUM_CLASS(InterpMode,(Nearest)(Linear)(Cubic)(Area)(Lanczos4))
Enum for image resizing modes.
virtual void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)=0
Report what happened in last process() to console/output video/GUI.
JEVOIS_DECLARE_PARAMETER(mean, cv::Scalar, "Mean values subtracted from input image, in the same RGB/BGR " "order as the network's input", cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg)
Parameter.
virtual std::vector< cv::Mat > process(cv::Mat const &img, bool isrgb, std::vector< vsi_nn_tensor_attr_t > const &attrs, std::vector< cv::Rect > &crops)=0
Extract blobs from input image.
JEVOIS_DECLARE_PARAMETER(details, bool, "Show more details about the pre-processing steps", false, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(interp, InterpMode, "Image interpolation to use when resizing the input image " "from camera to network input dims", InterpMode::Nearest, InterpMode_Values, ParamCateg)
Parameter.
JEVOIS_DECLARE_PARAMETER(letterbox, bool, "When true, extract the largest possible box from the input image " "with same aspect ratio as the network's input tensor, and then rescale it to that " "tensor's width and height (hence with cropping but no distortion). Otherwise, use " "the whole image and rescale it to the network's input width and height with some " "possible stretching.", false, ParamCateg)
Parameter.
std::vector< cv::Mat > process(jevois::RawImage const &img, std::vector< vsi_nn_tensor_attr_t > const &attrs)
Extract blobs from input image.
JEVOIS_DECLARE_PARAMETER(stdev, cv::Scalar, "Input image is divided by stdev after mean subtraction and scale " "factor are applied. This is rarely used. Same RGB/BGR order as the network's input", cv::Scalar(1.0F, 1.0F, 1.0F), ParamCateg)
Parameter.
std::vector< cv::Mat > const & blobs() const
Access the last computed blobs (or empty if process() has not yet been called)
cv::Size const & imagesize() const
Access the last processed image size.
virtual ~PreProcessor()
Destructor.
void b2i(float &x, float &y, size_t blobnum=0)
Convert coordinates from blob back to original image.
cv::Size blobsize(size_t num) const
Access the width and height of a given blob, accounting for NCHW or NHWC.
Main namespace for all JeVois classes and functions.
Definition Concepts.dox:2
A category to which multiple ParameterDef definitions can belong.