JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessor.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <opencv2/core/core.hpp>
22 #include <jevois/GPU/GUIhelper.H>
23 #include <jevois/Types/Enum.H>
24 
25 namespace jevois
26 {
27  class StdModule;
28  class RawImage;
29 
30  namespace dnn
31  {
32  class PreProcessor;
33 
34  namespace postprocessor
35  {
36  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
37  // will use different subsets of all available parameters:
38  static jevois::ParameterCategory const ParamCateg("DNN Post-Processing Options");
39 
40  //! Parameter \relates jevois::PostProcessorClassify
41  JEVOIS_DECLARE_PARAMETER(classoffset, int, "Offset added to model output when looking up class name. Useful if "
42  "your model uses a background class but your class file does not (use -1), or if your "
43  "model does not use a background class but your class file has one (use 1). If unsure, "
44  "use 0 and check whether reported class names are off.",
45  0, ParamCateg);
46 
47  //! Parameter \relates jevois::PostProcessorClassify
48  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(classes, std::string, "Path to text file with names of object classes",
49  "", ParamCateg);
50 
51  //! Parameter \relates jevois::PostProcessorClassify
52  JEVOIS_DECLARE_PARAMETER(top, unsigned int, "Max number of top-scoring predictions that score above "
53  "thresh to report",
54  5, ParamCateg);
55 
56  //! Parameter \relates jevois::PostProcessorClassify
57  JEVOIS_DECLARE_PARAMETER(thresh, float, "Threshold (in percent confidence) above which predictions will be "
58  "reported",
59  20.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
60 
61  //! Parameter \relates jevois::PostProcessorClassify
62  JEVOIS_DECLARE_PARAMETER(softmax, bool, "Apply a softmax to classification outputs",
63  false, ParamCateg);
64 
65  //! Parameter \relates jevois::PostProcessorClassify
66  JEVOIS_DECLARE_PARAMETER(scorescale, float, "Scaling factors applied to recognition scores, useful for "
67  "InceptionV3 and possibly other networks",
68  1.0F, ParamCateg);
69 
70  //! Enum \relates jevois::PostProcessorDetect
71  JEVOIS_DEFINE_ENUM_CLASS(DetectType, (FasterRCNN) (YOLO) (SSD) (TPUSSD) (RAWYOLOface) (RAWYOLOv2)
72  (RAWYOLOv3) (RAWYOLOv4) (RAWYOLOv3tiny) )
73 
74  //! Parameter \relates jevois::PostProcessorDetect
75  JEVOIS_DECLARE_PARAMETER(detecttype, DetectType, "Type of detection output format",
76  DetectType::YOLO, DetectType_Values, ParamCateg);
77 
78  //! Parameter \relates jevois::PostProcessorDetect
79  JEVOIS_DECLARE_PARAMETER(nms, float, "Non-maximum suppression intersection-over-union threshold in percent",
80  45.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
81 
82  //! Parameter \relates jevois::PostProcessorDetect
83  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(anchors, std::string, "For YOLO-type detection models with raw outputs, "
84  "list of anchors. Should be formatted as: w1, h1, w2, h2, ... ; ww1, hh1, ww2, hh2, "
85  "... ; ... where individual entries for a given YOLO layer are separated by commas, "
86  "and successive YOLO layers (in the order in which they appear in the Darknet .cfg "
87  "file) are separated by semicolons. Leave empty for other models. If your anchors are "
88  "the same for all YOLO layers, you may just specify them once.",
89  "", ParamCateg);
90 
91  //! Parameter \relates jevois::PostProcessorDetect \relates jevois::PostProcessorSegment
92  JEVOIS_DECLARE_PARAMETER(alpha, unsigned char, "Alpha channel value for drawn results",
93  64, ParamCateg);
94 
95  //! Parameter \relates jevois::PostProcessorSegment
96  JEVOIS_DECLARE_PARAMETER(bgid, unsigned char, "Class ID for the background, will show as fully transparent in "
97  "semantic segmentation overlays",
98  0, ParamCateg);
99 
100  //! Enum \relates jevois::PostProcessorSegment
101  JEVOIS_DEFINE_ENUM_CLASS(SegType, (Classes) (Classes2) (ArgMax) );
102 
103  //! Parameter \relates jevois::PostProcessorSegment
104  JEVOIS_DECLARE_PARAMETER(segtype, SegType, "Type of segmentation network output. If Classes, output is HxWxN "
105  "where N is the number of classes and we get one score per class, and we will show "
106  "the top scoring class for each pixel (e.g., UNet-MobileNet on TPU). If Classes2, "
107  "output is NxHxW and the rest is as for Classes (e.g., DeepLabV3 OpenCV). If ArgMax, "
108  "output is HxW and contains the class ID for each pixel (e.g., DeepLabV3 on TPU).",
109  SegType::Classes, SegType_Values, ParamCateg);
110  }
111 
112  //! Post-Processor for neural network pipeline
113  /*! This is the last step in a deep neural network processing Pipeline. \ingroup dnn */
115  {
116  public:
117 
118  //! Inherited constructor ok
120 
121  //! Destructor
122  virtual ~PostProcessor();
123 
124  //! Freeze/unfreeze parameters that users should not change while running
125  virtual void freeze(bool doit) = 0;
126 
127  //! Process outputs
128  virtual void process(std::vector<cv::Mat> const & outs, PreProcessor * preproc) = 0;
129 
130  //! Report what happened in last process() to console/output video/GUI
131  virtual void report(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
132  jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false) = 0;
133  };
134 
135  } // namespace dnn
136 } // namespace jevois
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::ParameterRegistry::Component
friend class Component
Allow Component to access our registry data, everyone else is locked out.
Definition: ParameterRegistry.H:51
jevois::Range
A generic range class.
Definition: Range.H:80
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
jevois::dnn::postprocessor::DetectType_Values
Type of detection output DetectType_Values
Definition: PostProcessor.H:76
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290))
Enum for different sensor models.
jevois::Component
A component of a model hierarchy.
Definition: Component.H:180
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::dnn::postprocessor::DetectType
DetectType
Definition: PostProcessor.H:75
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::imu::file
Data collection mode RAW means that the latest available raw data is returned each time hence timing may not be very accurate depending on how regularly grate into a FIFO and and accumulates resulting output data into the IMU s internal FIFO buffer at a fixed rate This parameter can only be set in a module s params cfg file
Definition: ICM20948.H:67
jevois::dnn::postprocessor::format
Type of detection output format
Definition: PostProcessor.H:75
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois
Definition: Concepts.dox:1
F
float F
Definition: GUIhelper.C:2150
Component.H
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:76
jevois::dnn::softmax
void softmax(float const *input, size_t n, float fac, float *output)
Apply softmax to a float vector.
Definition: Utils.C:443
jevois::dnn::PostProcessor
Post-Processor for neural network pipeline.
Definition: PostProcessor.H:114
jevois::module::number
Prepend standardized serial messages with a frame number
Definition: Module.H:216
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
GUIhelper.H
Enum.H