JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessor.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
20 #include <jevois/Component/Component.H>
21 #include <opencv2/core/core.hpp>
22 #include <jevois/GPU/GUIhelper.H>
23 #include <jevois/Types/Enum.H>
24 
25 namespace jevois
26 {
27  class StdModule;
28  class RawImage;
29 
30  namespace dnn
31  {
32  class PreProcessor;
33 
34  namespace postprocessor
35  {
36  // We define all parameters for all derived classes here to avoid duplicate definitions. Different derived classes
37  // will use different subsets of all available parameters:
38  static jevois::ParameterCategory const ParamCateg("DNN Post-Processing Options");
39 
40  //! Parameter \relates jevois::PostProcessorClassify
41  JEVOIS_DECLARE_PARAMETER(classoffset, int, "Offset added to model output when looking up class name. Useful if "
42  "your model uses a background class but your class file does not (use -1), or if your "
43  "model does not use a background class but your class file has one (use 1). If unsure, "
44  "use 0 and check whether reported class names are off.",
45  0, ParamCateg);
46 
47  //! Parameter \relates jevois::PostProcessorClassify
48  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(classes, std::string, "Path to text file with names of object classes",
49  "", ParamCateg);
50 
51  //! Parameter \relates jevois::PostProcessorClassify
52  JEVOIS_DECLARE_PARAMETER(top, unsigned int, "Max number of top-scoring predictions that score above "
53  "threshold to report",
54  5, ParamCateg);
55 
56  //! Parameter \relates jevois::PostProcessorDetect
57  JEVOIS_DECLARE_PARAMETER(maxnbox, unsigned int, "Max number of top-scoring boxes to report (for YOLO flavors, "
58  "this is the max for each scale)",
59  500, ParamCateg);
60 
61  //! Parameter \relates jevois::PostProcessorClassify \relates jevois::PostProcessorDetect
62  JEVOIS_DECLARE_PARAMETER(cthresh, float, "Classification threshold (in percent confidence) above which "
63  "predictions will be reported",
64  20.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
65 
66  //! Parameter \relates jevois::PostProcessorDetect
67  JEVOIS_DECLARE_PARAMETER(dthresh, float, "Detection box threshold (in percent confidence) above which "
68  "predictions will be reported. Not all networks use a separate box threshold, "
69  "many only use one threshold confidence threshold (cthresh parameter). The YOLO "
70  "family is an example that uses both box and classification confidences",
71  15.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
72 
73  //! Parameter \relates jevois::PostProcessorClassify
74  JEVOIS_DECLARE_PARAMETER(softmax, bool, "Apply a softmax to classification outputs",
75  false, ParamCateg);
76 
77  //! Parameter \relates jevois::PostProcessorClassify
78  JEVOIS_DECLARE_PARAMETER(scorescale, float, "Scaling factors applied to recognition scores. Mainly "
79  "for debugging if your scores seem too high or too low. If too high, usually "
80  "that means that you should turn on parameter softmax instead.",
81  1.0F, ParamCateg);
82 
83  //! Enum \relates jevois::PostProcessorDetect
84  JEVOIS_DEFINE_ENUM_CLASS(DetectType, (FasterRCNN) (YOLO) (SSD) (TPUSSD) (RAWYOLO) );
85 
86  //! Parameter \relates jevois::PostProcessorDetect
87  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(detecttype, DetectType, "Type of detection output format",
88  DetectType::YOLO, DetectType_Values, ParamCateg);
89 
90  //! Parameter \relates jevois::PostProcessorDetect
91  JEVOIS_DECLARE_PARAMETER(nms, float, "Non-maximum suppression intersection-over-union threshold in percent",
92  45.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
93 
94  //! Parameter \relates jevois::PostProcessorDetect
95  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(anchors, std::string, "For YOLO-type detection models with raw outputs, "
96  "list of anchors. Should be formatted as: w1, h1, w2, h2, ... ; ww1, hh1, ww2, hh2, "
97  "... ; ... where individual entries for a given YOLO layer are separated by commas, "
98  "and successive YOLO layers (from large to small, e.g., first the anchors for 52x52, "
99  "then for 26x26, then for 13x13) are separated by semicolons. Leave empty "
100  "for other models.",
101  "", ParamCateg);
102 
103  //! Parameter \relates jevois::PostProcessorDetect \relates jevois::PostProcessorSegment
104  JEVOIS_DECLARE_PARAMETER(alpha, unsigned char, "Alpha channel value for drawn results",
105  64, ParamCateg);
106 
107  //! Parameter \relates jevois::PostProcessorSegment
108  JEVOIS_DECLARE_PARAMETER(bgid, unsigned char, "Class ID for the background, will show as fully transparent in "
109  "semantic segmentation overlays",
110  0, ParamCateg);
111 
112  //! Enum \relates jevois::PostProcessorSegment
113  JEVOIS_DEFINE_ENUM_CLASS(SegType, (ClassesHWC) (ClassesCHW) (ArgMax) );
114 
115  //! Parameter \relates jevois::PostProcessorSegment
116  JEVOIS_DECLARE_PARAMETER(segtype, SegType, "Type of segmentation network output. ClassesHWC: output is 1xHxWxC "
117  "for C classes and we get one score per class; we will show "
118  "the top scoring class for each pixel (e.g., UNet-MobileNet on TPU). ClassesCHW: "
119  "output is 1xCxHxW and the rest is as for ClassesHWC (e.g., DeepLabV3 OpenCV). ArgMax: "
120  "output is HxW, 1xHxW, or 1xHxWx1 and contains the class ID for each pixel "
121  "(e.g., DeepLabV3 on TPU).",
122  SegType::ClassesHWC, SegType_Values, ParamCateg);
123 
124  //! Parameter \relates jevois::PostProcessorPython
125  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pypost, std::string, "Path below " JEVOIS_SHARE_PATH "/ of the python "
126  "post-processor file. Name of class defined in the file must match "
127  "the file name without the trailing '.py'",
128  "", ParamCateg);
129 
130  //! Parameter \relates jevois::PostProcessorDetectYOLO
131  JEVOIS_DECLARE_PARAMETER(scalexy, float, "If 0, use old-style YOLO boxes (YOLOv2/v3/v4); otherwise, this is "
132  "the scale_xy factor for new-style YOLO coordinates (YOLOv5/v7; value is usually 2.0 "
133  "but check the yolo layer in the model's .cfg file)",
134  0.0F, ParamCateg);
135 
136  //! Parameter \relates jevois::PostProcessorDetectYOLO
137  JEVOIS_DECLARE_PARAMETER(sigmoid, bool, "Apply sigmoid to raw YOLO outputs, use when the last conv layers "
138  "just before yolo/detection/region layers have linear activation (most "
139  "YOLOv2/v3/v4 models, but not YOLOv5/v7 which have logistic activation on their "
140  "last conv)",
141  true, ParamCateg);
142  }
143 
144  //! Post-Processor for neural network pipeline
145  /*! This is the last step in a deep neural network processing Pipeline. \ingroup dnn */
146  class PostProcessor : public jevois::Component
147  {
148  public:
149 
150  //! Inherited constructor ok
151  using jevois::Component::Component;
152 
153  //! Destructor
154  virtual ~PostProcessor();
155 
156  //! Freeze/unfreeze parameters that users should not change while running
157  virtual void freeze(bool doit) = 0;
158 
159  //! Process outputs
160  virtual void process(std::vector<cv::Mat> const & outs, PreProcessor * preproc) = 0;
161 
162  //! Report what happened in last process() to console/output video/GUI
163  virtual void report(jevois::StdModule * mod, jevois::RawImage * outimg = nullptr,
164  jevois::OptGUIhelper * helper = nullptr, bool overlay = true, bool idle = false) = 0;
165  };
166 
167  } // namespace dnn
168 } // namespace jevois
jevois::ParameterRegistry::Component
friend class Component
Allow Component and DynamicParameter to access our registry data, everyone else is locked out.
Definition: ParameterRegistry.H:51
jevois::Range
A generic range class.
Definition: Range.H:80
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
jevois::Component
A component of a model hierarchy.
Definition: Component.H:181
jevois::dnn::softmax
size_t softmax(float const *input, size_t const n, size_t const stride, float const fac, float *output, bool maxonly)
Apply softmax to a float vector.
Definition: Utils.C:589
jevois::dnn::PostProcessor::freeze
virtual void freeze(bool doit)=0
Freeze/unfreeze parameters that users should not change while running.
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois
Definition: Concepts.dox:1
F
float F
Definition: GUIhelper.C:1968
Component.H
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
jevois::dnn::PostProcessor::report
virtual void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false)=0
Report what happened in last process() to console/output video/GUI.
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:102
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290)(os08a10))
Enum for different sensor models.
jevois::dnn::PostProcessor::~PostProcessor
virtual ~PostProcessor()
Destructor.
Definition: PostProcessor.C:21
jevois::dnn::PostProcessor
Post-Processor for neural network pipeline.
Definition: PostProcessor.H:146
jevois::dnn::PostProcessor::process
virtual void process(std::vector< cv::Mat > const &outs, PreProcessor *preproc)=0
Process outputs.
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
GUIhelper.H
Enum.H