JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Pipeline.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <jevois/GPU/GUIhelper.H>
22 #include <jevois/Debug/Timer.H>
23 #include <jevois/Types/Enum.H>
24 #include <ovxlib/vsi_nn_pub.h> // for data types and quantization types
25 
26 namespace jevois
27 {
28  class StdModule;
29  class RawImage;
30 
31  namespace dnn
32  {
33  class PreProcessor;
34  class Network;
35  class PostProcessor;
36 
37  namespace pipeline
38  {
39  static jevois::ParameterCategory const ParamCateg("DNN Pipeline Options");
40 
41  //! Parameter \relates jevois::Pipeline
42  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(zooroot, std::string, "Path where to find zoo files (.yml). "
43  "If not absolute, it is relative to this module's path",
44  std::string(JEVOIS_SHARE_PATH) + "/dnn", ParamCateg);
45 
46  //! Parameter \relates jevois::Pipeline
47  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(zoo, std::string, "Filename for neural network zoo file (.yml). "
48  "If not absolute, it is relative to zooroot",
49  "models.yml", ParamCateg);
50  //! Enum \relates jevois::Pipeline
51  JEVOIS_DEFINE_ENUM_CLASS(Filter, (All) (OpenCV) (TPU) (NPU) (VPU) );
52 
53  //! Parameter \relates jevois::Pipeline
54  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(filter, Filter, "Filter to possibly only show as options in "
55  "the 'pipe' parameter some class of models from the zoo",
56  Filter::All, Filter_Values, ParamCateg);
57 
58  //! Parameter \relates jevois::Pipeline
59  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(pipe, std::string, "Pipeline to use, which should correspond to a "
60  "top-level entry in the zoo file",
61  "", ParamCateg);
62 
63  //! Enum \relates jevois::Pipeline
64  JEVOIS_DEFINE_ENUM_CLASS(PreProc, (Blob) (Custom) );
65 
66  //! Parameter \relates jevois::Pipeline
67  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(preproc, PreProc, "Pre-Processor to use, usually set automatically "
68  "by selecting a pipeline from the zoo file",
69  PreProc::Blob, PreProc_Values, ParamCateg);
70  //! Enum \relates jevois::Pipeline
71 #ifdef JEVOIS_PRO
72  JEVOIS_DEFINE_ENUM_CLASS(NetType, (OpenCV) (NPU) (TPU) (Custom) );
73 #else
74  JEVOIS_DEFINE_ENUM_CLASS(NetType, (OpenCV) (Custom) );
75 #endif
76 
77  //! Parameter \relates jevois::Pipeline
78  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(nettype, NetType, "Network runtime framework to use, usually set "
79  "automatically by selecting a pipeline from the zoo file",
80  NetType::OpenCV, NetType_Values, ParamCateg);
81  //! Enum \relates jevois::Pipeline
82  JEVOIS_DEFINE_ENUM_CLASS(PostProc, (Classify) (Detect) (Segment) (Custom) );
83 
84  //! Parameter \relates jevois::Pipeline
85  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(postproc, PostProc, "Post-Processor to use, usually set automatically "
86  "by selecting a pipeline from the zoo file",
87  PostProc::Classify, PostProc_Values, ParamCateg);
88  //! Enum \relates jevois::Pipeline
89  JEVOIS_DEFINE_ENUM_CLASS(Processing, (Sync) (Async) );
90 
91  //! Parameter \relates jevois::Pipeline
92  JEVOIS_DECLARE_PARAMETER(processing, Processing, "Type of processing: Sync runs pre-processing, "
93  "network, and post-processing sequentially for every frame. Use for fast "
94  "networks. Async runs the network in a thread and should be used for "
95  "networks slower than the camera framerate.",
96  Processing::Async, Processing_Values, ParamCateg);
97 
98  //! Parameter \relates jevois::Pipeline
99  JEVOIS_DECLARE_PARAMETER(overlay, bool, "Show some pipeline info as an overlay over output or GUI video",
100  true, ParamCateg);
101  }
102 
103  //! Neural processing pipeline
104  /*! A neural processing pipeline in JeVois consists of:
105 
106  - pre-processing an input image to extract one or more blobs used as input tensors to a deep neural network
107  - processing the input blobs through a deep network to obtain output blobs
108  - post-processing the output blobs to display results and send serial messages
109 
110  A pipeline is typically configured by parsing a YAML config file (zoo file) that determines what kind of
111  pre-processing, network, and post-processing to use, and that sets the parameters for those. \ingroup dnn */
112  class Pipeline : public jevois::Component,
113  public jevois::Parameter<pipeline::zooroot, pipeline::zoo, pipeline::filter, pipeline::pipe,
114  pipeline::processing, pipeline::preproc,
115  pipeline::nettype, pipeline::postproc, pipeline::overlay>
116  {
117  public:
118  //! Constructor
119  Pipeline(std::string const & instance);
120 
121  //! Destructor
122  virtual ~Pipeline();
123 
124  //! Returns true when all three or preproc, net, and postproc are ready
125  bool ready() const;
126 
127  //! Process an input image, send results to serial/image/gui
128  /*! If the network is not ready, no processing will occur. When helper is not null (i.e., using GUI display),
129  hide the information window when idle is true. This function catches all exceptions and reports them. */
130  void process(jevois::RawImage const & inimg, jevois::StdModule * mod,
131  jevois::RawImage * outimg, jevois::OptGUIhelper * helper, bool idle = false);
132 
133  //! Freeze/unfreeze parameters that users should not change while running
134  void freeze(bool doit);
135 
136  //! Set a custom pre-processor
137  /*! To create a custom pre-processor, create a new class that derives from jevois::dnn::PreProcessor and that
138  implements the pure virtual functions of that base class. Then, in your module, you should hold a
139  shared_ptr<jevois::Pipeline> which we will call itsPipeline, and in your constructor you instantiate it and
140  add it as a sub-component of your module (see constructor of \jvmod{DNN} for an example). Then proceed as
141  follows in your constructor or postInit() or parameter callback:
142 
143  \code
144  // Create an instance of our custom pre-processor and attach it as a sub-component of our pipeline
145  auto sptr = itsPipeline->addSubComponent<MyCustomPreProc>("preproc");
146  itsPipeline->setCustomPreProcessor(std::static_pointer_cast<jevois::dnn::PreProcessor>(sptr);
147  \endcode
148 
149  And you are ready to rock. */
150  void setCustomPreProcessor(std::shared_ptr<PreProcessor> pp);
151 
152  // Set a custom network type
153  /*! See doc of setCustomPreProcessor() for instructions, but use the name "network" instead of "preproc". */
154  void setCustomNetwork(std::shared_ptr<Network> n);
155 
156  // Set a custom post-processor
157  /*! See doc of setCustomPreProcessor() for instructions, but use the name "postproc" instead of "preproc". */
158  void setCustomPostProcessor(std::shared_ptr<PostProcessor> pp);
159 
160  protected:
161  void postInit() override;
162  void preUninit() override;
163  std::shared_ptr<PreProcessor> itsPreProcessor;
164  std::shared_ptr<Network> itsNetwork;
165  std::shared_ptr<PostProcessor> itsPostProcessor;
166 
167  private:
168  jevois::TimerOne itsTpre, itsTnet, itsTpost;
169 
170  bool itsZooChanged = false;
171  void onParamChange(pipeline::zooroot const & param, std::string const & val) override;
172  void onParamChange(pipeline::zoo const & param, std::string const & val) override;
173  void onParamChange(pipeline::filter const & param, pipeline::Filter const & val) override;
174  void onParamChange(pipeline::pipe const & param, std::string const & val) override;
175  void onParamChange(pipeline::nettype const & param, pipeline::NetType const & val) override;
176  void onParamChange(pipeline::preproc const & param, pipeline::PreProc const & val) override;
177  void onParamChange(pipeline::postproc const & param, pipeline::PostProc const & val) override;
178 
179  std::future<std::vector<cv::Mat>> itsNetFut;
180  std::array<std::string, 3> itsProcTimes { "PreProc: -", "Network: -", "PstProc: -" };
181  std::array<double, 3> itsProcSecs { 0.0, 0.0, 0.0 };
182  std::vector<cv::Mat> itsBlobs, itsOuts;
183  std::vector<vsi_nn_tensor_attr_t> itsInputAttrs;
184  std::vector<std::string> itsNetInfo, itsAsyncNetInfo;
185  std::string itsAsyncNetworkTime = "Network: -";
186  double itsAsyncNetworkSecs = 0.0;
187  void showInfo(std::vector<std::string> const & info, jevois::StdModule * mod,
188  jevois::RawImage * outimg, jevois::OptGUIhelper * helper, bool ovl, bool idle);
189  void asyncNetWait();
190  bool checkAsyncNetComplete();
191  double itsSecsSum = 0.0, itsSecsAvg = 0.0;
192  int itsSecsSumNum = 0;
193  bool itsPipeThrew = false;
194  void scanZoo(std::string const & zoofile, std::string const & filt, std::vector<std::string> & pipes,
195  std::string const & indent);
196  bool selectPipe(std::string const & zoofile, std::vector<std::string> const & tok);
197  void setZooParam(cv::FileNode & item, std::string const & zf, cv::FileNode const & node);
198  int itsOutImgY = 0;
199 
200  std::map<std::string, size_t> itsAccelerators;
201  };
202 
203  } // namespace dnn
204 } // namespace jevois
jevois::dnn::Pipeline::preUninit
void preUninit() override
Called before all sub-Components are uninit()ed.
Definition: Pipeline.C:72
jevois::dnn::Pipeline
Neural processing pipeline.
Definition: Pipeline.H:112
jevois::dnn::Pipeline::setCustomNetwork
void setCustomNetwork(std::shared_ptr< Network > n)
Definition: Pipeline.C:423
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
jevois::dnn::Pipeline::itsPostProcessor
std::shared_ptr< PostProcessor > itsPostProcessor
Definition: Pipeline.H:165
jevois::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(CameraSensor,(any)(ov9650)(ov2640)(ov7725)(ar0135)(imx290))
Enum for different sensor models.
jevois::Component
A component of a model hierarchy.
Definition: Component.H:180
jevois::dnn::Pipeline::Pipeline
Pipeline(std::string const &instance)
Constructor.
Definition: Pipeline.C:40
jevois::dnn::Pipeline::postInit
void postInit() override
Called after all sub-Components are init()ed.
Definition: Pipeline.C:65
jevois::dnn::Pipeline::ready
bool ready() const
Returns true when all three or preproc, net, and postproc are ready.
Definition: Pipeline.C:477
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::ParameterCategory
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
jevois::TimerOne
Simple one-shot timer class.
Definition: Timer.H:69
jevois::dnn::Pipeline::~Pipeline
virtual ~Pipeline()
Destructor.
Definition: Pipeline.C:79
jevois::dnn::Pipeline::itsNetwork
std::shared_ptr< Network > itsNetwork
Definition: Pipeline.H:164
jevois::dnn::Pipeline::setCustomPostProcessor
void setCustomPostProcessor(std::shared_ptr< PostProcessor > pp)
Definition: Pipeline.C:465
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois
Definition: Concepts.dox:1
Component.H
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
jevois::dnn::Pipeline::freeze
void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition: Pipeline.C:53
jevois::dnn::Pipeline::setCustomPreProcessor
void setCustomPreProcessor(std::shared_ptr< PreProcessor > pp)
Set a custom pre-processor.
Definition: Pipeline.C:372
Timer.H
jevois::dnn::Pipeline::process
void process(jevois::RawImage const &inimg, jevois::StdModule *mod, jevois::RawImage *outimg, jevois::OptGUIhelper *helper, bool idle=false)
Process an input image, send results to serial/image/gui.
Definition: Pipeline.C:498
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
GUIhelper.H
jevois::dnn::Pipeline::itsPreProcessor
std::shared_ptr< PreProcessor > itsPreProcessor
Definition: Pipeline.H:163
Enum.H