JeVoisBase  1.18
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
MultiDNN.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
20 #include <jevois/DNN/Pipeline.H>
21 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
23 
24 // icon from opencv
25 
26 static jevois::ParameterCategory const ParamCateg("MultiDNN Options");
27 
28 //! Parameter \relates MultiDNN
29 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(grid, cv::Size, "Grid of networks to run and display",
30  cv::Size(2, 2), ParamCateg);
31 
32 
33 //! Run multiple neural networks in parallel with a tiled display
34 /*! See \jvmod{DNN} for more details about the JeVois DNN framework.
35 
36  Edit the module's \b params.cfg file to select which models to run.
37 
38  You can load any model at runtime by setting the \p pipe parameter for each pipeline.
39 
40  @author Laurent Itti
41 
42  @displayname MultiDNN
43  @videomapping NONE 0 0 0.0 YUYV 640 480 15.0 JeVois MultiDNN
44  @videomapping YUYV 640 498 15.0 YUYV 640 480 15.0 JeVois MultiDNN
45  @email itti\@usc.edu
46  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
47  @copyright Copyright (C) 2018 by Laurent Itti, iLab and the University of Southern California
48  @mainurl http://jevois.org
49  @supporturl http://jevois.org/doc
50  @otherurl http://iLab.usc.edu
51  @license GPL v3
52  @distribution Unrestricted
53  @restrictions None
54  \ingroup modules */
55 class MultiDNN : public jevois::StdModule,
56  public jevois::Parameter<grid>
57 {
58  public:
59  // ####################################################################################################
60  //! Constructor
61  // ####################################################################################################
62  using StdModule::StdModule;
63 
64  // ####################################################################################################
65  //! Virtual destructor for safe inheritance
66  // ####################################################################################################
67  virtual ~MultiDNN()
68  { }
69 
70  // ####################################################################################################
71  //! Processing function implementation
72  // ####################################################################################################
73  void doprocess(std::shared_ptr<jevois::dnn::Pipeline> pipe, jevois::InputFrame const & inframe,
74  jevois::RawImage * outimg, jevois::OptGUIhelper * helper, bool idle)
75  {
76  // If we have a second (scaled) image, assume this is the one we want to process:
77  jevois::RawImage const & inimg = inframe.hasScaledImage() ? inframe.get2() : inframe.get();
78 
79  // Ok, process it:
80  pipe->process(inimg, this, outimg, helper, idle);
81  }
82 
83  // ####################################################################################################
84  //! Processing function, no video output
85  // ####################################################################################################
86  virtual void process(jevois::InputFrame && inframe) override
87  {
88  LFATAL("todo");
89  }
90 
91  // ####################################################################################################
92  //! Processing function with video output to USB
93  // ####################################################################################################
94  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
95  {
96  LFATAL("todo");
97  }
98 
99 #ifdef JEVOIS_PRO
100  // ####################################################################################################
101  //! Processing function with zero-copy and GUI on JeVois-Pro
102  // ####################################################################################################
103  virtual void process(jevois::InputFrame && inframe, jevois::GUIhelper & helper) override
104  {
105  // Compute overall frame rate, CPU usage, etc:
106  static jevois::Timer timer("main", 20, LOG_DEBUG);
107  std::string const & fpscpu = timer.stop();
108  timer.start();
109 
110  // Start the display frame:
111  unsigned short winw, winh;
112  bool idle = helper.startFrame(winw, winh);
113 
114  // Compute size and offset for our tiled video frames:
115  jevois::RawImage const & img = inframe.get();
116  cv::Size const g = grid::get();
117  if (g.width == 0 || g.height == 0) LFATAL("Cannot handle zero rows or columns of models");
118  unsigned int totw = img.width * g.width, toth = img.height * g.height;
119  jevois::applyLetterBox(totw, toth, winw, winh, false);
120  int const xoff = (winw - totw) / 2;
121  int const yoff = (winh - toth) / 2;
122  unsigned short iw = totw / g.width;
123  unsigned short ih = toth / g.height;
124 
125  // Loop over all pipelines and run and display each:
126  int x = xoff, y = yoff; int n = 0;
127 
128  for (auto & pipe : itsPipes)
129  {
130  // Display the camera input frame:
131  helper.drawInputFrame((std::to_string(n)+"c").c_str(), inframe, x, y, iw, ih, true);
132 
133  // Process:
134  doprocess(pipe, inframe, nullptr, &helper, idle);
135 
136  // Get ready for the next pipe:
137  ++n; x += iw; if (x > totw - iw/2) { x = xoff; y += ih; }
138  }
139 
140  // Show overall frame rate and camera and display info:
141  helper.iinfo(inframe, fpscpu, winw, winh);
142 
143  // Render the image and GUI:
144  helper.endFrame();
145  }
146 #endif
147 
148  // ####################################################################################################
149  protected:
150  void onParamChange(grid const & JEVOIS_UNUSED_PARAM(param), cv::Size const & newval) override
151  {
152  // Try to preserve as many existing pipes as we can:
153  size_t const num = newval.width * newval.height;
154  if (num == 0) LFATAL("Cannot handle zero rows or columns of models");
155 
156  while (itsPipes.size() < num)
157  itsPipes.emplace_back(addSubComponent<jevois::dnn::Pipeline>("p"+std::to_string(itsPipes.size())));
158 
159  while (num < itsPipes.size()) { removeSubComponent(itsPipes.back()); itsPipes.pop_back(); }
160  }
161 
162  std::vector<std::shared_ptr<jevois::dnn::Pipeline>> itsPipes;
163 
164 };
165 
166 // Allow the module to be loaded as a shared object (.so) file:
jevois::GUIhelper
Pipeline.H
MultiDNN::onParamChange
void onParamChange(grid const &JEVOIS_UNUSED_PARAM(param), cv::Size const &newval) override
Definition: MultiDNN.C:150
jevois::OutputFrame
jevois::GUIhelper::startFrame
bool startFrame(unsigned short &w, unsigned short &h)
Timer.H
Module.H
jevois::GUIhelper::endFrame
void endFrame()
jevois::RawImage
jevois::Component::removeSubComponent
void removeSubComponent(std::shared_ptr< Comp > &component)
jevois::ParameterCategory
MultiDNN::process
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output to USB.
Definition: MultiDNN.C:94
jevois::InputFrame::get
const RawImage & get(bool casync=false) const
jevois::GUIhelper
MultiDNN::itsPipes
std::vector< std::shared_ptr< jevois::dnn::Pipeline > > itsPipes
Definition: MultiDNN.C:162
jevois::RawImage::width
unsigned int width
jevois::InputFrame::hasScaledImage
bool hasScaledImage() const
JEVOIS_REGISTER_MODULE
JEVOIS_REGISTER_MODULE(MultiDNN)
MultiDNN
Run multiple neural networks in parallel with a tiled display.
Definition: MultiDNN.C:55
MultiDNN::~MultiDNN
virtual ~MultiDNN()
Virtual destructor for safe inheritance.
Definition: MultiDNN.C:67
jevois::InputFrame::get2
const RawImage & get2(bool casync=false) const
jevois::applyLetterBox
void applyLetterBox(unsigned int &imw, unsigned int &imh, unsigned int const winw, unsigned int const winh, bool noalias)
LFATAL
#define LFATAL(msg)
RawImageOps.H
jevois::GUIhelper::drawInputFrame
void drawInputFrame(char const *name, InputFrame const &frame, int &x, int &y, unsigned short &w, unsigned short &h, bool noalias=false, bool casync=false)
jevois::RawImage::height
unsigned int height
jevois::GUIhelper::iinfo
void iinfo(jevois::InputFrame const &inframe, std::string const &fpscpu, unsigned short winw=0, unsigned short winh=0)
to_string
std::string to_string(T const &val)
jevois::InputFrame
MultiDNN::process
virtual void process(jevois::InputFrame &&inframe) override
Processing function, no video output.
Definition: MultiDNN.C:86
Utils.H
jevois::dnn::Pipeline::process
void process(jevois::RawImage const &inimg, jevois::StdModule *mod, jevois::RawImage *outimg, jevois::OptGUIhelper *helper, bool idle=false)
MultiDNN::doprocess
void doprocess(std::shared_ptr< jevois::dnn::Pipeline > pipe, jevois::InputFrame const &inframe, jevois::RawImage *outimg, jevois::OptGUIhelper *helper, bool idle)
Processing function implementation.
Definition: MultiDNN.C:73
jevois::StdModule
Darknet::JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. " "If empty, use the module's path.", JEVOIS_SHARE_PATH "/darknet/single", ParamCateg)
Parameter.
jevois::Timer