JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DemoGPU.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Types/Enum.H>
20 #include <jevois/Debug/Timer.H>
21 
23 
25 #include <linux/videodev2.h>
26 
27 static jevois::ParameterCategory const ParamCateg("DemoGPU Parameters");
28 
29 // Note: we use NoEffect here as None seems to be defined a s anumeric constant somewhere, when compiling on host
30 
31 //! Parameter \relates DemoGPU
32 JEVOIS_DEFINE_ENUM_CLASS(Effect, (NoEffect) (Blur) (Sobel) (Median) (Mult) (Thresh) (Dilate) (Erode) (Twirl) (Dewarp));
33 
34 //! Parameter \relates DemoGPU
35 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(effect, Effect, "GPU image processing effect to apply",
36  Effect::Twirl, Effect_Values, ParamCateg);
37 
38 // icon by by Freepik in technology at flaticon
39 
40 //! Simple image filtering demo using OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU
41 /*! This class first copies the given input image into an OpenGL texture, then applies OpenGL-ES vertex and fragment
42  shaders to achieve some image processing, and finally gets the resulting pixels back into an image.
43 
44  This code is inspired somewhat from the tutorial and code examples found on this page:
45  http://robotblogging.blogspot.com/2013/10/gpu-accelerated-camera-processing-on.html
46 
47  But an important distinction is that we render to a framebuffer with an RGB565 renderbuffer attached, which
48  accelerates processing and transfer of the results a lot.
49 
50  Dewarp algorithm contributed by JeVois user [Ali AlSaibie](https://github.com/alsaibie). It is to be used with a
51  modified JeVois camera sensor that has a wide-angle lens. See http://jevois.org/qa/index.php?qa=153 for details.
52 
53  \youtube{rxsWR3I_LnM}
54 
55  Using this module
56  -----------------
57 
58  Unfortunately, Mac OSX computers refuse to detect the JeVois smart camera as soon as it exposes one or more RGB565
59  video modes. Thus, you cannot use this module with Macs, and the module is disabled by default. In addition, RGB565
60  does not seem to work in \c guvcview either, on Ubuntu prior to 17.04! Proceed as follows to enable and use this
61  module on a Linux host:
62 
63  Edit <b>JEVOIS:/jevois/config/videomappings.cfg</b> and look for the line that mentions DemoGPU. It is commented
64  out, so just remove the leading \b # sign. The line should then look like this:
65 
66  \verbatim
67  RGB565 320 240 22.0 YUYV 320 240 22.0 JeVois DemoGPU
68  \endverbatim
69 
70  Restart JeVois and run this on your Linux host if older than Ubuntu 17.04:
71 
72  \verbatim
73  sudo apt install ffmpeg
74  ffplay /dev/video0 -pixel_format rgb565 -video_size 320x240
75  \endverbatim
76 
77  or, with Ubuntu 17.04 and later:
78 
79  \verbatim
80  guvcview -f RGBP -x 320x240
81  \endverbatim
82 
83 
84  @author Laurent Itti
85 
86  @displayname Demo GPU
87  @videomapping RGB565 320 240 22.0 YUYV 320 240 22.0 JeVois DemoGPU
88  @email itti\@usc.edu
89  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
90  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
91  @mainurl http://jevois.org
92  @supporturl http://jevois.org/doc
93  @otherurl http://iLab.usc.edu
94  @license GPL v3
95  @distribution Unrestricted
96  @restrictions None
97  \ingroup modules */
98 class DemoGPU : public jevois::Module,
99  public jevois::Parameter<effect>
100 {
101  public:
102  //! Constrctor
103  DemoGPU(std::string const & instance) : jevois::Module(instance)
104  {
105  itsFilter = addSubComponent<FilterGPU>("gpu");
106  }
107 
108  //! Virtual destructor for safe inheritance
109  virtual ~DemoGPU() { }
110 
111  //! Processing function
112  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
113  {
114  static jevois::Timer timer("DemoGPU");
115 
116  // Wait for next available camera image:
117  jevois::RawImage const inimg = inframe.get();
118 
119  timer.start();
120 
121  // Convert input image to RGBA:
122  cv::Mat inimgcv = jevois::rawimage::convertToCvRGBA(inimg);
123 
124  // Let camera know we are done processing the input image:
125  inframe.done();
126 
127  // Wait for an image from our gadget driver into which we will put our results:
128  jevois::RawImage outimg = outframe.get();
129  outimg.require("output", outimg.width, outimg.height, V4L2_PIX_FMT_RGB565);
130  cv::Mat outimgcv = jevois::rawimage::cvImage(outimg);
131 
132  // Process input to output, going from YUYV to RGBA internally, to RGB565 rendering:
133  itsFilter->process(inimgcv, outimgcv);
134 
135  // Show processing fps:
136  std::string const & fpscpu = timer.stop();
137  jevois::rawimage::writeText(outimg, fpscpu, 3, outimg.height - 13, jevois::rgb565::White);
138  std::ostringstream oss; oss << "JeVois DemoGPU - Effect: " << effect::get();
139  jevois::rawimage::writeText(outimg, oss.str(), 3, 3, jevois::rgb565::White);
140 
141  // Send the output image with our processing results to the host over USB:
142  outframe.send();
143  }
144 
145 
146  // ####################################################################################################
147  void onParamChange(effect const & JEVOIS_UNUSED_PARAM(param), Effect const & newval)
148  {
149  switch (newval)
150  {
151  case Effect::NoEffect:
152  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/simplefragshader.glsl");
153  break;
154 
155  case Effect::Blur:
156  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/blurfragshader.glsl");
157  break;
158 
159  case Effect::Sobel:
160  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/sobelfragshader.glsl");
161  break;
162 
163  case Effect::Median:
164  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/medianfragshader.glsl");
165  break;
166 
167  case Effect::Mult:
168  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/multfragshader.glsl");
169  break;
170 
171  case Effect::Thresh:
172  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/threshfragshader.glsl");
173  break;
174 
175  case Effect::Dilate:
176  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/dilatefragshader.glsl");
177  break;
178 
179  case Effect::Erode:
180  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/erodefragshader.glsl");
181  break;
182 
183  case Effect::Twirl:
184  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/twirlfragshader.glsl");
185  itsFilter->setProgramParam1f("twirlamount", 2.0F);
186  break;
187 
188  // See fragment shader dewarpfragshader.glsl file for details
189  case Effect::Dewarp:
190  itsFilter->setProgram("shaders/dewarpvertshader.glsl", "shaders/dewarpfragshader.glsl");
191  break;
192  }
193 
194  // These parameters are used by the vertex shader and hence apply to most demo programs:
195  itsFilter->setProgramParam2f("offset", -1.0F, -1.0F);
196  itsFilter->setProgramParam2f("scale", 2.0F, 2.0F);
197 
198  // Crop Dewarped Image - change According to dewarped image
199  itsFilter->setProgramParam2f("offsetd", -1.2F, -1.2F);
200  itsFilter->setProgramParam2f("scaled", 2.4F, 2.4F);
201  }
202 
203  private:
204  std::shared_ptr<FilterGPU> itsFilter;
205 };
206 
207 // Allow the module to be loaded as a shared object (.so) file:
cv::Mat convertToCvRGBA(RawImage const &src)
cv::Mat cvImage(RawImage const &src)
friend friend class Module
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
unsigned int height
JEVOIS_DEFINE_ENUM_CLASS(Dict,(AR_MATRIX_CODE_3x3)(AR_MATRIX_CODE_3x3_HAMMING63)(AR_MATRIX_CODE_3x3_PARITY65))
Enum for parameter.
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. " "If empty, use the module's path.", JEVOIS_SHARE_PATH "/darknet/single", ParamCateg)
Parameter.
std::string const & stop()
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DemoGPU.C:112
JEVOIS_REGISTER_MODULE(DemoGPU)
void onParamChange(effect const &JEVOIS_UNUSED_PARAM(param), Effect const &newval)
Definition: DemoGPU.C:147
virtual ~DemoGPU()
Virtual destructor for safe inheritance.
Definition: DemoGPU.C:109
DemoGPU(std::string const &instance)
Constrctor.
Definition: DemoGPU.C:103
Simple image filtering demo using OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU.
Definition: DemoGPU.C:98
unsigned int width
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const