JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DemoGPU.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Types/Enum.H>
20 #include <jevois/Debug/Timer.H>
21 
23 
25 #include <linux/videodev2.h>
26 
27 static jevois::ParameterCategory const ParamCateg("DemoGPU Parameters");
28 
29 // Note: we use NoEffect here as None seems to be defined a s anumeric constant somewhere, when compiling on host
30 
31 //! Parameter \relates DemoGPU
32 JEVOIS_DEFINE_ENUM_CLASS(Effect, (NoEffect) (Blur) (Sobel) (Median) (Mult) (Thresh) (Dilate) (Erode) (Twirl) (Dewarp));
33 
34 //! Parameter \relates DemoGPU
35 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(effect, Effect, "GPU image processing effect to apply",
36  Effect::Twirl, Effect_Values, ParamCateg);
37 
38 // icon by by Freepik in technology at flaticon
39 
40 //! Simple image filtering demo using OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU
41 /*! This class first copies the given input image into an OpenGL texture, then applies OpenGL-ES vertex and fragment
42  shaders to achieve some image processing, and finally gets the resulting pixels back into an image.
43 
44  This code is inspired somewhat from the tutorial and code examples found on this page:
45  http://robotblogging.blogspot.com/2013/10/gpu-accelerated-camera-processing-on.html
46 
47  But an important distinction is that we render to a framebuffer with an RGB565 renderbuffer attached, which
48  accelerates processing and transfer of the results a lot.
49 
50  Dewarp algorithm contributed by JeVois user [Ali AlSaibie](https://github.com/alsaibie). It is to be used with a
51  modified JeVois camera sensor that has a wide-angle lens. See http://jevois.org/qa/index.php?qa=153 for details.
52 
53  @author Laurent Itti
54 
55  @displayname Demo GPU
56  @videomapping RGB565 320 240 22.0 YUYV 320 240 22.0 JeVois DemoGPU
57  @email itti\@usc.edu
58  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
59  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
60  @mainurl http://jevois.org
61  @supporturl http://jevois.org/doc
62  @otherurl http://iLab.usc.edu
63  @license GPL v3
64  @distribution Unrestricted
65  @restrictions None
66  \ingroup modules */
67 class DemoGPU : public jevois::Module,
68  public jevois::Parameter<effect>
69 {
70  public:
71  //! Constrctor
72  DemoGPU(std::string const & instance) : jevois::Module(instance)
73  {
74  itsFilter = addSubComponent<FilterGPU>("gpu");
75  }
76 
77  //! Virtual destructor for safe inheritance
78  virtual ~DemoGPU() { }
79 
80  //! Processing function
81  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
82  {
83  static jevois::Timer timer("DemoGPU");
84 
85  // Wait for next available camera image:
86  jevois::RawImage const inimg = inframe.get();
87 
88  timer.start();
89 
90  // Convert input image to RGBA:
91  cv::Mat inimgcv = jevois::rawimage::convertToCvRGBA(inimg);
92 
93  // Let camera know we are done processing the input image:
94  inframe.done();
95 
96  // Wait for an image from our gadget driver into which we will put our results:
97  jevois::RawImage outimg = outframe.get();
98  outimg.require("output", outimg.width, outimg.height, V4L2_PIX_FMT_RGB565);
99  cv::Mat outimgcv = jevois::rawimage::cvImage(outimg);
100 
101  // Process input to output, going from YUYV to RGBA internally, to RGB565 rendering:
102  itsFilter->process(inimgcv, outimgcv);
103 
104  // Show processing fps:
105  std::string const & fpscpu = timer.stop();
106  jevois::rawimage::writeText(outimg, fpscpu, 3, outimg.height - 13, jevois::rgb565::White);
107  std::ostringstream oss; oss << "JeVois DemoGPU - Effect: " << effect::get();
108  jevois::rawimage::writeText(outimg, oss.str(), 3, 3, jevois::rgb565::White);
109 
110  // Send the output image with our processing results to the host over USB:
111  outframe.send();
112  }
113 
114 
115  // ####################################################################################################
116  void onParamChange(effect const & JEVOIS_UNUSED_PARAM(param), Effect const & newval)
117  {
118  switch (newval)
119  {
120  case Effect::NoEffect:
121  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/simplefragshader.glsl");
122  break;
123 
124  case Effect::Blur:
125  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/blurfragshader.glsl");
126  break;
127 
128  case Effect::Sobel:
129  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/sobelfragshader.glsl");
130  break;
131 
132  case Effect::Median:
133  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/medianfragshader.glsl");
134  break;
135 
136  case Effect::Mult:
137  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/multfragshader.glsl");
138  break;
139 
140  case Effect::Thresh:
141  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/threshfragshader.glsl");
142  break;
143 
144  case Effect::Dilate:
145  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/dilatefragshader.glsl");
146  break;
147 
148  case Effect::Erode:
149  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/erodefragshader.glsl");
150  break;
151 
152  case Effect::Twirl:
153  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/twirlfragshader.glsl");
154  itsFilter->setProgramParam1f("twirlamount", 2.0F);
155  break;
156 
157  // See fragment shader dewarpfragshader.glsl file for details
158  case Effect::Dewarp:
159  itsFilter->setProgram("shaders/dewarpvertshader.glsl", "shaders/dewarpfragshader.glsl");
160  break;
161  }
162 
163  // These parameters are used by the vertex shader and hence apply to most demo programs:
164  itsFilter->setProgramParam2f("offset", -1.0F, -1.0F);
165  itsFilter->setProgramParam2f("scale", 2.0F, 2.0F);
166 
167  // Crop Dewarped Image - change According to dewarped image
168  itsFilter->setProgramParam2f("offsetd", -1.2F, -1.2F);
169  itsFilter->setProgramParam2f("scaled", 2.4F, 2.4F);
170  }
171 
172  private:
173  std::shared_ptr<FilterGPU> itsFilter;
174 };
175 
176 // Allow the module to be loaded as a shared object (.so) file:
cv::Mat convertToCvRGBA(RawImage const &src)
cv::Mat cvImage(RawImage const &src)
friend friend class Module
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
unsigned int height
JEVOIS_DEFINE_ENUM_CLASS(Dict,(AR_MATRIX_CODE_3x3)(AR_MATRIX_CODE_3x3_HAMMING63)(AR_MATRIX_CODE_3x3_PARITY65))
Enum for parameter.
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. " "If empty, use the module's path.", JEVOIS_SHARE_PATH "/darknet/single", ParamCateg)
Parameter.
std::string const & stop()
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DemoGPU.C:81
JEVOIS_REGISTER_MODULE(DemoGPU)
void onParamChange(effect const &JEVOIS_UNUSED_PARAM(param), Effect const &newval)
Definition: DemoGPU.C:116
virtual ~DemoGPU()
Virtual destructor for safe inheritance.
Definition: DemoGPU.C:78
DemoGPU(std::string const &instance)
Constrctor.
Definition: DemoGPU.C:72
Simple image filtering demo using OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU.
Definition: DemoGPU.C:67
unsigned int width
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const