JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
DemoCPUGPU.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Types/Enum.H>
20 
24 
26 #include <linux/videodev2.h>
27 
28 #include <opencv2/imgproc/imgproc.hpp>
29 #include <future>
30 
31 // icon by Freepik in computer at flaticon
32 
33 //! Simple image filtering demo using 4-core CPU processing and OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU
34 /*! In this demo, we compte saliency and gist over our 4 CPU cores while we also compute 4 different image filters over
35  the GPU, finally combining all results into a single grayscale image.
36 
37  @author Laurent Itti
38 
39  @displayname Demo CPU GPU
40  @videomapping GREY 160 495 60.0 YUYV 160 120 60.0 JeVois DemoCPUGPU
41  @email itti\@usc.edu
42  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
43  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
44  @mainurl http://jevois.org
45  @supporturl http://jevois.org/doc
46  @otherurl http://iLab.usc.edu
47  @license GPL v3
48  @distribution Unrestricted
49  @restrictions None
50  \ingroup modules */
51 class DemoCPUGPU : public jevois::Module
52 {
53  public:
54  //! Constrctor
55  DemoCPUGPU(std::string const & instance) : jevois::Module(instance)
56  {
57  itsFilter = addSubComponent<FilterGPU>("gpu");
58  itsSaliency = addSubComponent<Saliency>("saliency");
59  itsKF = addSubComponent<Kalman2D>("kalman");
60 
61  // Use some fairly large saliency and feature maps so we can see them:
62  itsSaliency->centermin::set(1);
63  itsSaliency->smscale::set(3);
64  }
65 
66  //! Virtual destructor for safe inheritance
67  virtual ~DemoCPUGPU() { }
68 
69  //! Set our GPU program after we are fully constructed and our Component path has been set
70  void postInit() override
71  {
72  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/combofragshader.glsl");
73  itsFilter->setProgramParam2f("offset", -1.0F, -1.0F);
74  itsFilter->setProgramParam2f("scale", 2.0F, 2.0F);
75  }
76 
77  //! Processing function
78  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
79  {
80  // Wait for next available camera image:
81  jevois::RawImage const inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
82  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
83 
84  // In this demo, the GPU completes earlier than the CPU. Hence, we are going to wait for the output frame in the
85  // main thread (which runs the GPU code). We use a mutex to signal to the saliency thread when the output image is
86  // available. Using a mutex and unique_lock here ensures that we will be exception-safe (which may be trickier
87  // with a condition variable or such):
88  jevois::RawImage outimg;
89  std::unique_lock<std::mutex> lck(itsOutMtx); // mutex will be released by main thread when outimg is available
90 
91  // Launch the saliency computation in a thread:
92  auto sal_fut = std::async(std::launch::async, [&](){
93  // Compute saliency and gist:
94  itsSaliency->process(inimg, true);
95 
96  // Find most salient point:
97  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
98 
99  // Compute attended location in original frame coordinates:
100  int const smlev = itsSaliency->smscale::get();
101  int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
102  unsigned int const dmx = (mx << smlev) + smadj;
103  unsigned int const dmy = (my << smlev) + smadj;
104  unsigned int const mapw = itsSaliency->salmap.dims.w, maph = itsSaliency->salmap.dims.h;
105  unsigned int const gistsize = itsSaliency->gist_size;
106  unsigned int const gistw = w - 6 * mapw; // width for gist block
107  unsigned int const gisth = (gistsize + gistw - 1) / gistw; // divide gist_size by w and round up
108 
109  // Filter over time the salient location coordinates:
110  itsKF->set(dmx, dmy, w, h);
111  float kfxraw, kfyraw; itsKF->get(kfxraw, kfyraw);
112 
113  // Send kalman-filtered most-salient-point info to serial port (for arduino, etc):
114  sendSerial("T2D " + std::to_string(int(kfxraw + 0.4999F)) + ' ' + std::to_string(int(kfyraw + 0.4999F)));
115 
116  // Wait for output image to be available:
117  std::lock_guard<std::mutex> _(itsOutMtx);
118 
119  // Paste saliency and feature maps:
120  unsigned int offset = 0;
121  pasteGrayMap(outimg, itsSaliency->salmap, offset, h*4, 20);
122  pasteGrayMap(outimg, itsSaliency->color, offset, h*4, 18);
123  pasteGrayMap(outimg, itsSaliency->intens, offset, h*4, 18);
124  pasteGrayMap(outimg, itsSaliency->ori, offset, h*4, 18);
125  pasteGrayMap(outimg, itsSaliency->flicker, offset, h*4, 18);
126  pasteGrayMap(outimg, itsSaliency->motion, offset, h*4, 18);
127 
128  // Paste gist. Note: at 160x120 we will only end up pasting the first 600 gist entries. This code may fail at
129  // different resolutions, beware:
130  unsigned char * d = outimg.pixelsw<unsigned char>() + 4*w*h + 6*mapw;
131  for (int i = 0; i < maph; ++i) memcpy(d + i*w, itsSaliency->gist + i*gistw, gistw);
132  });
133 
134  // Convert input image to grayscale:
135  cv::Mat grayimg = jevois::rawimage::convertToCvGray(inimg);
136 
137  // Once saliency is done using the input image, let camera know we are done with it:
138  itsSaliency->waitUntilDoneWithInput();
139  inframe.done();
140 
141  // Upload the greyscale image to GPU and apply 4 grayscale GPU filters, storing the 4 results into an RGBA image:
142  cv::Mat gpuout(h, w, CV_8UC4);
143  itsFilter->process(grayimg, gpuout);
144 
145  // Wait for an image from our gadget driver into which we will put our results:
146  outimg = outframe.get();
147  lck.unlock(); // let saliency thread proceed with using the output image
148  outimg.require("output", w, h * 4 + (h >> itsSaliency->smscale::get()), V4L2_PIX_FMT_GREY);
149 
150  // Unpack the GPU output into 4 gray images, starting from top-left of the output image:
152 
153  // Wait until saliency computation is complete:
154  sal_fut.get();
155 
156  // Send the output image with our processing results to the host over USB:
157  outframe.send();
158  }
159 
160  // ####################################################################################################
161  //! Paste a map and add its width to the dx offset
162  /*! Beware this is for a gray outimg only. */
163  void pasteGrayMap(jevois::RawImage & outimg, env_image const & fmap, unsigned int & dx, unsigned int dy,
164  unsigned int bitshift)
165  {
166  env_size_t const fw = fmap.dims.w, fh = fmap.dims.h;
167  unsigned int const ow = outimg.width, oh = outimg.height;
168 
169  if (dy + fh > oh) LFATAL("Map would extend past output image bottom");
170  if (fw + dx > ow) LFATAL("Map would extend past output image width");
171 
172  unsigned int const stride = ow - fw;
173 
174  intg32 * s = fmap.pixels; unsigned char * d = outimg.pixelsw<unsigned char>() + dx + dy * ow;
175 
176  for (unsigned int j = 0; j < fh; ++j)
177  {
178  for (unsigned int i = 0; i < fw; ++i)
179  {
180  intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
181  *d++ = (unsigned char)(v);
182  }
183  d += stride;
184  }
185 
186  // Add the map width to the dx offset:
187  dx += fw;
188  }
189 
190  private:
191  std::shared_ptr<FilterGPU> itsFilter;
192  std::shared_ptr<Saliency> itsSaliency;
193  std::shared_ptr<Kalman2D> itsKF;
194  std::mutex itsOutMtx;
195 };
196 
197 // Allow the module to be loaded as a shared object (.so) file:
Simple image filtering demo using 4-core CPU processing and OpenGL-ES 2.0 shaders on the Mali-400MP2 ...
Definition: DemoCPUGPU.C:51
friend friend class Module
env_size_t w
The width.
Definition: env_types.h:82
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
JEVOIS_REGISTER_MODULE(DemoCPUGPU)
Basic image class.
Definition: env_image.h:43
struct env_dims dims
Definition: env_image.h:45
env_size_t h
The height.
Definition: env_types.h:83
unsigned long env_size_t
Definition: env_types.h:71
DemoCPUGPU(std::string const &instance)
Constrctor.
Definition: DemoCPUGPU.C:55
cv::Mat convertToCvGray(RawImage const &src)
intg32 * pixels
Definition: env_image.h:46
virtual void sendSerial(std::string const &str)
#define LFATAL(msg)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DemoCPUGPU.C:78
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
virtual ~DemoCPUGPU()
Virtual destructor for safe inheritance.
Definition: DemoCPUGPU.C:67
void unpackCvRGBAtoGrayRawImage(cv::Mat const &src, RawImage &dst)
std::string to_string(T const &val)
void postInit() override
Set our GPU program after we are fully constructed and our Component path has been set...
Definition: DemoCPUGPU.C:70
void pasteGrayMap(jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int dy, unsigned int bitshift)
Paste a map and add its width to the dx offset.
Definition: DemoCPUGPU.C:163
unsigned int width