JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
BurnTest.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Timer.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 #include <stdlib.h>
33 
34 // Neon-related:
35 #include <Ne10/inc/NE10_imgproc.h>
36 
37 // GPU-related:
39 
40 // icon by Vectors Market in nature at www.flaticon.com
41 
42 //! This is a burn test: run the quad-core saliency demo while also loading up CPU, GPU and NEON in the background
43 /*! This burn test exercises all aspects of your JeVois smart camera to the maximum, namely:
44 
45  - launch two instances of whetstone (floating point benchmark test) running in the background
46  - launch two instances of dhrystone (integer benchmark test) running in the background
47  - grab frames from the camera sensor
48  - run the quad-core visual attention algorithm
49  - in parallel, run the NEON demo
50  - in parallel, run the GPU demo that processes the video through 4 image filters (shaders)
51  - stream attention video results over USB
52  - issue messages over the serial port
53 
54  This burn test is useful to test JeVois hardware for any malfunction. It should run forever without crashing on
55  JeVois hardware.
56 
57  @author Laurent Itti
58 
59  @videomapping YUYV 640 300 10.0 YUYV 320 240 10.0 JeVois BurnTest
60  @email itti\@usc.edu
61  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
62  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
63  @mainurl http://jevois.org
64  @supporturl http://jevois.org/doc
65  @otherurl http://iLab.usc.edu
66  @license GPL v3
67  @distribution Unrestricted
68  @restrictions None
69  \ingroup modules */
70 class BurnTest : public jevois::Module
71 {
72  public:
73  //! Constructor
74  BurnTest(std::string const & instance) :
75  jevois::Module(instance), itsTimer("BurnTest", 30, LOG_DEBUG)
76  {
77  itsSaliency = addSubComponent<Saliency>("saliency");
78  itsKF = addSubComponent<Kalman2D>("kalman");
79  itsFilter = addSubComponent<FilterGPU>("gpu");
80  }
81 
82  //! Set our GPU program after we are fully constructed and our Component path has been set
83  void postInit() override
84  {
85  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/combofragshader.glsl");
86  itsFilter->setProgramParam2f("offset", -1.0F, -1.0F);
87  itsFilter->setProgramParam2f("scale", 2.0F, 2.0F);
88  itsRunning.store(true);
89  }
90 
91  //! Kill our external processes on uninit
92  void postUninit() override
93  {
94  itsRunning.store(false);
95  system("/bin/rm /tmp/jevois-burntest");
96  system("killall -9 dhrystone");
97  system("killall -9 whetstone");
98  try { itsGPUfut.get(); } catch (...) { }
99  try { itsNEONfut.get(); } catch (...) { }
100  }
101 
102  //! Virtual destructor for safe inheritance
103  virtual ~BurnTest() { }
104 
105  //! Processing function
106  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
107  {
108  // Wait for next available camera image:
109  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
110  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
111 
112  itsTimer.start();
113 
114  // Check whether the input image size is small, in which case we will scale the maps up one notch for the purposes
115  // of this demo:
116  if (w < 170) { itsSaliency->centermin::set(1); itsSaliency->smscale::set(3); }
117  else { itsSaliency->centermin::set(2); itsSaliency->smscale::set(4); }
118 
119  // Launch the saliency computation in a thread:
120  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
121 
122  // While computing, wait for an image from our gadget driver into which we will put our results:
123  jevois::RawImage outimg = outframe.get();
124 
125  // Paste the original image to the top-left corner of the display:
126  jevois::rawimage::paste(inimg, outimg, 0, 0);
127  jevois::rawimage::writeText(outimg, "JeVois CPU+GPU+NEON BurnTest", 3, 3, jevois::yuyv::White);
128 
129  // Once saliency is done using the input image, let camera know we are done with it:
130  itsSaliency->waitUntilDoneWithInput();
131  inframe.done();
132 
133  // If our grayimg is empty, compute it and launch the GPU and NEON threads:
134  if (itsGrayImg.empty())
135  {
136  // Create a temp file:
137  system("touch /tmp/jevois-burntest");
138 
139  // start a couple of whetstones:
140  system("( while [ -f /tmp/jevois-burntest ]; do whetstone 2000000000; done ) &");
141  system("( while [ -f /tmp/jevois-burntest ]; do whetstone 2000000000; done ) &");
142 
143  // dhrystne too
144  system("( while [ -f /tmp/jevois-burntest ]; do dhrystone 2000000000; done ) &");
145  system("( while [ -f /tmp/jevois-burntest ]; do dhrystone 2000000000; done ) &");
146 
147  // then load the GPU
149  itsGPUfut = std::async(std::launch::async, [&](unsigned int ww, unsigned int hh) {
150  cv::Mat gpuout(hh, ww, CV_8UC4);
151  while (itsRunning.load()) itsFilter->process(itsGrayImg, gpuout);
152  }, w, h);
153 
154  // and load NEON too
156 
157  itsNEONfut = std::async(std::launch::async, [&](unsigned int ww, unsigned int hh) {
158  cv::Mat neonresult(hh, ww, CV_8UC4);
159  ne10_size_t src_size { ww, hh }, kernel_size { 5, 5 };
160  while (itsRunning.load())
161  {
162 #ifdef __ARM_NEON__
163  // Neon version:
164  ne10_img_boxfilter_rgba8888_neon(itsRGBAimg.data, neonresult.data, src_size, ww * 4, ww * 4, kernel_size);
165 #else
166  // On non-ARM/NEON host, revert to CPU version again:
167  ne10_img_boxfilter_rgba8888_c(itsRGBAimg.data, neonresult.data, src_size, ww * 4, ww * 4, kernel_size);
168 #endif
169  }
170  }, w, h);
171  }
172 
173  // Wait until saliency computation is complete:
174  sal_fut.get();
175 
176  // Get some info from the saliency computation:
177  int const smlev = itsSaliency->smscale::get();
178  int const smfac = (1 << smlev);
179  int const roihw = (smfac * 3) / 2; // roi half width and height
180  int const mapdrawfac = smfac / 4; // factor by which we enlarge the feature maps for drawing
181  int const mapdw = (w >> smlev) * mapdrawfac; // width of the drawn feature maps
182  int const mapdh = (h >> smlev) * mapdrawfac; // height of the drawn feature maps
183 
184  // Enforce the correct output image size and format:
185  outimg.require("output", w + (w & ~(smfac-1)), h + mapdh, V4L2_PIX_FMT_YUYV);
186 
187  // Find most salient point:
188  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
189 
190  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
191  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
192  unsigned int const dmy = (my << smlev) + (smfac >> 2);
193  int rx = std::min(int(w) - roihw, std::max(roihw, int(dmx + 1 + smfac/4)));
194  int ry = std::min(int(h) - roihw, std::max(roihw, int(dmy + 1 + smfac/4)));
195 
196  // Asynchronously launch a bunch of saliency drawings and filter the attended locations
197  auto draw_fut =
198  std::async(std::launch::async, [&]() {
199  // Filter the attended locations:
200  itsKF->set(dmx, dmy, w, h);
201  float kfxraw, kfyraw, kfximg, kfyimg; itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, w, h);
202 
203  // Draw a circle around the kalman-filtered attended location:
204  jevois::rawimage::drawCircle(outimg, int(kfximg + 0.5F), int(kfyimg + 0.5F),
205  20, 1, jevois::yuyv::LightGreen);
206 
207  // Send kalman-filtered most-salient-point coords to serial port (for arduino, etc):
208  //sendSerial("T2D " + std::to_string(int(kfxraw + 0.4999F)) + ' ' +
209  // std::to_string(int(kfyraw + 0.4999F)));
210 
211  // Paste the saliency map:
212  drawMap(outimg, &itsSaliency->salmap, w, 0, smfac, 20);
213  jevois::rawimage::writeText(outimg, "Saliency Map", w*2 - 12*6-4, 3, jevois::yuyv::White);
214  });
215 
216  // Paste the feature maps:
217  unsigned int dx = 0; // drawing x offset for each feature map
218  drawMap(outimg, &itsSaliency->color, dx, h, mapdrawfac, 18);
219  jevois::rawimage::writeText(outimg, "Color", dx+3, h+3, jevois::yuyv::White);
220  dx += mapdw;
221 
222  drawMap(outimg, &itsSaliency->intens, dx, h, mapdrawfac, 18);
223  jevois::rawimage::writeText(outimg, "Intensity", dx+3, h+3, jevois::yuyv::White);
224  dx += mapdw;
225 
226  drawMap(outimg, &itsSaliency->ori, dx, h, mapdrawfac, 18);
227  jevois::rawimage::writeText(outimg, "Orientation", dx+3, h+3, jevois::yuyv::White);
228  dx += mapdw;
229 
230  drawMap(outimg, &itsSaliency->flicker, dx, h, mapdrawfac, 18);
231  jevois::rawimage::writeText(outimg, "Flicker", dx+3, h+3, jevois::yuyv::White);
232  dx += mapdw;
233 
234  drawMap(outimg, &itsSaliency->motion, dx, h, mapdrawfac, 18);
235  jevois::rawimage::writeText(outimg, "Motion", dx+3, h+3, jevois::yuyv::White);
236  dx += mapdw;
237 
238  // Blank out free space in bottom-right corner, we will then draw the gist (which may only partially occupy that
239  // available space):
240  unsigned int const gw = outimg.width - dx, gh = outimg.height - h;
241  jevois::rawimage::drawFilledRect(outimg, dx, h, gw, gh, 0x8000);
242 
243  // Draw the gist vector, picking a zoom factor to maximize the area filled:
244  unsigned int const gscale = int(sqrt((gw * gh) / itsSaliency->gist_size));
245 
246  drawGist(outimg, itsSaliency->gist, itsSaliency->gist_size, dx, h, gw / gscale, gscale);
247  jevois::rawimage::drawRect(outimg, dx, h, gw, gh, 0x80a0);
248  jevois::rawimage::writeText(outimg, "Gist", dx+3, h+3, jevois::yuyv::White);
249 
250  // Wait for all drawings to complete:
251  draw_fut.get();
252 
253  // Draw a small square at most salient location in image and in saliency map:
254  jevois::rawimage::drawFilledRect(outimg, dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
255  jevois::rawimage::drawFilledRect(outimg, w + dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
256 
257  // Draw an ROI box around the most salient point:
258  jevois::rawimage::drawRect(outimg, rx - roihw, ry - roihw, roihw*2, roihw*2, 0xf0f0);
259  jevois::rawimage::drawRect(outimg, rx - roihw + 1, ry - roihw + 1, roihw*2-2, roihw*2-2, 0xf0f0);
260 
261  // Show processing fps:
262  std::string const & fpscpu = itsTimer.stop();
263  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
264 
265  // Send the output image with our processing results to the host over USB:
266  outframe.send();
267  }
268 
269  protected:
270  std::shared_ptr<Saliency> itsSaliency;
271  std::shared_ptr<Kalman2D> itsKF;
273  std::future<void> itsGPUfut;
274  std::shared_ptr<FilterGPU> itsFilter;
275  cv::Mat itsGrayImg;
276  std::future<void> itsNEONfut;
277  cv::Mat itsRGBAimg;
278  std::atomic<bool> itsRunning;
279 };
280 
281 // Allow the module to be loaded as a shared object (.so) file:
cv::Mat convertToCvRGBA(RawImage const &src)
friend friend class Module
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
void postUninit() override
Kill our external processes on uninit.
Definition: BurnTest.C:92
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: BurnTest.C:106
std::future< void > itsGPUfut
Definition: BurnTest.C:273
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Definition: Saliency.C:771
cv::Mat itsGrayImg
Definition: BurnTest.C:275
jevois::Timer itsTimer
Definition: BurnTest.C:272
cv::Mat itsRGBAimg
Definition: BurnTest.C:277
void postInit() override
Set our GPU program after we are fully constructed and our Component path has been set...
Definition: BurnTest.C:83
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Definition: Saliency.C:709
virtual ~BurnTest()
Virtual destructor for safe inheritance.
Definition: BurnTest.C:103
std::shared_ptr< FilterGPU > itsFilter
Definition: BurnTest.C:274
cv::Mat convertToCvGray(RawImage const &src)
std::future< void > itsNEONfut
Definition: BurnTest.C:276
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
std::atomic< bool > itsRunning
Definition: BurnTest.C:278
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
JEVOIS_REGISTER_MODULE(BurnTest)
std::shared_ptr< Saliency > itsSaliency
Definition: BurnTest.C:270
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
BurnTest(std::string const &instance)
Constructor.
Definition: BurnTest.C:74
This is a burn test: run the quad-core saliency demo while also loading up CPU, GPU and NEON in the b...
Definition: BurnTest.C:70
unsigned int width
std::shared_ptr< Kalman2D > itsKF
Definition: BurnTest.C:271
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)