JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
BurnTest.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Timer.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 #include <stdlib.h>
33 
34 // Neon-related:
35 #include <Ne10/inc/NE10_imgproc.h>
36 
37 // GPU-related:
39 
40 // icon by Vectors Market in nature at www.flaticon.com
41 
42 //! This is a burn test: run the quad-core saliency demo while also loading up CPU, GPU and NEON in the background
43 /*! This burn test exercises all aspects of your JeVois smart camera to the maximum, namely:
44 
45  - launch two instances of whetstone (floating point benchmark test) running in the background
46  - launch two instances of dhrystone (integer benchmark test) running in the background
47  - grab frames from the camera sensor
48  - run the quad-core visual attention algorithm
49  - in parallel, run the NEON demo
50  - in parallel, run the GPU demo that processes the video through 4 image filters (shaders)
51  - stream attention video results over USB
52  - issue messages over the serial port
53 
54  This burn test is useful to test JeVois hardware for any malfunction. It should run forever without crashing on
55  JeVois hardware. Demo display layout and markings are the same as for the DemoSaliency module.
56 
57  This burn test is one of the tests that every JeVois camera produced is tested with at the factory, before the unit
58  is shipped out.
59 
60  Things to try
61  -------------
62 
63  Select the burntest video mode (note that it is 640x300 \@ 10fps, while the default MicroSD card also includes a
64  mode with 640x300 \@ 60fps that runs the DemoSaliency module instead). Observe the CPU temperature at the bottom of
65  the live video window. If it ever reaches 75C (which it should not under normal conditions given the high power fan
66  on the JeVois smart camera), the CPU frequency also shown next to the temperature will drop down below 1344 MHz,
67  and will then come back up as the CPU temperature drops below 75C.
68 
69  Connect your JeVois camera to your host computer through a USB Tester device that measures voltage, current, and
70  power. You should reach about 3.7 Watts under the burn test, which is the maximum we have ever been able to achieve
71  with a JeVois unit.
72 
73 
74  @author Laurent Itti
75 
76  @videomapping YUYV 640 300 10.0 YUYV 320 240 10.0 JeVois BurnTest
77  @email itti\@usc.edu
78  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
79  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
80  @mainurl http://jevois.org
81  @supporturl http://jevois.org/doc
82  @otherurl http://iLab.usc.edu
83  @license GPL v3
84  @distribution Unrestricted
85  @restrictions None
86  \ingroup modules */
87 class BurnTest : public jevois::Module
88 {
89  public:
90  //! Constructor
91  BurnTest(std::string const & instance) :
92  jevois::Module(instance), itsTimer("BurnTest", 30, LOG_DEBUG)
93  {
94  itsSaliency = addSubComponent<Saliency>("saliency");
95  itsKF = addSubComponent<Kalman2D>("kalman");
96  itsFilter = addSubComponent<FilterGPU>("gpu");
97  }
98 
99  //! Set our GPU program after we are fully constructed and our Component path has been set
100  void postInit() override
101  {
102  itsFilter->setProgram("shaders/simplevertshader.glsl", "shaders/combofragshader.glsl");
103  itsFilter->setProgramParam2f("offset", -1.0F, -1.0F);
104  itsFilter->setProgramParam2f("scale", 2.0F, 2.0F);
105  itsRunning.store(true);
106  }
107 
108  //! Kill our external processes on uninit
109  void postUninit() override
110  {
111  itsRunning.store(false);
112  system("/bin/rm /tmp/jevois-burntest");
113  system("killall -9 dhrystone");
114  system("killall -9 whetstone");
115  try { itsGPUfut.get(); } catch (...) { }
116  try { itsNEONfut.get(); } catch (...) { }
117  }
118 
119  //! Virtual destructor for safe inheritance
120  virtual ~BurnTest() { }
121 
122  //! Processing function
123  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
124  {
125  // Wait for next available camera image:
126  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
127  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
128 
129  itsTimer.start();
130 
131  // Check whether the input image size is small, in which case we will scale the maps up one notch for the purposes
132  // of this demo:
133  if (w < 170) { itsSaliency->centermin::set(1); itsSaliency->smscale::set(3); }
134  else { itsSaliency->centermin::set(2); itsSaliency->smscale::set(4); }
135 
136  // Launch the saliency computation in a thread:
137  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
138 
139  // While computing, wait for an image from our gadget driver into which we will put our results:
140  jevois::RawImage outimg = outframe.get();
141 
142  // Paste the original image to the top-left corner of the display:
143  jevois::rawimage::paste(inimg, outimg, 0, 0);
144  jevois::rawimage::writeText(outimg, "JeVois CPU+GPU+NEON BurnTest", 3, 3, jevois::yuyv::White);
145 
146  // Once saliency is done using the input image, let camera know we are done with it:
147  itsSaliency->waitUntilDoneWithInput();
148  inframe.done();
149 
150  // If our grayimg is empty, compute it and launch the GPU and NEON threads:
151  if (itsGrayImg.empty())
152  {
153  // Create a temp file:
154  system("touch /tmp/jevois-burntest");
155 
156  // start a couple of whetstones:
157  system("( while [ -f /tmp/jevois-burntest ]; do whetstone 2000000000; done ) &");
158  system("( while [ -f /tmp/jevois-burntest ]; do whetstone 2000000000; done ) &");
159 
160  // dhrystne too
161  system("( while [ -f /tmp/jevois-burntest ]; do dhrystone 2000000000; done ) &");
162  system("( while [ -f /tmp/jevois-burntest ]; do dhrystone 2000000000; done ) &");
163 
164  // then load the GPU
166  itsGPUfut = std::async(std::launch::async, [&](unsigned int ww, unsigned int hh) {
167  cv::Mat gpuout(hh, ww, CV_8UC4);
168  while (itsRunning.load()) itsFilter->process(itsGrayImg, gpuout);
169  }, w, h);
170 
171  // and load NEON too
173 
174  itsNEONfut = std::async(std::launch::async, [&](unsigned int ww, unsigned int hh) {
175  cv::Mat neonresult(hh, ww, CV_8UC4);
176  ne10_size_t src_size { ww, hh }, kernel_size { 5, 5 };
177  while (itsRunning.load())
178  {
179 #ifdef __ARM_NEON__
180  // Neon version:
181  ne10_img_boxfilter_rgba8888_neon(itsRGBAimg.data, neonresult.data, src_size, ww * 4, ww * 4, kernel_size);
182 #else
183  // On non-ARM/NEON host, revert to CPU version again:
184  ne10_img_boxfilter_rgba8888_c(itsRGBAimg.data, neonresult.data, src_size, ww * 4, ww * 4, kernel_size);
185 #endif
186  }
187  }, w, h);
188  }
189 
190  // Wait until saliency computation is complete:
191  sal_fut.get();
192 
193  // Get some info from the saliency computation:
194  int const smlev = itsSaliency->smscale::get();
195  int const smfac = (1 << smlev);
196  int const roihw = (smfac * 3) / 2; // roi half width and height
197  int const mapdrawfac = smfac / 4; // factor by which we enlarge the feature maps for drawing
198  int const mapdw = (w >> smlev) * mapdrawfac; // width of the drawn feature maps
199  int const mapdh = (h >> smlev) * mapdrawfac; // height of the drawn feature maps
200 
201  // Enforce the correct output image size and format:
202  outimg.require("output", w + (w & ~(smfac-1)), h + mapdh, V4L2_PIX_FMT_YUYV);
203 
204  // Find most salient point:
205  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
206 
207  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
208  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
209  unsigned int const dmy = (my << smlev) + (smfac >> 2);
210  int rx = std::min(int(w) - roihw, std::max(roihw, int(dmx + 1 + smfac/4)));
211  int ry = std::min(int(h) - roihw, std::max(roihw, int(dmy + 1 + smfac/4)));
212 
213  // Asynchronously launch a bunch of saliency drawings and filter the attended locations
214  auto draw_fut =
215  std::async(std::launch::async, [&]() {
216  // Filter the attended locations:
217  itsKF->set(dmx, dmy, w, h);
218  float kfxraw, kfyraw, kfximg, kfyimg;
219  itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, inimg.width, inimg.height, 1.0F, 1.0F);
220 
221  // Draw a circle around the kalman-filtered attended location:
222  jevois::rawimage::drawCircle(outimg, int(kfximg), int(kfyimg), 20, 1, jevois::yuyv::LightGreen);
223 
224  // Send saliency info to serial port (for arduino, etc):
225  //sendSerial(jevois::sformat("T2D %d %d", int(kfxraw), int(kfyraw)));
226 
227  // Paste the saliency map:
228  drawMap(outimg, &itsSaliency->salmap, w, 0, smfac, 20);
229  jevois::rawimage::writeText(outimg, "Saliency Map", w*2 - 12*6-4, 3, jevois::yuyv::White);
230  });
231 
232  // Paste the feature maps:
233  unsigned int dx = 0; // drawing x offset for each feature map
234  drawMap(outimg, &itsSaliency->color, dx, h, mapdrawfac, 18);
235  jevois::rawimage::writeText(outimg, "Color", dx+3, h+3, jevois::yuyv::White);
236  dx += mapdw;
237 
238  drawMap(outimg, &itsSaliency->intens, dx, h, mapdrawfac, 18);
239  jevois::rawimage::writeText(outimg, "Intensity", dx+3, h+3, jevois::yuyv::White);
240  dx += mapdw;
241 
242  drawMap(outimg, &itsSaliency->ori, dx, h, mapdrawfac, 18);
243  jevois::rawimage::writeText(outimg, "Orientation", dx+3, h+3, jevois::yuyv::White);
244  dx += mapdw;
245 
246  drawMap(outimg, &itsSaliency->flicker, dx, h, mapdrawfac, 18);
247  jevois::rawimage::writeText(outimg, "Flicker", dx+3, h+3, jevois::yuyv::White);
248  dx += mapdw;
249 
250  drawMap(outimg, &itsSaliency->motion, dx, h, mapdrawfac, 18);
251  jevois::rawimage::writeText(outimg, "Motion", dx+3, h+3, jevois::yuyv::White);
252  dx += mapdw;
253 
254  // Blank out free space in bottom-right corner, we will then draw the gist (which may only partially occupy that
255  // available space):
256  unsigned int const gw = outimg.width - dx, gh = outimg.height - h;
257  jevois::rawimage::drawFilledRect(outimg, dx, h, gw, gh, 0x8000);
258 
259  // Draw the gist vector, picking a zoom factor to maximize the area filled:
260  unsigned int const gscale = int(sqrt((gw * gh) / itsSaliency->gist_size));
261 
262  drawGist(outimg, itsSaliency->gist, itsSaliency->gist_size, dx, h, gw / gscale, gscale);
263  jevois::rawimage::drawRect(outimg, dx, h, gw, gh, 0x80a0);
264  jevois::rawimage::writeText(outimg, "Gist", dx+3, h+3, jevois::yuyv::White);
265 
266  // Wait for all drawings to complete:
267  draw_fut.get();
268 
269  // Draw a small square at most salient location in image and in saliency map:
270  jevois::rawimage::drawFilledRect(outimg, dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
271  jevois::rawimage::drawFilledRect(outimg, w + dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
272 
273  // Draw an ROI box around the most salient point:
274  jevois::rawimage::drawRect(outimg, rx - roihw, ry - roihw, roihw*2, roihw*2, 0xf0f0);
275  jevois::rawimage::drawRect(outimg, rx - roihw + 1, ry - roihw + 1, roihw*2-2, roihw*2-2, 0xf0f0);
276 
277  // Show processing fps:
278  std::string const & fpscpu = itsTimer.stop();
279  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
280 
281  // Send the output image with our processing results to the host over USB:
282  outframe.send();
283  }
284 
285  protected:
286  std::shared_ptr<Saliency> itsSaliency;
287  std::shared_ptr<Kalman2D> itsKF;
289  std::future<void> itsGPUfut;
290  std::shared_ptr<FilterGPU> itsFilter;
291  cv::Mat itsGrayImg;
292  std::future<void> itsNEONfut;
293  cv::Mat itsRGBAimg;
294  std::atomic<bool> itsRunning;
295 };
296 
297 // Allow the module to be loaded as a shared object (.so) file:
cv::Mat convertToCvRGBA(RawImage const &src)
friend friend class Module
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
void postUninit() override
Kill our external processes on uninit.
Definition: BurnTest.C:109
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: BurnTest.C:123
std::future< void > itsGPUfut
Definition: BurnTest.C:289
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Definition: Saliency.C:771
cv::Mat itsGrayImg
Definition: BurnTest.C:291
jevois::Timer itsTimer
Definition: BurnTest.C:288
cv::Mat itsRGBAimg
Definition: BurnTest.C:293
void postInit() override
Set our GPU program after we are fully constructed and our Component path has been set...
Definition: BurnTest.C:100
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Definition: Saliency.C:709
virtual ~BurnTest()
Virtual destructor for safe inheritance.
Definition: BurnTest.C:120
std::shared_ptr< FilterGPU > itsFilter
Definition: BurnTest.C:290
cv::Mat convertToCvGray(RawImage const &src)
std::future< void > itsNEONfut
Definition: BurnTest.C:292
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
std::atomic< bool > itsRunning
Definition: BurnTest.C:294
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
JEVOIS_REGISTER_MODULE(BurnTest)
std::shared_ptr< Saliency > itsSaliency
Definition: BurnTest.C:286
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
BurnTest(std::string const &instance)
Constructor.
Definition: BurnTest.C:91
This is a burn test: run the quad-core saliency demo while also loading up CPU, GPU and NEON in the b...
Definition: BurnTest.C:87
unsigned int width
std::shared_ptr< Kalman2D > itsKF
Definition: BurnTest.C:287
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)