JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
DemoSaliency.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Timer.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 
33 // icon by Freepik in other at flaticon
34 
35 //! Simple demo of the visual saliency algorithm of Itti et al., IEEE PAMI, 1998
36 /*! Visual saliency algorithm as described at http://ilab.usc.edu/bu/
37 
38  @author Laurent Itti
39 
40  @videomapping YUYV 176 90 120.0 YUYV 88 72 120.0 JeVois DemoSaliency
41  @videomapping YUYV 320 150 60.0 YUYV 160 120 60.0 JeVois DemoSaliency
42  @videomapping YUYV 352 180 120.0 YUYV 176 144 120.0 JeVois DemoSaliency
43  @videomapping YUYV 352 180 100.0 YUYV 176 144 100.0 JeVois DemoSaliency
44  @videomapping YUYV 640 300 60.0 YUYV 320 240 60.0 JeVois DemoSaliency
45  @videomapping YUYV 704 360 30.0 YUYV 352 288 30.0 JeVois DemoSaliency
46  @videomapping YUYV 1280 600 15.0 YUYV 640 480 15.0 JeVois DemoSaliency
47  @videomapping YUYV 320 260 30.0 YUYV 320 240 30.0 JeVois DemoArUco
48  @email itti\@usc.edu
49  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
50  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
51  @mainurl http://jevois.org
52  @supporturl http://jevois.org/doc
53  @otherurl http://iLab.usc.edu
54  @license GPL v3
55  @distribution Unrestricted
56  @restrictions None
57  \ingroup modules */
59 {
60  public:
61  //! Constructor
62  DemoSaliency(std::string const & instance) :
63  jevois::Module(instance), itsTimer("DemoSaliency")
64  {
65  itsSaliency = addSubComponent<Saliency>("saliency");
66  itsKF = addSubComponent<Kalman2D>("kalman");
67  }
68 
69  //! Virtual destructor for safe inheritance
70  virtual ~DemoSaliency() { }
71 
72  //! Processing function
73  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
74  {
75  // Wait for next available camera image:
76  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
77  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
78 
79  itsTimer.start();
80 
81  // Check whether the input image size is small, in which case we will scale the maps up one notch for the purposes
82  // of this demo:
83  if (w < 170) { itsSaliency->centermin::set(1); itsSaliency->smscale::set(3); }
84  else { itsSaliency->centermin::set(2); itsSaliency->smscale::set(4); }
85 
86  // Launch the saliency computation in a thread:
87  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
88 
89  // While computing, wait for an image from our gadget driver into which we will put our results:
90  jevois::RawImage outimg = outframe.get();
91 
92  // Paste the original image to the top-left corner of the display:
93  jevois::rawimage::paste(inimg, outimg, 0, 0);
94  jevois::rawimage::writeText(outimg, "JeVois Saliency + Gist Demo", 3, 3, jevois::yuyv::White);
95 
96  // Once saliency is done using the input image, let camera know we are done with it:
97  itsSaliency->waitUntilDoneWithInput();
98  inframe.done();
99 
100  // Wait until saliency computation is complete:
101  sal_fut.get();
102 
103  // Get some info from the saliency computation:
104  int const smlev = itsSaliency->smscale::get();
105  int const smfac = (1 << smlev);
106  int const roihw = (smfac * 3) / 2; // roi half width and height
107  int const mapdrawfac = smfac / 4; // factor by which we enlarge the feature maps for drawing
108  int const mapdw = (w >> smlev) * mapdrawfac; // width of the drawn feature maps
109  int const mapdh = (h >> smlev) * mapdrawfac; // height of the drawn feature maps
110 
111  // Enforce the correct output image size and format:
112  outimg.require("output", w + (w & ~(smfac-1)), h + mapdh, V4L2_PIX_FMT_YUYV);
113 
114  // Find most salient point:
115  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
116 
117  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
118  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
119  unsigned int const dmy = (my << smlev) + (smfac >> 2);
120  int rx = std::min(int(w) - roihw, std::max(roihw, int(dmx + 1 + smfac/4)));
121  int ry = std::min(int(h) - roihw, std::max(roihw, int(dmy + 1 + smfac/4)));
122 
123  // Asynchronously launch a bunch of saliency drawings and filter the attended locations
124  auto draw_fut =
125  std::async(std::launch::async, [&]() {
126  // Filter the attended locations:
127  itsKF->set(dmx, dmy, w, h);
128  float kfxraw, kfyraw, kfximg, kfyimg; itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, w, h);
129 
130  // Draw a circle around the kalman-filtered attended location:
131  jevois::rawimage::drawCircle(outimg, int(kfximg + 0.5F), int(kfyimg + 0.5F),
132  20, 1, jevois::yuyv::LightGreen);
133 
134  // Send kalman-filtered most-salient-point coords to serial port (for arduino, etc):
135  sendSerial("T2D " + std::to_string(int(kfxraw + 0.4999F)) + ' ' +
136  std::to_string(int(kfyraw + 0.4999F)));
137 
138  // Paste the saliency map:
139  drawMap(outimg, &itsSaliency->salmap, w, 0, smfac, 20);
140  jevois::rawimage::writeText(outimg, "Saliency Map", w*2 - 12*6-4, 3, jevois::yuyv::White);
141  });
142 
143  // Paste the feature maps:
144  unsigned int dx = 0; // drawing x offset for each feature map
145  drawMap(outimg, &itsSaliency->color, dx, h, mapdrawfac, 18);
146  jevois::rawimage::writeText(outimg, "Color", dx+3, h+3, jevois::yuyv::White);
147  dx += mapdw;
148 
149  drawMap(outimg, &itsSaliency->intens, dx, h, mapdrawfac, 18);
150  jevois::rawimage::writeText(outimg, "Intensity", dx+3, h+3, jevois::yuyv::White);
151  dx += mapdw;
152 
153  drawMap(outimg, &itsSaliency->ori, dx, h, mapdrawfac, 18);
154  jevois::rawimage::writeText(outimg, "Orientation", dx+3, h+3, jevois::yuyv::White);
155  dx += mapdw;
156 
157  drawMap(outimg, &itsSaliency->flicker, dx, h, mapdrawfac, 18);
158  jevois::rawimage::writeText(outimg, "Flicker", dx+3, h+3, jevois::yuyv::White);
159  dx += mapdw;
160 
161  drawMap(outimg, &itsSaliency->motion, dx, h, mapdrawfac, 18);
162  jevois::rawimage::writeText(outimg, "Motion", dx+3, h+3, jevois::yuyv::White);
163  dx += mapdw;
164 
165  // Blank out free space in bottom-right corner, we will then draw the gist (which may only partially occupy that
166  // available space):
167  unsigned int const gw = outimg.width - dx, gh = outimg.height - h;
168  jevois::rawimage::drawFilledRect(outimg, dx, h, gw, gh, 0x8000);
169 
170  // Draw the gist vector, picking a zoom factor to maximize the area filled:
171  unsigned int const gscale = int(sqrt((gw * gh) / itsSaliency->gist_size));
172 
173  drawGist(outimg, itsSaliency->gist, itsSaliency->gist_size, dx, h, gw / gscale, gscale);
174  jevois::rawimage::drawRect(outimg, dx, h, gw, gh, 0x80a0);
175  jevois::rawimage::writeText(outimg, "Gist", dx+3, h+3, jevois::yuyv::White);
176 
177  // Wait for all drawings to complete:
178  draw_fut.get();
179 
180  // Draw a small square at most salient location in image and in saliency map:
181  jevois::rawimage::drawFilledRect(outimg, dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
182  jevois::rawimage::drawFilledRect(outimg, w + dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
183 
184  // Draw an ROI box around the most salient point:
185  jevois::rawimage::drawRect(outimg, rx - roihw, ry - roihw, roihw*2, roihw*2, 0xf0f0);
186  jevois::rawimage::drawRect(outimg, rx - roihw + 1, ry - roihw + 1, roihw*2-2, roihw*2-2, 0xf0f0);
187 
188  // Show processing fps:
189  std::string const & fpscpu = itsTimer.stop();
190  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
191 
192  // Send the output image with our processing results to the host over USB:
193  outframe.send();
194  }
195 
196  protected:
197  std::shared_ptr<Saliency> itsSaliency;
198  std::shared_ptr<Kalman2D> itsKF;
200 };
201 
202 // Allow the module to be loaded as a shared object (.so) file:
std::shared_ptr< Saliency > itsSaliency
Definition: DemoSaliency.C:197
friend friend class Module
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
JEVOIS_REGISTER_MODULE(DemoSaliency)
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Definition: Saliency.C:771
Simple demo of the visual saliency algorithm of Itti et al., IEEE PAMI, 1998.
Definition: DemoSaliency.C:58
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DemoSaliency.C:73
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Definition: Saliency.C:709
DemoSaliency(std::string const &instance)
Constructor.
Definition: DemoSaliency.C:62
virtual void sendSerial(std::string const &str)
jevois::Timer itsTimer
Definition: DemoSaliency.C:199
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
virtual ~DemoSaliency()
Virtual destructor for safe inheritance.
Definition: DemoSaliency.C:70
std::shared_ptr< Kalman2D > itsKF
Definition: DemoSaliency.C:198
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
std::string to_string(T const &val)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
unsigned int width
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)