JeVoisBase  1.2
JeVois Smart Embedded Machine Vision Toolkit Base Modules
DemoSaliency.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Timer.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 
33 // icon by Freepik in other at flaticon
34 
35 //! Simple demo of the visual saliency algorithm of Itti et al., IEEE PAMI, 1998
36 /*! Visual saliency algorithm as described at http://ilab.usc.edu/bu/
37 
38  @author Laurent Itti
39 
40  @videomapping YUYV 176 90 120.0 YUYV 88 72 120.0 JeVois DemoSaliency
41  @videomapping YUYV 320 150 60.0 YUYV 160 120 60.0 JeVois DemoSaliency
42  @videomapping YUYV 352 180 120.0 YUYV 176 144 120.0 JeVois DemoSaliency
43  @videomapping YUYV 352 180 100.0 YUYV 176 144 100.0 JeVois DemoSaliency
44  @videomapping YUYV 640 300 60.0 YUYV 320 240 60.0 JeVois DemoSaliency
45  @videomapping YUYV 704 360 30.0 YUYV 352 288 30.0 JeVois DemoSaliency
46  @videomapping YUYV 1280 600 15.0 YUYV 640 480 15.0 JeVois DemoSaliency
47  @videomapping YUYV 320 260 30.0 YUYV 320 240 30.0 JeVois DemoArUco
48  @email itti\@usc.edu
49  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
50  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
51  @mainurl http://jevois.org
52  @supporturl http://jevois.org/doc
53  @otherurl http://iLab.usc.edu
54  @license GPL v3
55  @distribution Unrestricted
56  @restrictions None
57  \ingroup modules */
59 {
60  public:
61  //! Constructor
62  DemoSaliency(std::string const & instance) :
63  jevois::Module(instance), itsTimer("DemoSaliency")
64  {
65  itsSaliency = addSubComponent<Saliency>("saliency");
66  itsKF = addSubComponent<Kalman2D>("kalman");
67  }
68 
69  //! Virtual destructor for safe inheritance
70  virtual ~DemoSaliency() { }
71 
72  //! Processing function
73  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
74  {
75  // Wait for next available camera image:
76  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
77  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
78 
79  itsTimer.start();
80 
81  // Check whether the input image size is small, in which case we will scale the maps up one notch for the purposes
82  // of this demo:
83  if (w < 170) { itsSaliency->centermin::set(1); itsSaliency->smscale::set(3); }
84  else { itsSaliency->centermin::set(2); itsSaliency->smscale::set(4); }
85 
86  // Launch the saliency computation in a thread:
87  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
88 
89  // While computing, wait for an image from our gadget driver into which we will put our results:
90  jevois::RawImage outimg = outframe.get();
91 
92  // Paste the original image to the top-left corner of the display:
93  jevois::rawimage::paste(inimg, outimg, 0, 0);
94  jevois::rawimage::writeText(outimg, "JeVois Saliency + Gist Demo", 3, 3, jevois::yuyv::White);
95 
96  // Once saliency is done using the input image, let camera know we are done with it:
97  itsSaliency->waitUntilDoneWithInput();
98  inframe.done();
99 
100  // Wait until saliency computation is complete:
101  sal_fut.get();
102 
103  // Get some info from the saliency computation:
104  int const smlev = itsSaliency->smscale::get();
105  int const smfac = (1 << smlev);
106  int const roihw = (smfac * 3) / 2; // roi half width and height
107  int const mapdrawfac = smfac / 4; // factor by which we enlarge the feature maps for drawing
108  int const mapdw = (w >> smlev) * mapdrawfac; // width of the drawn feature maps
109  int const mapdh = (h >> smlev) * mapdrawfac; // height of the drawn feature maps
110 
111  // Enforce the correct output image size and format:
112  outimg.require("output", w + (w & ~(smfac-1)), h + mapdh, V4L2_PIX_FMT_YUYV);
113 
114  // Find most salient point:
115  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
116 
117  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
118  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
119  unsigned int const dmy = (my << smlev) + (smfac >> 2);
120  int rx = std::min(int(w) - roihw, std::max(roihw, int(dmx + 1 + smfac/4)));
121  int ry = std::min(int(h) - roihw, std::max(roihw, int(dmy + 1 + smfac/4)));
122 
123  // Asynchronously launch a bunch of saliency drawings and filter the attended locations
124  auto draw_fut =
125  std::async(std::launch::async, [&]() {
126  // Filter the attended locations:
127  itsKF->set(dmx, dmy, w, h);
128  float kfxraw, kfyraw, kfximg, kfyimg;
129  itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, inimg.width, inimg.height, 1.0F, 1.0F);
130 
131  // Draw a circle around the kalman-filtered attended location:
132  jevois::rawimage::drawCircle(outimg, int(kfximg), int(kfyimg), 20, 1, jevois::yuyv::LightGreen);
133 
134  // Send kalman-filtered most-salient-point coords to serial port (for arduino, etc):
135  sendSerial(jevois::sformat("T2D %d %d", int(kfxraw), int(kfyraw)));
136 
137  // Paste the saliency map:
138  drawMap(outimg, &itsSaliency->salmap, w, 0, smfac, 20);
139  jevois::rawimage::writeText(outimg, "Saliency Map", w*2 - 12*6-4, 3, jevois::yuyv::White);
140  });
141 
142  // Paste the feature maps:
143  unsigned int dx = 0; // drawing x offset for each feature map
144  drawMap(outimg, &itsSaliency->color, dx, h, mapdrawfac, 18);
145  jevois::rawimage::writeText(outimg, "Color", dx+3, h+3, jevois::yuyv::White);
146  dx += mapdw;
147 
148  drawMap(outimg, &itsSaliency->intens, dx, h, mapdrawfac, 18);
149  jevois::rawimage::writeText(outimg, "Intensity", dx+3, h+3, jevois::yuyv::White);
150  dx += mapdw;
151 
152  drawMap(outimg, &itsSaliency->ori, dx, h, mapdrawfac, 18);
153  jevois::rawimage::writeText(outimg, "Orientation", dx+3, h+3, jevois::yuyv::White);
154  dx += mapdw;
155 
156  drawMap(outimg, &itsSaliency->flicker, dx, h, mapdrawfac, 18);
157  jevois::rawimage::writeText(outimg, "Flicker", dx+3, h+3, jevois::yuyv::White);
158  dx += mapdw;
159 
160  drawMap(outimg, &itsSaliency->motion, dx, h, mapdrawfac, 18);
161  jevois::rawimage::writeText(outimg, "Motion", dx+3, h+3, jevois::yuyv::White);
162  dx += mapdw;
163 
164  // Blank out free space in bottom-right corner, we will then draw the gist (which may only partially occupy that
165  // available space):
166  unsigned int const gw = outimg.width - dx, gh = outimg.height - h;
167  jevois::rawimage::drawFilledRect(outimg, dx, h, gw, gh, 0x8000);
168 
169  // Draw the gist vector, picking a zoom factor to maximize the area filled:
170  unsigned int const gscale = int(sqrt((gw * gh) / itsSaliency->gist_size));
171 
172  drawGist(outimg, itsSaliency->gist, itsSaliency->gist_size, dx, h, gw / gscale, gscale);
173  jevois::rawimage::drawRect(outimg, dx, h, gw, gh, 0x80a0);
174  jevois::rawimage::writeText(outimg, "Gist", dx+3, h+3, jevois::yuyv::White);
175 
176  // Wait for all drawings to complete:
177  draw_fut.get();
178 
179  // Draw a small square at most salient location in image and in saliency map:
180  jevois::rawimage::drawFilledRect(outimg, dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
181  jevois::rawimage::drawFilledRect(outimg, w + dmx + 1, dmy + 1, smfac/2, smfac/2, 0xffff);
182 
183  // Draw an ROI box around the most salient point:
184  jevois::rawimage::drawRect(outimg, rx - roihw, ry - roihw, roihw*2, roihw*2, 0xf0f0);
185  jevois::rawimage::drawRect(outimg, rx - roihw + 1, ry - roihw + 1, roihw*2-2, roihw*2-2, 0xf0f0);
186 
187  // Show processing fps:
188  std::string const & fpscpu = itsTimer.stop();
189  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
190 
191  // Send the output image with our processing results to the host over USB:
192  outframe.send();
193  }
194 
195  protected:
196  std::shared_ptr<Saliency> itsSaliency;
197  std::shared_ptr<Kalman2D> itsKF;
199 };
200 
201 // Allow the module to be loaded as a shared object (.so) file:
std::shared_ptr< Saliency > itsSaliency
Definition: DemoSaliency.C:196
friend friend class Module
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
JEVOIS_REGISTER_MODULE(DemoSaliency)
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Definition: Saliency.C:771
Simple demo of the visual saliency algorithm of Itti et al., IEEE PAMI, 1998.
Definition: DemoSaliency.C:58
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DemoSaliency.C:73
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Definition: Saliency.C:709
DemoSaliency(std::string const &instance)
Constructor.
Definition: DemoSaliency.C:62
virtual void sendSerial(std::string const &str)
jevois::Timer itsTimer
Definition: DemoSaliency.C:198
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
virtual ~DemoSaliency()
Virtual destructor for safe inheritance.
Definition: DemoSaliency.C:70
std::shared_ptr< Kalman2D > itsKF
Definition: DemoSaliency.C:197
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
unsigned int width
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)