JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
SaliencyGist.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Profiler.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 
33 // icon by Freepik in other at flaticon
34 
35 //! Simple saliency map and gist computation module
36 /*! Computes a saliency map and gist, intended for use by machines.
37 
38  See the DemoSaliency modute for more explanations about saliency and gist algorithms and for a demo output intended
39  for human viewing.
40 
41  What is returned depends on the selected output image resolution; it should always be grayscale, and can contain any
42  of:
43 
44  - saliency map only
45  - saliency + gist
46  - saliency + feature maps
47  - saliency + feature maps + gist
48 
49  @author Laurent Itti
50 
51  @videomapping GREY 120 25 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps + gist
52  @videomapping GREY 120 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps
53  @videomapping GREY 20 73 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + gist
54  @videomapping GREY 20 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency only
55  @videomapping GREY 72 16 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # gist only
56  @email itti\@usc.edu
57  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
58  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
59  @mainurl http://jevois.org
60  @supporturl http://jevois.org/doc
61  @otherurl http://iLab.usc.edu
62  @license GPL v3
63  @distribution Unrestricted
64  @restrictions None
65  \ingroup modules */
67 {
68  public:
69  //! Constructor
70  SaliencyGist(std::string const & instance) : jevois::Module(instance)
71  {
72  itsSaliency = addSubComponent<Saliency>("saliency");
73  itsKF = addSubComponent<Kalman2D>("kalman");
74  }
75 
76  //! Virtual destructor for safe inheritance
77  virtual ~SaliencyGist() { }
78 
79  //! Processing function with video output
80  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
81  {
82  // Wait for next available camera image:
83  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
84  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
85 
86  // Launch the saliency computation in a thread:
87  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
88 
89  // While computing, wait for an image from our gadget driver into which we will put our results:
90  jevois::RawImage outimg = outframe.get();
91 
92  // Enforce the correct output image pixel format (will enforce size later):
93  unsigned int const ow = outimg.width, oh = outimg.height;
94  outimg.require("output", ow, oh, V4L2_PIX_FMT_GREY);
95 
96  // Once saliency is done using the input image, let camera know we are done with it:
97  itsSaliency->waitUntilDoneWithInput();
98  inframe.done();
99 
100  // Wait until saliency computation is complete:
101  sal_fut.get();
102 
103  // Find most salient point:
104  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
105 
106  // Compute attended location in original frame coordinates:
107  int const smlev = itsSaliency->smscale::get();
108  int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
109  unsigned int const dmx = (mx << smlev) + smadj;
110  unsigned int const dmy = (my << smlev) + smadj;
111 
112  // Filter these locations:
113  itsKF->set(dmx, dmy, w, h);
114  float kfxraw, kfyraw; itsKF->get(kfxraw, kfyraw, 1.0F); // round to int for serial
115 
116  // Send kalman-filtered most-salient-point info to serial port (for arduino, etc):
117  sendSerial(jevois::sformat("T2D %d %d", int(kfxraw), int(kfyraw)));
118 
119  // Paste results into the output image, first check for valid output dims:
120  unsigned int const mapw = itsSaliency->salmap.dims.w, maph = itsSaliency->salmap.dims.h;
121  unsigned int const gistonlyw = 72; unsigned int const gistsize = itsSaliency->gist_size;
122  unsigned int gisth = (gistsize + ow - 1) / ow; // divide gist_size by ow and round up
123 
124  if (false == ( (ow == mapw && oh == maph) ||
125  (ow == mapw * 6 && oh == maph) ||
126  (ow == gistonlyw && oh == gistsize / gistonlyw) ||
127  (ow == mapw && oh == maph + (gistsize + mapw - 1) / mapw) ||
128  (ow == mapw * 6 && oh == maph + (gistsize + mapw*6 - 1) / (mapw*6)) ) )
129  LFATAL("Incorrect output size. With current saliency parameters, valid sizes are: " <<
130  mapw << 'x' << maph << " (saliency map only), " <<
131  mapw * 6 << 'x' << maph << " (saliency map + feature maps), " <<
132  gistonlyw << 'x' << gistsize / gistonlyw << " (gist only), " <<
133  mapw << 'x' << maph + (gistsize + mapw - 1) / mapw << " (saliency map + gist), " <<
134  mapw * 6 << 'x' << maph + (gistsize + mapw*6 - 1) / (mapw*6) << " (saliency + feature maps + gist).");
135 
136  // Paste saliency and feature maps if desired:
137  unsigned int offset = 0;
138  if (oh == maph || oh == maph + gisth)
139  {
140  pasteGrayMap(outimg, itsSaliency->salmap, offset, 20);
141  if (ow == mapw * 6)
142  {
143  pasteGrayMap(outimg, itsSaliency->color, offset, 18);
144  pasteGrayMap(outimg, itsSaliency->intens, offset, 18);
145  pasteGrayMap(outimg, itsSaliency->ori, offset, 18);
146  pasteGrayMap(outimg, itsSaliency->flicker, offset, 18);
147  pasteGrayMap(outimg, itsSaliency->motion, offset, 18);
148  }
149  }
150 
151  // Paste gist if desired:
152  if (oh == gisth || oh == maph + gisth)
153  {
154  unsigned char * d = outimg.pixelsw<unsigned char>(); if (oh == maph + gisth) d += ow * maph;
155  memcpy(d, itsSaliency->gist, gistsize);
156 
157  // Possibly blank out the remainder of the last line of gist:
158  int const rem = ow * gisth - gistsize;
159  if (rem > 0) memset(d + gistsize, 0, rem);
160  }
161 
162  // Send the output image with our processing results to the host over USB:
163  outframe.send();
164  }
165 
166  // ####################################################################################################
167  //! Paste a map and add its width to the dx offset
168  /*! Beware this is for a gray outimg only. */
169  void pasteGrayMap(jevois::RawImage & outimg, env_image const & fmap, unsigned int & dx, unsigned int bitshift)
170  {
171  env_size_t const fw = fmap.dims.w, fh = fmap.dims.h;
172  unsigned int const ow = outimg.width, oh = outimg.height;
173 
174  if (fh > oh) LFATAL("Map would extend past output image bottom");
175  if (fw + dx > ow) LFATAL("Map would extend past output image width");
176 
177  unsigned int const stride = ow - fw;
178 
179  intg32 * s = fmap.pixels; unsigned char * d = outimg.pixelsw<unsigned char>() + dx;
180 
181  for (unsigned int j = 0; j < fh; ++j)
182  {
183  for (unsigned int i = 0; i < fw; ++i)
184  {
185  intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
186  *d++ = (unsigned char)(v);
187  }
188  d += stride;
189  }
190 
191  // Add the map width to the dx offset:
192  dx += fw;
193  }
194 
195  protected:
196  std::shared_ptr<Saliency> itsSaliency;
197  std::shared_ptr<Kalman2D> itsKF;
198 };
199 
200 // Allow the module to be loaded as a shared object (.so) file:
friend friend class Module
env_size_t w
The width.
Definition: env_types.h:82
virtual ~SaliencyGist()
Virtual destructor for safe inheritance.
Definition: SaliencyGist.C:77
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
Simple saliency map and gist computation module.
Definition: SaliencyGist.C:66
unsigned int height
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output.
Definition: SaliencyGist.C:80
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Basic image class.
Definition: env_image.h:43
std::shared_ptr< Kalman2D > itsKF
Definition: SaliencyGist.C:197
struct env_dims dims
Definition: env_image.h:45
std::shared_ptr< Saliency > itsSaliency
Definition: SaliencyGist.C:196
env_size_t h
The height.
Definition: env_types.h:83
SaliencyGist(std::string const &instance)
Constructor.
Definition: SaliencyGist.C:70
JEVOIS_REGISTER_MODULE(SaliencyGist)
unsigned long env_size_t
Definition: env_types.h:71
intg32 * pixels
Definition: env_image.h:46
virtual void sendSerial(std::string const &str)
#define LFATAL(msg)
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
unsigned int width
void pasteGrayMap(jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int bitshift)
Paste a map and add its width to the dx offset.
Definition: SaliencyGist.C:169