JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
SaliencyGist.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Profiler.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 
33 // icon by Freepik in other at flaticon
34 
35 //! Simple saliency map and gist computation module
36 /*! Computes a saliency map and gist, intended for use by machines.
37 
38  See the \jvmod{DemoSaliency} modute for more explanations about saliency and gist algorithms and for a demo output
39  intended for human viewing.
40 
41  What is returned depends on the selected output image resolution; it should always be grayscale, and can contain any
42  of:
43 
44  - saliency map only
45  - saliency + gist
46  - saliency + feature maps
47  - saliency + feature maps + gist
48 
49  See the example video mappings provided with this module for sample resolutions and the associated elements that are
50  returned.
51 
52  Serial Messages
53  ---------------
54 
55  This module can send standardized serial messages as described in \ref UserSerialStyle. One message is issued on
56  every video frame at the temporally filtered attended location. The \p id field in the messages simply is \b salient
57  for all messages.
58 
59 
60  @author Laurent Itti
61 
62  @videomapping GREY 120 25 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps + gist
63  @videomapping GREY 120 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps
64  @videomapping GREY 20 73 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + gist
65  @videomapping GREY 20 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency only
66  @videomapping GREY 72 16 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # gist only
67  @email itti\@usc.edu
68  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
69  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
70  @mainurl http://jevois.org
71  @supporturl http://jevois.org/doc
72  @otherurl http://iLab.usc.edu
73  @license GPL v3
74  @distribution Unrestricted
75  @restrictions None
76  \ingroup modules */
78 {
79  public:
80  //! Constructor
81  SaliencyGist(std::string const & instance) : jevois::StdModule(instance)
82  {
83  itsSaliency = addSubComponent<Saliency>("saliency");
84  itsKF = addSubComponent<Kalman2D>("kalman");
85  }
86 
87  //! Virtual destructor for safe inheritance
88  virtual ~SaliencyGist() { }
89 
90  //! Processing function with video output
91  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
92  {
93  // Wait for next available camera image:
94  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
95  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
96 
97  // Launch the saliency computation in a thread:
98  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
99 
100  // While computing, wait for an image from our gadget driver into which we will put our results:
101  jevois::RawImage outimg = outframe.get();
102 
103  // Enforce the correct output image pixel format (will enforce size later):
104  unsigned int const ow = outimg.width, oh = outimg.height;
105  outimg.require("output", ow, oh, V4L2_PIX_FMT_GREY);
106 
107  // Once saliency is done using the input image, let camera know we are done with it:
108  itsSaliency->waitUntilDoneWithInput();
109  inframe.done();
110 
111  // Wait until saliency computation is complete:
112  sal_fut.get();
113 
114  // Find most salient point:
115  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
116 
117  // Compute attended location in original frame coordinates:
118  int const smlev = itsSaliency->smscale::get();
119  int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
120  unsigned int const dmx = (mx << smlev) + smadj;
121  unsigned int const dmy = (my << smlev) + smadj;
122 
123  // Filter these locations:
124  itsKF->set(dmx, dmy, w, h);
125  float kfxraw, kfyraw; itsKF->get(kfxraw, kfyraw, 1.0F); // round to int for serial
126 
127  // Send kalman-filtered most-salient-point info to serial port (for arduino, etc):
128  sendSerialStd2D(kfxraw, kfyraw, 0.0F, 0.0F, "salient");
129 
130  // Paste results into the output image, first check for valid output dims:
131  unsigned int const mapw = itsSaliency->salmap.dims.w, maph = itsSaliency->salmap.dims.h;
132  unsigned int const gistonlyw = 72; unsigned int const gistsize = itsSaliency->gist_size;
133  unsigned int gisth = (gistsize + ow - 1) / ow; // divide gist_size by ow and round up
134 
135  if (false == ( (ow == mapw && oh == maph) ||
136  (ow == mapw * 6 && oh == maph) ||
137  (ow == gistonlyw && oh == gistsize / gistonlyw) ||
138  (ow == mapw && oh == maph + (gistsize + mapw - 1) / mapw) ||
139  (ow == mapw * 6 && oh == maph + (gistsize + mapw*6 - 1) / (mapw*6)) ) )
140  LFATAL("Incorrect output size. With current saliency parameters, valid sizes are: " <<
141  mapw << 'x' << maph << " (saliency map only), " <<
142  mapw * 6 << 'x' << maph << " (saliency map + feature maps), " <<
143  gistonlyw << 'x' << gistsize / gistonlyw << " (gist only), " <<
144  mapw << 'x' << maph + (gistsize + mapw - 1) / mapw << " (saliency map + gist), " <<
145  mapw * 6 << 'x' << maph + (gistsize + mapw*6 - 1) / (mapw*6) << " (saliency + feature maps + gist).");
146 
147  // Paste saliency and feature maps if desired:
148  unsigned int offset = 0;
149  if (oh == maph || oh == maph + gisth)
150  {
151  pasteGrayMap(outimg, itsSaliency->salmap, offset, 20);
152  if (ow == mapw * 6)
153  {
154  pasteGrayMap(outimg, itsSaliency->color, offset, 18);
155  pasteGrayMap(outimg, itsSaliency->intens, offset, 18);
156  pasteGrayMap(outimg, itsSaliency->ori, offset, 18);
157  pasteGrayMap(outimg, itsSaliency->flicker, offset, 18);
158  pasteGrayMap(outimg, itsSaliency->motion, offset, 18);
159  }
160  }
161 
162  // Paste gist if desired:
163  if (oh == gisth || oh == maph + gisth)
164  {
165  unsigned char * d = outimg.pixelsw<unsigned char>(); if (oh == maph + gisth) d += ow * maph;
166  memcpy(d, itsSaliency->gist, gistsize);
167 
168  // Possibly blank out the remainder of the last line of gist:
169  int const rem = ow * gisth - gistsize;
170  if (rem > 0) memset(d + gistsize, 0, rem);
171  }
172 
173  // Send the output image with our processing results to the host over USB:
174  outframe.send();
175  }
176 
177  // ####################################################################################################
178  //! Paste a map and add its width to the dx offset
179  /*! Beware this is for a gray outimg only. */
180  void pasteGrayMap(jevois::RawImage & outimg, env_image const & fmap, unsigned int & dx, unsigned int bitshift)
181  {
182  env_size_t const fw = fmap.dims.w, fh = fmap.dims.h;
183  unsigned int const ow = outimg.width, oh = outimg.height;
184 
185  if (fh > oh) LFATAL("Map would extend past output image bottom");
186  if (fw + dx > ow) LFATAL("Map would extend past output image width");
187 
188  unsigned int const stride = ow - fw;
189 
190  intg32 * s = fmap.pixels; unsigned char * d = outimg.pixelsw<unsigned char>() + dx;
191 
192  for (unsigned int j = 0; j < fh; ++j)
193  {
194  for (unsigned int i = 0; i < fw; ++i)
195  {
196  intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
197  *d++ = (unsigned char)(v);
198  }
199  d += stride;
200  }
201 
202  // Add the map width to the dx offset:
203  dx += fw;
204  }
205 
206  protected:
207  std::shared_ptr<Saliency> itsSaliency;
208  std::shared_ptr<Kalman2D> itsKF;
209 };
210 
211 // Allow the module to be loaded as a shared object (.so) file:
env_size_t w
The width.
Definition: env_types.h:82
virtual ~SaliencyGist()
Virtual destructor for safe inheritance.
Definition: SaliencyGist.C:88
Simple saliency map and gist computation module.
Definition: SaliencyGist.C:77
unsigned int height
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output.
Definition: SaliencyGist.C:91
Basic image class.
Definition: env_image.h:43
std::shared_ptr< Kalman2D > itsKF
Definition: SaliencyGist.C:208
StdModule(std::string const &instance)
struct env_dims dims
Definition: env_image.h:45
std::shared_ptr< Saliency > itsSaliency
Definition: SaliencyGist.C:207
env_size_t h
The height.
Definition: env_types.h:83
SaliencyGist(std::string const &instance)
Constructor.
Definition: SaliencyGist.C:81
void sendSerialStd2D(float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
JEVOIS_REGISTER_MODULE(SaliencyGist)
unsigned long env_size_t
Definition: env_types.h:71
intg32 * pixels
Definition: env_image.h:46
#define LFATAL(msg)
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
unsigned int width
void pasteGrayMap(jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int bitshift)
Paste a map and add its width to the dx offset.
Definition: SaliencyGist.C:180
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const