JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
SaliencyGist.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Profiler.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 
33 // icon by Freepik in other at flaticon
34 
35 //! Simple saliency map and gist computation module
36 /*! Computes a saliency map and gist, intended for use by machines.
37 
38  See the DemoSaliency modute for more explanations about saliency and gist algorithms and for a demo output intended
39  for human viewing.
40 
41  What is returned depends on the selected output image resolution; it should always be grayscale, and can contain any
42  of:
43 
44  - saliency map only
45  - saliency + gist
46  - saliency + feature maps
47  - saliency + feature maps + gist
48 
49  Serial Messages
50  ---------------
51 
52  This module can send standardized serial messages as described in \ref UserSerialStyle. One message is issued for on
53  every video frame at the temporally filtered attended location. The \p id field in the messages simply is \b salient
54  for all messages.
55 
56  @author Laurent Itti
57 
58  @videomapping GREY 120 25 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps + gist
59  @videomapping GREY 120 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps
60  @videomapping GREY 20 73 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + gist
61  @videomapping GREY 20 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency only
62  @videomapping GREY 72 16 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # gist only
63  @email itti\@usc.edu
64  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
65  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
66  @mainurl http://jevois.org
67  @supporturl http://jevois.org/doc
68  @otherurl http://iLab.usc.edu
69  @license GPL v3
70  @distribution Unrestricted
71  @restrictions None
72  \ingroup modules */
74 {
75  public:
76  //! Constructor
77  SaliencyGist(std::string const & instance) : jevois::StdModule(instance)
78  {
79  itsSaliency = addSubComponent<Saliency>("saliency");
80  itsKF = addSubComponent<Kalman2D>("kalman");
81  }
82 
83  //! Virtual destructor for safe inheritance
84  virtual ~SaliencyGist() { }
85 
86  //! Processing function with video output
87  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
88  {
89  // Wait for next available camera image:
90  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
91  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
92 
93  // Launch the saliency computation in a thread:
94  auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
95 
96  // While computing, wait for an image from our gadget driver into which we will put our results:
97  jevois::RawImage outimg = outframe.get();
98 
99  // Enforce the correct output image pixel format (will enforce size later):
100  unsigned int const ow = outimg.width, oh = outimg.height;
101  outimg.require("output", ow, oh, V4L2_PIX_FMT_GREY);
102 
103  // Once saliency is done using the input image, let camera know we are done with it:
104  itsSaliency->waitUntilDoneWithInput();
105  inframe.done();
106 
107  // Wait until saliency computation is complete:
108  sal_fut.get();
109 
110  // Find most salient point:
111  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
112 
113  // Compute attended location in original frame coordinates:
114  int const smlev = itsSaliency->smscale::get();
115  int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
116  unsigned int const dmx = (mx << smlev) + smadj;
117  unsigned int const dmy = (my << smlev) + smadj;
118 
119  // Filter these locations:
120  itsKF->set(dmx, dmy, w, h);
121  float kfxraw, kfyraw; itsKF->get(kfxraw, kfyraw, 1.0F); // round to int for serial
122 
123  // Send kalman-filtered most-salient-point info to serial port (for arduino, etc):
124  sendSerialStd2D(kfxraw, kfyraw, 0.0F, 0.0F, "salient");
125 
126  // Paste results into the output image, first check for valid output dims:
127  unsigned int const mapw = itsSaliency->salmap.dims.w, maph = itsSaliency->salmap.dims.h;
128  unsigned int const gistonlyw = 72; unsigned int const gistsize = itsSaliency->gist_size;
129  unsigned int gisth = (gistsize + ow - 1) / ow; // divide gist_size by ow and round up
130 
131  if (false == ( (ow == mapw && oh == maph) ||
132  (ow == mapw * 6 && oh == maph) ||
133  (ow == gistonlyw && oh == gistsize / gistonlyw) ||
134  (ow == mapw && oh == maph + (gistsize + mapw - 1) / mapw) ||
135  (ow == mapw * 6 && oh == maph + (gistsize + mapw*6 - 1) / (mapw*6)) ) )
136  LFATAL("Incorrect output size. With current saliency parameters, valid sizes are: " <<
137  mapw << 'x' << maph << " (saliency map only), " <<
138  mapw * 6 << 'x' << maph << " (saliency map + feature maps), " <<
139  gistonlyw << 'x' << gistsize / gistonlyw << " (gist only), " <<
140  mapw << 'x' << maph + (gistsize + mapw - 1) / mapw << " (saliency map + gist), " <<
141  mapw * 6 << 'x' << maph + (gistsize + mapw*6 - 1) / (mapw*6) << " (saliency + feature maps + gist).");
142 
143  // Paste saliency and feature maps if desired:
144  unsigned int offset = 0;
145  if (oh == maph || oh == maph + gisth)
146  {
147  pasteGrayMap(outimg, itsSaliency->salmap, offset, 20);
148  if (ow == mapw * 6)
149  {
150  pasteGrayMap(outimg, itsSaliency->color, offset, 18);
151  pasteGrayMap(outimg, itsSaliency->intens, offset, 18);
152  pasteGrayMap(outimg, itsSaliency->ori, offset, 18);
153  pasteGrayMap(outimg, itsSaliency->flicker, offset, 18);
154  pasteGrayMap(outimg, itsSaliency->motion, offset, 18);
155  }
156  }
157 
158  // Paste gist if desired:
159  if (oh == gisth || oh == maph + gisth)
160  {
161  unsigned char * d = outimg.pixelsw<unsigned char>(); if (oh == maph + gisth) d += ow * maph;
162  memcpy(d, itsSaliency->gist, gistsize);
163 
164  // Possibly blank out the remainder of the last line of gist:
165  int const rem = ow * gisth - gistsize;
166  if (rem > 0) memset(d + gistsize, 0, rem);
167  }
168 
169  // Send the output image with our processing results to the host over USB:
170  outframe.send();
171  }
172 
173  // ####################################################################################################
174  //! Paste a map and add its width to the dx offset
175  /*! Beware this is for a gray outimg only. */
176  void pasteGrayMap(jevois::RawImage & outimg, env_image const & fmap, unsigned int & dx, unsigned int bitshift)
177  {
178  env_size_t const fw = fmap.dims.w, fh = fmap.dims.h;
179  unsigned int const ow = outimg.width, oh = outimg.height;
180 
181  if (fh > oh) LFATAL("Map would extend past output image bottom");
182  if (fw + dx > ow) LFATAL("Map would extend past output image width");
183 
184  unsigned int const stride = ow - fw;
185 
186  intg32 * s = fmap.pixels; unsigned char * d = outimg.pixelsw<unsigned char>() + dx;
187 
188  for (unsigned int j = 0; j < fh; ++j)
189  {
190  for (unsigned int i = 0; i < fw; ++i)
191  {
192  intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
193  *d++ = (unsigned char)(v);
194  }
195  d += stride;
196  }
197 
198  // Add the map width to the dx offset:
199  dx += fw;
200  }
201 
202  protected:
203  std::shared_ptr<Saliency> itsSaliency;
204  std::shared_ptr<Kalman2D> itsKF;
205 };
206 
207 // Allow the module to be loaded as a shared object (.so) file:
env_size_t w
The width.
Definition: env_types.h:82
virtual ~SaliencyGist()
Virtual destructor for safe inheritance.
Definition: SaliencyGist.C:84
Simple saliency map and gist computation module.
Definition: SaliencyGist.C:73
unsigned int height
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output.
Definition: SaliencyGist.C:87
Basic image class.
Definition: env_image.h:43
std::shared_ptr< Kalman2D > itsKF
Definition: SaliencyGist.C:204
StdModule(std::string const &instance)
struct env_dims dims
Definition: env_image.h:45
std::shared_ptr< Saliency > itsSaliency
Definition: SaliencyGist.C:203
env_size_t h
The height.
Definition: env_types.h:83
SaliencyGist(std::string const &instance)
Constructor.
Definition: SaliencyGist.C:77
void sendSerialStd2D(float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
JEVOIS_REGISTER_MODULE(SaliencyGist)
unsigned long env_size_t
Definition: env_types.h:71
intg32 * pixels
Definition: env_image.h:46
#define LFATAL(msg)
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
unsigned int width
void pasteGrayMap(jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int bitshift)
Paste a map and add its width to the dx offset.
Definition: SaliencyGist.C:176
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const