JeVoisBase  1.17
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
SaliencyGist.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Profiler.H>
26 
27 #include <opencv2/core/core.hpp>
28 #include <opencv2/imgproc/imgproc.hpp>
29 
30 #include <future>
31 #include <linux/videodev2.h> // for v4l2 pixel types
32 
33 // icon by Freepik in other at flaticon
34 
35 //! Simple saliency map and gist computation module
36 /*! Computes a saliency map and gist, intended for use by machines.
37 
38  See the \jvmod{DemoSaliency} modute for more explanations about saliency and gist algorithms and for a demo output
39  intended for human viewing.
40 
41  What is returned depends on the selected output image resolution; it should always be grayscale, and can contain any
42  of:
43 
44  - saliency map only
45  - saliency + gist
46  - saliency + feature maps
47  - saliency + feature maps + gist
48 
49  See the example video mappings provided with this module for sample resolutions and the associated elements that are
50  returned.
51 
52  Serial Messages
53  ---------------
54 
55  This module can send standardized serial messages as described in \ref UserSerialStyle. One message is issued on
56  every video frame at the temporally filtered attended location. The \p id field in the messages simply is \b salient
57  for all messages.
58 
59  See \ref UserSerialStyle for more on standardized serial messages, and \ref coordhelpers for more info on
60  standardized coordinates.
61 
62 
63  @author Laurent Itti
64 
65  @videomapping GREY 120 25 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps + gist
66  @videomapping GREY 120 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + feature maps
67  @videomapping GREY 20 73 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency + gist
68  @videomapping GREY 20 15 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # saliency only
69  @videomapping GREY 72 16 60.0 YUYV 320 240 60.0 JeVois SaliencyGist # gist only
70  @email itti\@usc.edu
71  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
72  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
73  @mainurl http://jevois.org
74  @supporturl http://jevois.org/doc
75  @otherurl http://iLab.usc.edu
76  @license GPL v3
77  @distribution Unrestricted
78  @restrictions None
79  \ingroup modules */
81 {
82  public:
83  //! Constructor
84  SaliencyGist(std::string const & instance) : jevois::StdModule(instance)
85  {
86  itsSaliency = addSubComponent<Saliency>("saliency");
87  itsKF = addSubComponent<Kalman2D>("kalman");
88  }
89 
90  //! Virtual destructor for safe inheritance
91  virtual ~SaliencyGist() { }
92 
93  //! Processing function with video output
94  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
95  {
96  // Wait for next available camera image:
97  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
98  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
99 
100  // Launch the saliency computation in a thread:
101  auto sal_fut = jevois::async([&](){ itsSaliency->process(inimg, true); });
102 
103  // While computing, wait for an image from our gadget driver into which we will put our results:
104  jevois::RawImage outimg = outframe.get();
105 
106  // Enforce the correct output image pixel format (will enforce size later):
107  unsigned int const ow = outimg.width, oh = outimg.height;
108  outimg.require("output", ow, oh, V4L2_PIX_FMT_GREY);
109 
110  // Once saliency is done using the input image, let camera know we are done with it:
111  itsSaliency->waitUntilDoneWithInput();
112  inframe.done();
113 
114  // Wait until saliency computation is complete:
115  sal_fut.get();
116 
117  // Find most salient point:
118  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
119 
120  // Compute attended location in original frame coordinates:
121  int const smlev = itsSaliency->smscale::get();
122  int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
123  unsigned int const dmx = (mx << smlev) + smadj;
124  unsigned int const dmy = (my << smlev) + smadj;
125 
126  // Filter these locations:
127  itsKF->set(dmx, dmy, w, h);
128  float kfxraw, kfyraw; itsKF->get(kfxraw, kfyraw, 1.0F); // round to int for serial
129 
130  // Send kalman-filtered most-salient-point info to serial port (for arduino, etc):
131  sendSerialStd2D(kfxraw, kfyraw, 0.0F, 0.0F, "salient");
132 
133  // Paste results into the output image, first check for valid output dims:
134  unsigned int const mapw = itsSaliency->salmap.dims.w, maph = itsSaliency->salmap.dims.h;
135  unsigned int const gistonlyw = 72; unsigned int const gistsize = itsSaliency->gist_size;
136  unsigned int gisth = (gistsize + ow - 1) / ow; // divide gist_size by ow and round up
137 
138  if (false == ( (ow == mapw && oh == maph) ||
139  (ow == mapw * 6 && oh == maph) ||
140  (ow == gistonlyw && oh == gistsize / gistonlyw) ||
141  (ow == mapw && oh == maph + (gistsize + mapw - 1) / mapw) ||
142  (ow == mapw * 6 && oh == maph + (gistsize + mapw*6 - 1) / (mapw*6)) ) )
143  LFATAL("Incorrect output size. With current saliency parameters, valid sizes are: " <<
144  mapw << 'x' << maph << " (saliency map only), " <<
145  mapw * 6 << 'x' << maph << " (saliency map + feature maps), " <<
146  gistonlyw << 'x' << gistsize / gistonlyw << " (gist only), " <<
147  mapw << 'x' << maph + (gistsize + mapw - 1) / mapw << " (saliency map + gist), " <<
148  mapw * 6 << 'x' << maph + (gistsize + mapw*6 - 1) / (mapw*6) << " (saliency + feature maps + gist).");
149 
150  // Paste saliency and feature maps if desired:
151  unsigned int offset = 0;
152  if (oh == maph || oh == maph + gisth)
153  {
154  pasteGrayMap(outimg, itsSaliency->salmap, offset, 20);
155  if (ow == mapw * 6)
156  {
157  pasteGrayMap(outimg, itsSaliency->color, offset, 18);
158  pasteGrayMap(outimg, itsSaliency->intens, offset, 18);
159  pasteGrayMap(outimg, itsSaliency->ori, offset, 18);
160  pasteGrayMap(outimg, itsSaliency->flicker, offset, 18);
161  pasteGrayMap(outimg, itsSaliency->motion, offset, 18);
162  }
163  }
164 
165  // Paste gist if desired:
166  if (oh == gisth || oh == maph + gisth)
167  {
168  unsigned char * d = outimg.pixelsw<unsigned char>(); if (oh == maph + gisth) d += ow * maph;
169  memcpy(d, itsSaliency->gist, gistsize);
170 
171  // Possibly blank out the remainder of the last line of gist:
172  int const rem = ow * gisth - gistsize;
173  if (rem > 0) memset(d + gistsize, 0, rem);
174  }
175 
176  // Send the output image with our processing results to the host over USB:
177  outframe.send();
178  }
179 
180  // ####################################################################################################
181  //! Paste a map and add its width to the dx offset
182  /*! Beware this is for a gray outimg only. */
183  void pasteGrayMap(jevois::RawImage & outimg, env_image const & fmap, unsigned int & dx, unsigned int bitshift)
184  {
185  env_size_t const fw = fmap.dims.w, fh = fmap.dims.h;
186  unsigned int const ow = outimg.width, oh = outimg.height;
187 
188  if (fh > oh) LFATAL("Map would extend past output image bottom");
189  if (fw + dx > ow) LFATAL("Map would extend past output image width");
190 
191  unsigned int const stride = ow - fw;
192 
193  intg32 * s = fmap.pixels; unsigned char * d = outimg.pixelsw<unsigned char>() + dx;
194 
195  for (unsigned int j = 0; j < fh; ++j)
196  {
197  for (unsigned int i = 0; i < fw; ++i)
198  {
199  intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
200  *d++ = (unsigned char)(v);
201  }
202  d += stride;
203  }
204 
205  // Add the map width to the dx offset:
206  dx += fw;
207  }
208 
209  protected:
210  std::shared_ptr<Saliency> itsSaliency;
211  std::shared_ptr<Kalman2D> itsKF;
212 };
213 
214 // Allow the module to be loaded as a shared object (.so) file:
Profiler.H
env_dims::w
env_size_t w
The width.
Definition: env_types.h:82
jevois::OutputFrame
SaliencyGist
Simple saliency map and gist computation module.
Definition: SaliencyGist.C:80
Module.H
env_size_t
unsigned long env_size_t
Definition: env_types.h:71
Log.H
jevois::RawImage
SaliencyGist::~SaliencyGist
virtual ~SaliencyGist()
Virtual destructor for safe inheritance.
Definition: SaliencyGist.C:91
jevois::StdModule::sendSerialStd2D
void sendSerialStd2D(float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
jevois::RawImage::require
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
jevois::RawImage::width
unsigned int width
jevois::RawImage::pixelsw
T * pixelsw()
SaliencyGist::process
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output.
Definition: SaliencyGist.C:94
jevois
F
float F
env_dims::h
env_size_t h
The height.
Definition: env_types.h:83
SaliencyGist::itsKF
std::shared_ptr< Kalman2D > itsKF
Definition: SaliencyGist.C:211
env_image::dims
struct env_dims dims
Definition: env_image.h:45
Kalman2D.H
jevois::StdModule::StdModule
StdModule(std::string const &instance)
LFATAL
#define LFATAL(msg)
SaliencyGist::itsSaliency
std::shared_ptr< Saliency > itsSaliency
Definition: SaliencyGist.C:210
SaliencyGist::SaliencyGist
SaliencyGist(std::string const &instance)
Constructor.
Definition: SaliencyGist.C:84
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
RawImageOps.H
jevois::RawImage::height
unsigned int height
env_image::pixels
intg32 * pixels
Definition: env_image.h:46
jevois::InputFrame
SaliencyGist::pasteGrayMap
void pasteGrayMap(jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int bitshift)
Paste a map and add its width to the dx offset.
Definition: SaliencyGist.C:183
h
int h
env_image
Basic image class.
Definition: env_image.h:43
jevois::StdModule
Saliency.H
JEVOIS_REGISTER_MODULE
JEVOIS_REGISTER_MODULE(SaliencyGist)
ColorConversion.h
intg32
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52