JeVoisBase  1.2
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Saliency.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
21 #include <jevois/Image/RawImage.H>
22 #include <jevois/Debug/Profiler.H>
23 
24 #include <opencv2/core/core.hpp>
25 
31 
32 #include <mutex>
33 #include <condition_variable>
34 
35 namespace saliency
36 {
37  static jevois::ParameterCategory const ParamCateg("Saliency/Gist Options");
38 
39  //! Parameter \relates Saliency
40  JEVOIS_DECLARE_PARAMETER(cweight, byte, "Color channel weight", 255, ParamCateg);
41 
42  //! Parameter \relates Saliency
43  JEVOIS_DECLARE_PARAMETER(iweight, byte, "Intensity channel weight", 255, ParamCateg);
44 
45  //! Parameter \relates Saliency
46  JEVOIS_DECLARE_PARAMETER(oweight, byte, "Orientation channel weight", 255, ParamCateg);
47 
48  //! Parameter \relates Saliency
49  JEVOIS_DECLARE_PARAMETER(fweight, byte, "Flicker channel weight", 255, ParamCateg);
50 
51  //! Parameter \relates Saliency
52  JEVOIS_DECLARE_PARAMETER(mweight, byte, "Motion channel weight", 255, ParamCateg);
53 
54  //! Parameter \relates Saliency
55  JEVOIS_DECLARE_PARAMETER(centermin, size_t, "Lowest (finest) of the 3 center scales", 2, ParamCateg);
56 
57  //! Parameter \relates Saliency
58  JEVOIS_DECLARE_PARAMETER(deltamin, size_t, "Lowest (finest) of the 2 center-surround delta scales", 3, ParamCateg);
59 
60  //! Parameter \relates Saliency
61  JEVOIS_DECLARE_PARAMETER(smscale, size_t, "Scale of the saliency map", 4, ParamCateg);
62 
63  //! Parameter \relates Saliency
64  JEVOIS_DECLARE_PARAMETER(mthresh, byte, "Motion threshold", 0, ParamCateg);
65 
66  //! Parameter \relates Saliency
67  JEVOIS_DECLARE_PARAMETER(fthresh, byte, "Flicker threshold", 0, ParamCateg);
68 
69  //! Parameter \relates Saliency
70  JEVOIS_DECLARE_PARAMETER(msflick, bool, "Use multiscale flicker computation", false, ParamCateg);
71 }
72 
73 //! Simple wrapper class around Rob Peter's C-optimized, fixed-point-math visual saliency code
74 /*! This component implements a visual saliency detector using the algorithm from Itti et al., IEEE Trans PAMI,
75  1998. The saliency map highlights locations in an video stream that stand out and would attract the attention of a
76  human. In addition, this component computes the gist of the scene, a low-dimentional statistical summary of the
77  whole scene that can be used for scene classification.
78 
79  Note that the Saliency component here imposes some restrictions on the saliency computations:
80 
81  - we enforce that always 6 feature maps are computed for each channel, which is obtained by using 3 center scales
82  and 2 center-surround scale deltas. In this component, only the finest center scale and smallest delta can be
83  specified, as well as the saliency map scale. This is so that the gist vector can have a fixed size.
84 
85  - number of orientations in the orientation channel is fixed at 4, number of directions in the motion channel is
86  fixed at 4. This is again to obtain a fixed gist vector size.
87 
88  - we always consider all of C, I O, F and M channels as opposed to having a more dynamic collection of channels as
89  done in other implementations of this model (see, e.g., http://iLab.usc.edu/toolkit/). This is again so that we
90  have fixed gist size and available output maps. Note that some channels will not be computed if their weight is
91  set to zero, and instead the maps will be empty and the gist entries will be zeroed out.
92 
93  See the research paper at http://ilab.usc.edu/publications/doc/Itti_etal98pami.pdf
94  \ingroup components*/
95 class Saliency : public jevois::Component,
96  public jevois::Parameter<saliency::cweight, saliency::iweight, saliency::oweight, saliency::fweight,
97  saliency::mweight, saliency::centermin, saliency::deltamin, saliency::smscale,
98  saliency::mthresh, saliency::fthresh, saliency::msflick>
99 {
100  public:
101  //! Constructor
102  Saliency(std::string const & instance);
103 
104  //! Destructor
105  virtual ~Saliency();
106 
107  //! Process a raw YUYV image. Results are stored in the Saliency class.
108  void process(jevois::RawImage const & input, bool do_gist);
109 
110  //! Process an RGB image. Results are stored in the Saliency class.
111  void process(cv::Mat const & input, bool do_gist);
112 
113  //! Wait until process() is done using the input image
114  /*! This assumes that you are running process() in a different thread and here just want to wait until the initial
115  processing that uses the input image is complete, so you can return that input image to the camera driver. */
116  void waitUntilDoneWithInput() const;
117 
118  struct env_image salmap; //!< The saliency map
119 
120  //! Get location and value of max point in the saliency map
121  void getSaliencyMax(int & x, int & y, intg32 & value);
122 
123  //! Inhibit the saliency map around a point, sigma is in pixels at the sacle of the map
124  void inhibitionOfReturn(int const x, int const y, float const sigma);
125 
127  struct env_image color;
128  struct env_image ori;
131 
132  //! Gist vector has 1152 entries, 72 feature maps * 16 values/map
133  /*! The 16 values are in raster order. If a feature weight is zero, the corresponding gist values are all
134  zero. The feature maps are in order:
135 
136  red/green (6*16), blue/yellow (6*16), intens (6*16), ori (6*4*16), flicker (6*16), motion (6*4*16)
137 
138  so the offsets are:
139  red/green: offset 0 len 6*16
140  blue/yellow: offset 1*6*16 len 6*16
141  intens: offset 2*6*16 len 6*16
142  ori0: offset 3*6*16 len 6*16
143  ori1: offset 4*6*16 len 6*16
144  ori2: offset 5*6*16 len 6*16
145  ori3: offset 6*6*16 len 6*16
146  flicker: offset 7*6*16 len 6*16
147  motion0: offset 8*6*16 len 6*16
148  motion1: offset 9*6*16 len 6*16
149  motion2: offset 10*6*16 len 6*16
150  motion3: offset 11*6*16 len 6*16 */
151  unsigned char * gist;
152  size_t const gist_size;
153 
154  // Helper struct for gist computation, of no use to end users
155  struct visitor_data { unsigned char * gist; size_t gist_size; env_params * envp; };
156 
157  private:
158  struct env_params envp;
159 
160  void combine_output(struct env_image* chanOut, const intg32 iweight, struct env_image* result);
161  struct env_math imath;
162  struct env_image prev_input;
163  struct env_pyr prev_lowpass5;
164  struct env_motion_channel motion_chan;
165  std::mutex itsMtx;
166 
167  // locally rewritten to use our thread pool
168  void env_mt_chan_orientation(const char* tagName, const struct env_image* img, env_chan_status_func* status_func,
169  void* status_userdata, struct env_image* result);
170 
171  // locally rewritten to use our thread pool
172  void env_mt_motion_channel_input(struct env_motion_channel* chan, const char* tagName,
173  const struct env_dims inputdims, struct env_pyr* lowpass5,
174  env_chan_status_func* status_func, void* status_userdata,
175  struct env_image* result);
176 
177  void processStart(struct env_dims const & dims, bool do_gist);
178 
179  visitor_data itsVisitorData;
180  jevois::Profiler itsProfiler;
181 
182  //! A mutex used to signal when the raw image is not needed anymore by process() (RawImage version)
183  mutable std::mutex itsRawImageMtx;
184 
185  //! A condition variable that gets notified during process(RawImage...) when raw image not needed anymore
186  mutable std::condition_variable itsRawImageCond;
187  mutable bool itsInputDone;
188 };
189 
190 //! Draw a saliency map or feature map in a YUYV image
191 /*! \relates Saliency */
192 void drawMap(jevois::RawImage & img, env_image const * fmap, unsigned int xoff, unsigned int yoff, unsigned int scale);
193 
194 //! Draw a saliency map or feature map in a YUYV image, applying some right bitshift to values
195 /*! \relates Saliency */
196 void drawMap(jevois::RawImage & img, env_image const * fmap, unsigned int xoff, unsigned int yoff, unsigned int scale,
197  unsigned int bitshift);
198 
199 //! Draw a gist vector in a YUYV image as a rectangle of width width*scale and correct height
200 /*! \relates Saliency */
201 void drawGist(jevois::RawImage & img, unsigned char const * gist, size_t gistsize, unsigned int xoff, unsigned int yoff,
202  unsigned int width, unsigned int scale);
203 
void process(jevois::RawImage const &input, bool do_gist)
Process a raw YUYV image. Results are stored in the Saliency class.
Definition: Saliency.C:339
size_t const gist_size
Definition: Saliency.H:152
struct env_image motion
Definition: Saliency.H:130
std::string value
struct env_image ori
Definition: Saliency.H:128
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Draw a saliency map or feature map in a YUYV image.
Definition: Saliency.C:709
JEVOIS_DECLARE_PARAMETER(thresh1, double,"First threshold for hysteresis", 50.0, ParamCateg)
void getSaliencyMax(int &x, int &y, intg32 &value)
Get location and value of max point in the saliency map.
Definition: Saliency.C:672
Basic image class.
Definition: env_image.h:43
A composite channel containing a set of direction channels.
struct env_image flicker
Definition: Saliency.H:129
Saliency(std::string const &instance)
Constructor.
Definition: Saliency.C:81
struct env_image color
Definition: Saliency.H:127
struct env_image intens
Definition: Saliency.H:126
virtual ~Saliency()
Destructor.
Definition: Saliency.C:106
type_with_N_bits< unsigned char, 8 >::type byte
8-bit unsigned integer
Definition: Types.H:86
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
struct env_image salmap
The saliency map.
Definition: Saliency.H:118
unsigned char * gist
Definition: Saliency.H:155
Simple wrapper class around Rob Peter's C-optimized, fixed-point-math visual saliency code...
Definition: Saliency.H:95
A simple struct to hold a pair of width/height dimensions.
Definition: env_types.h:80
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Draw a gist vector in a YUYV image as a rectangle of width width*scale and correct height...
Definition: Saliency.C:771
This class implements a set of images, often used as a dyadic pyramid.
Definition: env_pyr.h:45
unsigned char * gist
Gist vector has 1152 entries, 72 feature maps * 16 values/map.
Definition: Saliency.H:151
void waitUntilDoneWithInput() const
Wait until process() is done using the input image.
Definition: Saliency.C:217
void inhibitionOfReturn(int const x, int const y, float const sigma)
Inhibit the saliency map around a point, sigma is in pixels at the sacle of the map.
Definition: Saliency.C:686
void( env_chan_status_func)(void *userdata, const char *tagName, const struct env_image *img)
Definition: env_channel.h:55
env_params * envp
Definition: Saliency.H:155