JeVoisBase  1.6
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
SalientRegions.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Profiler.H>
25 
26 #include <opencv2/core/core.hpp>
27 #include <opencv2/imgproc/imgproc.hpp>
28 
29 #include <future>
30 #include <linux/videodev2.h> // for v4l2 pixel types
31 
32 // icon by Freepik in other at flaticon
33 
34 // Module parameters:
35 static jevois::ParameterCategory const ParamCateg("Salient Regions Options");
36 
37 //! Parameter \relates SalientRegions
38 JEVOIS_DECLARE_PARAMETER(inhsigma, float, "Sigma (pixels) used for inhibition of return", 32.0F, ParamCateg);
39 
40 //! Extract the most salient regions and send them out over USB
41 /*! This module extracts cropped images around the N most salient (i.e., conspicuous, or attention-grabbing) locations
42  in the JeVois camera's field of view. These regions are then streamed over USB, one of top of another. This module
43  thus produces an output mainly intended for machine use: a host computer might grab the regions detected by JeVois,
44  and run, for example, some object recognition algorithm on each region.
45 
46  See \jvmod{DemoSaliency} for more information about visual attention and saliency.
47 
48  The number of regions extracted (N) is decided by the height of the desired USB output image, while the (square)
49  region size (width and height) is determined by the output image width. Note that region width and height must be a
50  multiple of 4.
51 
52  The most salient region is on top, the second most salient region is below the first one, etc.
53 
54  Example use
55  -----------
56 
57  With video mapping
58 
59  \verbatim
60  YUYV 64 192 25.0 YUYV 320 240 25.0 JeVois SalientRegions
61  \endverbatim
62 
63  in <b>JEVOIS:/config/videomappings.cfg</b>, this module will extract three 64x64 salient regions, and will send them
64  over USB one on top of the other (since USB video width is 64, which determines region size, and USB video height is
65  3x64 = 192, which determines the number of regions).
66 
67 
68  @author Laurent Itti
69 
70  @videomapping YUYV 64 192 25.0 YUYV 320 240 25.0 JeVois SalientRegions
71  @videomapping YUYV 100 400 10.0 YUYV 640 480 10.0 JeVois SalientRegions
72  @email itti\@usc.edu
73  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
74  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
75  @mainurl http://jevois.org
76  @supporturl http://jevois.org/doc
77  @otherurl http://iLab.usc.edu
78  @license GPL v3
79  @distribution Unrestricted
80  @restrictions None
81  \ingroup modules */
83  public jevois::Parameter<inhsigma>
84 {
85  public:
86  //! Constructor
87  SalientRegions(std::string const & instance) : jevois::Module(instance)
88  { itsSaliency = addSubComponent<Saliency>("saliency"); }
89 
90  //! Virtual destructor for safe inheritance
91  virtual ~SalientRegions() { }
92 
93  //! Processing function
94  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
95  {
96  // Wait for next available camera image:
97  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
98  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
99 
100  // Compute the saliency map, no gist:
101  itsSaliency->process(inimg, false);
102 
103  // Wait for an image from our gadget driver into which we will put our results:
104  jevois::RawImage outimg = outframe.get();
105  int const rwh = outimg.width & (~3); // output image width sets region width and height, must be multiple of 4
106  int const nr = outimg.height / rwh; // output image height sets the number of regions
107  outimg.require("output", rwh, nr * rwh, V4L2_PIX_FMT_YUYV);
108 
109  // Get some info from the saliency computation:
110  int const smlev = itsSaliency->smscale::get();
111  int const smfac = (1 << smlev);
112 
113  // Copy each region to output:
114  for (int i = 0; i < nr; ++i)
115  {
116  // Find most salient point:
117  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
118 
119  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
120  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
121  unsigned int const dmy = (my << smlev) + (smfac >> 2);
122  int rx = (std::min(int(w) - rwh/2, std::max(rwh/2, int(dmx + 1 + (smfac >> 2))))) & (~1);
123  int ry = (std::min(int(h) - rwh/2, std::max(rwh/2, int(dmy + 1 + (smfac >> 2))))) & (~1);
124 
125  // Paste the roi:
126  jevois::rawimage::roipaste(inimg, rx - rwh/2, ry - rwh/2, rwh, rwh, outimg, 0, i * rwh);
127 
128  // Inhibit this salient location so we move to the next one:
129  itsSaliency->inhibitionOfReturn(mx, my, inhsigma::get() / smfac);
130  }
131 
132  // Let camera know we are done processing the raw YUV input image:
133  inframe.done();
134 
135  // Send the output image with our processing results to the host over USB:
136  outframe.send();
137  }
138 
139  protected:
140  std::shared_ptr<Saliency> itsSaliency;
141 };
142 
143 // Allow the module to be loaded as a shared object (.so) file:
friend friend class Module
unsigned int height
Extract the most salient regions and send them out over USB.
JEVOIS_REGISTER_MODULE(SalientRegions)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
void roipaste(RawImage const &src, int x, int y, unsigned int w, unsigned int h, RawImage &dest, int dx, int dy)
JEVOIS_DECLARE_PARAMETER(camparams, std::string, "File stem of camera parameters, or empty. Camera resolution " "will be appended, as well as a .cfg extension. For example, specifying 'camera_para' " "here and running the camera sensor at 320x240 will attempt to load " "camera_para320x240.dat from within the module's directory.", "camera_para", ParamCateg)
Parameter.
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
SalientRegions(std::string const &instance)
Constructor.
std::shared_ptr< Saliency > itsSaliency
unsigned int width
virtual ~SalientRegions()
Virtual destructor for safe inheritance.
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const