JeVoisBase  1.3
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
SalientRegions.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Debug/Profiler.H>
25 
26 #include <opencv2/core/core.hpp>
27 #include <opencv2/imgproc/imgproc.hpp>
28 
29 #include <future>
30 #include <linux/videodev2.h> // for v4l2 pixel types
31 
32 // icon by Freepik in other at flaticon
33 
34 // Module parameters:
35 static jevois::ParameterCategory const ParamCateg("Salient Regions Options");
36 
37 //! Parameter \relates SalientRegions
38 JEVOIS_DECLARE_PARAMETER(inhsigma, float, "Sigma (pixels) used for inhibition of return", 32.0F, ParamCateg);
39 
40 //! Extract the most salient regions and send them out
41 /*! This module produces an output intended for machine use, and extracts cropped images around the N most salient
42  locations. N is decided by the height of the desired USB output image. See DemoSaliency for more information about
43  visual attention and saliency.
44 
45  Here we show how the output image size chosen in the videomappings.cfg config file can be used to decide what output
46  to compute. Namely, here we assume square regions stacked vertically. So the width of the outut image sets region
47  size, and the height divided by that size sets the number of regions.
48 
49  @author Laurent Itti
50 
51  @videomapping YUYV 64 192 25.0 YUYV 320 240 25.0 JeVois SalientRegions
52  @videomapping YUYV 100 400 10.0 YUYV 640 480 10.0 JeVois SalientRegions
53  @email itti\@usc.edu
54  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
55  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
56  @mainurl http://jevois.org
57  @supporturl http://jevois.org/doc
58  @otherurl http://iLab.usc.edu
59  @license GPL v3
60  @distribution Unrestricted
61  @restrictions None
62  \ingroup modules */
64  public jevois::Parameter<inhsigma>
65 {
66  public:
67  //! Constructor
68  SalientRegions(std::string const & instance) : jevois::Module(instance)
69  { itsSaliency = addSubComponent<Saliency>("saliency"); }
70 
71  //! Virtual destructor for safe inheritance
72  virtual ~SalientRegions() { }
73 
74  //! Processing function
75  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
76  {
77  // Wait for next available camera image:
78  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
79  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // accept any image size but require YUYV pixels
80 
81  // Compute the saliency map, no gist:
82  itsSaliency->process(inimg, false);
83 
84  // Wait for an image from our gadget driver into which we will put our results:
85  jevois::RawImage outimg = outframe.get();
86  int const rwh = outimg.width & (~3); // output image width sets region width and height, must be multiple of 4
87  int const nr = outimg.height / rwh; // output image height sets the number of regions
88  outimg.require("output", rwh, nr * rwh, V4L2_PIX_FMT_YUYV);
89 
90  // Get some info from the saliency computation:
91  int const smlev = itsSaliency->smscale::get();
92  int const smfac = (1 << smlev);
93 
94  // Copy each region to output:
95  for (int i = 0; i < nr; ++i)
96  {
97  // Find most salient point:
98  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
99 
100  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
101  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
102  unsigned int const dmy = (my << smlev) + (smfac >> 2);
103  int rx = (std::min(int(w) - rwh/2, std::max(rwh/2, int(dmx + 1 + (smfac >> 2))))) & (~1);
104  int ry = (std::min(int(h) - rwh/2, std::max(rwh/2, int(dmy + 1 + (smfac >> 2))))) & (~1);
105 
106  // Paste the roi:
107  jevois::rawimage::roipaste(inimg, rx - rwh/2, ry - rwh/2, rwh, rwh, outimg, 0, i * rwh);
108 
109  // Inhibit this salient location so we move to the next one:
110  itsSaliency->inhibitionOfReturn(mx, my, inhsigma::get() / smfac);
111  }
112 
113  // Let camera know we are done processing the raw YUV input image:
114  inframe.done();
115 
116  // Send the output image with our processing results to the host over USB:
117  outframe.send();
118  }
119 
120  protected:
121  std::shared_ptr<Saliency> itsSaliency;
122 };
123 
124 // Allow the module to be loaded as a shared object (.so) file:
friend friend class Module
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
Extract the most salient regions and send them out.
JEVOIS_REGISTER_MODULE(SalientRegions)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
void roipaste(RawImage const &src, int x, int y, unsigned int w, unsigned int h, RawImage &dest, int dx, int dy)
JEVOIS_DECLARE_PARAMETER(camparams, std::string,"File stem of camera parameters, or empty. Camera resolution ""will be appended, as well as a .cfg extension. For example, specifying 'camera_para' ""here and running the camera sensor at 320x240 will attempt to load ""camera_para320x240.dat from within the module's directory.","camera_para", ParamCateg)
Parameter.
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
SalientRegions(std::string const &instance)
Constructor.
std::shared_ptr< Saliency > itsSaliency
unsigned int width
virtual ~SalientRegions()
Virtual destructor for safe inheritance.