JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
SaliencySURF.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 
19 #include <jevois/Core/Module.H>
20 #include <jevois/Debug/Log.H>
21 #include <jevois/Util/Utils.H>
23 #include <jevois/Debug/Timer.H>
25 
26 #include <linux/videodev2.h>
29 #include <opencv2/opencv.hpp>
30 
31 // icon by Freepik in people at flaticon
32 
33 // Module parameters:
34 #define PATHPREFIX "/jevois/data/saliencysurf/"
35 static jevois::ParameterCategory const ParamCateg("Salient Regions Options");
36 
37 //! Parameter \relates SaliencySURF
38 JEVOIS_DECLARE_PARAMETER(inhsigma, float, "Sigma (pixels) used for inhibition of return", 32.0F, ParamCateg);
39 
40 //! Parameter \relates SaliencySURF
41 JEVOIS_DECLARE_PARAMETER(regions, size_t, "Number of salient regions", 2, ParamCateg);
42 
43 //! Parameter \relates SaliencySURF
44 JEVOIS_DECLARE_PARAMETER(rsiz, size_t, "Width and height (pixels) of salient regions", 64, ParamCateg);
45 
46 //! Parameter \relates SaliencySURF
47 JEVOIS_DECLARE_PARAMETER(save, bool, "Save regions when true, useful to create a training set. They will be saved to "
48  PATHPREFIX, false, ParamCateg);
49 
50 //! Simple salient region detection and identification using keypoint matching
51 /*! This module finds objects by matching keypoint descriptors between a current set of salient regions and a set of
52  training images. Here we use SURF keypoints and descriptors as provided by OpenCV. The algorithm is quite slow and
53  consists of 3 phases: detect keypoint locations, compute keypoint descriptors, and match descriptors from current
54  image to training image descriptors. Here, we alternate between computing keypoints and descriptors on one frame (or
55  more, depending on how slow that gets), and doing the matching on the next frame. This module also provides an
56  example of letting some computation happen even after we exit the process() function. Here, we keep detecting
57  keypoints and computing descriptors even outside process(). The itsKPfut future is our handle to that thread, and we
58  also use it to alternate between detection and matching on alternating frames.
59 
60  Also see the ObjectDetect module for a related algorithm (without attention).
61 
62  Training: Simply add images of the objects you want to detect in JEVOIS:/modules/JeVois/SaliencySURF/images/ on your
63  JeVois microSD card. Those will be processed when the module starts. The names of recognized objects returned by
64  this module are simply the file names of the pictures you have added in that directory. No additional trainign
65  procedure is needed. Beware that the more images you add, the slower the algorithm will run, and the higher your
66  chances of confusions among several of your objects.
67 
68  @author Laurent Itti
69 
70  @displayname Saliency SURF
71  @videomapping YUYV 320 288 30.0 YUYV 320 240 30.0 JeVois SaliencySURF
72  @email itti\@usc.edu
73  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
74  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
75  @mainurl http://jevois.org
76  @supporturl http://jevois.org/doc
77  @otherurl http://iLab.usc.edu
78  @license GPL v3
79  @distribution Unrestricted
80  @restrictions None
81  \ingroup modules */
82 class SaliencySURF : public jevois::Module, public jevois::Parameter<inhsigma, regions, rsiz, save>
83 {
84  public:
85  // ####################################################################################################
86  //! Constructor
87  // ####################################################################################################
88  SaliencySURF(std::string const & instance) : jevois::Module(instance), itsBuf(1000)
89  {
90  itsSaliency = addSubComponent<Saliency>("saliency");
91  itsMatcher = addSubComponent<ObjectMatcher>("surf");
92  }
93 
94  // ####################################################################################################
95  //! Virtual destructor for safe inheritance
96  // ####################################################################################################
97  virtual ~SaliencySURF()
98  { }
99 
100  // ####################################################################################################
101  //! Get started
102  // ####################################################################################################
103  void postInit() override
104  {
105  // Get our run() thread going, it is in charge of compresing and saving frames:
106  itsRunFut = std::async(std::launch::async, &SaliencySURF::run, this);
107 
108  LINFO("Using " << itsMatcher->numtrain() << " Training Images.");
109  }
110 
111  // ####################################################################################################
112  //! Get stopped
113  // ####################################################################################################
114  void postUninit() override
115  {
116  // Push an empty frame into our buffer to signal the end of video to our thread:
117  itsBuf.push(cv::Mat());
118 
119  // Wait for the thread to complete:
120  LINFO("Waiting for writer thread to complete, " << itsBuf.filled_size() << " frames to go...");
121  try { itsRunFut.get(); } catch (...) { jevois::warnAndIgnoreException(); }
122  LINFO("Writer thread completed. Syncing disk...");
123  if (std::system("/bin/sync")) LERROR("Error syncing disk -- IGNORED");
124  }
125 
126  // ####################################################################################################
127  //! Processing function
128  // ####################################################################################################
129  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
130  {
131  static jevois::Timer timer("processing", 30, LOG_DEBUG);
132 
133  // Wait for next available camera image. Any resolution ok, but require YUYV since we assume it for drawings:
134  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
135  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV);
136 
137  timer.start();
138 
139  // While we process it, start a thread to wait for output frame and paste the input image into it:
140  jevois::RawImage outimg; // main thread should not use outimg until paste thread is complete
141  cv::Mat grayimg;
142 
143  auto paste_fut = std::async(std::launch::async, [&]() {
144  // Convert input image to greyscale:
145  grayimg = jevois::rawimage::convertToCvGray(inimg);
146 
147  // Wait output frame and paste input into it:
148  outimg = outframe.get();
149  outimg.require("output", w, h + 4*12, inimg.fmt);
150  jevois::rawimage::paste(inimg, outimg, 0, 0);
151  jevois::rawimage::writeText(outimg, "JeVois Saliency + SURF Demo", 3, 3, jevois::yuyv::White);
152  jevois::rawimage::drawFilledRect(outimg, 0, h, w, outimg.height-h, 0x8000);
153  });
154 
155  // Compute the saliency map, no gist:
156  itsSaliency->process(inimg, false);
157 
158  // Get some info from the saliency computation:
159  int const smlev = itsSaliency->smscale::get();
160  int const smfac = (1 << smlev);
161  int const rwh = rsiz::get();
162 
163  // We need the grayscale and output images to proceed:
164  paste_fut.get();
165 
166  // Process each region:
167  int k = 0;
168  for (size_t i = 0; i < regions::get(); ++i)
169  {
170  // Find most salient point:
171  int mx, my; intg32 msal; itsSaliency->getSaliencyMax(mx, my, msal);
172 
173  // Compute attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
174  unsigned int const dmx = (mx << smlev) + (smfac >> 2);
175  unsigned int const dmy = (my << smlev) + (smfac >> 2);
176  int rx = (std::min(int(w) - rwh/2, std::max(rwh/2, int(dmx + 1 + (smfac >> 2))))) & (~1);
177  int ry = (std::min(int(h) - rwh/2, std::max(rwh/2, int(dmy + 1 + (smfac >> 2))))) & (~1);
178  unsigned short col = jevois::yuyv::White;
179 
180  // Grab the ROI:
181  cv::Mat roi = grayimg(cv::Rect(rx - rwh/2, ry - rwh/2, rwh, rwh));
182 
183  // Save it if desired:
184  if (save::get()) itsBuf.push(roi);
185 
186  // Process it through our matcher:
187  size_t trainidx;
188  double dist = itsMatcher->process(roi, trainidx);
189  if (dist < 100.0)
190  {
191  jevois::rawimage::writeText(outimg, std::string("Detected: ") + itsMatcher->traindata(trainidx).name +
192  " avg distance " + std::to_string(dist), 3, h + k*12 + 2, jevois::yuyv::White);
193  col = jevois::yuyv::LightGreen;
194  ++k;
195  }
196 
197  // Draw the ROI:
198  jevois::rawimage::drawRect(outimg, rx - rwh/2, ry - rwh/2, rwh, rwh, 1, col);
199 
200  // Inhibit this salient location so we move to the next one:
201  itsSaliency->inhibitionOfReturn(mx, my, inhsigma::get() / smfac);
202  }
203 
204  // Show processing fps:
205  std::string const & fpscpu = timer.stop();
206  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
207 
208  // Send the output image with our processing results to the host over USB:
209  outframe.send();
210  }
211 
212  private:
213  // ####################################################################################################
214  void run() // Runs in a thread to save regions as images, for training
215  {
216  size_t frame = 0; char tmp[2048];
217 
218  // Create directory just in case it does not exist:
219  std::string const cmd = "/bin/mkdir -p " PATHPREFIX;
220  if (std::system(cmd.c_str())) LERROR("Error running [" << cmd << "] -- IGNORED");
221 
222  while (true)
223  {
224  // Get next frame from the buffer:
225  cv::Mat im = itsBuf.pop();
226 
227  // An empty image will be pushed when we are ready to unload the module:
228  if (im.empty()) break;
229 
230  // Write the frame:
231  std::snprintf(tmp, 2047, "%s/frame%06zu.png", PATHPREFIX, frame);
232  cv::imwrite(tmp, im);
233 
234  // Report what is going on once in a while:
235  if ((++frame % 100) == 0) LINFO("Saved " << frame << " salient regions.");
236  }
237  }
238 
239  std::shared_ptr<ObjectMatcher> itsMatcher;
240  std::shared_ptr<Saliency> itsSaliency;
241  std::future<void> itsRunFut;
243 };
244 
245 // Allow the module to be loaded as a shared object (.so) file:
std::string warnAndIgnoreException()
size_t filled_size() const
friend friend class Module
#define PATHPREFIX
Definition: SaliencySURF.C:34
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
void push(T const &val)
#define JEVOIS_DECLARE_PARAMETER(ParamName, ParamType,...)
Parameter.
unsigned int fmt
#define LERROR(msg)
void postUninit() override
Get stopped.
Definition: SaliencySURF.C:114
SaliencySURF(std::string const &instance)
Constructor.
Definition: SaliencySURF.C:88
JEVOIS_REGISTER_MODULE(SaliencySURF)
cv::Mat convertToCvGray(RawImage const &src)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: SaliencySURF.C:129
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
Simple salient region detection and identification using keypoint matching.
Definition: SaliencySURF.C:82
std::string to_string(T const &val)
#define LINFO(msg)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
virtual ~SaliencySURF()
Virtual destructor for safe inheritance.
Definition: SaliencySURF.C:97
void postInit() override
Get started.
Definition: SaliencySURF.C:103
unsigned int width
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)