JeVoisBase  1.6
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DenseSift.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
23 
24 #include <linux/videodev2.h>
25 #include <opencv2/core/core.hpp>
26 #include <opencv2/imgproc/imgproc.hpp>
27 
28 #include <vlfeat/vl/dsift.h>
29 
30 // Module parameters: allow user to play with step and binsize:
31 static jevois::ParameterCategory const ParamCateg("Dense Sift Options");
32 
33 //! Parameter \relates DenseSift
34 JEVOIS_DECLARE_PARAMETER(step, unsigned int, "Keypoint step (pixels)", 11, ParamCateg);
35 
36 //! Parameter \relates DenseSift
37 JEVOIS_DECLARE_PARAMETER(binsize, unsigned int, "Descriptor bin size", 8, ParamCateg);
38 
39 // icon by Pixel Buddha in interface at flaticon
40 
41 //! Simple demo of dense SIFT feature descriptors extraction
42 /*! Compute SIFT keypoint descriptors on a regular grid over the input image.
43 
44  This module is useful when using JeVois as a pre-processor, delivering a dense array of keypoint descriptors to a
45  host computer, where the array is disguised as a grayscale video frame. Upon receiving the array of descriptors, the
46  host computer can further process them. For example, the host computer may compute camera motion in space by
47  matching descriptors across successive frames, or may attempt to detect and identify objects based on the
48  descriptors.
49 
50  Beware that changing the values for the \p step and \p binsize parameters changes the output image size, so you need
51  to adjust your video mappings accordingly. Hence, setting those parameters is best done once and for all in the
52  module's optional \b params.cfg or \b script.cfg file.
53 
54  This module can either have a color YUYV output, which shows the original camera image, keypoint locations, and
55  descriptor values; or a greyscale output, which is just the descriptor values.
56 
57  This algorithm is implemented using the VLfeat library. It is quite slow, maybe because this library is a bit old
58  and appears to be single-threaded.
59 
60 
61  @author Laurent Itti
62 
63  @displayname Dense SIFT
64  @videomapping YUYV 288 120 5.0 YUYV 160 120 5.0 JeVois DenseSift
65  @videomapping GREY 128 117 5.0 YUYV 160 120 5.0 JeVois DenseSift
66  @email itti\@usc.edu
67  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
68  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
69  @mainurl http://jevois.org
70  @supporturl http://jevois.org/doc
71  @otherurl http://iLab.usc.edu
72  @license GPL v3
73  @distribution Unrestricted
74  @restrictions None
75  \ingroup modules */
76 class DenseSift : public jevois::Module,
77  public jevois::Parameter<step, binsize>
78 {
79  public:
80  //! Default base class constructor ok
82 
83  //! Virtual destructor for safe inheritance
84  virtual ~DenseSift() { }
85 
86  //! Processing function
87  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
88  {
89  static jevois::Timer timer("processing");
90 
91  // Wait for next available camera image:
92  jevois::RawImage inimg = inframe.get();
93  unsigned int const w = inimg.width, h = inimg.height;
94  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // any image size but require YUYV pixels
95  bool demodisplay = false;
96 
97  timer.start();
98 
99  // Create the dense sift filter:
100  VlDsiftFilter * vlds = vl_dsift_new_basic(w, h, step::get(), binsize::get());
101  int const descsize = vl_dsift_get_descriptor_size(vlds);
102  int const numkp = vl_dsift_get_keypoint_num(vlds);
103 
104  // Everything from here on is in a try-catch so we de-allocate vlds on exception:
105  try
106  {
107  // While we convert it, start a thread to wait for out frame and paste the input into it:
108  jevois::RawImage outimg;
109  auto paste_fut = std::async(std::launch::async, [&]() {
110  // Get next output video frame:
111  outimg = outframe.get();
112 
113  // Do we want color (demo) or grey (raw data) output:
114  switch (outimg.fmt)
115  {
116  case V4L2_PIX_FMT_YUYV:
117  demodisplay = true;
118  outimg.require("output", w + 128, h, V4L2_PIX_FMT_YUYV);
119  jevois::rawimage::paste(inimg, outimg, 0, 0);
120  jevois::rawimage::writeText(outimg, "JeVois Dense SIFT Demo", 3, 3, jevois::yuyv::White);
121 
122  // if the number of keypoints (based on step) is smaller than image height, blank out bottom:
123  if (numkp < int(h))
124  jevois::rawimage::drawFilledRect(outimg, w, numkp, descsize, h-numkp, jevois::yuyv::DarkGrey);
125  break;
126 
127  case V4L2_PIX_FMT_GREY:
128  demodisplay = false;
129  outimg.require("output", descsize, numkp, V4L2_PIX_FMT_GREY);
130  break;
131 
132  default: LFATAL("This module only supports YUYV or GREY output images");
133  }
134  });
135 
136  // Convert input frame to gray byte first:
137  cv::Mat grayimgcv = jevois::rawimage::convertToCvGray(inimg);
138 
139  // Then we need it as floats for vlfeat:
140  cv::Mat floatimgcv; grayimgcv.convertTo(floatimgcv, CV_32F, 1.0, 0.0);
141 
142  // Wait for paste to finish up:
143  paste_fut.get();
144 
145  // Let camera know we are done processing the input image:
146  inframe.done();
147 
148  // Process the float gray image:
149  vl_dsift_process(vlds, reinterpret_cast<float const *>(floatimgcv.data));
150 
151  // Get the descriptors: size is descsize * numkp:
152  float const * descriptors = vl_dsift_get_descriptors(vlds);
153 
154  // Convert them to byte using opencv. The conversion factor should be 255, but for demo display the descriptors
155  // look mostly black, so we use a higher factor for demo display:
156  cv::Mat dfimg(numkp, descsize, CV_32FC1, reinterpret_cast<void *>(const_cast<float *>(descriptors)));
157  cv::Mat bdfimg; dfimg.convertTo(bdfimg, CV_8U, (demodisplay ? 512.0 : 255.0), 0.0);
158 
159  std::string const & fpscpu = timer.stop();
160 
161  if (demodisplay)
162  {
163  // Paste into our output image:
164  jevois::rawimage::pasteGreyToYUYV(bdfimg, outimg, w, 0);
165  jevois::rawimage::writeText(outimg, "SIFT", w + 3, 3, jevois::yuyv::LightGreen);
166 
167  // Draw the keypoint locations and scale:
168  VlDsiftKeypoint const * keypoints = vl_dsift_get_keypoints(vlds);
169 
170  for (int i = 0; i < numkp; ++i)
171  {
172  VlDsiftKeypoint const & kp = keypoints[i];
173  unsigned int s = (unsigned int)(kp.s / 150.0 + 1.499); if (s >20) s = 20;
174  jevois::rawimage::drawDisk(outimg, int(kp.x + 0.499), int(kp.y + 0.499), s, jevois::yuyv::LightPink);
175  }
176 
177  // Show processing fps:
178  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
179  }
180  else
181  {
182  // Just copy the byte-converted descriptors to the output image:
183  memcpy(outimg.buf->data(), bdfimg.data, outimg.width * outimg.height);
184  }
185 
186  // Send the output image with our processing results to the host over USB:
187  outframe.send();
188  }
189  catch (...) { jevois::warnAndIgnoreException(); }
190 
191  // Nuke the dense sift computer:
192  vl_dsift_delete(vlds);
193  }
194 };
195 
196 // Allow the module to be loaded as a shared object (.so) file:
std::string warnAndIgnoreException()
friend friend class Module
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
unsigned int height
unsigned int fmt
std::shared_ptr< VideoBuf > buf
Simple demo of dense SIFT feature descriptors extraction.
Definition: DenseSift.C:76
cv::Mat convertToCvGray(RawImage const &src)
std::string const & stop()
JEVOIS_DECLARE_PARAMETER(camparams, std::string, "File stem of camera parameters, or empty. Camera resolution " "will be appended, as well as a .cfg extension. For example, specifying 'camera_para' " "here and running the camera sensor at 320x240 will attempt to load " "camera_para320x240.dat from within the module's directory.", "camera_para", ParamCateg)
Parameter.
virtual ~DenseSift()
Virtual destructor for safe inheritance.
Definition: DenseSift.C:84
#define LFATAL(msg)
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
void pasteGreyToYUYV(cv::Mat const &src, RawImage &dest, int dx, int dy)
JEVOIS_REGISTER_MODULE(DenseSift)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DenseSift.C:87
void drawDisk(RawImage &img, int x, int y, unsigned int rad, unsigned int col)
unsigned int width
void paste(RawImage const &src, RawImage &dest, int dx, int dy)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const