JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
DenseSift.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
23 
24 #include <linux/videodev2.h>
25 #include <opencv2/core/core.hpp>
26 #include <opencv2/imgproc/imgproc.hpp>
27 
28 #include <vlfeat/vl/dsift.h>
29 
30 // Module parameters: allow user to play with step and binsize:
31 static jevois::ParameterCategory const ParamCateg("Dense Sift Options");
32 
33 //! Parameter \relates DenseSift
34 JEVOIS_DECLARE_PARAMETER(step, unsigned int, "Keypoint step (pixels)", 11, ParamCateg);
35 
36 //! Parameter \relates DenseSift
37 JEVOIS_DECLARE_PARAMETER(binsize, unsigned int, "Descriptor bin size", 8, ParamCateg);
38 
39 // icon by Pixel Buddha in interface at flaticon
40 
41 //! Simple demo of dense SIFT feature descriptors extraction
42 /*! Compute SIFT keypoint descriptors on a regular grid over the input image. Beware that changing the values for the
43  step and binsize parameters changes the output image size, so you need to adjust your video mappings
44  accordingly. Hence, setting those parameters is best done once and for all in the module's optional params.cfg
45  file. This module can either have a color YUYV output, which shows the original camera image, keypoint locations,
46  and descriptor values; or a greyscale output, which is just the descriptor values.
47 
48  This algorithm is implemengted using the VLfeat library. It is quite slow, maybe because this library is a bit old
49  and appears to be single-threaded.
50 
51  @author Laurent Itti
52 
53  @displayname Dense SIFT
54  @videomapping YUYV 288 120 5.0 YUYV 160 120 5.0 JeVois DenseSift
55  @videomapping GREY 128 117 5.0 YUYV 160 120 5.0 JeVois DenseSift
56  @email itti\@usc.edu
57  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
58  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
59  @mainurl http://jevois.org
60  @supporturl http://jevois.org/doc
61  @otherurl http://iLab.usc.edu
62  @license GPL v3
63  @distribution Unrestricted
64  @restrictions None
65  \ingroup modules */
66 class DenseSift : public jevois::Module,
67  public jevois::Parameter<step, binsize>
68 {
69  public:
70  //! Default base class constructor ok
72 
73  //! Virtual destructor for safe inheritance
74  virtual ~DenseSift() { }
75 
76  //! Processing function
77  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
78  {
79  static jevois::Timer timer("processing");
80 
81  // Wait for next available camera image:
82  jevois::RawImage inimg = inframe.get();
83  unsigned int const w = inimg.width, h = inimg.height;
84  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV); // any image size but require YUYV pixels
85  bool demodisplay = false;
86 
87  timer.start();
88 
89  // Create the dense sift filter:
90  VlDsiftFilter * vlds = vl_dsift_new_basic(w, h, step::get(), binsize::get());
91  int const descsize = vl_dsift_get_descriptor_size(vlds);
92  int const numkp = vl_dsift_get_keypoint_num(vlds);
93 
94  // Everything from here on is in a try-catch so we de-allocate vlds on exception:
95  try
96  {
97  // While we convert it, start a thread to wait for out frame and paste the input into it:
98  jevois::RawImage outimg;
99  auto paste_fut = std::async(std::launch::async, [&]() {
100  // Get next output video frame:
101  outimg = outframe.get();
102 
103  // Do we want color (demo) or grey (raw data) output:
104  switch (outimg.fmt)
105  {
106  case V4L2_PIX_FMT_YUYV:
107  demodisplay = true;
108  outimg.require("output", w + 128, h, V4L2_PIX_FMT_YUYV);
109  jevois::rawimage::paste(inimg, outimg, 0, 0);
110  jevois::rawimage::writeText(outimg, "JeVois Dense SIFT Demo", 3, 3, jevois::yuyv::White);
111 
112  // if the number of keypoints (based on step) is smaller than image height, blank out bottom:
113  if (numkp < int(h))
114  jevois::rawimage::drawFilledRect(outimg, w, numkp, descsize, h-numkp, jevois::yuyv::DarkGrey);
115  break;
116 
117  case V4L2_PIX_FMT_GREY:
118  demodisplay = false;
119  outimg.require("output", descsize, numkp, V4L2_PIX_FMT_GREY);
120  break;
121 
122  default: LFATAL("This module only supports YUYV or GREY output images");
123  }
124  });
125 
126  // Convert input frame to gray byte first:
127  cv::Mat grayimgcv = jevois::rawimage::convertToCvGray(inimg);
128 
129  // Then we need it as floats for vlfeat:
130  cv::Mat floatimgcv; grayimgcv.convertTo(floatimgcv, CV_32F, 1.0, 0.0);
131 
132  // Wait for paste to finish up:
133  paste_fut.get();
134 
135  // Let camera know we are done processing the input image:
136  inframe.done();
137 
138  // Process the float gray image:
139  vl_dsift_process(vlds, reinterpret_cast<float const *>(floatimgcv.data));
140 
141  // Get the descriptors: size is descsize * numkp:
142  float const * descriptors = vl_dsift_get_descriptors(vlds);
143 
144  // Convert them to byte using opencv. The conversion factor should be 255, but for demo display the descriptors
145  // look mostly black, so we use a higher factor for demo display:
146  cv::Mat dfimg(numkp, descsize, CV_32FC1, reinterpret_cast<void *>(const_cast<float *>(descriptors)));
147  cv::Mat bdfimg; dfimg.convertTo(bdfimg, CV_8U, (demodisplay ? 512.0 : 255.0), 0.0);
148 
149  std::string const & fpscpu = timer.stop();
150 
151  if (demodisplay)
152  {
153  // Paste into our output image:
154  jevois::rawimage::pasteGreyToYUYV(bdfimg, outimg, w, 0);
155  jevois::rawimage::writeText(outimg, "SIFT", w + 3, 3, jevois::yuyv::LightGreen);
156 
157  // Draw the keypoint locations and scale:
158  VlDsiftKeypoint const * keypoints = vl_dsift_get_keypoints(vlds);
159 
160  for (int i = 0; i < numkp; ++i)
161  {
162  VlDsiftKeypoint const & kp = keypoints[i];
163  unsigned int s = (unsigned int)(kp.s / 150.0 + 1.499); if (s >20) s = 20;
164  jevois::rawimage::drawDisk(outimg, int(kp.x + 0.499), int(kp.y + 0.499), s, jevois::yuyv::LightPink);
165  }
166 
167  // Show processing fps:
168  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
169  }
170  else
171  {
172  // Just copy the byte-converted descriptors to the output image:
173  memcpy(outimg.buf->data(), bdfimg.data, outimg.width * outimg.height);
174  }
175 
176  // Send the output image with our processing results to the host over USB:
177  outframe.send();
178  }
179  catch (...) { jevois::warnAndIgnoreException(); }
180 
181  // Nuke the dense sift computer:
182  vl_dsift_delete(vlds);
183  }
184 };
185 
186 // Allow the module to be loaded as a shared object (.so) file:
std::string warnAndIgnoreException()
friend friend class Module
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
unsigned int height
unsigned int fmt
JEVOIS_DECLARE_PARAMETER(camparams, std::string,"Filename of camera parameters, or empty","", ParamCateg)
Parameter.
std::shared_ptr< VideoBuf > buf
Simple demo of dense SIFT feature descriptors extraction.
Definition: DenseSift.C:66
cv::Mat convertToCvGray(RawImage const &src)
virtual ~DenseSift()
Virtual destructor for safe inheritance.
Definition: DenseSift.C:74
#define LFATAL(msg)
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
void pasteGreyToYUYV(cv::Mat const &src, RawImage &dest, int dx, int dy)
JEVOIS_REGISTER_MODULE(DenseSift)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
Definition: DenseSift.C:77
void drawDisk(RawImage &img, int x, int y, unsigned int rad, unsigned int col)
unsigned int width
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)