JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
ObjectTracker.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
24 
25 #include <linux/videodev2.h>
26 #include <opencv2/core/core.hpp>
27 #include <opencv2/imgproc/imgproc.hpp>
28 #include <string.h>
29 
30 // This code was loosely inspired by:
31 
32 // https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp
33 
34 // Written by Kyle Hounslow 2013
35 
36 // Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
37 // documentation files (the "Software") , to deal in the Software without restriction, including without limitation the
38 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
39 // permit persons to whom the Software is furnished to do so, subject to the following conditions:
40 //
41 // The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
42 // Software.
43 //
44 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
45 // WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
46 // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
47 // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 
49 // icon by Roundicons in miscellaneous at flaticon
50 
51 static jevois::ParameterCategory const ParamCateg("ObjectTracker Options");
52 
53 //! Parameter \relates ObjectTracker
54 JEVOIS_DECLARE_PARAMETER(hrange, jevois::Range<unsigned char>, "Range of H values for HSV window",
55  jevois::Range<unsigned char>(10, 245), ParamCateg);
56 
57 //! Parameter \relates ObjectTracker
58 JEVOIS_DECLARE_PARAMETER(srange, jevois::Range<unsigned char>, "Range of S values for HSV window",
59  jevois::Range<unsigned char>(10, 245), ParamCateg);
60 
61 //! Parameter \relates ObjectTracker
62 JEVOIS_DECLARE_PARAMETER(vrange, jevois::Range<unsigned char>, "Range of V values for HSV window",
63  jevois::Range<unsigned char>(10, 245), ParamCateg);
64 
65 //! Parameter \relates ObjectTracker
66 JEVOIS_DECLARE_PARAMETER(maxnumobj, size_t, "Max number of objects to declare a clean image",
67  10, ParamCateg);
68 
69 //! Parameter \relates ObjectTracker
70 JEVOIS_DECLARE_PARAMETER(objectarea, jevois::Range<unsigned int>, "Range of object area (in pixels) to track",
71  jevois::Range<unsigned int>(20*20, 100*100), ParamCateg);
72 
73 //! Parameter \relates ObjectTracker
74 JEVOIS_DECLARE_PARAMETER(erodesize, size_t, "Erosion structuring element size (pixels)",
75  3, ParamCateg);
76 
77 //! Parameter \relates ObjectTracker
78 JEVOIS_DECLARE_PARAMETER(dilatesize, size_t, "Dilation structuring element size (pixels)",
79  8, ParamCateg);
80 
81 //! Parameter \relates ObjectTracker
82 JEVOIS_DECLARE_PARAMETER(debug, bool, "Show contours of all object candidates if true",
83  true, ParamCateg);
84 
85 
86 //! Simple color-based object detection/tracking
87 /*! This modules isolates pixels within a given HSV range (hue, saturation, and value of color pixels), does some
88  cleanups, and extracts object contrours. It sends information about object centers over serial. This module usually
89  works best with the camera sensor set to manual exposure, gain, color balance, etc so that HSV color values are
90  reliable. See the cript.cfg file in this module's directory for an example of how to set the camera settigs each
91  time this module is loaded.
92 
93  This code was loosely inspired by:
94  https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp written
95  by Kyle Hounslow, 2013.
96 
97  Serial output
98  -------------
99 
100  This module sends serial messages of the form
101 
102  \verbatim
103  T2D x y
104  \endverbatim
105 
106  where \a x and \a y are the standardized coordinates of the center of each detected and good object (good objects
107  have a pixel area within the range specified by \p objectarea, and are only reported when the image is clean enough
108  according to \p maxnumobj. See \ref coordhelpers for standardized coordinates. One message is sent for every good
109  object on every frame.
110 
111  Trying it out
112  -------------
113 
114  The default parameter settings (which are set in \b script.cfg explained below) attempt to detect light blue
115  objects. Present a light blue object to the JeVois camera and see whether it is detected. When detected and good
116  enough according to \p objectarea and \p maxnumobj, a green circle will be drawn at the center of each good object.
117 
118  Tuning
119  ------
120 
121  You should adjust parameters hrange, srange, and vrange to isolate the range of Hue, Saturation, and Value
122  (respectively) that correspond to the objects you want to detect. Note that there is a params.cfg file in this
123  module's directory that provides a range tuned to a lighgt blue object, as shown in the demo screenshot.
124 
125  Tuning the parameters is best done interactively by connecting to your JeVois camera while it is looking at some
126  object of the desired color. Once you have achieved a tuning, you may want to set the hrange, srange, and vrange
127  parameters in your \b script.cfg file for this module on the microSD card (see below).
128 
129  Typically, you would start by narrowing down on the hue, then the value, and finally the saturation. Make sure you
130  also move your camera around and show it typical background clutter so check for false positives (detections of
131  things which you are not interested, which can happen if your ranges are too wide).
132 
133  Config file
134  -----------
135 
136  JeVois allows you to store parameter settings and commands in a file named \b script.cfg stored in the directory of
137  a module. The file \b script.cfg may contain any sequence of commands as you would type them interactively in the
138  JeVois command-line interface. For the ObjectTracker module, a default script is provided that sets the camera to
139  manual color, gain, and exposure mode (for more reliable color values), and to setup communication with a pan/tilt
140  head as described in \ref ArduinoTutorial.
141 
142  The \b script.cfg file for ObjectTracker is stored on your microSD at
143  <b>JEVOIS:/modules/JeVois/ObjectTracker/script.cfg</b> and is shown in \ref ArduinoTutorial as an example.
144 
145 
146  @author Laurent Itti
147 
148  @videomapping YUYV 320 254 60.0 YUYV 320 240 60.0 JeVois ObjectTracker
149  @videomapping NONE 0 0 0.0 YUYV 320 240 60.0 JeVois ObjectTracker
150  @email itti\@usc.edu
151  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
152  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
153  @mainurl http://jevois.org
154  @supporturl http://jevois.org/doc
155  @otherurl http://iLab.usc.edu
156  @license GPL v3
157  @distribution Unrestricted
158  @restrictions None
159  \ingroup modules */
161  public jevois::Parameter<hrange, srange, vrange, maxnumobj, objectarea, erodesize,
162  dilatesize, debug>
163 {
164  public:
165  //! Default base class constructor ok
167 
168  //! Virtual destructor for safe inheritance
169  virtual ~ObjectTracker() { }
170 
171  //! Processing function, no USB video output
172  virtual void process(jevois::InputFrame && inframe) override
173  {
174  // Wait for next available camera image. Any resolution and format ok:
175  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
176 
177  // Convert input image to BGR24, then to HSV:
178  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
179  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
180 
181  // Let camera know we are done processing the input image:
182  inframe.done();
183 
184  // Threshold the HSV image to only keep pixels within the desired HSV range:
185  cv::Mat imgth;
186  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
187  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
188 
189  // Apply morphological operations to cleanup the image noise:
190  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
191  cv::erode(imgth, imgth, erodeElement);
192 
193  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
194  cv::dilate(imgth, imgth, dilateElement);
195 
196  // Detect objects by finding contours:
197  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
198  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
199 
200  // Identify the "good" objects:
201  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
202  {
203  double refArea = 0.0; int x = 0, y = 0;
204 
205  for (int index = 0; index >= 0; index = hierarchy[index][0])
206  {
207  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
208  double area = moment.m00;
209  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea)
210  { x = moment.m10 / area + 0.4999; y = moment.m01 / area + 0.4999; refArea = area; }
211  }
212 
213  if (refArea > 0.0)
214  {
215  // Standardize the coordinates to -1000 ... 1000:
216  float xx = x, yy = y; jevois::coords::imgToStd(xx, yy, w, h, 1.0F);
217 
218  // Send coords to serial port (for arduino, etc):
219  sendSerial(jevois::sformat("T2D %d %d", int(xx), int(yy)));
220  }
221  }
222  }
223 
224  //! Processing function, with USB video output
225  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
226  {
227  static jevois::Timer timer("processing");
228 
229  // Wait for next available camera image. Any resolution ok, but require YUYV since we assume it for drawings:
230  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
231  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV);
232 
233  timer.start();
234 
235  // While we process it, start a thread to wait for output frame and paste the input image into it:
236  jevois::RawImage outimg; // main thread should not use outimg until paste thread is complete
237  auto paste_fut = std::async(std::launch::async, [&]() {
238  outimg = outframe.get();
239  outimg.require("output", w, h + 14, inimg.fmt);
240  jevois::rawimage::paste(inimg, outimg, 0, 0);
241  jevois::rawimage::writeText(outimg, "JeVois Color Object Tracker", 3, 3, jevois::yuyv::White);
242  jevois::rawimage::drawFilledRect(outimg, 0, h, w, outimg.height-h, 0x8000);
243  });
244 
245  // Convert input image to BGR24, then to HSV:
246  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
247  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
248 
249  // Threshold the HSV image to only keep pixels within the desired HSV range:
250  cv::Mat imgth;
251  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
252  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
253 
254  // Wait for paste to finish up:
255  paste_fut.get();
256 
257  // Let camera know we are done processing the input image:
258  inframe.done();
259 
260  // Apply morphological operations to cleanup the image noise:
261  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
262  cv::erode(imgth, imgth, erodeElement);
263 
264  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
265  cv::dilate(imgth, imgth, dilateElement);
266 
267  // Detect objects by finding contours:
268  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
269  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
270 
271  // If desired, draw all contours in a thread:
272  std::future<void> draw_fut;
273  if (debug::get())
274  draw_fut = std::async(std::launch::async, [&]() {
275  // We reinterpret the top portion of our YUYV output image as an opencv 8UC2 image:
276  cv::Mat outuc2(imgth.rows, imgth.cols, CV_8UC2, outimg.pixelsw<unsigned char>()); // pixel data shared
277  for (size_t i = 0; i < contours.size(); ++i)
278  cv::drawContours(outuc2, contours, i, jevois::yuyv::LightPink, 2, 8, hierarchy);
279  });
280 
281  // Identify the "good" objects:
282  int numobj = 0;
283  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
284  {
285  double refArea = 0.0; int x = 0, y = 0;
286 
287  for (int index = 0; index >= 0; index = hierarchy[index][0])
288  {
289  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
290  double area = moment.m00;
291  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea)
292  { x = moment.m10 / area + 0.4999; y = moment.m01 / area + 0.4999; refArea = area; }
293  }
294 
295  if (refArea > 0.0)
296  {
297  ++numobj;
298  jevois::rawimage::drawCircle(outimg, x, y, 20, 1, jevois::yuyv::LightGreen);
299 
300  // Standardize the coordinates to -1000 ... 1000:
301  float xx = x, yy = y; jevois::coords::imgToStd(xx, yy, w, h, 1.0F);
302 
303  // Send coords to serial port (for arduino, etc):
304  sendSerial(jevois::sformat("T2D %d %d", int(xx), int(yy)));
305  }
306  }
307 
308  // Show number of detected objects:
309  jevois::rawimage::writeText(outimg, "Detected " + std::to_string(numobj) + " objects.",
310  3, h + 2, jevois::yuyv::White);
311 
312  // Show processing fps:
313  std::string const & fpscpu = timer.stop();
314  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
315 
316  // Possibly wait until all contours are drawn, if they had been requested:
317  if (draw_fut.valid()) draw_fut.get();
318 
319  // Send the output image with our processing results to the host over USB:
320  outframe.send();
321  }
322 };
323 
324 // Allow the module to be loaded as a shared object (.so) file:
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function, with USB video output.
friend friend class Module
void imgToStd(float &x, float &y, RawImage const &camimg, float const eps=0.1F)
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
cv::Mat convertToCvBGR(RawImage const &src)
unsigned int height
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
JEVOIS_REGISTER_MODULE(ObjectTracker)
unsigned int fmt
JEVOIS_DECLARE_PARAMETER(camparams, std::string,"Filename of camera parameters, or empty","", ParamCateg)
Parameter.
virtual ~ObjectTracker()
Virtual destructor for safe inheritance.
virtual void process(jevois::InputFrame &&inframe) override
Processing function, no USB video output.
Simple color-based object detection/tracking.
double area(const std::vector< Point2D< T > > &polygon, const bool getsigned=false)
What is the area of a polygon?
Definition: Point2D.H:422
virtual void sendSerial(std::string const &str)
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
std::string to_string(T const &val)
unsigned int width
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)