JeVoisBase  1.0
JeVois Smart Embedded Machine Vision Toolkit Base Modules
ObjectTracker.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
23 
24 #include <linux/videodev2.h>
25 #include <opencv2/core/core.hpp>
26 #include <opencv2/imgproc/imgproc.hpp>
27 #include <string.h>
28 
29 // This code was loosely inspired by:
30 
31 // https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp
32 
33 // Written by Kyle Hounslow 2013
34 
35 // Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
36 // documentation files (the "Software") , to deal in the Software without restriction, including without limitation the
37 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
38 // permit persons to whom the Software is furnished to do so, subject to the following conditions:
39 //
40 // The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
41 // Software.
42 //
43 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
44 // WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
45 // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
46 // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
47 
48 // icon by Roundicons in miscellaneous at flaticon
49 
50 static jevois::ParameterCategory const ParamCateg("ObjectTracker Options");
51 
52 //! Parameter \relates ObjectTracker
53 JEVOIS_DECLARE_PARAMETER(hrange, jevois::Range<unsigned char>, "Range of H values for HSV window",
54  jevois::Range<unsigned char>(10, 245), ParamCateg);
55 
56 //! Parameter \relates ObjectTracker
57 JEVOIS_DECLARE_PARAMETER(srange, jevois::Range<unsigned char>, "Range of S values for HSV window",
58  jevois::Range<unsigned char>(10, 245), ParamCateg);
59 
60 //! Parameter \relates ObjectTracker
61 JEVOIS_DECLARE_PARAMETER(vrange, jevois::Range<unsigned char>, "Range of V values for HSV window",
62  jevois::Range<unsigned char>(10, 245), ParamCateg);
63 
64 //! Parameter \relates ObjectTracker
65 JEVOIS_DECLARE_PARAMETER(maxnumobj, size_t, "Max number of objects to declare a clean image",
66  10, ParamCateg);
67 
68 //! Parameter \relates ObjectTracker
69 JEVOIS_DECLARE_PARAMETER(objectarea, jevois::Range<unsigned int>, "Range of object area (in pixels) to track",
70  jevois::Range<unsigned int>(20*20, 100*100), ParamCateg);
71 
72 //! Parameter \relates ObjectTracker
73 JEVOIS_DECLARE_PARAMETER(erodesize, size_t, "Erosion structuring element size (pixels)",
74  3, ParamCateg);
75 
76 //! Parameter \relates ObjectTracker
77 JEVOIS_DECLARE_PARAMETER(dilatesize, size_t, "Dilation structuring element size (pixels)",
78  8, ParamCateg);
79 
80 //! Parameter \relates ObjectTracker
81 JEVOIS_DECLARE_PARAMETER(debug, bool, "Show contours of all object candidates if true",
82  true, ParamCateg);
83 
84 
85 //! Simple color-based object detection/tracking
86 /*! This modules isolates pixels within a given HSV range, does some cleanups, and extract object contrours. It sends
87  information about object centers over serial. This module usually works best with the camera sensor set to manual
88  exposure, gain, color balance, etc so that HSV color values are reliable. See the cript.cfg file in this module's
89  directory for an example of how to set the camera settigs each time this module is loaded.
90 
91  This code was loosely inspired by:
92  https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp written
93  by Kyle Hounslow, 2013.
94 
95  Tuning
96  ------
97 
98  You should adjust parameters hrange, srange, and vrange to isolate the range of Hue, Saturation, and Value
99  (respectively) that correspond to the objects you want to detect. Note that there is a params.cfg file in this
100  module's directory that provides a range tuned to a lighgt blue object, as shown in the demo screenshot.
101 
102  Tuning the parameters is best done interactively by connecting to your JeVois camera while it is looking at some
103  object of the desired color. Once you have achieved a tuning, you may want to set the hrange, srange, and vrange
104  parameters in your script.cfg file for this module on the microSD card (see below).
105 
106  Typically, you would start by narrowing down on the hue, then the value, and finally the saturation. Make sure you
107  also move your camera around and show it typical background clutter so check for false positives (detections of
108  things which you are not interested, which can happen if your ranges are too wide).
109 
110  Config file
111  -----------
112 
113  JeVois allows you to store parameter settings and commands in a file named \c script.cfg stored in the directory of
114  a module. The file \c script.cfg may contain any sequence of commands as you would type them interactively in the
115  JeVois command-line interface. For the ObjectTracker module, a default script is provided that sets the camera to
116  manual color, gain, and exposure mode (for more reliable color values), and to setup communication with a pan/tilt
117  head as described at http://jevois.org/doc/ArduinoTutorial.html
118 
119  The \c script.cfg file for ObjectTracker is stored on your microSD at
120  JEVOIS:/modules/JeVois/ObjectTracker/script.cfg and is shown in this JeVois tutorial:
121  http://jevois.org/doc/ArduinoTutorial.html as an example.
122 
123 
124  @author Laurent Itti
125 
126  @videomapping YUYV 320 254 60.0 YUYV 320 240 60.0 JeVois ObjectTracker
127  @email itti\@usc.edu
128  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
129  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
130  @mainurl http://jevois.org
131  @supporturl http://jevois.org/doc
132  @otherurl http://iLab.usc.edu
133  @license GPL v3
134  @distribution Unrestricted
135  @restrictions None
136  \ingroup modules */
138  public jevois::Parameter<hrange, srange, vrange, maxnumobj, objectarea, erodesize,
139  dilatesize, debug>
140 {
141  public:
142  //! Default base class constructor ok
144 
145  //! Virtual destructor for safe inheritance
146  virtual ~ObjectTracker() { }
147 
148  //! Processing function
149  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
150  {
151  static jevois::Timer timer("processing");
152 
153  // Wait for next available camera image. Any resolution ok, but require YUYV since we assume it for drawings:
154  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
155  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV);
156 
157  timer.start();
158 
159  // While we process it, start a thread to wait for output frame and paste the input image into it:
160  jevois::RawImage outimg; // main thread should not use outimg until paste thread is complete
161  auto paste_fut = std::async(std::launch::async, [&]() {
162  outimg = outframe.get();
163  outimg.require("output", w, h + 14, inimg.fmt);
164  jevois::rawimage::paste(inimg, outimg, 0, 0);
165  jevois::rawimage::writeText(outimg, "JeVois Color Object Tracker", 3, 3, jevois::yuyv::White);
166  jevois::rawimage::drawFilledRect(outimg, 0, h, w, outimg.height-h, 0x8000);
167  });
168 
169  // Convert input image to BGR24, then to HSV:
170  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
171  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
172 
173  // Threshold the HSV image to only keep pixels within the desired HSV range:
174  cv::Mat imgth;
175  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
176  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
177 
178  // Wait for paste to finish up:
179  paste_fut.get();
180 
181  // Let camera know we are done processing the input image:
182  inframe.done();
183 
184  // Apply morphological operations to cleanup the image noise:
185  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
186  cv::erode(imgth, imgth, erodeElement);
187 
188  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
189  cv::dilate(imgth, imgth, dilateElement);
190 
191  // Detect objects by finding contours:
192  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
193  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
194 
195  // If desired, draw all contours in a thread:
196  std::future<void> draw_fut;
197  if (debug::get())
198  draw_fut = std::async(std::launch::async, [&]() {
199  // We reinterpret the top portion of our YUYV output image as an opencv 8UC2 image:
200  cv::Mat outuc2(imgth.rows, imgth.cols, CV_8UC2, outimg.pixelsw<unsigned char>()); // pixel data shared
201  for (size_t i = 0; i < contours.size(); ++i)
202  cv::drawContours(outuc2, contours, i, jevois::yuyv::LightPink, 2, 8, hierarchy);
203  });
204 
205  // Identify the "good" objects:
206  int numobj = 0;
207  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
208  {
209  double refArea = 0.0; int x = 0, y = 0;
210 
211  for (int index = 0; index >= 0; index = hierarchy[index][0])
212  {
213  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
214  double area = moment.m00;
215  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea)
216  { x = moment.m10 / area + 0.4999; y = moment.m01 / area + 0.4999; refArea = area; }
217  }
218 
219  if (refArea > 0.0)
220  {
221  ++numobj;
222  jevois::rawimage::drawCircle(outimg, x, y, 20, 1, jevois::yuyv::LightGreen);
223 
224  // Send coords to serial port (for arduino, etc), normalizing to -1000...1000:
225  sendSerial("T2D " + std::to_string(int((x - 0.5F * w) * 2000.0F / w)) + ' ' +
226  std::to_string(int((y - 0.5F * h) * 2000.0F / h)));
227 
228  }
229  }
230 
231  // Show number of detected objects:
232  jevois::rawimage::writeText(outimg, "Detected " + std::to_string(numobj) + " objects.",
233  3, h + 2, jevois::yuyv::White);
234 
235  // Show processing fps:
236  std::string const & fpscpu = timer.stop();
237  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
238 
239  // Possibly wait until all contours are drawn, if they had been requested:
240  if (draw_fut.valid()) draw_fut.get();
241 
242  // Send the output image with our processing results to the host over USB:
243  outframe.send();
244  }
245 };
246 
247 // Allow the module to be loaded as a shared object (.so) file:
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
friend friend class Module
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
cv::Mat convertToCvBGR(RawImage const &src)
unsigned int height
#define JEVOIS_DECLARE_PARAMETER(ParamName, ParamType,...)
Parameter.
JEVOIS_REGISTER_MODULE(ObjectTracker)
unsigned int fmt
virtual ~ObjectTracker()
Virtual destructor for safe inheritance.
Simple color-based object detection/tracking.
double area(const std::vector< Point2D< T > > &polygon, const bool getsigned=false)
What is the area of a polygon?
Definition: Point2D.H:422
virtual void sendSerial(std::string const &str)
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
std::string to_string(T const &val)
unsigned int width
std::string const & stop()
void paste(RawImage const &src, RawImage &dest, int dx, int dy)