JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
ObjectTracker.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
24 
25 #include <linux/videodev2.h>
26 #include <opencv2/core/core.hpp>
27 #include <opencv2/imgproc/imgproc.hpp>
28 #include <string.h>
29 
30 // This code was loosely inspired by:
31 
32 // https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp
33 
34 // Written by Kyle Hounslow 2013
35 
36 // Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
37 // documentation files (the "Software") , to deal in the Software without restriction, including without limitation the
38 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
39 // permit persons to whom the Software is furnished to do so, subject to the following conditions:
40 //
41 // The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
42 // Software.
43 //
44 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
45 // WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
46 // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
47 // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 
49 // icon by Roundicons in miscellaneous at flaticon
50 
51 static jevois::ParameterCategory const ParamCateg("ObjectTracker Options");
52 
53 //! Parameter \relates ObjectTracker
54 JEVOIS_DECLARE_PARAMETER(hrange, jevois::Range<unsigned char>, "Range of H values for HSV window",
55  jevois::Range<unsigned char>(10, 245), ParamCateg);
56 
57 //! Parameter \relates ObjectTracker
58 JEVOIS_DECLARE_PARAMETER(srange, jevois::Range<unsigned char>, "Range of S values for HSV window",
59  jevois::Range<unsigned char>(10, 245), ParamCateg);
60 
61 //! Parameter \relates ObjectTracker
62 JEVOIS_DECLARE_PARAMETER(vrange, jevois::Range<unsigned char>, "Range of V values for HSV window",
63  jevois::Range<unsigned char>(10, 245), ParamCateg);
64 
65 //! Parameter \relates ObjectTracker
66 JEVOIS_DECLARE_PARAMETER(maxnumobj, size_t, "Max number of objects to declare a clean image",
67  10, ParamCateg);
68 
69 //! Parameter \relates ObjectTracker
70 JEVOIS_DECLARE_PARAMETER(objectarea, jevois::Range<unsigned int>, "Range of object area (in pixels) to track",
71  jevois::Range<unsigned int>(20*20, 100*100), ParamCateg);
72 
73 //! Parameter \relates ObjectTracker
74 JEVOIS_DECLARE_PARAMETER(erodesize, size_t, "Erosion structuring element size (pixels)",
75  3, ParamCateg);
76 
77 //! Parameter \relates ObjectTracker
78 JEVOIS_DECLARE_PARAMETER(dilatesize, size_t, "Dilation structuring element size (pixels)",
79  8, ParamCateg);
80 
81 //! Parameter \relates ObjectTracker
82 JEVOIS_DECLARE_PARAMETER(debug, bool, "Show contours of all object candidates if true",
83  true, ParamCateg);
84 
85 
86 //! Simple color-based object detection/tracking
87 /*! This modules isolates pixels within a given HSV range (hue, saturation, and value of color pixels), does some
88  cleanups, and extracts object contours. It sends information about object centers over serial.
89 
90  This module usually works best with the camera sensor set to manual exposure, manual gain, manual color balance, etc
91  so that HSV color values are reliable. See the file \b script.cfg file in this module's directory for an example of
92  how to set the camera settings each time this module is loaded.
93 
94  This code was loosely inspired by:
95  https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp written
96  by Kyle Hounslow, 2013.
97 
98  Serial Messages
99  ---------------
100 
101  This module can send standardized serial messages as described in \ref UserSerialStyle. One message is issued on
102  every video frame for each detected and good object (good objects have a pixel area within the range specified by \p
103  objectarea, and are only reported when the image is clean enough according to \p maxnumobj). The \p id field in the
104  messages simply is \b blob for all messages.
105 
106  - Serial message type: \b 2D
107  - `id`: always \b blob
108  - `x`, `y`, or vertices: standardized 2D coordinates of blob center or of corners of bounding box
109  (depending on \p serstyle)
110  - `w`, `h`: standardized object size
111  - `extra`: none (empty string)
112 
113  Trying it out
114  -------------
115 
116  The default parameter settings (which are set in \b script.cfg explained below) attempt to detect light blue
117  objects. Present a light blue object to the JeVois camera and see whether it is detected. When detected and good
118  enough according to \p objectarea and \p maxnumobj, a green circle will be drawn at the center of each good object.
119 
120  For further use of this module, you may want to check out the following tutorials:
121 
122  - [Tuning the color-based object tracker using a python graphical
123  interface](http://jevois.org/tutorials/UserColorTracking.html)
124  - [Making a motorized pan-tilt head for JeVois and tracking
125  objects](http://jevois.org/tutorials/UserPanTilt.html)
126  - \ref ArduinoTutorial
127 
128  Tuning
129  ------
130 
131  You should adjust parameters \p hrange, \p srange, and \p vrange to isolate the range of Hue, Saturation, and Value
132  (respectively) that correspond to the objects you want to detect. Note that there is a \b script.cfg file in this
133  module's directory that provides a range tuned to a lighgt blue object, as shown in the demo screenshot.
134 
135  Tuning the parameters is best done interactively by connecting to your JeVois camera while it is looking at some
136  object of the desired color. Once you have achieved a tuning, you may want to set the hrange, srange, and vrange
137  parameters in your \b script.cfg file for this module on the microSD card (see below).
138 
139  Typically, you would start by narrowing down on the hue, then the value, and finally the saturation. Make sure you
140  also move your camera around and show it typical background clutter so check for false positives (detections of
141  things which you are not interested, which can happen if your ranges are too wide).
142 
143  Config file
144  -----------
145 
146  JeVois allows you to store parameter settings and commands in a file named \b script.cfg stored in the directory of
147  a module. The file \b script.cfg may contain any sequence of commands as you would type them interactively in the
148  JeVois command-line interface. For the \jvmod{ObjectTracker} module, a default script is provided that sets the
149  camera to manual color, gain, and exposure mode (for more reliable color values), and to setup communication with a
150  pan/tilt head as described in \ref ArduinoTutorial.
151 
152  The \b script.cfg file for ObjectTracker is stored on your microSD at
153  <b>JEVOIS:/modules/JeVois/ObjectTracker/script.cfg</b> and is shown in \ref ArduinoTutorial as an example.
154 
155 
156  @author Laurent Itti
157 
158  @videomapping YUYV 320 254 60.0 YUYV 320 240 60.0 JeVois ObjectTracker
159  @videomapping NONE 0 0 0.0 YUYV 320 240 60.0 JeVois ObjectTracker
160  @email itti\@usc.edu
161  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
162  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
163  @mainurl http://jevois.org
164  @supporturl http://jevois.org/doc
165  @otherurl http://iLab.usc.edu
166  @license GPL v3
167  @distribution Unrestricted
168  @restrictions None
169  \ingroup modules */
171  public jevois::Parameter<hrange, srange, vrange, maxnumobj, objectarea, erodesize,
172  dilatesize, debug>
173 {
174  public:
175  //! Default base class constructor ok
177 
178  //! Virtual destructor for safe inheritance
179  virtual ~ObjectTracker() { }
180 
181  //! Processing function, no USB video output
182  virtual void process(jevois::InputFrame && inframe) override
183  {
184  // Wait for next available camera image. Any resolution and format ok:
185  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
186 
187  // Convert input image to BGR24, then to HSV:
188  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
189  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
190 
191  // Let camera know we are done processing the input image:
192  inframe.done();
193 
194  // Threshold the HSV image to only keep pixels within the desired HSV range:
195  cv::Mat imgth;
196  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
197  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
198 
199  // Apply morphological operations to cleanup the image noise:
200  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
201  cv::erode(imgth, imgth, erodeElement);
202 
203  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
204  cv::dilate(imgth, imgth, dilateElement);
205 
206  // Detect objects by finding contours:
207  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
208  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
209 
210  // Identify the "good" objects:
211  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
212  {
213  double refArea = 0.0; int x = 0, y = 0; int refIdx = 0;
214 
215  for (int index = 0; index >= 0; index = hierarchy[index][0])
216  {
217  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
218  double area = moment.m00;
219  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea) { refArea = area; refIdx = index; }
220  }
221 
222  // Send coords to serial port (for arduino, etc):
223  if (refArea > 0.0) sendSerialContour2D(w, h, contours[refIdx], "blob");
224  }
225  }
226 
227  //! Processing function, with USB video output
228  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
229  {
230  static jevois::Timer timer("processing");
231 
232  // Wait for next available camera image. Any resolution ok, but require YUYV since we assume it for drawings:
233  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
234  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV);
235 
236  timer.start();
237 
238  // While we process it, start a thread to wait for output frame and paste the input image into it:
239  jevois::RawImage outimg; // main thread should not use outimg until paste thread is complete
240  auto paste_fut = std::async(std::launch::async, [&]() {
241  outimg = outframe.get();
242  outimg.require("output", w, h + 14, inimg.fmt);
243  jevois::rawimage::paste(inimg, outimg, 0, 0);
244  jevois::rawimage::writeText(outimg, "JeVois Color Object Tracker", 3, 3, jevois::yuyv::White);
245  jevois::rawimage::drawFilledRect(outimg, 0, h, w, outimg.height-h, 0x8000);
246  });
247 
248  // Convert input image to BGR24, then to HSV:
249  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
250  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
251 
252  // Threshold the HSV image to only keep pixels within the desired HSV range:
253  cv::Mat imgth;
254  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
255  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
256 
257  // Wait for paste to finish up:
258  paste_fut.get();
259 
260  // Let camera know we are done processing the input image:
261  inframe.done();
262 
263  // Apply morphological operations to cleanup the image noise:
264  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
265  cv::erode(imgth, imgth, erodeElement);
266 
267  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
268  cv::dilate(imgth, imgth, dilateElement);
269 
270  // Detect objects by finding contours:
271  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
272  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
273 
274  // If desired, draw all contours in a thread:
275  std::future<void> draw_fut;
276  if (debug::get())
277  draw_fut = std::async(std::launch::async, [&]() {
278  // We reinterpret the top portion of our YUYV output image as an opencv 8UC2 image:
279  cv::Mat outuc2(imgth.rows, imgth.cols, CV_8UC2, outimg.pixelsw<unsigned char>()); // pixel data shared
280  for (size_t i = 0; i < contours.size(); ++i)
281  cv::drawContours(outuc2, contours, i, jevois::yuyv::LightPink, 2, 8, hierarchy);
282  });
283 
284  // Identify the "good" objects:
285  int numobj = 0;
286  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
287  {
288  double refArea = 0.0; int x = 0, y = 0; int refIdx = 0;
289 
290  for (int index = 0; index >= 0; index = hierarchy[index][0])
291  {
292  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
293  double area = moment.m00;
294  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea)
295  { x = moment.m10 / area + 0.4999; y = moment.m01 / area + 0.4999; refArea = area; refIdx = index; }
296  }
297 
298  if (refArea > 0.0)
299  {
300  ++numobj;
301  jevois::rawimage::drawCircle(outimg, x, y, 20, 1, jevois::yuyv::LightGreen);
302 
303  // Send coords to serial port (for arduino, etc):
304  sendSerialContour2D(w, h, contours[refIdx], "blob");
305  }
306  }
307 
308  // Show number of detected objects:
309  jevois::rawimage::writeText(outimg, "Detected " + std::to_string(numobj) + " objects.",
310  3, h + 2, jevois::yuyv::White);
311 
312  // Show processing fps:
313  std::string const & fpscpu = timer.stop();
314  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
315 
316  // Possibly wait until all contours are drawn, if they had been requested:
317  if (draw_fut.valid()) draw_fut.get();
318 
319  // Send the output image with our processing results to the host over USB:
320  outframe.send();
321  }
322 };
323 
324 // Allow the module to be loaded as a shared object (.so) file:
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function, with USB video output.
void sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
cv::Mat convertToCvBGR(RawImage const &src)
unsigned int height
JEVOIS_REGISTER_MODULE(ObjectTracker)
unsigned int fmt
virtual ~ObjectTracker()
Virtual destructor for safe inheritance.
StdModule(std::string const &instance)
virtual void process(jevois::InputFrame &&inframe) override
Processing function, no USB video output.
Simple color-based object detection/tracking.
double area(const std::vector< Point2D< T > > &polygon, const bool getsigned=false)
What is the area of a polygon?
Definition: Point2D.H:422
std::string const & stop()
JEVOIS_DECLARE_PARAMETER(camparams, std::string, "File stem of camera parameters, or empty. Camera resolution " "will be appended, as well as a .cfg extension. For example, specifying 'camera_para' " "here and running the camera sensor at 320x240 will attempt to load " "camera_para320x240.dat from within the module's directory.", "camera_para", ParamCateg)
Parameter.
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
std::string to_string(T const &val)
unsigned int width
void paste(RawImage const &src, RawImage &dest, int dx, int dy)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const