JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
ObjectTracker.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Debug/Timer.H>
24 
25 #include <linux/videodev2.h>
26 #include <opencv2/core/core.hpp>
27 #include <opencv2/imgproc/imgproc.hpp>
28 #include <string.h>
29 
30 // This code was loosely inspired by:
31 
32 // https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp
33 
34 // Written by Kyle Hounslow 2013
35 
36 // Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
37 // documentation files (the "Software") , to deal in the Software without restriction, including without limitation the
38 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
39 // permit persons to whom the Software is furnished to do so, subject to the following conditions:
40 //
41 // The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
42 // Software.
43 //
44 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
45 // WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
46 // COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
47 // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
48 
49 // icon by Roundicons in miscellaneous at flaticon
50 
51 static jevois::ParameterCategory const ParamCateg("ObjectTracker Options");
52 
53 //! Parameter \relates ObjectTracker
54 JEVOIS_DECLARE_PARAMETER(hrange, jevois::Range<unsigned char>, "Range of H values for HSV window",
55  jevois::Range<unsigned char>(10, 245), ParamCateg);
56 
57 //! Parameter \relates ObjectTracker
58 JEVOIS_DECLARE_PARAMETER(srange, jevois::Range<unsigned char>, "Range of S values for HSV window",
59  jevois::Range<unsigned char>(10, 245), ParamCateg);
60 
61 //! Parameter \relates ObjectTracker
62 JEVOIS_DECLARE_PARAMETER(vrange, jevois::Range<unsigned char>, "Range of V values for HSV window",
63  jevois::Range<unsigned char>(10, 245), ParamCateg);
64 
65 //! Parameter \relates ObjectTracker
66 JEVOIS_DECLARE_PARAMETER(maxnumobj, size_t, "Max number of objects to declare a clean image",
67  10, ParamCateg);
68 
69 //! Parameter \relates ObjectTracker
70 JEVOIS_DECLARE_PARAMETER(objectarea, jevois::Range<unsigned int>, "Range of object area (in pixels) to track",
71  jevois::Range<unsigned int>(20*20, 100*100), ParamCateg);
72 
73 //! Parameter \relates ObjectTracker
74 JEVOIS_DECLARE_PARAMETER(erodesize, size_t, "Erosion structuring element size (pixels)",
75  3, ParamCateg);
76 
77 //! Parameter \relates ObjectTracker
78 JEVOIS_DECLARE_PARAMETER(dilatesize, size_t, "Dilation structuring element size (pixels)",
79  8, ParamCateg);
80 
81 //! Parameter \relates ObjectTracker
82 JEVOIS_DECLARE_PARAMETER(debug, bool, "Show contours of all object candidates if true",
83  true, ParamCateg);
84 
85 
86 //! Simple color-based object detection/tracking
87 /*! This modules isolates pixels within a given HSV range (hue, saturation, and value of color pixels), does some
88  cleanups, and extracts object contrours. It sends information about object centers over serial. This module usually
89  works best with the camera sensor set to manual exposure, gain, color balance, etc so that HSV color values are
90  reliable. See the cript.cfg file in this module's directory for an example of how to set the camera settigs each
91  time this module is loaded.
92 
93  This code was loosely inspired by:
94  https://raw.githubusercontent.com/kylehounslow/opencv-tuts/master/object-tracking-tut/objectTrackingTut.cpp written
95  by Kyle Hounslow, 2013.
96 
97  Serial Messages
98  ---------------
99 
100  This module can send standardized serial messages as described in \ref UserSerialStyle. One message is issued for on
101  every video frame for each detected and good object (good objects have a pixel area within the range specified by \p
102  objectarea, and are only reported when the image is clean enough according to \p maxnumobj). The \p id field in the
103  messages simply is \b blob for all messages.
104 
105  - Serial message type: \b 2D
106  - `id`: always \b blob
107  - `x`, `y`, or vertices: standardized 2D coordinates of blob center or of corners of bounding box
108  (depending on \p serstyle)
109  - `w`, `h`: standardized object size
110  - `extra`: none (empty string)
111 
112  Trying it out
113  -------------
114 
115  The default parameter settings (which are set in \b script.cfg explained below) attempt to detect light blue
116  objects. Present a light blue object to the JeVois camera and see whether it is detected. When detected and good
117  enough according to \p objectarea and \p maxnumobj, a green circle will be drawn at the center of each good object.
118 
119  Tuning
120  ------
121 
122  You should adjust parameters hrange, srange, and vrange to isolate the range of Hue, Saturation, and Value
123  (respectively) that correspond to the objects you want to detect. Note that there is a params.cfg file in this
124  module's directory that provides a range tuned to a lighgt blue object, as shown in the demo screenshot.
125 
126  Tuning the parameters is best done interactively by connecting to your JeVois camera while it is looking at some
127  object of the desired color. Once you have achieved a tuning, you may want to set the hrange, srange, and vrange
128  parameters in your \b script.cfg file for this module on the microSD card (see below).
129 
130  Typically, you would start by narrowing down on the hue, then the value, and finally the saturation. Make sure you
131  also move your camera around and show it typical background clutter so check for false positives (detections of
132  things which you are not interested, which can happen if your ranges are too wide).
133 
134  Config file
135  -----------
136 
137  JeVois allows you to store parameter settings and commands in a file named \b script.cfg stored in the directory of
138  a module. The file \b script.cfg may contain any sequence of commands as you would type them interactively in the
139  JeVois command-line interface. For the ObjectTracker module, a default script is provided that sets the camera to
140  manual color, gain, and exposure mode (for more reliable color values), and to setup communication with a pan/tilt
141  head as described in \ref ArduinoTutorial.
142 
143  The \b script.cfg file for ObjectTracker is stored on your microSD at
144  <b>JEVOIS:/modules/JeVois/ObjectTracker/script.cfg</b> and is shown in \ref ArduinoTutorial as an example.
145 
146 
147  @author Laurent Itti
148 
149  @videomapping YUYV 320 254 60.0 YUYV 320 240 60.0 JeVois ObjectTracker
150  @videomapping NONE 0 0 0.0 YUYV 320 240 60.0 JeVois ObjectTracker
151  @email itti\@usc.edu
152  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
153  @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
154  @mainurl http://jevois.org
155  @supporturl http://jevois.org/doc
156  @otherurl http://iLab.usc.edu
157  @license GPL v3
158  @distribution Unrestricted
159  @restrictions None
160  \ingroup modules */
162  public jevois::Parameter<hrange, srange, vrange, maxnumobj, objectarea, erodesize,
163  dilatesize, debug>
164 {
165  public:
166  //! Default base class constructor ok
168 
169  //! Virtual destructor for safe inheritance
170  virtual ~ObjectTracker() { }
171 
172  //! Processing function, no USB video output
173  virtual void process(jevois::InputFrame && inframe) override
174  {
175  // Wait for next available camera image. Any resolution and format ok:
176  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
177 
178  // Convert input image to BGR24, then to HSV:
179  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
180  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
181 
182  // Let camera know we are done processing the input image:
183  inframe.done();
184 
185  // Threshold the HSV image to only keep pixels within the desired HSV range:
186  cv::Mat imgth;
187  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
188  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
189 
190  // Apply morphological operations to cleanup the image noise:
191  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
192  cv::erode(imgth, imgth, erodeElement);
193 
194  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
195  cv::dilate(imgth, imgth, dilateElement);
196 
197  // Detect objects by finding contours:
198  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
199  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
200 
201  // Identify the "good" objects:
202  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
203  {
204  double refArea = 0.0; int x = 0, y = 0; int refIdx = 0;
205 
206  for (int index = 0; index >= 0; index = hierarchy[index][0])
207  {
208  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
209  double area = moment.m00;
210  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea) { refArea = area; refIdx = index; }
211  }
212 
213  // Send coords to serial port (for arduino, etc):
214  if (refArea > 0.0) sendSerialContour2D(w, h, contours[refIdx], "blob");
215  }
216  }
217 
218  //! Processing function, with USB video output
219  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
220  {
221  static jevois::Timer timer("processing");
222 
223  // Wait for next available camera image. Any resolution ok, but require YUYV since we assume it for drawings:
224  jevois::RawImage inimg = inframe.get(); unsigned int const w = inimg.width, h = inimg.height;
225  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV);
226 
227  timer.start();
228 
229  // While we process it, start a thread to wait for output frame and paste the input image into it:
230  jevois::RawImage outimg; // main thread should not use outimg until paste thread is complete
231  auto paste_fut = std::async(std::launch::async, [&]() {
232  outimg = outframe.get();
233  outimg.require("output", w, h + 14, inimg.fmt);
234  jevois::rawimage::paste(inimg, outimg, 0, 0);
235  jevois::rawimage::writeText(outimg, "JeVois Color Object Tracker", 3, 3, jevois::yuyv::White);
236  jevois::rawimage::drawFilledRect(outimg, 0, h, w, outimg.height-h, 0x8000);
237  });
238 
239  // Convert input image to BGR24, then to HSV:
240  cv::Mat imgbgr = jevois::rawimage::convertToCvBGR(inimg);
241  cv::Mat imghsv; cv::cvtColor(imgbgr, imghsv, cv::COLOR_BGR2HSV);
242 
243  // Threshold the HSV image to only keep pixels within the desired HSV range:
244  cv::Mat imgth;
245  cv::inRange(imghsv, cv::Scalar(hrange::get().min(), srange::get().min(), vrange::get().min()),
246  cv::Scalar(hrange::get().max(), srange::get().max(), vrange::get().max()), imgth);
247 
248  // Wait for paste to finish up:
249  paste_fut.get();
250 
251  // Let camera know we are done processing the input image:
252  inframe.done();
253 
254  // Apply morphological operations to cleanup the image noise:
255  cv::Mat erodeElement = getStructuringElement(cv::MORPH_RECT, cv::Size(erodesize::get(), erodesize::get()));
256  cv::erode(imgth, imgth, erodeElement);
257 
258  cv::Mat dilateElement = getStructuringElement(cv::MORPH_RECT, cv::Size(dilatesize::get(), dilatesize::get()));
259  cv::dilate(imgth, imgth, dilateElement);
260 
261  // Detect objects by finding contours:
262  std::vector<std::vector<cv::Point> > contours; std::vector<cv::Vec4i> hierarchy;
263  cv::findContours(imgth, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
264 
265  // If desired, draw all contours in a thread:
266  std::future<void> draw_fut;
267  if (debug::get())
268  draw_fut = std::async(std::launch::async, [&]() {
269  // We reinterpret the top portion of our YUYV output image as an opencv 8UC2 image:
270  cv::Mat outuc2(imgth.rows, imgth.cols, CV_8UC2, outimg.pixelsw<unsigned char>()); // pixel data shared
271  for (size_t i = 0; i < contours.size(); ++i)
272  cv::drawContours(outuc2, contours, i, jevois::yuyv::LightPink, 2, 8, hierarchy);
273  });
274 
275  // Identify the "good" objects:
276  int numobj = 0;
277  if (hierarchy.size() > 0 && hierarchy.size() <= maxnumobj::get())
278  {
279  double refArea = 0.0; int x = 0, y = 0; int refIdx = 0;
280 
281  for (int index = 0; index >= 0; index = hierarchy[index][0])
282  {
283  cv::Moments moment = cv::moments((cv::Mat)contours[index]);
284  double area = moment.m00;
285  if (objectarea::get().contains(int(area + 0.4999)) && area > refArea)
286  { x = moment.m10 / area + 0.4999; y = moment.m01 / area + 0.4999; refArea = area; refIdx = index; }
287  }
288 
289  if (refArea > 0.0)
290  {
291  ++numobj;
292  jevois::rawimage::drawCircle(outimg, x, y, 20, 1, jevois::yuyv::LightGreen);
293 
294  // Send coords to serial port (for arduino, etc):
295  sendSerialContour2D(w, h, contours[refIdx], "blob");
296  }
297  }
298 
299  // Show number of detected objects:
300  jevois::rawimage::writeText(outimg, "Detected " + std::to_string(numobj) + " objects.",
301  3, h + 2, jevois::yuyv::White);
302 
303  // Show processing fps:
304  std::string const & fpscpu = timer.stop();
305  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
306 
307  // Possibly wait until all contours are drawn, if they had been requested:
308  if (draw_fut.valid()) draw_fut.get();
309 
310  // Send the output image with our processing results to the host over USB:
311  outframe.send();
312  }
313 };
314 
315 // Allow the module to be loaded as a shared object (.so) file:
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function, with USB video output.
void sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
cv::Mat convertToCvBGR(RawImage const &src)
unsigned int height
JEVOIS_REGISTER_MODULE(ObjectTracker)
unsigned int fmt
virtual ~ObjectTracker()
Virtual destructor for safe inheritance.
StdModule(std::string const &instance)
virtual void process(jevois::InputFrame &&inframe) override
Processing function, no USB video output.
Simple color-based object detection/tracking.
double area(const std::vector< Point2D< T > > &polygon, const bool getsigned=false)
What is the area of a polygon?
Definition: Point2D.H:422
std::string const & stop()
JEVOIS_DECLARE_PARAMETER(camparams, std::string, "File stem of camera parameters, or empty. Camera resolution " "will be appended, as well as a .cfg extension. For example, specifying 'camera_para' " "here and running the camera sensor at 320x240 will attempt to load " "camera_para320x240.dat from within the module's directory.", "camera_para", ParamCateg)
Parameter.
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
std::string to_string(T const &val)
unsigned int width
void paste(RawImage const &src, RawImage &dest, int dx, int dy)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const