ObjectDetect Class Reference

Simple object detection using keypoint matching. More...

Inheritance diagram for ObjectDetect:

Collaboration diagram for ObjectDetect:

Public Member Functions
	ObjectDetect (std::string const &instance)
	Constructor.
virtual	~ObjectDetect ()
	Virtual destructor for safe inheritance.
void	onParamChange (win const &JEVOIS_UNUSED_PARAM(param), floatpair const &newval) override
	Parameter callback.
virtual void	process (jevois::InputFrame &&inframe) override
	Processing function with no USB output.
virtual void	process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
	Processing function with USB output.
void	parseSerial (std::string const &str, std::shared_ptr< jevois::UserInterface > s) override
	Receive a string from a serial port which contains a user command.
void	supportedCommands (std::ostream &os) override
	Human-readable description of this Module's supported custom commands.
Public Member Functions inherited from jevois::StdModule
	StdModule (std::string const &instance)
virtual	~StdModule ()
void	sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
void	sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")
void	sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")
void	sendSerialObjReco (std::vector< ObjReco > const &res)
void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)
void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetectOBB const &det)
	JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine))
	JEVOIS_DECLARE_PARAMETER (serstyle, SerStyle, "Style for standardized serial messages as defined in " "http://jevois.org/doc/UserSerialStyle.html", SerStyle::Terse, SerStyle_Values, ParamCateg)
	JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)
	JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime))
	JEVOIS_DECLARE_PARAMETER (serstamp, SerStamp, "Prepend standardized serial messages with a frame number, " "time, frame+time, or frame+date+time. See details in " "http://jevois.org/doc/UserSerialStyle.html", SerStamp::None, SerStamp_Values, ParamCateg)
	JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both))
	JEVOIS_DECLARE_PARAMETER (sermark, SerMark, "Send serial message to mark the beginning (MARK START) of the " "processing of a video frame from the camera sensor, the end (MARK STOP), or both. " "Useful, among others, if one needs to know when no results were sent over serial " "on a given frame. Combine with parameter serstamp if you need to know the frame number.", SerMark::None, SerMark_Values, ParamCateg)
Public Member Functions inherited from jevois::Module
	Module (std::string const &instance)
virtual	~Module ()
virtual void	process (InputFrame &&inframe, GUIhelper &helper)
virtual void	sendSerial (std::string const &str)
Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)
virtual	~Component ()
std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)
void	removeSubComponent (std::shared_ptr< Comp > &component)
void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const
bool	isTopLevel () const
bool	initialized () const
std::string const &	className () const
std::string const &	instanceName () const
std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)
void	setParamValUnique (std::string const &paramdescriptor, T const &val)
std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const
T	getParamValUnique (std::string const &paramdescriptor) const
std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)
void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const
std::string	getParamStringUnique (std::string const &paramdescriptor) const
void	freezeParam (std::string const &paramdescriptor, bool doit)
void	freezeAllParams (bool doit)
std::string	descriptor () const
void	setParamsFromFile (std::string const &filename)
std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)
virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)
std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)
void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)
void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)
void	setPath (std::string const &path)
std::filesystem::path	absolutePath (std::filesystem::path const &path="")
std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)
void	removeSubComponent (std::shared_ptr< Comp > &component)
void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const
bool	isTopLevel () const
bool	initialized () const
std::string const &	className () const
std::string const &	instanceName () const
std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)
void	setParamValUnique (std::string const &paramdescriptor, T const &val)
std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const
T	getParamValUnique (std::string const &paramdescriptor) const
std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)
void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const
std::string	getParamStringUnique (std::string const &paramdescriptor) const
void	freezeParam (std::string const &paramdescriptor, bool doit)
void	freezeAllParams (bool doit)
std::string	descriptor () const
void	setParamsFromFile (std::string const &filename)
std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)
virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)
std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)
void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)
void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)
void	setPath (std::string const &path)
std::filesystem::path	absolutePath (std::filesystem::path const &path="")
Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Related Symbols
(Note that these are not member symbols.)
	JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK (win, floatpair, "Width and height (in percent of image size, with valid percentages between " "10.0 and 100.0) of the window used to interactively save objects", floatpair(50.0F, 50.0F), ParamCateg)
	Parameter.
	JEVOIS_DECLARE_PARAMETER (showwin, bool, "Show the interactive image capture window when true", false, ParamCateg)
	Parameter.

Additional Inherited Members
Protected Member Functions inherited from jevois::StdModule
void	sendSerialMarkStart ()
void	sendSerialMarkStop ()
std::string	getStamp () const
Protected Member Functions inherited from jevois::Component
virtual void	preInit ()
virtual void	postInit ()
virtual void	preUninit ()
virtual void	postUninit ()
virtual void	preInit ()
virtual void	postInit ()
virtual void	preUninit ()
virtual void	postUninit ()
Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)
void	removeParameter (ParameterBase *const param)
void	callbackInitCall ()

Detailed Description

Simple object detection using keypoint matching.

This module finds objects by matching keypoint descriptors between the current image and a set of training images. Here we use SURF keypoints and descriptors as provided by OpenCV.

The algorithm consists of 4 phases:

• detect keypoint locations, typically corners or other distinctive texture elements or markings;
• compute keypoint descriptors, which are summary representations of the image neighborhood around each keypoint;
• match descriptors from current image to descriptors previously extracted from training images;
• if enough matches are found between the current image and a given training image, and they are of good enough quality, compute the homography (geometric transformation) between keypoint locations in that training image and locations of the matching keypoints in the current image. If it is well conditioned (i.e., a 3D viewpoint change could well explain how the keypoints moved between the training and current images), declare that a match was found, and draw a green rectangle around the detected object.

The algorithm comes by default with one training image, for the Priority Mail logo of the U.S. Postal Service. Search for "USPS priority mail" on the web and point JeVois to a picture of the logo on your screen to recognize it. See the screenshots of this module for examples of how that logo looks.

Offline training

Simply add images of the objects you want to detect in JEVOIS:/modules/JeVois/ObjectDetect/images/ on your JeVois microSD card. Those will be processed when the module starts. The names of recognized objects returned by this module are simply the file names of the pictures you have added in that directory. No additional training procedure is needed. Beware that the more images you add, the slower the algorithm will run, and the higher your chances of confusions among several of your objects.

With JeVois v1.1 or later, you do not need to eject the microSD from JeVois, and you can instead add images live by exporting the microSD inside JeVois using the usbsd command. See MicroSD card organization and files (last section) for details. When you are done adding new images or deleting unwanted ones, properly eject the virtual USB flash drive, and JeVois will restart and load the new training data.

Live training

With JeVois v1.2 and later you can train this algorithm live by telling JeVois to capture and save an image of an object, which can be used later to identify this object again.

First, enable display of a training window using:

setpar showwin true

You should now see a gray rectangle. You can adjust the window size and aspect ratio using the win parameter. By default, the algorithm will train new objects that occupy half the width and height of the camera image.

Point your JeVois camera to a clean view of an object you want to learn (if possible, with a blank, featureless background, as this algorithm does not attempt to segment objects and would otherwise also learn features of the background as part of the object). Make sure the objects fits inside the gray rectangle and fills as much of it as possible. You should adjust the distance between the object and the camera, and the grey rectangle, to roughly match the distance at which you want to detect that object in the future. Then issue the command:

save somename

over a serial connection to JeVois, where somename is the name you want to give to this object. This will grab the current camera image, crop it using the gray rectangle, and save the crop as a new training image somename.png for immediate use. The algorithm will immediately re-train on all objects, including the new one. You should see the object being detected shortly after you send your save command. Note that we save the image as grayscale since this algorithm does not use color anyway.

You can see the list of current images by using command:

list

Finally, you can delete an image using command:

del somename

where somename is the object name without extension, and a .png extension will be added. The image will immediately be deleted and that object will not be recognized anymore.

For more information, see JeVois tutorial Live training of the Object Detection module and the associated video:

Serial Messages

This module can send standardized serial messages as described in Standardized serial messages formatting. One message is issued on every video frame for the best detected object (highest score).

• Serial message type: 2D
• id: filename of the recognized object
• x, y: standardized 2D coordinates of the object center
• w, h, or vertices: Standardized bounding box around the object
• extra: none (empty string)

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Programmer notes

This algorithm is quite slow. So, here, we alternate between computing keypoints and descriptors on one frame (or more, depending on how slow that gets), and doing the matching on the next frame. This module also provides an example of letting some computation happen even after we exit the process() function. Here, we keep detecting keypoints and computing descriptors even outside process(). The itsKPfut future is our handle to that thread, and we also use it to alternate between detection and matching on alternating frames.

Author

Laurent Itti

Videomapping:

YUYV 320 252 30.0 YUYV 320 240 30.0 JeVois ObjectDetect

Custom module command:

list - show current list of training images

Custom module command:

save somename - grab current frame and save as new training image somename.png

Custom module command:

del somename - delete training image somename.png

Email:

itti@usc.edu

Address:

University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA

Copyright

Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California

Main URL:

http://jevois.org

Support URL:

http://jevois.org/doc

Other URL:

http://iLab.usc.edu

License:

GPL v3

Distribution:

Unrestricted

Restrictions:

None

Definition at line 169 of file ObjectDetect.C.

Constructor & Destructor Documentation

ObjectDetect()

ObjectDetect::ObjectDetect ( std::string const &  instance )

inline

Constructor.

Definition at line 176 of file ObjectDetect.C.

~ObjectDetect()

virtual ObjectDetect::~ObjectDetect ( )

inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 182 of file ObjectDetect.C.

Member Function Documentation

onParamChange()

void ObjectDetect::onParamChange ( win const &  JEVOIS_UNUSED_PARAMparam, floatpair const &  newval  )

inlineoverride

Parameter callback.

Definition at line 187 of file ObjectDetect.C.

References JEVOIS_UNUSED_PARAM.

parseSerial()

void ObjectDetect::parseSerial ( std::string const &  str, std::shared_ptr< jevois::UserInterface >  s  )

inlineoverridevirtual

Receive a string from a serial port which contains a user command.

Reimplemented from jevois::Module.

Definition at line 318 of file ObjectDetect.C.

References jevois::Component::absolutePath(), jevois::Component::removeSubComponent(), jevois::split(), and jevois::system().

process() [1/2]

virtual void ObjectDetect::process ( jevois::InputFrame &&  inframe )

inlineoverridevirtual

Processing function with no USB output.

Reimplemented from jevois::Module.

Definition at line 197 of file ObjectDetect.C.

References jevois::StdModule::sendSerialContour2D(), jevois::Timer::start(), and jevois::Timer::stop().

process() [2/2]

virtual void ObjectDetect::process ( jevois::InputFrame &&  inframe, jevois::OutputFrame &&  outframe  )

inlineoverridevirtual

Processing function with USB output.

Reimplemented from jevois::Module.

Definition at line 220 of file ObjectDetect.C.

References jevois::async(), jevois::rawimage::convertToCvGray(), jevois::rawimage::drawFilledRect(), jevois::rawimage::drawLine(), jevois::rawimage::drawRect(), jevois::RawImage::fmt, h, jevois::RawImage::height, jevois::yuyv::LightGreen, jevois::yuyv::MedGrey, jevois::rawimage::paste(), jevois::RawImage::require(), jevois::StdModule::sendSerialContour2D(), jevois::Timer::start(), jevois::Timer::stop(), jevois::yuyv::White, jevois::RawImage::width, and jevois::rawimage::writeText().

supportedCommands()

void ObjectDetect::supportedCommands ( std::ostream &  os )

inlineoverridevirtual

Human-readable description of this Module's supported custom commands.

Reimplemented from jevois::Module.

Definition at line 374 of file ObjectDetect.C.

Friends And Related Symbol Documentation

JEVOIS_DECLARE_PARAMETER()

JEVOIS_DECLARE_PARAMETER ( showwin  , bool  , "Show the interactive image capture window when true"  , false  , ParamCateg    )

related

Parameter.

JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK()

JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK ( win  , floatpair  , "Width and height (in percent of image size, with valid percentages between " "10.0 and 100.0) of the window used to interactively save objects"  , floatpair(50.0F, 50.0F)  , ParamCateg    )

related

Parameter.

---

The documentation for this class was generated from the following file:

• src/Modules/ObjectDetect/ObjectDetect.C