Detect multiple objects in scenes using the Darknet YOLO deep neural network. More...

Inheritance diagram for DarknetYOLO:

[legend]

Collaboration diagram for DarknetYOLO:

[legend]

Public Member Functions
	DarknetYOLO (std::string const &instance)
	Constructor. More...

virtual	~DarknetYOLO ()
	Virtual destructor for safe inheritance. More...

virtual void	postUninit () override
	Un-initialization. More...

virtual void	process (jevois::InputFrame &&inframe) override
	Processing function, no video output. More...

virtual void	process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
	Processing function with video output to USB. More...

Public Member Functions inherited from jevois::StdModule
	StdModule (std::string const &instance)

virtual	~StdModule ()

void	sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")

void	sendSerialObjReco (std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)

	JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle

	JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)

	JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime)) JEVOIS_DECLARE_PARAMETER(serstamp

	JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both)) JEVOIS_DECLARE_PARAMETER(sermark

Public Member Functions inherited from jevois::Module
	Module (std::string const &instance)

virtual	~Module ()

virtual void	process (InputFrame &&inframe, GUIhelper &helper)

virtual void	sendSerial (std::string const &str)

virtual void	parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)

virtual void	supportedCommands (std::ostream &os)

Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)

virtual	~Component ()

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

Engine *	engine () const

bool	initialized () const

const std::string &	className () const

const std::string &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor)

void	unFreezeParam (std::string const &paramdescriptor)

void	freezeAllParams ()

void	unFreezeAllParams ()

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

Engine *	engine () const

bool	initialized () const

const std::string &	className () const

const std::string &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor)

void	unFreezeParam (std::string const &paramdescriptor)

void	freezeAllParams ()

void	unFreezeAllParams ()

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Protected Attributes
std::shared_ptr< Yolo >	itsYolo

std::future< float >	itsPredictFut

cv::Mat	itsRawInputCv

cv::Mat	itsRawPrevOutputCv

cv::Mat	itsNetInput

Related Functions
(Note that these are not member functions.)
	JEVOIS_DECLARE_PARAMETER (netin, cv::Size, "Width and height (in pixels) of the neural network input layer, or [0 0] " "to make it match camera frame size. NOTE: for YOLO v3 sizes must be multiples of 32.", cv::Size(320, 224), ParamCateg)
	Parameter. More...

Additional Inherited Members
Protected Member Functions inherited from jevois::StdModule
void	sendSerialMarkStart ()

void	sendSerialMarkStop ()

std::string	getStamp () const

Protected Member Functions inherited from jevois::Component
virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)

void	removeParameter (ParameterBase *const param)

void	callbackInitCall ()

Detailed Description

Detect multiple objects in scenes using the Darknet YOLO deep neural network.

Darknet is a popular neural network framework, and YOLO is a very interesting network that detects all objects in a scene in one pass. This module detects all instances of any of the objects it knows about (determined by the network structure, labels, dataset used for training, and weights obtained) in the image that is given to it.

See https://pjreddie.com/darknet/yolo/

This module runs a YOLO network and shows all detections obtained. The YOLO network is currently quite slow, hence it is only run once in a while. Point your camera towards some interesting scene, keep it stable, and wait for YOLO to tell you what it found. The framerate figures shown at the bottom left of the display reflect the speed at which each new video frame from the camera is processed, but in this module this just amounts to converting the image to RGB, sending it to the neural network for processing in a separate thread, and creating the demo display. Actual network inference speed (time taken to compute the predictions on one image) is shown at the bottom right. See below for how to trade-off speed and accuracy.

Note that by default this module runs tiny-YOLO V3 which can detect and recognize 80 different kinds of objects from the Microsoft COCO dataset. This module can also run tiny-YOLO V2 for COCO, or tiny-YOLO V2 for the Pascal-VOC dataset with 20 object categories. See the module's params.cfg file to switch network.

The 80 COCO object categories are: person, bicycle, car, motorbike, aeroplane, bus, train, truck, boat, traffic, fire, stop, parking, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra, giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard, sports, kite, baseball, baseball, skateboard, surfboard, tennis, bottle, wine, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange, broccoli, carrot, hot, pizza, donut, cake, chair, sofa, pottedplant, bed, diningtable, toilet, tvmonitor, laptop, mouse, remote, keyboard, cell, microwave, oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy, hair, toothbrush.
The 20 Pascal-VOC object categories are: aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow, diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor.

Sometimes it will make mistakes! The performance of yolov3-tiny is about 33.1% correct (mean average precision) on the COCO test set.

Speed and network size

The parameter netin allows you to rescale the neural network to the specified size. Beware that this will only work if the network used is fully convolutional (as is the case of the default tiny-yolo network). This not only allows you to adjust processing speed (and, conversely, accuracy), but also to better match the network to the input images (e.g., the default size for tiny-yolo is 416x416, and, thus, passing it a input image of size 640x480 will result in first scaling that input to 416x312, then letterboxing it by adding gray borders on top and bottom so that the final input to the network is 416x416). This letterboxing can be completely avoided by just resizing the network to 320x240.

Here are expected processing speeds for yolov2-tiny-voc:

when netin = [0 0], processes letterboxed 416x416 inputs, about 2450ms/image
when netin = [320 240], processes 320x240 inputs, about 1350ms/image
when netin = [160 120], processes 160x120 inputs, about 695ms/image

YOLO V3 is faster, more accurate, uses less memory, and can detect 80 COCO categories:

when netin = [320 240], processes 320x240 inputs, about 870ms/image

Serial messages

When detections are found which are above threshold, one message will be sent for each detected object (i.e., for each box that gets drawn when USB outputs are used), using a standardized 2D message:

Serial message type: 2D
id: the category of the recognized object, followed by ':' and the confidence score in percent
x, y, or vertices: standardized 2D coordinates of object center or corners
w, h: standardized object size
extra: any number of additional category:score pairs which had an above-threshold score for that box

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Author: Laurent Itti

Display Name:: Darknet YOLO

Videomapping:: NONE 0 0 0.0 YUYV 640 480 0.4 JeVois DarknetYOLO

Videomapping:: YUYV 1280 480 15.0 YUYV 640 480 15.0 JeVois DarknetYOLO

Email:: itti@usc.edu

Address:: University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA

Copyright: Copyright (C) 2017 by Laurent Itti, iLab and the University of Southern California

Main URL:: http://jevois.org

Support URL:: http://jevois.org/doc

Other URL:: http://iLab.usc.edu

License:: GPL v3

Distribution:: Unrestricted

Restrictions:: None

Definition at line 119 of file DarknetYOLO.C.

Constructor & Destructor Documentation

◆ DarknetYOLO()

DarknetYOLO::DarknetYOLO ( std::string const & instance )

inline

Constructor.

Definition at line 126 of file DarknetYOLO.C.

References itsYolo.

◆ ~DarknetYOLO()

virtual DarknetYOLO::~DarknetYOLO ( )

inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 134 of file DarknetYOLO.C.

Member Function Documentation

◆ postUninit()

virtual void DarknetYOLO::postUninit ( )

inlineoverridevirtual

Un-initialization.

Reimplemented from jevois::Component.

Definition at line 140 of file DarknetYOLO.C.

References itsPredictFut.

◆ process() [1/2]

virtual void DarknetYOLO::process ( jevois::InputFrame && inframe )

inlineoverridevirtual

Processing function, no video output.

Reimplemented from jevois::Module.

Definition at line 148 of file DarknetYOLO.C.

References jevois::rawimage::convertToCvRGB(), h, jevois::RawImage::height, itsNetInput, itsYolo, LINFO, jevois::rescaleCv(), demo::w, and jevois::RawImage::width.

◆ process() [2/2]

virtual void DarknetYOLO::process	(	jevois::InputFrame &&	inframe,
		jevois::OutputFrame &&	outframe
	)

inlineoverridevirtual

Processing function with video output to USB.

Reimplemented from jevois::Module.

Definition at line 195 of file DarknetYOLO.C.

References jevois::async(), jevois::rawimage::convertToCvRGB(), jevois::rawimage::cvImage(), jevois::rawimage::drawFilledRect(), jevois::RawImage::fmt, h, jevois::RawImage::height, itsNetInput, itsPredictFut, itsRawInputCv, itsRawPrevOutputCv, itsYolo, jevois::rawimage::paste(), jevois::RawImage::require(), jevois::rescaleCv(), jevois::Timer::start(), jevois::Timer::stop(), success, to_string(), demo::w, jevois::RawImage::width, and jevois::rawimage::writeText().

Friends And Related Function Documentation

◆ JEVOIS_DECLARE_PARAMETER()

JEVOIS_DECLARE_PARAMETER	(	netin	,
		cv::Size	,
		"Width and height (in pixels) of the neural network input	layer,
		or " "to make it match camera frame size. NOTE:for YOLO v3 sizes must be multiples of 32."	[0 0],
		cv::Size(320, 224)	,
		ParamCateg
	)

Member Data Documentation

◆ itsNetInput

cv::Mat DarknetYOLO::itsNetInput

protected

Definition at line 337 of file DarknetYOLO.C.

Referenced by process().

◆ itsPredictFut

std::future<float> DarknetYOLO::itsPredictFut

protected

Definition at line 334 of file DarknetYOLO.C.

Referenced by postUninit(), and process().

◆ itsRawInputCv

cv::Mat DarknetYOLO::itsRawInputCv

protected

Definition at line 335 of file DarknetYOLO.C.

Referenced by process().

◆ itsRawPrevOutputCv

cv::Mat DarknetYOLO::itsRawPrevOutputCv

protected

Definition at line 336 of file DarknetYOLO.C.

Referenced by process().

◆ itsYolo

std::shared_ptr<Yolo> DarknetYOLO::itsYolo

protected

Definition at line 333 of file DarknetYOLO.C.

Referenced by DarknetYOLO(), and process().

The documentation for this class was generated from the following file:

src/Modules/DarknetYOLO/DarknetYOLO.C

Public Member Functions

Protected Attributes

Related Functions

Additional Inherited Members

Detailed Description

Speed and network size

Serial messages

Constructor & Destructor Documentation

◆ DarknetYOLO()

◆ ~DarknetYOLO()

Member Function Documentation

◆ postUninit()

◆ process() [1/2]

◆ process() [2/2]

Friends And Related Function Documentation

◆ JEVOIS_DECLARE_PARAMETER()

Member Data Documentation

◆ itsNetInput

◆ itsPredictFut

◆ itsRawInputCv

◆ itsRawPrevOutputCv

◆ itsYolo