JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DarknetSingle Class Reference

Identify objects using Darknet deep neural network. More...

Inheritance diagram for DarknetSingle:
Collaboration diagram for DarknetSingle:

Public Member Functions

 DarknetSingle (std::string const &instance)
 Constructor. More...
 
virtual ~DarknetSingle ()
 Virtual destructor for safe inheritance. More...
 
virtual void postUninit () override
 Un-initialization. More...
 
void sendAllSerial ()
 Send serial messages. More...
 
virtual void process (jevois::InputFrame &&inframe) override
 Processing function, no video output. More...
 
virtual void process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
 Processing function with video output to USB. More...
 
- Public Member Functions inherited from jevois::Module
 Module (std::string const &instance)
 
virtual ~Module ()
 
virtual void sendSerial (std::string const &str)
 
virtual void parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)
 
virtual void supportedCommands (std::ostream &os)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle
 
 JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, ParamCateg)
 
- Public Member Functions inherited from jevois::Component
 Component (std::string const &instance)
 
virtual ~Component ()
 
std::shared_ptr< ComponentgetSubComponent (std::string const &instance) const
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
- Public Member Functions inherited from jevois::ParameterRegistry
virtual ~ParameterRegistry ()
 

Protected Attributes

std::shared_ptr< DarknetitsDarknet
 
std::vector< Darknet::predresultitsResults
 
std::future< float > itsPredictFut
 
cv::Mat itsRawInputCv
 
cv::Mat itsCvImg
 
cv::Mat itsRawPrevOutputCv
 
unsigned long itsFrame
 

Additional Inherited Members

- Protected Member Functions inherited from jevois::Component
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
- Protected Member Functions inherited from jevois::ParameterRegistry
void addParameter (ParameterBase *const param)
 
void removeParameter (ParameterBase *const param)
 
void callbackInitCall ()
 

Detailed Description

Identify objects using Darknet deep neural network.

Darknet is a popular neural network framework. This module identifies the object in a square region in the center of the camera field of view using a deep convolutional neural network. The deep network analyzes the image by filtering it using many different filter kernels, and several passes (network layers). This essentially amounts to detecting the presence of both simple and complex parts of known objects in the image (e.g., detecting edges in lower layers to detecting car wheels or even whole cars in higher layers). The last layer of the network is reduced to vector with one entry per known kind of object (object class). This module returns the class names of the top scoring candidates in the outut vector which (if any) have scored above a minimum confidence threshold.

Darknet is a great alternative to popular neural network frameworks like Caffe, TensorFlow, MxNet, pyTorch, Theano, etc as it features: 1) small footprint which is great for small embedded systems; 2) hardware acceleration using ARM NEON instructions; 3) support for large GPUs when compiled on expensive servers, which is useful to train the neural networks on big servers, then copying the trained weights directly to JeVois for use with live video.

See https://pjreddie.com/darknet for more details about darknet.

This module runs a Darknet network and shows the top-scoring results. The network is currently a bit slow, hence it is only run once in a while. Point your camera towards some interesting object, make the object fit in the picture shown at right (which will be fed to the neural network), keep it stable, and wait for Darknet to tell you what it found.

Note that by default this module runs the Imagenet1k tiny Darknet (it can also run the slightly slower but a bit more accurate Darknet Reference network; see parameters). There are 1000 different kinds of objects (object classes) that this network can recognize (too long to list here). The input layer of these two networks is 224x224 pixels. This modules takes a crop at the center of the video image, with size determined by the network input size. With the defult network parameters, this module hence requires at least 320x240 camera sensor resolution. The networks provided on the JeVois microSD image have been trained on large clusters of GPUs, typically using 1.2 million training images from the ImageNet dataset.

Sometimes this module will make mistakes! The performance of darknet-tiny is about 58.7% correct (mean average precision) on the test set, and Darknet Reference is about 61.1% correct on the test set, using the default 224x224 network input layer size.

Neural network size and speed

When using a video mapping with USB output, the network is automatically resized to a square size that is the difference between the USB output video width and the camera input width (e.g., when USB video mode is 544x240 and camera sensor mode is 320x240, the network will be resized to 224x224 where 224 is 544-320).

The network size direcly affects both speed and accuracy. Larger networks run slower but are more accurate.

For example:

  • with USB output 544x240 (network size 224x224), this module runs at about 450ms/prediction.
  • with USB output 448x240 (network size 128x128), this module runs at about 180ms/prediction.

When using a videomapping with no USB output, the network is not resized (since we would not know what to resize it to). You can still change its native size by changing the network's config file, for example, change the width and height fields in JEVOIS:/share/darknet/single/cfg/tiny.cfg.

Note that network dims must always be such that they fit inside the camera input image.

Serial messages

  • On every frame where detection results were obtained, this module sends a message
      DKF framenum
    where framenum is the frame number (starts at 0).
  • In addition, when detections are found, up to top messages will be sent, for those category candidates that have scored above thresh:
      DKR category score
    where category is the category name (from namefile) and score is the confidence score from 0.0 to 100.0
Author
Laurent Itti
Display Name:
Darknet Single
Videomapping:
NONE 0 0 0.0 YUYV 320 240 2.1 JeVois DarknetSingle
Videomapping:
YUYV 544 240 15.0 YUYV 320 240 15.0 JeVois DarknetSingle
Videomapping:
YUYV 448 240 15.0 YUYV 320 240 15.0 JeVois DarknetSingle
Email:
itti@usc.edu
Address:
University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
Main URL:
http://jevois.org
Support URL:
http://jevois.org/doc
Other URL:
http://iLab.usc.edu
License:
GPL v3
Distribution:
Unrestricted
Restrictions:
None

Definition at line 114 of file DarknetSingle.C.

Constructor & Destructor Documentation

◆ DarknetSingle()

DarknetSingle::DarknetSingle ( std::string const &  instance)
inline

Constructor.

Definition at line 120 of file DarknetSingle.C.

References itsDarknet.

◆ ~DarknetSingle()

virtual DarknetSingle::~DarknetSingle ( )
inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 128 of file DarknetSingle.C.

Member Function Documentation

◆ postUninit()

virtual void DarknetSingle::postUninit ( )
inlineoverridevirtual

Un-initialization.

Reimplemented from jevois::Component.

Definition at line 134 of file DarknetSingle.C.

References itsPredictFut.

◆ process() [1/2]

virtual void DarknetSingle::process ( jevois::InputFrame &&  inframe)
inlineoverridevirtual

Processing function, no video output.

Reimplemented from jevois::Module.

Definition at line 151 of file DarknetSingle.C.

References jevois::rawimage::cvImage(), jevois::RawImage::height, itsCvImg, itsDarknet, itsFrame, itsResults, LINFO, sendAllSerial(), and jevois::RawImage::width.

◆ process() [2/2]

◆ sendAllSerial()

void DarknetSingle::sendAllSerial ( )
inline

Send serial messages.

Definition at line 142 of file DarknetSingle.C.

References itsFrame, itsResults, jevois::Module::sendSerial(), jevois::sformat(), and to_string().

Referenced by process().

Member Data Documentation

◆ itsCvImg

cv::Mat DarknetSingle::itsCvImg
protected

Definition at line 338 of file DarknetSingle.C.

Referenced by process().

◆ itsDarknet

std::shared_ptr<Darknet> DarknetSingle::itsDarknet
protected

Definition at line 334 of file DarknetSingle.C.

Referenced by DarknetSingle(), and process().

◆ itsFrame

unsigned long DarknetSingle::itsFrame
protected

Definition at line 340 of file DarknetSingle.C.

Referenced by process(), and sendAllSerial().

◆ itsPredictFut

std::future<float> DarknetSingle::itsPredictFut
protected

Definition at line 336 of file DarknetSingle.C.

Referenced by postUninit(), and process().

◆ itsRawInputCv

cv::Mat DarknetSingle::itsRawInputCv
protected

Definition at line 337 of file DarknetSingle.C.

Referenced by process().

◆ itsRawPrevOutputCv

cv::Mat DarknetSingle::itsRawPrevOutputCv
protected

Definition at line 339 of file DarknetSingle.C.

Referenced by process().

◆ itsResults

std::vector<Darknet::predresult> DarknetSingle::itsResults
protected

Definition at line 335 of file DarknetSingle.C.

Referenced by process(), and sendAllSerial().


The documentation for this class was generated from the following file: