JeVoisBase  1.20
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DarknetSaliency Class Reference

Detect salient objects and identify them using Darknet deep neural network. More...

Inheritance diagram for DarknetSaliency:
Collaboration diagram for DarknetSaliency:

Public Member Functions

 DarknetSaliency (std::string const &instance)
 Constructor. More...
 
virtual ~DarknetSaliency ()
 Virtual destructor for safe inheritance. More...
 
virtual void postUninit () override
 Un-initialization. More...
 
virtual void getSalROI (jevois::RawImage const &inimg, int &rx, int &ry, int &rw, int &rh)
 Helper function: compute saliency ROI in a thread, return top-left corner and size. More...
 
virtual void process (jevois::InputFrame &&inframe) override
 Processing function, no video output. More...
 
virtual void process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
 Processing function with video output to USB. More...
 
- Public Member Functions inherited from jevois::StdModule
 StdModule (std::string const &instance)
 
virtual ~StdModule ()
 
void sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")
 
void sendSerialObjReco (std::vector< ObjReco > const &res)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle
 
 JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime)) JEVOIS_DECLARE_PARAMETER(serstamp
 
 JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both)) JEVOIS_DECLARE_PARAMETER(sermark
 
- Public Member Functions inherited from jevois::Module
 Module (std::string const &instance)
 
virtual ~Module ()
 
virtual void process (InputFrame &&inframe, GUIhelper &helper)
 
virtual void sendSerial (std::string const &str)
 
virtual void parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)
 
virtual void supportedCommands (std::ostream &os)
 
- Public Member Functions inherited from jevois::Component
 Component (std::string const &instance)
 
virtual ~Component ()
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
Engineengine () const
 
bool initialized () const
 
const std::string & className () const
 
const std::string & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)
 
void setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)
 
void removeDynamicParameter (std::string const &name)
 
void setPath (std::string const &path)
 
std::filesystem::path absolutePath (std::filesystem::path const &path="")
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
Engineengine () const
 
bool initialized () const
 
const std::string & className () const
 
const std::string & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)
 
void setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)
 
void removeDynamicParameter (std::string const &name)
 
void setPath (std::string const &path)
 
std::filesystem::path absolutePath (std::filesystem::path const &path="")
 
- Public Member Functions inherited from jevois::ParameterRegistry
virtual ~ParameterRegistry ()
 

Protected Attributes

std::shared_ptr< SaliencyitsSaliency
 
std::shared_ptr< DarknetitsDarknet
 
std::vector< jevois::ObjRecoitsResults
 
std::future< float > itsPredictFut
 
cv::Mat itsRawInputCv
 
cv::Mat itsCvImg
 
cv::Mat itsRawPrevOutputCv
 

Related Functions

(Note that these are not member functions.)

 JEVOIS_DECLARE_PARAMETER (foa, cv::Size, "Width and height (in pixels) of the focus of attention. " "This is the size of the image crop that is taken around the most salient " "location in each frame. The foa size must fit within the camera input frame size.", cv::Size(128, 128), ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (netin, cv::Size, "Width and height (in pixels) of the neural network input " "layer. This is the size to which the image crop taken around the most salient " "location in each frame will be rescaled before feeding to the neural network.", cv::Size(128, 128), ParamCateg)
 Parameter. More...
 

Additional Inherited Members

- Protected Member Functions inherited from jevois::StdModule
void sendSerialMarkStart ()
 
void sendSerialMarkStop ()
 
std::string getStamp () const
 
- Protected Member Functions inherited from jevois::Component
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
- Protected Member Functions inherited from jevois::ParameterRegistry
void addParameter (ParameterBase *const param)
 
void removeParameter (ParameterBase *const param)
 
void callbackInitCall ()
 

Detailed Description

Detect salient objects and identify them using Darknet deep neural network.

Darknet is a popular neural network framework. This module first finds the most conspicuous (salient) object in the scene, then identifies it using a deep neural network. It returns the top scoring candidates.

See http://ilab.usc.edu/bu/ for more information about saliency detection, and https://pjreddie.com/darknet for more information about the Darknet deep neural network framework.

This module runs a Darknet network on an image window around the most salient point and shows the top-scoring results. The network is currently a bit slow, hence it is only run once in a while. Point your camera towards some interesting object, and wait for Darknet to tell you what it found. The framerate figures shown at the bottom left of the display reflect the speed at which each new video frame from the camera is processed, but in this module this just amounts to computing the saliency map from the camera input, converting the input image to RGB, cropping it around the most salient location, sending it to the neural network for processing in a separate thread, and creating the demo display. Actual network inference speed (time taken to compute the predictions on one image crop) is shown at the bottom right. See below for how to trade-off speed and accuracy.

Note that by default this module runs the Imagenet1k tiny Darknet (it can also run the slightly slower but a bit more accurate Darknet Reference network; see parameters). There are 1000 different kinds of objects (object classes) that this network can recognize (too long to list here).

Sometimes it will make mistakes! The performance of darknet-tiny is about 58.7% correct (mean average precision) on the test set, and Darknet Reference is about 61.1% correct on the test set. This is when running these networks at 224x224 network input resolution (see parameter netin below).

Neural network size and speed

When using networks that are fully convolutional (as is the case for the default networks provided with this module), one can resize the network to any desired input size. The network size direcly affects both speed and accuracy. Larger networks run slower but are more accurate.

This module provides two parameters that allow you to adjust this tradeoff:

  • foa determines the size of a region of interest that is cropped around the most salient location
  • netin determines the size to which that region of interest is rescaled and fed to the neural network

For example:

  • with netin = (224 224), this module runs at about 450ms/prediction.
  • with netin = (128 128), this module runs at about 180ms/prediction.

Finally note that, when using video mappings with USB output, irrespective of foa and netin, the crop around the most salient image region (with size given by foa) will always also be rescaled so that, when placed to the right of the input image, it fills the desired USB output dims. For example, if camera mode is 320x240 and USB output size is 544x240, then the attended and recognized object will be rescaled to 224x224 (since 224 = 544-320) for display purposes only. This is so that one does not need to change USB video resolution while playing with different values of foa and netin live.

Serial messages

On every frame where detection results were obtained that are above thresh, this module sends a standardized 2D message as specified in UserSerialStyle:

  • Serial message type: 2D
  • id: top-scoring category name of the recognized object, followed by ':' and the confidence score in percent
  • x, y, or vertices: standardized 2D coordinates of object center or corners
  • w, h: standardized object size
  • extra: any number of additional category:score pairs which had an above-threshold score, in order of decreasing score where category is the category name (from namefile) and score is the confidence score from 0.0 to 100.0

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Author
Laurent Itti
Display Name:
Darknet Saliency
Videomapping:
NONE 0 0 0.0 YUYV 320 240 5.0 JeVois DarknetSaliency
Videomapping:
YUYV 460 240 15.0 YUYV 320 240 15.0 JeVois DarknetSaliency # not for mac (width not multiple of 16)
Videomapping:
YUYV 560 240 15.0 YUYV 320 240 15.0 JeVois DarknetSaliency
Videomapping:
YUYV 880 480 15.0 YUYV 640 480 15.0 JeVois DarknetSaliency # set foa param to 256 256
Email:
itti@usc.edu
Address:
University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
Main URL:
http://jevois.org
Support URL:
http://jevois.org/doc
Other URL:
http://iLab.usc.edu
License:
GPL v3
Distribution:
Unrestricted
Restrictions:
None

Definition at line 126 of file DarknetSaliency.C.

Constructor & Destructor Documentation

◆ DarknetSaliency()

DarknetSaliency::DarknetSaliency ( std::string const &  instance)
inline

Constructor.

Definition at line 133 of file DarknetSaliency.C.

References itsDarknet, and itsSaliency.

◆ ~DarknetSaliency()

virtual DarknetSaliency::~DarknetSaliency ( )
inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 142 of file DarknetSaliency.C.

Member Function Documentation

◆ getSalROI()

virtual void DarknetSaliency::getSalROI ( jevois::RawImage const &  inimg,
int &  rx,
int &  ry,
int &  rw,
int &  rh 
)
inlinevirtual

Helper function: compute saliency ROI in a thread, return top-left corner and size.

Definition at line 156 of file DarknetSaliency.C.

References h, jevois::RawImage::height, demo::int, itsSaliency, demo::w, and jevois::RawImage::width.

Referenced by process().

◆ postUninit()

virtual void DarknetSaliency::postUninit ( )
inlineoverridevirtual

Un-initialization.

Reimplemented from jevois::Component.

Definition at line 148 of file DarknetSaliency.C.

References itsPredictFut.

◆ process() [1/2]

virtual void DarknetSaliency::process ( jevois::InputFrame &&  inframe)
inlineoverridevirtual

◆ process() [2/2]

Friends And Related Function Documentation

◆ JEVOIS_DECLARE_PARAMETER() [1/2]

JEVOIS_DECLARE_PARAMETER ( foa  ,
cv::Size  ,
"Width and height (in pixels) of the focus of attention. " "This is the size of the image crop that is taken around the most salient " "location in each frame. The foa size must fit within the camera input frame size."  ,
cv::Size(128, 128)  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [2/2]

JEVOIS_DECLARE_PARAMETER ( netin  ,
cv::Size  ,
"Width and height (in pixels) of the neural network input " "layer. This is the size to which the image crop taken around the most salient " "location in each frame will be rescaled before feeding to the neural network."  ,
cv::Size(128, 128)  ,
ParamCateg   
)
related

Parameter.

Member Data Documentation

◆ itsCvImg

cv::Mat DarknetSaliency::itsCvImg
protected

Definition at line 382 of file DarknetSaliency.C.

Referenced by process().

◆ itsDarknet

std::shared_ptr<Darknet> DarknetSaliency::itsDarknet
protected

Definition at line 378 of file DarknetSaliency.C.

Referenced by DarknetSaliency(), and process().

◆ itsPredictFut

std::future<float> DarknetSaliency::itsPredictFut
protected

Definition at line 380 of file DarknetSaliency.C.

Referenced by postUninit(), and process().

◆ itsRawInputCv

cv::Mat DarknetSaliency::itsRawInputCv
protected

Definition at line 381 of file DarknetSaliency.C.

Referenced by process().

◆ itsRawPrevOutputCv

cv::Mat DarknetSaliency::itsRawPrevOutputCv
protected

Definition at line 383 of file DarknetSaliency.C.

Referenced by process().

◆ itsResults

std::vector<jevois::ObjReco> DarknetSaliency::itsResults
protected

Definition at line 379 of file DarknetSaliency.C.

Referenced by process().

◆ itsSaliency

std::shared_ptr<Saliency> DarknetSaliency::itsSaliency
protected

Definition at line 377 of file DarknetSaliency.C.

Referenced by DarknetSaliency(), and getSalROI().


The documentation for this class was generated from the following file: