JeVoisBase  1.16
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DetectionDNN Class Reference

Detect and recognize multiple objects in scenes using OpenCV Deep Neural Nets (DNN) More...

Inheritance diagram for DetectionDNN:
Collaboration diagram for DetectionDNN:

Public Member Functions

 DetectionDNN (std::string const &instance)
 Constructor. More...
 
virtual ~DetectionDNN ()
 Virtual destructor for safe inheritance. More...
 
void onParamChange (model const &param, Model const &val) override
 Parameter callback: set the selected model. More...
 
void load ()
 Load and initialize a model. More...
 
void postprocess (cv::Mat const &frame, std::vector< cv::Mat > const &outs, jevois::RawImage *outframe=nullptr)
 Post-processing to extract boxes from network outputs. More...
 
virtual void process (jevois::InputFrame &&inframe) override
 Processing function, no video output. More...
 
virtual void process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
 Processing function with video output to USB. More...
 
- Public Member Functions inherited from jevois::StdModule
 StdModule (std::string const &instance)
 
virtual ~StdModule ()
 
void sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")
 
void sendSerialObjReco (std::vector< ObjReco > const &res)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle
 
 JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime)) JEVOIS_DECLARE_PARAMETER(serstamp
 
 JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both)) JEVOIS_DECLARE_PARAMETER(sermark
 
- Public Member Functions inherited from jevois::Module
 Module (std::string const &instance)
 
virtual ~Module ()
 
virtual void process (InputFrame &&inframe, GUIhelper &helper)
 
virtual void sendSerial (std::string const &str)
 
virtual void parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)
 
virtual void supportedCommands (std::ostream &os)
 
size_t frameNum () const
 
- Public Member Functions inherited from jevois::Component
 Component (std::string const &instance)
 
virtual ~Component ()
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
Engineengine () const
 
bool initialized () const
 
const std::string & className () const
 
const std::string & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
Engineengine () const
 
bool initialized () const
 
const std::string & className () const
 
const std::string & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
- Public Member Functions inherited from jevois::ParameterRegistry
virtual ~ParameterRegistry ()
 

Protected Attributes

std::vector< std::string > itsClasses
 
cv::dnn::Net itsNet
 
std::vector< cv::String > itsOutNames
 
std::vector< int > itsOutLayers
 
std::string itsOutLayerType
 

Related Functions

(Note that these are not member functions.)

 JEVOIS_DEFINE_ENUM_CLASS (Model,(Custom)(Face)(MobileNetSSDvoc)(MobileNetSSDcoco)(MobileNet2SSDcoco)(TinyYOLOv3)(TinyYOLOv2))
 Enum. More...
 
 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK (model, Model, "Shortcut parameter to load a model. This sets parameters " "classnames, configname, modelname, etc for the selected network. When " "the selected model is Custom, those other parameters will be set instead " "from the module's params.cfg config file.", Model::Custom, Model_Values, ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (classnames, std::string, "Path to a text file with names of classes to label detected objects", JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.classes", ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (configname, std::string, "Path to a text file that contains network configuration. " "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet).", JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.prototxt", ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (modelname, std::string, "Path to a binary file of model contains trained weights. " "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), " "or .weights (Darknet).", JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.caffemodel", ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (netin, cv::Size, "Width and height (in pixels) of the neural network input layer, or [0 0] " "to make it match camera frame size. NOTE: for YOLO v3 sizes must be multiples of 32.", cv::Size(160, 120), ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (thresh, float, "Detection threshold in percent confidence", 50.0F, jevois::Range< float >(0.0F, 100.0F), ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (nms, float, "Non-maximum suppression intersection-over-union threshold in percent", 45.0F, jevois::Range< float >(0.0F, 100.0F), ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (rgb, bool, "When true, model works with RGB input images instead BGR ones", true, ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (scale, float, "Value scaling factor applied to input pixels", 2.0F/255.0F, ParamCateg)
 Parameter. More...
 
 JEVOIS_DECLARE_PARAMETER (mean, cv::Scalar, "Mean BGR value subtracted from input image", cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg)
 Parameter. More...
 

Additional Inherited Members

- Protected Member Functions inherited from jevois::StdModule
void sendSerialMarkStart ()
 
void sendSerialMarkStop ()
 
std::string getStamp () const
 
- Protected Member Functions inherited from jevois::Component
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void postUninit ()
 
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void postUninit ()
 
- Protected Member Functions inherited from jevois::ParameterRegistry
void addParameter (ParameterBase *const param)
 
void removeParameter (ParameterBase *const param)
 
void callbackInitCall ()
 

Detailed Description

Detect and recognize multiple objects in scenes using OpenCV Deep Neural Nets (DNN)

This module runs an object detection deep neural network using the OpenCV DNN library. Detection networks analyze a whole scene and produce a number of bounding boxes around detected objects, together with identity labels and confidence scores for each detected box.

This module runs the selected deep neural network and shows all detections obtained.

Note that by default this module runs the OpenCV Face Detector DNN which can detect human faces.

Included with the standard JeVois distribution are the following networks:

  • OpenCV Face Detector, Caffe model
  • MobileNet + SSD trained on Pascal VOC (20 object classes), Caffe model
  • MobileNet + SSD trained on Coco (80 object classes), TensorFlow model
  • MobileNet v2 + SSD trained on Coco (80 object classes), TensorFlow model
  • Darknet Tiny YOLO v3 trained on Coco (80 object classes), Darknet model
  • Darknet Tiny YOLO v2 trained on Pascal VOC (20 object classes), Darknet model

See the module's params.cfg file to switch network. Object categories are as follows:

  • The 80 COCO object categories are: person, bicycle, car, motorbike, aeroplane, bus, train, truck, boat, traffic, fire, stop, parking, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra, giraffe, backpack, umbrella, handbag, tie, suitcase, frisbee, skis, snowboard, sports, kite, baseball, baseball, skateboard, surfboard, tennis, bottle, wine, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange, broccoli, carrot, hot, pizza, donut, cake, chair, sofa, pottedplant, bed, diningtable, toilet, tvmonitor, laptop, mouse, remote, keyboard, cell, microwave, oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy, hair, toothbrush.
  • The 20 Pascal-VOC object categories are: aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow, diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor.

Sometimes it will make mistakes! The performance of yolov3-tiny is about 33.1% correct (mean average precision) on the COCO test set. The OpenCV Face Detector is quite fast and robust!

Speed and network size

The parameter netin allows you to rescale the neural network to the specified size. Beware that this will only work if the network used is fully convolutional (as is the case with the default networks listed above). This not only allows you to adjust processing speed (and, conversely, accuracy), but also to better match the network to the input images (e.g., the default size for tiny-yolo is 416x416, and, thus, passing it a input image of size 640x480 will result in first scaling that input to 416x312, then letterboxing it by adding gray borders on top and bottom so that the final input to the network is 416x416). This letterboxing can be completely avoided by just resizing the network to 320x240.

Here are expected processing speeds for the OpenCV Face Detector:

  • when netin = [320 240], processes 320x240 inputs, about 650ms/image (1.5 frames/s)
  • when netin = [160 120], processes 160x120 inputs, about 190ms/image (5.0 frames/s)

Serial messages

When detections are found which are above threshold, one message will be sent for each detected object (i.e., for each box that gets drawn when USB outputs are used), using a standardized 2D message:

  • Serial message type: 2D
  • id: the category of the recognized object, followed by ':' and the confidence score in percent
  • x, y, or vertices: standardized 2D coordinates of object center or corners
  • w, h: standardized object size
  • extra: any number of additional category:score pairs which had an above-threshold score for that box

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

This code is heavily inspired from: https://github.com/opencv/opencv/blob/master/samples/dnn/object_detection.cpp

Author
Laurent Itti
Display Name:
Detection DNN
Videomapping:
NONE 0 0 0.0 YUYV 640 480 15.0 JeVois DetectionDNN
Videomapping:
YUYV 640 498 15.0 YUYV 640 480 15.0 JeVois DetectionDNN
Email:
itti@usc.edu
Address:
University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
Main URL:
http://jevois.org
Support URL:
http://jevois.org/doc
Other URL:
http://iLab.usc.edu
License:
GPL v3
Distribution:
Unrestricted
Restrictions:
None

Definition at line 171 of file DetectionDNN.C.

Constructor & Destructor Documentation

◆ DetectionDNN()

DetectionDNN::DetectionDNN ( std::string const &  instance)
inline

Constructor.

Definition at line 183 of file DetectionDNN.C.

◆ ~DetectionDNN()

virtual DetectionDNN::~DetectionDNN ( )
inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 189 of file DetectionDNN.C.

Member Function Documentation

◆ load()

void DetectionDNN::load ( )
inline

Load and initialize a model.

Definition at line 272 of file DetectionDNN.C.

References itsClasses, itsNet, itsOutLayers, itsOutLayerType, itsOutNames, LFATAL, and dknet::Net.

Referenced by onParamChange().

◆ onParamChange()

void DetectionDNN::onParamChange ( model const &  param,
Model const &  val 
)
inlineoverride

Parameter callback: set the selected model.

Definition at line 195 of file DetectionDNN.C.

References freeze(), load(), reset(), and unFreeze().

◆ postprocess()

void DetectionDNN::postprocess ( cv::Mat const &  frame,
std::vector< cv::Mat > const &  outs,
jevois::RawImage outframe = nullptr 
)
inline

◆ process() [1/2]

virtual void DetectionDNN::process ( jevois::InputFrame &&  inframe)
inlineoverridevirtual

Processing function, no video output.

Reimplemented from jevois::Module.

Definition at line 424 of file DetectionDNN.C.

References test-model::blob, jevois::rawimage::convertToCvBGR(), h, jevois::RawImage::height, itsNet, itsOutNames, postprocess(), and jevois::RawImage::width.

◆ process() [2/2]

Friends And Related Function Documentation

◆ JEVOIS_DECLARE_PARAMETER() [1/9]

JEVOIS_DECLARE_PARAMETER ( classnames  ,
std::string  ,
"Path to a text file with names of classes to label detected objects"  ,
JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.classes"  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [2/9]

JEVOIS_DECLARE_PARAMETER ( configname  ,
std::string  ,
"Path to a text file that contains network configuration. " "Can have extension .prototxt   Caffe,
pbtxtTensorFlow,
or .cfg(Darknet)."  ,
JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.prototxt"  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [3/9]

JEVOIS_DECLARE_PARAMETER ( mean  ,
cv::Scalar  ,
"Mean BGR value subtracted from input image"  ,
cv::Scalar(127.5F, 127.5F, 127.5F ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [4/9]

JEVOIS_DECLARE_PARAMETER ( modelname  ,
std::string  ,
"Path to a binary file of model contains trained weights. " "Can have extension .caffemodel   Caffe,
pbTensorFlow,
.t7 or .  netTorch,
" "or .weights(Darknet)."  ,
JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.caffemodel"  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [5/9]

JEVOIS_DECLARE_PARAMETER ( netin  ,
cv::Size  ,
"Width and height (in pixels) of the neural network input  layer,
or " "to make it match camera frame size. NOTE:for YOLO v3 sizes must be multiples of 32."  [0 0],
cv::Size(160, 120)  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [6/9]

JEVOIS_DECLARE_PARAMETER ( nms  ,
float  ,
"Non-maximum suppression intersection-over-union threshold in percent"  ,
45.  0F,
jevois::Range< float >  0.0F, 100.0F,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [7/9]

JEVOIS_DECLARE_PARAMETER ( rgb  ,
bool  ,
"When  true,
model works with RGB input images instead BGR ones"  ,
true  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [8/9]

JEVOIS_DECLARE_PARAMETER ( scale  ,
float  ,
"Value scaling factor applied to input pixels"  ,
2.0F/255.  0F,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER() [9/9]

JEVOIS_DECLARE_PARAMETER ( thresh  ,
float  ,
"Detection threshold in percent confidence"  ,
50.  0F,
jevois::Range< float >  0.0F, 100.0F,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK()

JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK ( model  ,
Model  ,
"Shortcut parameter to load a model. This sets parameters " "  classnames,
configname  ,
modelname  ,
etc for the selected network. When " "the selected model is  Custom,
those other parameters will be set instead " "from the module 's params.cfg config file."  ,
Model::Custom  ,
Model_Values  ,
ParamCateg   
)
related

Parameter.

◆ JEVOIS_DEFINE_ENUM_CLASS()

JEVOIS_DEFINE_ENUM_CLASS ( Model  ,
(Custom)(Face)(MobileNetSSDvoc)(MobileNetSSDcoco)(MobileNet2SSDcoco)(TinyYOLOv3)(TinyYOLOv2)   
)
related

Enum.

Member Data Documentation

◆ itsClasses

std::vector<std::string> DetectionDNN::itsClasses
protected

Definition at line 513 of file DetectionDNN.C.

Referenced by load(), and postprocess().

◆ itsNet

cv::dnn::Net DetectionDNN::itsNet
protected

Definition at line 514 of file DetectionDNN.C.

Referenced by load(), postprocess(), and process().

◆ itsOutLayers

std::vector<int> DetectionDNN::itsOutLayers
protected

Definition at line 516 of file DetectionDNN.C.

Referenced by load().

◆ itsOutLayerType

std::string DetectionDNN::itsOutLayerType
protected

Definition at line 517 of file DetectionDNN.C.

Referenced by load(), and postprocess().

◆ itsOutNames

std::vector<cv::String> DetectionDNN::itsOutNames
protected

Definition at line 515 of file DetectionDNN.C.

Referenced by load(), and process().


The documentation for this class was generated from the following file: