JeVoisBase
1.22
JeVois Smart Embedded Machine Vision Toolkit Base Modules
|
|
Detect and recognize multiple objects in scenes using OpenCV Deep Neural Nets (DNN) More...
Public Member Functions | |
DetectionDNN (std::string const &instance) | |
Constructor. | |
virtual | ~DetectionDNN () |
Virtual destructor for safe inheritance. | |
void | onParamChange (model const &, Model const &val) override |
Parameter callback: set the selected model. | |
void | load () |
Load and initialize a model. | |
void | postprocess (cv::Mat const &frame, std::vector< cv::Mat > const &outs, jevois::RawImage *outframe=nullptr) |
Post-processing to extract boxes from network outputs. | |
virtual void | process (jevois::InputFrame &&inframe) override |
Processing function, no video output. | |
virtual void | process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override |
Processing function with video output to USB. | |
Public Member Functions inherited from jevois::StdModule | |
StdModule (std::string const &instance) | |
virtual | ~StdModule () |
void | sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="") |
void | sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="") |
void | sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="") |
void | sendSerialObjReco (std::vector< ObjReco > const &res) |
void | sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res) |
void | sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det) |
void | sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetectOBB const &det) |
JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) | |
JEVOIS_DECLARE_PARAMETER (serstyle, SerStyle, "Style for standardized serial messages as defined in " "http://jevois.org/doc/UserSerialStyle.html", SerStyle::Terse, SerStyle_Values, ParamCateg) | |
JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg) | |
JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime)) | |
JEVOIS_DECLARE_PARAMETER (serstamp, SerStamp, "Prepend standardized serial messages with a frame number, " "time, frame+time, or frame+date+time. See details in " "http://jevois.org/doc/UserSerialStyle.html", SerStamp::None, SerStamp_Values, ParamCateg) | |
JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both)) | |
JEVOIS_DECLARE_PARAMETER (sermark, SerMark, "Send serial message to mark the beginning (MARK START) of the " "processing of a video frame from the camera sensor, the end (MARK STOP), or both. " "Useful, among others, if one needs to know when no results were sent over serial " "on a given frame. Combine with parameter serstamp if you need to know the frame number.", SerMark::None, SerMark_Values, ParamCateg) | |
Public Member Functions inherited from jevois::Module | |
Module (std::string const &instance) | |
virtual | ~Module () |
virtual void | process (InputFrame &&inframe, GUIhelper &helper) |
virtual void | sendSerial (std::string const &str) |
virtual void | parseSerial (std::string const &str, std::shared_ptr< UserInterface > s) |
virtual void | supportedCommands (std::ostream &os) |
Public Member Functions inherited from jevois::Component | |
Component (std::string const &instance) | |
virtual | ~Component () |
std::shared_ptr< Comp > | addSubComponent (std::string const &instance, Args &&...args) |
void | removeSubComponent (std::shared_ptr< Comp > &component) |
void | removeSubComponent (std::string const &instance, bool warnIfNotFound=true) |
std::shared_ptr< Comp > | getSubComponent (std::string const &instance) const |
bool | isTopLevel () const |
bool | initialized () const |
std::string const & | className () const |
std::string const & | instanceName () const |
std::vector< std::string > | setParamVal (std::string const ¶mdescriptor, T const &val) |
void | setParamValUnique (std::string const ¶mdescriptor, T const &val) |
std::vector< std::pair< std::string, T > > | getParamVal (std::string const ¶mdescriptor) const |
T | getParamValUnique (std::string const ¶mdescriptor) const |
std::vector< std::string > | setParamString (std::string const ¶mdescriptor, std::string const &val) |
void | setParamStringUnique (std::string const ¶mdescriptor, std::string const &val) |
std::vector< std::pair< std::string, std::string > > | getParamString (std::string const ¶mdescriptor) const |
std::string | getParamStringUnique (std::string const ¶mdescriptor) const |
void | freezeParam (std::string const ¶mdescriptor, bool doit) |
void | freezeAllParams (bool doit) |
std::string | descriptor () const |
void | setParamsFromFile (std::string const &filename) |
std::istream & | setParamsFromStream (std::istream &is, std::string const &absfile) |
virtual void | paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="") |
void | foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="") |
std::shared_ptr< DynamicParameter< T > > | addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category) |
std::shared_ptr< DynamicParameter< T > > | addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category) |
void | setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true) |
void | removeDynamicParameter (std::string const &name, bool throw_if_not_found=true) |
void | setPath (std::string const &path) |
std::filesystem::path | absolutePath (std::filesystem::path const &path="") |
std::shared_ptr< Comp > | addSubComponent (std::string const &instance, Args &&...args) |
void | removeSubComponent (std::shared_ptr< Comp > &component) |
void | removeSubComponent (std::string const &instance, bool warnIfNotFound=true) |
std::shared_ptr< Comp > | getSubComponent (std::string const &instance) const |
bool | isTopLevel () const |
bool | initialized () const |
std::string const & | className () const |
std::string const & | instanceName () const |
std::vector< std::string > | setParamVal (std::string const ¶mdescriptor, T const &val) |
void | setParamValUnique (std::string const ¶mdescriptor, T const &val) |
std::vector< std::pair< std::string, T > > | getParamVal (std::string const ¶mdescriptor) const |
T | getParamValUnique (std::string const ¶mdescriptor) const |
std::vector< std::string > | setParamString (std::string const ¶mdescriptor, std::string const &val) |
void | setParamStringUnique (std::string const ¶mdescriptor, std::string const &val) |
std::vector< std::pair< std::string, std::string > > | getParamString (std::string const ¶mdescriptor) const |
std::string | getParamStringUnique (std::string const ¶mdescriptor) const |
void | freezeParam (std::string const ¶mdescriptor, bool doit) |
void | freezeAllParams (bool doit) |
std::string | descriptor () const |
void | setParamsFromFile (std::string const &filename) |
std::istream & | setParamsFromStream (std::istream &is, std::string const &absfile) |
virtual void | paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="") |
void | foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="") |
std::shared_ptr< DynamicParameter< T > > | addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category) |
std::shared_ptr< DynamicParameter< T > > | addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category) |
void | setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true) |
void | removeDynamicParameter (std::string const &name, bool throw_if_not_found=true) |
void | setPath (std::string const &path) |
std::filesystem::path | absolutePath (std::filesystem::path const &path="") |
Public Member Functions inherited from jevois::ParameterRegistry | |
virtual | ~ParameterRegistry () |
Protected Attributes | |
std::vector< std::string > | itsClasses |
cv::dnn::Net | itsNet |
std::vector< cv::String > | itsOutNames |
std::vector< int > | itsOutLayers |
std::string | itsOutLayerType |
Related Symbols | |
(Note that these are not member symbols.) | |
JEVOIS_DEFINE_ENUM_CLASS (Model,(Custom)(Face)(MobileNetSSDvoc)(MobileNetSSDcoco)(MobileNet2SSDcoco)(TinyYOLOv3)(TinyYOLOv2)) | |
Enum. | |
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK (model, Model, "Shortcut parameter to load a model. This sets parameters " "classnames, configname, modelname, etc for the selected network. When " "the selected model is Custom, those other parameters will be set instead " "from the module's params.cfg config file.", Model::Custom, Model_Values, ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (classnames, std::string, "Path to a text file with names of classes to label detected objects", JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.classes", ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (configname, std::string, "Path to a text file that contains network configuration. " "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet).", JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.prototxt", ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (modelname, std::string, "Path to a binary file of model contains trained weights. " "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), " "or .weights (Darknet).", JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.caffemodel", ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (netin, cv::Size, "Width and height (in pixels) of the neural network input layer, or [0 0] " "to make it match camera frame size. NOTE: for YOLO v3 sizes must be multiples of 32.", cv::Size(160, 120), ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (thresh, float, "Detection threshold in percent confidence", 50.0F, jevois::Range< float >(0.0F, 100.0F), ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (nms, float, "Non-maximum suppression intersection-over-union threshold in percent", 45.0F, jevois::Range< float >(0.0F, 100.0F), ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (rgb, bool, "When true, model works with RGB input images instead BGR ones", true, ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (scale, float, "Value scaling factor applied to input pixels", 2.0F/255.0F, ParamCateg) | |
Parameter. | |
JEVOIS_DECLARE_PARAMETER (mean, cv::Scalar, "Mean BGR value subtracted from input image", cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg) | |
Parameter. | |
JEVOIS_DEFINE_ENUM_CLASS (Target,(CPU)(OpenCL)(OpenCL_FP16)) | |
Enum. | |
JEVOIS_DECLARE_PARAMETER (target, Target, "OpenCV compute target to use. Changes will take effect " "next time you load a different model.", Target::OpenCL, Target_Values, ParamCateg) | |
Parameter. | |
Additional Inherited Members | |
Protected Member Functions inherited from jevois::StdModule | |
void | sendSerialMarkStart () |
void | sendSerialMarkStop () |
std::string | getStamp () const |
Protected Member Functions inherited from jevois::Component | |
virtual void | preInit () |
virtual void | postInit () |
virtual void | preUninit () |
virtual void | postUninit () |
virtual void | preInit () |
virtual void | postInit () |
virtual void | preUninit () |
virtual void | postUninit () |
Protected Member Functions inherited from jevois::ParameterRegistry | |
void | addParameter (ParameterBase *const param) |
void | removeParameter (ParameterBase *const param) |
void | callbackInitCall () |
Detect and recognize multiple objects in scenes using OpenCV Deep Neural Nets (DNN)
This module runs an object detection deep neural network using the OpenCV DNN library. Detection networks analyze a whole scene and produce a number of bounding boxes around detected objects, together with identity labels and confidence scores for each detected box.
This module runs the selected deep neural network and shows all detections obtained.
Note that by default this module runs the OpenCV Face Detector DNN which can detect human faces.
Included with the standard JeVois distribution are the following networks:
See the module's params.cfg file to switch network. Object categories are as follows:
Sometimes it will make mistakes! The performance of yolov3-tiny is about 33.1% correct (mean average precision) on the COCO test set. The OpenCV Face Detector is quite fast and robust!
The parameter netin
allows you to rescale the neural network to the specified size. Beware that this will only work if the network used is fully convolutional (as is the case with the default networks listed above). This not only allows you to adjust processing speed (and, conversely, accuracy), but also to better match the network to the input images (e.g., the default size for tiny-yolo is 416x416, and, thus, passing it a input image of size 640x480 will result in first scaling that input to 416x312, then letterboxing it by adding gray borders on top and bottom so that the final input to the network is 416x416). This letterboxing can be completely avoided by just resizing the network to 320x240.
Here are expected processing speeds for the OpenCV Face Detector:
When detections are found which are above threshold, one message will be sent for each detected object (i.e., for each box that gets drawn when USB outputs are used), using a standardized 2D message:
id
: the category of the recognized object, followed by ':' and the confidence score in percentx
, y
, or vertices: standardized 2D coordinates of object center or cornersw
, h
: standardized object sizeextra
: any number of additional category:score pairs which had an above-threshold score for that boxSee Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.
This code is heavily inspired from: https://github.com/opencv/opencv/blob/master/samples/dnn/object_detection.cpp
Definition at line 171 of file DetectionDNN.C.
|
inline |
Constructor.
Definition at line 183 of file DetectionDNN.C.
|
inlinevirtual |
Virtual destructor for safe inheritance.
Definition at line 189 of file DetectionDNN.C.
|
inline |
Load and initialize a model.
Definition at line 272 of file DetectionDNN.C.
References itsClasses, itsNet, itsOutLayers, itsOutLayerType, itsOutNames, and LFATAL.
Referenced by onParamChange().
|
inlineoverride |
Parameter callback: set the selected model.
Definition at line 195 of file DetectionDNN.C.
References JEVOIS_SHARE_PATH, and load().
|
inline |
Post-processing to extract boxes from network outputs.
Definition at line 308 of file DetectionDNN.C.
References jevois::rawimage::drawRect(), jevois::rawimage::Font10x20, itsClasses, itsNet, itsOutLayerType, LFATAL, jevois::yuyv::LightGreen, jevois::StdModule::sendSerialObjDetImg2D(), jevois::sformat(), and jevois::rawimage::writeText().
Referenced by process(), process(), PyDetectionDNN.PyDetectionDNN::process(), PyPostDAMOyolo.PyPostDAMOyolo::process(), and PyPostYOLOv8seg.PyPostYOLOv8seg::process().
|
inlineoverridevirtual |
Processing function, no video output.
Reimplemented from jevois::Module.
Definition at line 424 of file DetectionDNN.C.
References jevois::rawimage::convertToCvBGR(), itsNet, itsOutNames, and postprocess().
|
inlineoverridevirtual |
Processing function with video output to USB.
Reimplemented from jevois::Module.
Definition at line 450 of file DetectionDNN.C.
References jevois::async(), jevois::yuyv::Black, jevois::rawimage::convertToCvBGR(), jevois::rawimage::drawFilledRect(), jevois::RawImage::fmt, h, jevois::RawImage::height, itsNet, itsOutNames, jevois::rawimage::paste(), postprocess(), jevois::RawImage::require(), jevois::sformat(), jevois::Timer::start(), jevois::Timer::stop(), jevois::yuyv::White, jevois::RawImage::width, and jevois::rawimage::writeText().
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Parameter.
|
related |
Enum.
|
related |
Enum.
|
protected |
Definition at line 512 of file DetectionDNN.C.
Referenced by load(), and postprocess().
|
protected |
Definition at line 513 of file DetectionDNN.C.
Referenced by load(), postprocess(), process(), and process().
|
protected |
Definition at line 515 of file DetectionDNN.C.
Referenced by load().
|
protected |
Definition at line 516 of file DetectionDNN.C.
Referenced by load(), and postprocess().
|
protected |
Definition at line 514 of file DetectionDNN.C.