Detect and recognize multiple objects in scenes using OpenCV, NPU, TPU, or VPU Deep Neural Nets. More...

Inheritance diagram for DNN:

[legend]

Collaboration diagram for DNN:

[legend]

Public Member Functions
	DNN (std::string const &instance)
	Constructor.

virtual	~DNN ()
	Virtual destructor for safe inheritance.

void	doprocess (jevois::InputFrame const &inframe, jevois::RawImage outimg, jevois::OptGUIhelper helper, bool idle)
	Processing function implementation.

virtual void	process (jevois::InputFrame &&inframe) override
	Processing function, no video output.

virtual void	process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
	Processing function with video output to USB on JeVois-A33.

virtual void	process (jevois::InputFrame &&inframe, jevois::GUIhelper &helper) override
	Processing function with zero-copy and GUI on JeVois-Pro.

Public Member Functions inherited from jevois::StdModule
	StdModule (std::string const &instance)

virtual	~StdModule ()

void	sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")

void	sendSerialObjReco (std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetectOBB const &det)

	JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine))

	JEVOIS_DECLARE_PARAMETER (serstyle, SerStyle, "Style for standardized serial messages as defined in " "http://jevois.org/doc/UserSerialStyle.html", SerStyle::Terse, SerStyle_Values, ParamCateg)

	JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)

	JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime))

	JEVOIS_DECLARE_PARAMETER (serstamp, SerStamp, "Prepend standardized serial messages with a frame number, " "time, frame+time, or frame+date+time. See details in " "http://jevois.org/doc/UserSerialStyle.html", SerStamp::None, SerStamp_Values, ParamCateg)

	JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both))

	JEVOIS_DECLARE_PARAMETER (sermark, SerMark, "Send serial message to mark the beginning (MARK START) of the " "processing of a video frame from the camera sensor, the end (MARK STOP), or both. " "Useful, among others, if one needs to know when no results were sent over serial " "on a given frame. Combine with parameter serstamp if you need to know the frame number.", SerMark::None, SerMark_Values, ParamCateg)

Public Member Functions inherited from jevois::Module
	Module (std::string const &instance)

virtual	~Module ()

virtual void	sendSerial (std::string const &str)

virtual void	parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)

virtual void	supportedCommands (std::ostream &os)

Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)

virtual	~Component ()

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

bool	initialized () const

std::string const &	className () const

std::string const &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor, bool doit)

void	freezeAllParams (bool doit)

void	hideParam (std::string const &paramdescriptor, bool doit)

void	hideAllParams (bool doit)

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

bool	initialized () const

std::string const &	className () const

std::string const &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor, bool doit)

void	freezeAllParams (bool doit)

void	hideParam (std::string const &paramdescriptor, bool doit)

void	hideAllParams (bool doit)

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Protected Attributes
std::shared_ptr< jevois::dnn::Pipeline >	itsPipeline

Additional Inherited Members
Protected Member Functions inherited from jevois::StdModule
void	sendSerialMarkStart ()

void	sendSerialMarkStop ()

std::string	getStamp () const

Protected Member Functions inherited from jevois::Component
virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

virtual void	postUninit ()

virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

virtual void	postUninit ()

Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)

void	removeParameter (ParameterBase *const param)

void	callbackInitCall ()

Detailed Description

Detect and recognize multiple objects in scenes using OpenCV, NPU, TPU, or VPU Deep Neural Nets.

This module runs a deep neural network using the OpenCV DNN library. Classification networks try to identify the whole object or scene in the field of view, and return the top scoring object classes. Detection networks analyze a scene and produce a number of bounding boxes around detected objects, together with identity labels and confidence scores for each detected box. Semantic segmentation networks create a pixel-by-pixel mask which assigns a class label to every location in the camera view.

To select a network, see parameter pipe of component Pipeline.

The following keys are used in the JeVois-Pro GUI (pipe parameter of Pipeline component):

OpenCV: network loaded by OpenCV DNN framework and running on CPU.
ORT: network loaded by ONNX-Runtime framework and running on CPU.
NPU: network running native on the JeVois-Pro integrated 5-TOPS NPU (neural processing unit).
SPU: network running on the optional 26-TOPS Hailo8 SPU accelerator (stream processing unit).
TPU: network running on the optional 4-TOPS Google Coral TPU accelerator (tensor processing unit).
VPU: network running on the optional 1-TOPS MyriadX VPU accelerator (vector processing unit).
NPUX: network loaded by OpenCV and running on NPU via the TIM-VX OpenCV extension. To run efficiently, network should have been quantized to int8, otherwise some slow CPU-based emulation will occur.
VPUX: network optimized for VPU but running on CPU if VPU is not available. Note that VPUX entries are automatically created by scanning all VPU entries and changing their target from Myriad to CPU, if a VPU accelerator is not detected. If a VPU is detected, then VPU models are listed and VPUX ones are not. VPUX emulation runs on the JeVois-Pro CPU using the Arm Compute Library to provide efficient implementation of various network layers and operations.

For expected network speed, see JeVois-Pro Deep Neural Network Benchmarks

Serial messages

For classification networks, when object classes are found with confidence scores above thresh, a message containing up to top category:score pairs will be sent per video frame. Exact message format depends on the current serstyle setting and is described in Standardized serial messages formatting. For example, when serstyle is Detail, this module sends:

DO category:score category:score ... category:score

where category is a category name (from namefile) and score is the confidence score from 0.0 to 100.0 that this category was recognized. The pairs are in order of decreasing score.

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

For object detection networks, when detections are found which are above threshold, one message will be sent for each detected object (i.e., for each box that gets drawn when USB outputs are used), using a standardized 2D message:

Serial message type: 2D
id: the category of the recognized object, followed by ':' and the confidence score in percent
x, y, or vertices: standardized 2D coordinates of object center or corners
w, h: standardized object size
extra: any number of additional category:score pairs which had an above-threshold score for that box

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Author: Laurent Itti

Display Name:: DNN

Videomapping:: NONE 0 0 0.0 YUYV 640 480 15.0 JeVois DNN

Videomapping:: YUYV 640 498 15.0 YUYV 640 480 15.0 JeVois DNN

Email:: itti@usc.edu

Address:: University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA

Copyright: Copyright (C) 2018 by Laurent Itti, iLab and the University of Southern California

Main URL:: http://jevois.org

Support URL:: http://jevois.org/doc

Other URL:: http://iLab.usc.edu

License:: GPL v3

Distribution:: Unrestricted

Restrictions:: None

Definition at line 97 of file DNN.C.

Constructor & Destructor Documentation

◆ DNN()

DNN::DNN ( std::string const & instance )

inline

Constructor.

Definition at line 103 of file DNN.C.

References itsPipeline.

◆ ~DNN()

virtual DNN::~DNN ( )

inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 111 of file DNN.C.

Member Function Documentation

◆ doprocess()

void DNN::doprocess	(	jevois::InputFrame const &	inframe,
		jevois::RawImage *	outimg,
		jevois::OptGUIhelper *	helper,
		bool	idle
	)

inline

Processing function implementation.

Definition at line 117 of file DNN.C.

References jevois::InputFrame::getp(), and itsPipeline.

Referenced by process(), process(), and process().

◆ process() [1/3]

virtual void DNN::process ( jevois::InputFrame && inframe )

inlineoverridevirtual

Processing function, no video output.

Reimplemented from jevois::Module.

Definition at line 130 of file DNN.C.

References doprocess().

◆ process() [2/3]

virtual void DNN::process	(	jevois::InputFrame &&	inframe,
		jevois::GUIhelper &	helper
	)

inlineoverridevirtual

Processing function with zero-copy and GUI on JeVois-Pro.

Reimplemented from jevois::Module.

Definition at line 164 of file DNN.C.

References doprocess(), jevois::GUIhelper::drawInputFrame(), jevois::GUIhelper::endFrame(), h, jevois::GUIhelper::iinfo(), jevois::Timer::start(), jevois::GUIhelper::startFrame(), and jevois::Timer::stop().

◆ process() [3/3]

virtual void DNN::process	(	jevois::InputFrame &&	inframe,
		jevois::OutputFrame &&	outframe
	)

inlineoverridevirtual

Processing function with video output to USB on JeVois-A33.

Reimplemented from jevois::Module.

Definition at line 138 of file DNN.C.

References doprocess(), jevois::RawImage::fmt, h, jevois::RawImage::height, jevois::rawimage::paste(), jevois::RawImage::require(), and jevois::RawImage::width.

Member Data Documentation

◆ itsPipeline

std::shared_ptr<jevois::dnn::Pipeline> DNN::itsPipeline

protected

Definition at line 193 of file DNN.C.

Referenced by DNN(), and doprocess().

The documentation for this class was generated from the following file:

src/Modules/DNN/DNN.C

Public Member Functions

Protected Attributes

Additional Inherited Members

Detailed Description

Serial messages

Constructor & Destructor Documentation

◆ DNN()

◆ ~DNN()

Member Function Documentation

◆ doprocess()

◆ process() [1/3]

◆ process() [2/3]

◆ process() [3/3]

Member Data Documentation

◆ itsPipeline