Detect salient objects and identify them using TensorFlow deep neural network. More...

Inheritance diagram for TensorFlowSaliency:

[legend]

Collaboration diagram for TensorFlowSaliency:

[legend]

Public Member Functions
	TensorFlowSaliency (std::string const &instance)
	Constructor.

virtual	~TensorFlowSaliency ()
	Virtual destructor for safe inheritance.

virtual void	postUninit () override
	Un-initialization.

virtual void	getSalROI (jevois::RawImage const &inimg)
	Helper function: compute saliency ROI in a thread, return top-left corner and size.

virtual void	process (jevois::InputFrame &&inframe) override
	Processing function, no video output.

virtual void	process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
	Processing function with video output to USB.

Public Member Functions inherited from jevois::StdModule
	StdModule (std::string const &instance)

virtual	~StdModule ()

void	sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")

void	sendSerialObjReco (std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetectOBB const &det)

	JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine))

	JEVOIS_DECLARE_PARAMETER (serstyle, SerStyle, "Style for standardized serial messages as defined in " "http://jevois.org/doc/UserSerialStyle.html", SerStyle::Terse, SerStyle_Values, ParamCateg)

	JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)

	JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime))

	JEVOIS_DECLARE_PARAMETER (serstamp, SerStamp, "Prepend standardized serial messages with a frame number, " "time, frame+time, or frame+date+time. See details in " "http://jevois.org/doc/UserSerialStyle.html", SerStamp::None, SerStamp_Values, ParamCateg)

	JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both))

	JEVOIS_DECLARE_PARAMETER (sermark, SerMark, "Send serial message to mark the beginning (MARK START) of the " "processing of a video frame from the camera sensor, the end (MARK STOP), or both. " "Useful, among others, if one needs to know when no results were sent over serial " "on a given frame. Combine with parameter serstamp if you need to know the frame number.", SerMark::None, SerMark_Values, ParamCateg)

Public Member Functions inherited from jevois::Module
	Module (std::string const &instance)

virtual	~Module ()

virtual void	process (InputFrame &&inframe, GUIhelper &helper)

virtual void	sendSerial (std::string const &str)

virtual void	parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)

virtual void	supportedCommands (std::ostream &os)

Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)

virtual	~Component ()

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

bool	initialized () const

std::string const &	className () const

std::string const &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor, bool doit)

void	freezeAllParams (bool doit)

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

bool	initialized () const

std::string const &	className () const

std::string const &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor, bool doit)

void	freezeAllParams (bool doit)

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Protected Attributes
std::shared_ptr< Saliency >	itsSaliency

std::shared_ptr< TensorFlow >	itsTensorFlow

std::vector< jevois::ObjReco >	itsResults

std::future< float >	itsPredictFut

cv::Mat	itsRawInputCv

cv::Mat	itsCvImg

cv::Mat	itsRawPrevOutputCv

int	itsRx

int	itsRy

int	itsRw

int	itsRh

Related Symbols
(Note that these are not member symbols.)
	JEVOIS_DECLARE_PARAMETER (foa, cv::Size, "Width and height (in pixels) of the focus of attention. " "This is the size of the image crop that is taken around the most salient " "location in each frame. The foa size must fit within the camera input frame size. To avoid " "rescaling, it is best to use here the size that the deep network expects as input.", cv::Size(128, 128), ParamCateg)
	Parameter.

Additional Inherited Members
Protected Member Functions inherited from jevois::StdModule
void	sendSerialMarkStart ()

void	sendSerialMarkStop ()

std::string	getStamp () const

Protected Member Functions inherited from jevois::Component
virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)

void	removeParameter (ParameterBase *const param)

void	callbackInitCall ()

Detailed Description

Detect salient objects and identify them using TensorFlow deep neural network.

TensorFlow is a popular neural network framework. This module first finds the most conspicuous (salient) object in the scene, then identifies it using a deep neural network. It returns the top scoring candidates.

See http://ilab.usc.edu/bu/ for more information about saliency detection, and https://www.tensorflow.org for more information about the TensorFlow deep neural network framework.

This module runs a TensorFlow network on an image window around the most salient point and shows the top-scoring results. We alternate, on every other frame, between updating the salient window crop location, and predicting what is in it. Actual network inference speed (time taken to compute the predictions on one image crop) is shown at the bottom right. See below for how to trade-off speed and accuracy.

Note that by default this module runs fast variant of MobileNets trained on the ImageNet dataset. There are 1000 different kinds of objects (object classes) that this network can recognize (too long to list here). It is possible to use bigger and more complex networks, but it will likely slow down the framerate.

For more information about MobileNets, see https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md

For more information about the ImageNet dataset used for training, see http://www.image-net.org/challenges/LSVRC/2012/

Sometimes this module will make mistakes! The performance of mobilenets is about 40% to 70% correct (mean average precision) on the test set, depending on network size (bigger networks are more accurate but slower).

Neural network size and speed

This module provides a parameter, foa, which determines the size of a region of interest that is cropped around the most salient location. This region will then be rescaled, if needed, to the neural network's expected input size. To avoid wasting time rescaling, it is hence best to select an foa size that is equal to the network's input size.

The network actual input size varies depending on which network is used; for example, mobilenet_v1_0.25_128_quant expects 128x128 input images, while mobilenet_v1_1.0_224 expects 224x224. We automatically rescale the cropped window to the network's desired input size. Note that there is a cost to rescaling, so, for best performance, you should match foa size to the network input size.

For example:

mobilenet_v1_0.25_128_quant (network size 128x128), runs at about 8ms/prediction (125 frames/s).
mobilenet_v1_0.5_128_quant (network size 128x128), runs at about 18ms/prediction (55 frames/s).
mobilenet_v1_0.25_224_quant (network size 224x224), runs at about 24ms/prediction (41 frames/s).
mobilenet_v1_1.0_224_quant (network size 224x224), runs at about 139ms/prediction (7 frames/s).

When using video mappings with USB output, irrespective of foa, the crop around the most salient image region (with size given by foa) will always also be rescaled so that, when placed to the right of the input image, it fills the desired USB output dims. For example, if camera mode is 320x240 and USB output size is 544x240, then the attended and recognized object will be rescaled to 224x224 (since 224 = 544-320) for display purposes only. This is so that one does not need to change USB video resolution while playing with different values of foa live.

Serial messages

On every frame where detection results were obtained that are above thresh, this module sends a standardized 2D message as specified in Standardized serial messages formatting

Serial message type: 2D
id: top-scoring category name of the recognized object, followed by ':' and the confidence score in percent
x, y, or vertices: standardized 2D coordinates of object center or corners
w, h: standardized object size
extra: any number of additional category:score pairs which had an above-threshold score, in order of decreasing score where category is the category name (from namefile) and score is the confidence score from 0.0 to 100.0

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Using your own network

For a step-by-step tutorial, see Training custom TensorFlow networks for JeVois.

This module supports RGB or grayscale inputs, byte or float32. You should create and train your network using fast GPUs, and then follow the instruction here to convert your trained network to TFLite format:

https://www.tensorflow.org/lite/

Then you just need to create a directory under JEVOIS:/share/tensorflow/ with the name of your network, and, in there, two files, labels.txt with the category labels, and model.tflite with your model converted to TensorFlow Lite (flatbuffer format). Finally, edit JEVOIS:/modules/JeVois/TensorFlowEasy/params.cfg to select your new network when the module is launched.

Author: Laurent Itti

Display Name:: TensorFlow Saliency

Videomapping:: NONE 0 0 0.0 YUYV 320 240 15.0 JeVois TensorFlowSaliency

Videomapping:: YUYV 448 240 30.0 YUYV 320 240 30.0 JeVois TensorFlowSaliency # recommended network size 128x128

Videomapping:: YUYV 512 240 30.0 YUYV 320 240 30.0 JeVois TensorFlowSaliency # recommended network size 192x192

Videomapping:: YUYV 544 240 30.0 YUYV 320 240 30.0 JeVois TensorFlowSaliency # recommended network size 224x224

Email:: itti@usc.edu

Address:: University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA

Copyright: Copyright (C) 2018 by Laurent Itti, iLab and the University of Southern California

Main URL:: http://jevois.org

Support URL:: http://jevois.org/doc

Other URL:: http://iLab.usc.edu

License:: GPL v3

Distribution:: Unrestricted

Restrictions:: None

Definition at line 139 of file TensorFlowSaliency.C.

Constructor & Destructor Documentation

◆ TensorFlowSaliency()

TensorFlowSaliency::TensorFlowSaliency ( std::string const & instance )

inline

Constructor.

Definition at line 146 of file TensorFlowSaliency.C.

References itsSaliency, and itsTensorFlow.

◆ ~TensorFlowSaliency()

virtual TensorFlowSaliency::~TensorFlowSaliency ( )

inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 156 of file TensorFlowSaliency.C.

Member Function Documentation

◆ getSalROI()

virtual void TensorFlowSaliency::getSalROI ( jevois::RawImage const & inimg )

inlinevirtual

Helper function: compute saliency ROI in a thread, return top-left corner and size.

Definition at line 170 of file TensorFlowSaliency.C.

References h, jevois::RawImage::height, itsRh, itsRw, itsRx, itsRy, itsSaliency, LFATAL, and jevois::RawImage::width.

Referenced by process(), and process().

◆ postUninit()

virtual void TensorFlowSaliency::postUninit ( )

inlineoverridevirtual

Un-initialization.

Reimplemented from jevois::Component.

Definition at line 162 of file TensorFlowSaliency.C.

References itsPredictFut.

◆ process() [1/2]

virtual void TensorFlowSaliency::process ( jevois::InputFrame && inframe )

inlineoverridevirtual

Processing function, no video output.

Reimplemented from jevois::Module.

Definition at line 204 of file TensorFlowSaliency.C.

References jevois::rawimage::cvImage(), getSalROI(), h, jevois::RawImage::height, itsResults, itsRh, itsRw, itsRx, itsRy, itsTensorFlow, LINFO, jevois::rescaleCv(), jevois::StdModule::sendSerialObjDetImg2D(), and jevois::RawImage::width.

◆ process() [2/2]

virtual void TensorFlowSaliency::process	(	jevois::InputFrame &&	inframe,
		jevois::OutputFrame &&	outframe
	)

inlineoverridevirtual

Processing function with video output to USB.

Reimplemented from jevois::Module.

Definition at line 246 of file TensorFlowSaliency.C.

References jevois::async(), jevois::yuyv::Black, jevois::rawimage::convertCvRGBtoCvYUYV(), jevois::rawimage::cvImage(), jevois::rawimage::drawFilledRect(), jevois::rawimage::drawRect(), getSalROI(), h, jevois::RawImage::height, itsRawInputCv, itsRawPrevOutputCv, itsResults, itsRh, itsRw, itsRx, itsRy, itsTensorFlow, jevois::yuyv::LightPink, jevois::rawimage::paste(), jevois::RawImage::require(), jevois::rescaleCv(), jevois::StdModule::sendSerialObjDetImg2D(), jevois::sformat(), jevois::Timer::start(), jevois::Timer::stop(), jevois::yuyv::White, jevois::RawImage::width, and jevois::rawimage::writeText().

Friends And Related Symbol Documentation

◆ JEVOIS_DECLARE_PARAMETER()

JEVOIS_DECLARE_PARAMETER	(	foa	,
		cv::Size	,
		"Width and height (in pixels) of the focus of attention. " "This is the size of the image crop that is taken around the most salient " "location in each frame. The foa size must fit within the camera input frame size. To avoid " "	rescaling,
		it is best to use here the size that the deep network expects as input."	,
		cv::Size(128, 128)	,
		ParamCateg
	)

Member Data Documentation

◆ itsCvImg

cv::Mat TensorFlowSaliency::itsCvImg

protected

Definition at line 379 of file TensorFlowSaliency.C.

◆ itsPredictFut

std::future<float> TensorFlowSaliency::itsPredictFut

protected

Definition at line 377 of file TensorFlowSaliency.C.

Referenced by postUninit().

◆ itsRawInputCv

cv::Mat TensorFlowSaliency::itsRawInputCv

protected

Definition at line 378 of file TensorFlowSaliency.C.

Referenced by process().

◆ itsRawPrevOutputCv

cv::Mat TensorFlowSaliency::itsRawPrevOutputCv

protected

Definition at line 380 of file TensorFlowSaliency.C.

Referenced by process().

◆ itsResults

std::vector<jevois::ObjReco> TensorFlowSaliency::itsResults

protected

Definition at line 376 of file TensorFlowSaliency.C.

Referenced by process(), and process().

◆ itsRh

int TensorFlowSaliency::itsRh

protected

Definition at line 381 of file TensorFlowSaliency.C.

Referenced by getSalROI(), process(), and process().

◆ itsRw

int TensorFlowSaliency::itsRw

protected

Definition at line 381 of file TensorFlowSaliency.C.

Referenced by getSalROI(), process(), and process().

◆ itsRx

int TensorFlowSaliency::itsRx

protected

Definition at line 381 of file TensorFlowSaliency.C.

Referenced by getSalROI(), process(), and process().

◆ itsRy

int TensorFlowSaliency::itsRy

protected

Definition at line 381 of file TensorFlowSaliency.C.

Referenced by getSalROI(), process(), and process().

◆ itsSaliency

std::shared_ptr<Saliency> TensorFlowSaliency::itsSaliency

protected

Definition at line 374 of file TensorFlowSaliency.C.

Referenced by getSalROI(), and TensorFlowSaliency().

◆ itsTensorFlow

std::shared_ptr<TensorFlow> TensorFlowSaliency::itsTensorFlow

protected

Definition at line 375 of file TensorFlowSaliency.C.

Referenced by process(), process(), and TensorFlowSaliency().

The documentation for this class was generated from the following file:

src/Modules/TensorFlowSaliency/TensorFlowSaliency.C

Public Member Functions

Protected Attributes

Related Symbols

Additional Inherited Members

Detailed Description

Neural network size and speed

Serial messages

Using your own network

Constructor & Destructor Documentation

◆ TensorFlowSaliency()

◆ ~TensorFlowSaliency()

Member Function Documentation

◆ getSalROI()

◆ postUninit()

◆ process() [1/2]

◆ process() [2/2]

Friends And Related Symbol Documentation

◆ JEVOIS_DECLARE_PARAMETER()

Member Data Documentation

◆ itsCvImg

◆ itsPredictFut

◆ itsRawInputCv

◆ itsRawPrevOutputCv

◆ itsResults

◆ itsRh

◆ itsRw

◆ itsRx

◆ itsRy

◆ itsSaliency

◆ itsTensorFlow