JeVoisBase  1.8
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlowSaliency Class Reference

Detect salient objects and identify them using TensorFlow deep neural network. More...

Inheritance diagram for TensorFlowSaliency:
Collaboration diagram for TensorFlowSaliency:

Public Member Functions

 TensorFlowSaliency (std::string const &instance)
 Constructor. More...
 
virtual ~TensorFlowSaliency ()
 Virtual destructor for safe inheritance. More...
 
virtual void postUninit () override
 Un-initialization. More...
 
void sendAllSerial (int inw, int inh, int salx, int saly, int roiw, int roih)
 Send serial messages. More...
 
virtual void getSalROI (jevois::RawImage const &inimg)
 Helper function: compute saliency ROI in a thread, return top-left corner and size. More...
 
virtual void process (jevois::InputFrame &&inframe) override
 Processing function, no video output. More...
 
virtual void process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
 Processing function with video output to USB. More...
 
- Public Member Functions inherited from jevois::StdModule
 StdModule (std::string const &instance)
 
virtual ~StdModule ()
 
void sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle
 
 JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime)) JEVOIS_DECLARE_PARAMETER(serstamp
 
- Public Member Functions inherited from jevois::Module
 Module (std::string const &instance)
 
virtual ~Module ()
 
virtual void sendSerial (std::string const &str)
 
virtual void parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)
 
virtual void supportedCommands (std::ostream &os)
 
size_t frameNum () const
 
- Public Member Functions inherited from jevois::Component
 Component (std::string const &instance)
 
virtual ~Component ()
 
std::shared_ptr< ComponentgetSubComponent (std::string const &instance) const
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
- Public Member Functions inherited from jevois::ParameterRegistry
virtual ~ParameterRegistry ()
 

Protected Attributes

std::shared_ptr< SaliencyitsSaliency
 
std::shared_ptr< TensorFlowitsTensorFlow
 
std::vector< TensorFlow::predresultitsResults
 
std::future< float > itsPredictFut
 
cv::Mat itsRawInputCv
 
cv::Mat itsCvImg
 
cv::Mat itsRawPrevOutputCv
 
unsigned long itsFrame
 
int itsRx
 
int itsRy
 
int itsRw
 
int itsRh
 

Related Functions

(Note that these are not member functions.)

 JEVOIS_DECLARE_PARAMETER (foa, cv::Size, "Width and height (in pixels) of the focus of attention. " "This is the size of the image crop that is taken around the most salient " "location in each frame. The foa size must fit within the camera input frame size. To avoid " "rescaling, it is best to use here the size that the deep network expects as input.", cv::Size(128, 128), ParamCateg)
 Parameter. More...
 

Additional Inherited Members

- Protected Member Functions inherited from jevois::StdModule
std::string getStamp () const
 
- Protected Member Functions inherited from jevois::Component
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
- Protected Member Functions inherited from jevois::ParameterRegistry
void addParameter (ParameterBase *const param)
 
void removeParameter (ParameterBase *const param)
 
void callbackInitCall ()
 

Detailed Description

Detect salient objects and identify them using TensorFlow deep neural network.

TensorFlow is a popular neural network framework. This module first finds the most conspicuous (salient) object in the scene, then identifies it using a deep neural network. It returns the top scoring candidates.

See http://ilab.usc.edu/bu/ for more information about saliency detection, and https://www.tensorflow.org for more information about the TensorFlow deep neural network framework.

This module runs a TensorFlow network on an image window around the most salient point and shows the top-scoring results. We alternate, on every other frame, between updating the salient window crop location, and predicting what is in it. Actual network inference speed (time taken to compute the predictions on one image crop) is shown at the bottom right. See below for how to trade-off speed and accuracy.

Note that by default this module runs fast variant of MobileNets trained on the ImageNet dataset. There are 1000 different kinds of objects (object classes) that this network can recognize (too long to list here). It is possible to use bigger and more complex networks, but it will likely slow down the framerate.

For more information about MobileNets, see https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md

For more information about the ImageNet dataset used for training, see http://www.image-net.org/challenges/LSVRC/2012/

Sometimes this module will make mistakes! The performance of mobilenets is about 40% to 70% correct (mean average precision) on the test set, depending on network size (bigger networks are more accurate but slower).

Neural network size and speed

This module provides a parameter, foa, which determines the size of a region of interest that is cropped around the most salient location. This region will then be rescaled, if needed, to the neural network's expected input size. To avoid wasting time rescaling, it is hence best to select an foa size that is equal to the network's input size.

The network actual input size varies depending on which network is used; for example, mobilenet_v1_0.25_128_quant expects 128x128 input images, while mobilenet_v1_1.0_224 expects 224x224. We automatically rescale the cropped window to the network's desired input size. Note that there is a cost to rescaling, so, for best performance, you should match foa size to the network input size.

For example:

  • mobilenet_v1_0.25_128_quant (network size 128x128), runs at about 12ms/prediction (83.3 frames/s).
  • mobilenet_v1_0.5_128_quant (network size 128x128), runs at about 26ms/prediction (38.5 frames/s).
  • mobilenet_v1_0.25_224_quant (network size 224x224), runs at about 35ms/prediction (28.5 frames/s).
  • mobilenet_v1_1.0_224_quant (network size 224x224), runs at about 185ms/prediction (5.4 frames/s).

When using video mappings with USB output, irrespective of foa, the crop around the most salient image region (with size given by foa) will always also be rescaled so that, when placed to the right of the input image, it fills the desired USB output dims. For example, if camera mode is 320x240 and USB output size is 544x240, then the attended and recognized object will be rescaled to 224x224 (since 224 = 544-320) for display purposes only. This is so that one does not need to change USB video resolution while playing with different values of foa live.

Serial messages

  • On every frame where detection results were obtained, this module sends a message
      TFS framenum
      T2 x y
    where framenum is the frame number (starts at 0). The T2 message is a standardized message about the location and size of the salient region of interest in which the object was found. The message can be customized, see Standardized serial messages formatting.
  • In addition, when detections are found which are above threshold, up to top messages will be sent, for those category candidates that have scored above thresh:
      TFR category score
    where category is the category name (from namefile) and score is the confidence score from 0.0 to 100.0

Using your own network

For a step-by-step tutorial, see Training custom TensorFlow networks for JeVois.

This module supports RGB or grayscale inputs, byte or float32. You should create and train your network using fast GPUs, and then follow the instruction here to convert your trained network to TFLite format:

https://www.tensorflow.org/mobile/tflite/

Then you just need to create a directory under JEVOIS:/share/tensorflow/ with the name of your network, and, in there, two files, labels.txt with the category labels, and model.tflite with your model converted to TensorFlow Lite (flatbuffer format). Finally, edit JEVOIS:/modules/JeVois/TensorFlowEasy/params.cfg to select your new network when the module is launched.

Author
Laurent Itti
Display Name:
TensorFlow Saliency
Videomapping:
NONE 0 0 0.0 YUYV 320 240 15.0 JeVois TensorFlowSaliency
Videomapping:
YUYV 448 240 30.0 YUYV 320 240 30.0 JeVois TensorFlowSaliency # recommended network size 128x128
Videomapping:
YUYV 512 240 30.0 YUYV 320 240 30.0 JeVois TensorFlowSaliency # recommended network size 192x192
Videomapping:
YUYV 544 240 30.0 YUYV 320 240 30.0 JeVois TensorFlowSaliency # recommended network size 224x224
Email:
itti@usc.edu
Address:
University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
Main URL:
http://jevois.org
Support URL:
http://jevois.org/doc
Other URL:
http://iLab.usc.edu
License:
GPL v3
Distribution:
Unrestricted
Restrictions:
None

Definition at line 142 of file TensorFlowSaliency.C.

Constructor & Destructor Documentation

◆ TensorFlowSaliency()

TensorFlowSaliency::TensorFlowSaliency ( std::string const &  instance)
inline

Constructor.

Definition at line 149 of file TensorFlowSaliency.C.

References itsSaliency, and itsTensorFlow.

◆ ~TensorFlowSaliency()

virtual TensorFlowSaliency::~TensorFlowSaliency ( )
inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 159 of file TensorFlowSaliency.C.

Member Function Documentation

◆ getSalROI()

virtual void TensorFlowSaliency::getSalROI ( jevois::RawImage const &  inimg)
inlinevirtual

Helper function: compute saliency ROI in a thread, return top-left corner and size.

Definition at line 188 of file TensorFlowSaliency.C.

References jevois::RawImage::height, itsRh, itsRw, itsRx, itsRy, itsSaliency, LFATAL, and jevois::RawImage::width.

Referenced by process().

◆ postUninit()

virtual void TensorFlowSaliency::postUninit ( )
inlineoverridevirtual

Un-initialization.

Reimplemented from jevois::Component.

Definition at line 165 of file TensorFlowSaliency.C.

References itsPredictFut.

◆ process() [1/2]

virtual void TensorFlowSaliency::process ( jevois::InputFrame &&  inframe)
inlineoverridevirtual

◆ process() [2/2]

◆ sendAllSerial()

void TensorFlowSaliency::sendAllSerial ( int  inw,
int  inh,
int  salx,
int  saly,
int  roiw,
int  roih 
)
inline

Send serial messages.

Definition at line 173 of file TensorFlowSaliency.C.

References itsFrame, itsResults, jevois::Module::sendSerial(), jevois::StdModule::sendSerialImg2D(), jevois::sformat(), and to_string().

Referenced by process().

Friends And Related Function Documentation

◆ JEVOIS_DECLARE_PARAMETER()

JEVOIS_DECLARE_PARAMETER ( foa  ,
cv::Size  ,
"Width and height (in pixels) of the focus of attention. " "This is the size of the image crop that is taken around the most salient " "location in each frame. The foa size must fit within the camera input frame size. To avoid " "  rescaling,
it is best to use here the size that the deep network expects as input."  ,
cv::Size(128, 128)  ,
ParamCateg   
)
related

Parameter.

Member Data Documentation

◆ itsCvImg

cv::Mat TensorFlowSaliency::itsCvImg
protected

Definition at line 402 of file TensorFlowSaliency.C.

◆ itsFrame

unsigned long TensorFlowSaliency::itsFrame
protected

Definition at line 404 of file TensorFlowSaliency.C.

Referenced by process(), and sendAllSerial().

◆ itsPredictFut

std::future<float> TensorFlowSaliency::itsPredictFut
protected

Definition at line 400 of file TensorFlowSaliency.C.

Referenced by postUninit().

◆ itsRawInputCv

cv::Mat TensorFlowSaliency::itsRawInputCv
protected

Definition at line 401 of file TensorFlowSaliency.C.

Referenced by process().

◆ itsRawPrevOutputCv

cv::Mat TensorFlowSaliency::itsRawPrevOutputCv
protected

Definition at line 403 of file TensorFlowSaliency.C.

Referenced by process().

◆ itsResults

std::vector<TensorFlow::predresult> TensorFlowSaliency::itsResults
protected

Definition at line 399 of file TensorFlowSaliency.C.

Referenced by process(), and sendAllSerial().

◆ itsRh

int TensorFlowSaliency::itsRh
protected

Definition at line 405 of file TensorFlowSaliency.C.

Referenced by getSalROI(), and process().

◆ itsRw

int TensorFlowSaliency::itsRw
protected

Definition at line 405 of file TensorFlowSaliency.C.

Referenced by getSalROI(), and process().

◆ itsRx

int TensorFlowSaliency::itsRx
protected

Definition at line 405 of file TensorFlowSaliency.C.

Referenced by getSalROI(), and process().

◆ itsRy

int TensorFlowSaliency::itsRy
protected

Definition at line 405 of file TensorFlowSaliency.C.

Referenced by getSalROI(), and process().

◆ itsSaliency

std::shared_ptr<Saliency> TensorFlowSaliency::itsSaliency
protected

Definition at line 397 of file TensorFlowSaliency.C.

Referenced by getSalROI(), and TensorFlowSaliency().

◆ itsTensorFlow

std::shared_ptr<TensorFlow> TensorFlowSaliency::itsTensorFlow
protected

Definition at line 398 of file TensorFlowSaliency.C.

Referenced by process(), and TensorFlowSaliency().


The documentation for this class was generated from the following file: