JeVoisBase  1.8
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlowEasy Class Reference

Identify objects using TensorFlow deep neural network. More...

Inheritance diagram for TensorFlowEasy:
Collaboration diagram for TensorFlowEasy:

Public Member Functions

 TensorFlowEasy (std::string const &instance)
 Constructor. More...
 
virtual ~TensorFlowEasy ()
 Virtual destructor for safe inheritance. More...
 
void sendAllSerial ()
 Send serial messages. More...
 
virtual void process (jevois::InputFrame &&inframe) override
 Processing function, no video output. More...
 
virtual void process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
 Processing function with video output to USB. More...
 
- Public Member Functions inherited from jevois::Module
 Module (std::string const &instance)
 
virtual ~Module ()
 
virtual void sendSerial (std::string const &str)
 
virtual void parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)
 
virtual void supportedCommands (std::ostream &os)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle
 
 JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, ParamCateg)
 
- Public Member Functions inherited from jevois::Component
 Component (std::string const &instance)
 
virtual ~Component ()
 
std::shared_ptr< ComponentgetSubComponent (std::string const &instance) const
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor)
 
void unFreezeParam (std::string const &paramdescriptor)
 
void freezeAllParams ()
 
void unFreezeAllParams ()
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
void setPath (std::string const &path)
 
std::string absolutePath (std::string const &path="")
 
- Public Member Functions inherited from jevois::ParameterRegistry
virtual ~ParameterRegistry ()
 

Protected Attributes

std::shared_ptr< TensorFlowitsTensorFlow
 
std::vector< TensorFlow::predresultitsResults
 
unsigned long itsFrame
 

Related Functions

(Note that these are not member functions.)

 JEVOIS_DECLARE_PARAMETER (foa, cv::Size, "Width and height (in pixels) of the fixed, central focus of attention. " "This is the size of the central image crop that is taken in each frame and fed to the " "deep neural network. If the foa size does not fit within the camera input frame size, " "it will be shrunk to fit. To avoid spending CPU resources on rescaling the selected " "image region, it is best to use here the size that the deep network expects as input.", cv::Size(128, 128), ParamCateg)
 Parameter. More...
 

Additional Inherited Members

- Protected Member Functions inherited from jevois::Component
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void postUninit ()
 
virtual void preInit ()
 
virtual void postInit ()
 
virtual void preUninit ()
 
virtual void postUninit ()
 
- Protected Member Functions inherited from jevois::ParameterRegistry
void addParameter (ParameterBase *const param)
 
void removeParameter (ParameterBase *const param)
 
void callbackInitCall ()
 

Detailed Description

Identify objects using TensorFlow deep neural network.

TensorFlow is a popular neural network framework. This module identifies the object in a square region in the center of the camera field of view using a deep convolutional neural network.

The deep network analyzes the image by filtering it using many different filter kernels, and several stacked passes (network layers). This essentially amounts to detecting the presence of both simple and complex parts of known objects in the image (e.g., from detecting edges in lower layers of the network to detecting car wheels or even whole cars in higher layers). The last layer of the network is reduced to a vector with one entry per known kind of object (object class). This module returns the class names of the top scoring candidates in the output vector, if any have scored above a minimum confidence threshold. When nothing is recognized with sufficiently high confidence, there is no output.

This module runs a TensorFlow network and shows the top-scoring results. In this module, we run the deep network on every video frame, so framerate will vary depending on network complexity (see below). Point your camera towards some interesting object, make the object fit within the grey box shown in the video (which will be fed to the neural network), keep it stable, and TensorFlow will tell you what it thinks this object is.

Note that by default this module runs different flavors of MobileNets trained on the ImageNet dataset. There are 1000 different kinds of objects (object classes) that these networks can recognize (too long to list here). The input layer of these networks is 299x299, 224x224, 192x192, 160x160, or 128x128 pixels by default, depending on the network used. The networks provided on the JeVois microSD image have been trained on large clusters of GPUs, using 1.2 million training images from the ImageNet dataset.

For more information about MobileNets, see https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md

For more information about the ImageNet dataset used for training, see http://www.image-net.org/challenges/LSVRC/2012/

Sometimes this module will make mistakes! The performance of mobilenets is about 40% to 70% correct (mean average precision) on the test set, depending on network size (bigger networks are more accurate but slower).

Neural network size and speed

This module takes a central image region of size given by the foa parameter. If necessary, this image region is then rescaled to match the deep network's expected input size. The network input size varies depending on which network is used; for example, mobilenet_v1_0.25_128_quant expects 128x128 input images, while mobilenet_v1_1.0_224 expects 224x224. Note that there is a CPU cost to rescaling, so, for best performance, you should match the foa size to the network's input size.

For example:

  • mobilenet_v1_0.25_128_quant (network size 128x128), runs at about 12ms/prediction (83.3 frames/s).
  • mobilenet_v1_0.5_128_quant (network size 128x128), runs at about 26ms/prediction (38.5 frames/s).
  • mobilenet_v1_0.25_224_quant (network size 224x224), runs at about 35ms/prediction (28.5 frames/s).
  • mobilenet_v1_1.0_224_quant (network size 224x224), runs at about 185ms/prediction (5.4 frames/s).

To easily select one of the available networks, see JEVOIS:/modules/JeVois/TensorFlowEasy/params.cfg on the microSD card of your JeVois camera.

Serial messages

  • On every frame where detection results were obtained, this module sends a message
      TEF framenum
    where framenum is the frame number (starts at 0).
  • In addition, when detections are found, up to top messages will be sent, for those category candidates that have scored above thresh:
      TER category score
    where category is the category name (from labels.txt in the network's data directory) and score is the confidence score from 0.0 to 100.0

Using your own network

For a step-by-step tutorial, see Training custom TensorFlow networks for JeVois.

This module supports RGB or grayscale inputs, byte or float32. You should create and train your network using fast GPUs, and then follow the instruction here to convert your trained network to TFLite format:

https://www.tensorflow.org/mobile/tflite/

Then you just need to create a directory under JEVOIS:/share/tensorflow/ with the name of your network, and, in there, two files, labels.txt with the category labels, and model.tflite with your model converted to TensorFlow Lite (flatbuffer format). Finally, edit JEVOIS:/modules/JeVois/TensorFlowEasy/params.cfg to select your new network when the module is launched.

Author
Laurent Itti
Display Name:
TensorFlow Easy
Videomapping:
NONE 0 0 0.0 YUYV 320 240 60.0 JeVois TensorFlowEasy
Videomapping:
YUYV 320 308 30.0 YUYV 320 240 30.0 JeVois TensorFlowEasy
Videomapping:
YUYV 640 548 30.0 YUYV 640 480 30.0 JeVois TensorFlowEasy
Videomapping:
YUYV 1280 1092 7.0 YUYV 1280 1024 7.0 JeVois TensorFlowEasy
Email:
itti@usc.edu
Address:
University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
Main URL:
http://jevois.org
Support URL:
http://jevois.org/doc
Other URL:
http://iLab.usc.edu
License:
GPL v3
Distribution:
Unrestricted
Restrictions:
None

Definition at line 140 of file TensorFlowEasy.C.

Constructor & Destructor Documentation

◆ TensorFlowEasy()

TensorFlowEasy::TensorFlowEasy ( std::string const &  instance)
inline

Constructor.

Definition at line 147 of file TensorFlowEasy.C.

References itsTensorFlow.

◆ ~TensorFlowEasy()

virtual TensorFlowEasy::~TensorFlowEasy ( )
inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 155 of file TensorFlowEasy.C.

Member Function Documentation

◆ process() [1/2]

virtual void TensorFlowEasy::process ( jevois::InputFrame &&  inframe)
inlineoverridevirtual

Processing function, no video output.

Reimplemented from jevois::Module.

Definition at line 170 of file TensorFlowEasy.C.

References jevois::rawimage::cvImage(), jevois::RawImage::height, itsFrame, itsResults, itsTensorFlow, LINFO, jevois::rescaleCv(), sendAllSerial(), and jevois::RawImage::width.

◆ process() [2/2]

◆ sendAllSerial()

void TensorFlowEasy::sendAllSerial ( )
inline

Send serial messages.

Definition at line 161 of file TensorFlowEasy.C.

References itsFrame, itsResults, jevois::Module::sendSerial(), jevois::sformat(), and to_string().

Referenced by process().

Friends And Related Function Documentation

◆ JEVOIS_DECLARE_PARAMETER()

JEVOIS_DECLARE_PARAMETER ( foa  ,
cv::Size  ,
"Width and height (in pixels) of the  fixed,
central focus of attention. " "This is the size of the central image crop that is taken in each frame and fed to the " "deep neural network. If the foa size does not fit within the camera input frame  size,
" "it will be shrunk to fit. To avoid spending CPU resources on rescaling the selected " "image  region,
it is best to use here the size that the deep network expects as input."  ,
cv::Size(128, 128)  ,
ParamCateg   
)
related

Parameter.

Member Data Documentation

◆ itsFrame

unsigned long TensorFlowEasy::itsFrame
protected

Definition at line 320 of file TensorFlowEasy.C.

Referenced by process(), and sendAllSerial().

◆ itsResults

std::vector<TensorFlow::predresult> TensorFlowEasy::itsResults
protected

Definition at line 319 of file TensorFlowEasy.C.

Referenced by process(), and sendAllSerial().

◆ itsTensorFlow

std::shared_ptr<TensorFlow> TensorFlowEasy::itsTensorFlow
protected

Definition at line 318 of file TensorFlowEasy.C.

Referenced by process(), and TensorFlowEasy().


The documentation for this class was generated from the following file: