JeVoisBase  1.17
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlow.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
22 #include <jevois/Types/Enum.H>
23 #include <jevois/Types/ObjReco.H>
24 #include <atomic>
25 
26 #include <tensorflow/lite/model.h>
27 #include <tensorflow/lite/interpreter.h>
28 
29 namespace tflow
30 {
31  static jevois::ParameterCategory const ParamCateg("TensorFlow Options");
32 
33  //! Parameter \relates TensorFlow
34  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(netdir, std::string, "Network to load. This should be the name of a "
35  "directory within JEVOIS:/share/tensorflow/ which should contain two files: "
36  "model.tflite and labels.txt",
37  "mobilenet_v1_224_android_quant_2017_11_08", ParamCateg);
38 
39  //! Parameter \relates TensorFlow
40  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. "
41  "If empty, use the module's path.",
42  JEVOIS_SHARE_PATH "/tensorflow", ParamCateg);
43 
44  //! Parameter \relates TensorFlow
45  JEVOIS_DECLARE_PARAMETER(top, unsigned int, "Max number of top-scoring predictions that score above thresh to return",
46  5, ParamCateg);
47 
48  //! Parameter \relates TensorFlow
49  JEVOIS_DECLARE_PARAMETER(thresh, float, "Threshold (in percent confidence) above which predictions will be reported",
50  20.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
51 
52  //! Parameter \relates TensorFlow
53  JEVOIS_DECLARE_PARAMETER(threads, int, "Number of parallel computation threads, or 0 for auto",
54  4, jevois::Range<int>(0, 1024), ParamCateg);
55 
56  //! Parameter \relates TensorFlow
57  JEVOIS_DECLARE_PARAMETER(scorescale, float, "Scaling factors applied to recognition scores, useful for InceptionV3",
58  1.0F, ParamCateg);
59 }
60 
61 //! Identify an object using TensorFlow deep neural network
62 /*! TensorFlow is a popular neural network framework. This component identifies the object in the given image crop. It
63  returns the top scoring candidates.
64 
65  See https://www.tensorflow.org
66 
67  TensorFlow is a great deep learning and deep neural network framework. TensorFlow Lite (used here) is optimized for
68  embedded systems like the small JeVois camera because it has a very small footprint and fewer dependencies than
69  other deep neural network frameworks like MXNet, Theano, Keras, PyTorch, etc. In addition, the port of TensorFlow to
70  JeVois includes acceleration using the ARM NEON multimedia instructions.
71 
72  \ingroup Components */
74  public jevois::Parameter<tflow::netdir, tflow::dataroot, tflow::top, tflow::thresh, tflow::threads,
75  tflow::scorescale>
76 {
77  public:
78  //! Constructor
79  TensorFlow(std::string const & instance);
80 
81  //! Initialize, configure and load the network in a thread
82  /*! Any call to predict() will simply throw until the network is loaded and ready */
83  void postInit() override;
84 
85  //! Virtual destructor for safe inheritance
86  virtual ~TensorFlow();
87 
88  //! Un-initialize and free resources
89  void postUninit() override;
90 
91  //! Processing function, results are stored internally in the underlying TensorFlow network object
92  /*! Expects an OpenCV byte RGB (CV_8UC3) or Gray (CV_8UC1) image, which may be further converted to float if that is
93  what the network wants as input. The image dims and number of channels must match the network's input dims and
94  channels. Returns the prediction time (neural net forward pass) in milliseconds. Throws std::logic_error if the
95  network is still loading and not ready. */
96  float predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results);
97 
98  //! Get input width, height, channels
99  /*! Throws std::logic_error if the network is still loading and not ready. */
100  void getInDims(int & w, int & h, int & c);
101 
102  // We leave these in the open in case one wants to access the probs, names, etc but just be careful with them
103  std::vector<std::string> labels;
104  size_t numlabels; // labels is padded to a multiple of 16; numlabels is the actual unpadded number of labels
105  std::unique_ptr<tflite::FlatBufferModel> model;
106  std::unique_ptr<tflite::Interpreter> interpreter;
107 
108  protected:
109  void onParamChange(tflow::netdir const & param, std::string const & newval) override;
110  void onParamChange(tflow::dataroot const & param, std::string const & newval) override;
111  void loadNet();
112  void readLabelsFile(std::string const & fname);
113 
114  template <class T>
115  void get_top_n(T * prediction, int prediction_size, std::vector<jevois::ObjReco> & top_results,
116  bool input_floating);
117 
118  std::future<void> itsReadyFut;
119  std::atomic<bool> itsReady;
120  std::atomic<bool> itsNeedReload;
121 
122  struct JeVoisReporter : public tflite::ErrorReporter
123  {
124  int Report(char const * format, va_list args) override;
125  };
126 
128 };
TensorFlow::~TensorFlow
virtual ~TensorFlow()
Virtual destructor for safe inheritance.
Definition: TensorFlow.C:56
jevois::Range
TensorFlow::JeVoisReporter
Definition: TensorFlow.H:122
JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
TensorFlow::TensorFlow
TensorFlow(std::string const &instance)
Constructor.
Definition: TensorFlow.C:51
jevois::Component
tflow
Definition: TensorFlow.H:29
TensorFlow::loadNet
void loadNet()
Definition: TensorFlow.C:134
TensorFlow::model
std::unique_ptr< tflite::FlatBufferModel > model
Definition: TensorFlow.H:105
jevois::ParameterCategory
TensorFlow::postUninit
void postUninit() override
Un-initialize and free resources.
Definition: TensorFlow.C:214
TensorFlow::labels
std::vector< std::string > labels
Definition: TensorFlow.H:103
ObjReco.H
TensorFlow
Identify an object using TensorFlow deep neural network.
Definition: TensorFlow.H:73
TensorFlow::predict
float predict(cv::Mat const &cvimg, std::vector< jevois::ObjReco > &results)
Processing function, results are stored internally in the underlying TensorFlow network object.
Definition: TensorFlow.C:220
format
Type of detection output format
TensorFlow::numlabels
size_t numlabels
Definition: TensorFlow.H:104
TensorFlow::readLabelsFile
void readLabelsFile(std::string const &fname)
Definition: TensorFlow.C:71
TensorFlow::postInit
void postInit() override
Initialize, configure and load the network in a thread.
Definition: TensorFlow.C:89
TensorFlow::onParamChange
void onParamChange(tflow::netdir const &param, std::string const &newval) override
Definition: TensorFlow.C:63
F
float F
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
Component.H
Enum.H
TensorFlow::JeVoisReporter::Report
int Report(char const *format, va_list args) override
Definition: TensorFlow.C:42
TensorFlow::itsNeedReload
std::atomic< bool > itsNeedReload
Definition: TensorFlow.H:120
dknet::dataroot
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:41
TensorFlow::itsReady
std::atomic< bool > itsReady
Definition: TensorFlow.H:119
RawImageOps.H
TensorFlow::interpreter
std::unique_ptr< tflite::Interpreter > interpreter
Definition: TensorFlow.H:106
TensorFlow::getInDims
void getInDims(int &w, int &h, int &c)
Get input width, height, channels.
Definition: TensorFlow.C:296
TensorFlow::itsErrorReporter
JeVoisReporter itsErrorReporter
Definition: TensorFlow.H:127
h
int h
TensorFlow::get_top_n
void get_top_n(T *prediction, int prediction_size, std::vector< jevois::ObjReco > &top_results, bool input_floating)
Definition: TensorFlow.C:97
TensorFlow::itsReadyFut
std::future< void > itsReadyFut
Definition: TensorFlow.H:118