JeVoisBase  1.8
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlow.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
22 #include <jevois/Types/Enum.H>
23 #include <atomic>
24 
25 #include <tensorflow/contrib/lite/model.h>
26 #include <tensorflow/contrib/lite/interpreter.h>
27 
28 namespace tflow
29 {
30  static jevois::ParameterCategory const ParamCateg("TensorFlow Options");
31 
32  //! Parameter \relates TensorFlow
33  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(netdir, std::string, "Network to load. This should be the name of a "
34  "directory within JEVOIS:/share/tensorflow/ which should contain two files: "
35  "model.tflite and labels.txt",
36  "mobilenet_v1_224_android_quant_2017_11_08", ParamCateg);
37 
38  //! Parameter \relates TensorFlow
39  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. "
40  "If empty, use the module's path.",
41  JEVOIS_SHARE_PATH "/tensorflow", ParamCateg);
42 
43  //! Parameter \relates TensorFlow
44  JEVOIS_DECLARE_PARAMETER(top, unsigned int, "Max number of top-scoring predictions that score above thresh to return",
45  5, ParamCateg);
46 
47  //! Parameter \relates TensorFlow
48  JEVOIS_DECLARE_PARAMETER(thresh, float, "Threshold (in percent confidence) above which predictions will be reported",
49  20.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
50 
51  //! Parameter \relates TensorFlow
52  JEVOIS_DECLARE_PARAMETER(threads, int, "Number of parallel computation threads, or 0 for auto",
53  4, jevois::Range<int>(0, 1024), ParamCateg);
54 
55  //! Parameter \relates TensorFlow
56  JEVOIS_DECLARE_PARAMETER(scorescale, float, "Scaling factors applied to recognition scores, useful for InceptionV3",
57  1.0F, ParamCateg);
58 }
59 
60 //! Identify an object using TensorFlow deep neural network
61 /*! TensorFlow is a popular neural network framework. This component identifies the object in the given image crop. It
62  returns the top scoring candidates.
63 
64  See https://www.tensorflow.org
65 
66  TensorFlow is a great deep learning and deep neural network framework. TensorFlow Lite (used here) is optimized for
67  embedded systems like the small JeVois camera because it has a very small footprint and fewer dependencies than
68  other deep neural network frameworks like MXNet, Theano, Keras, PyTorch, etc. In addition, the port of TensorFlow to
69  JeVois includes acceleration using the ARM NEON multimedia instructions.
70 
71  \ingroup Components */
73  public jevois::Parameter<tflow::netdir, tflow::dataroot, tflow::top, tflow::thresh, tflow::threads,
74  tflow::scorescale>
75 {
76  public:
77  //! Constructor
78  TensorFlow(std::string const & instance);
79 
80  //! Initialize, configure and load the network in a thread
81  /*! Any call to predict() will simply throw until the network is loaded and ready */
82  void postInit() override;
83 
84  //! Virtual destructor for safe inheritance
85  virtual ~TensorFlow();
86 
87  //! Un-initialize and free resources
88  void postUninit() override;
89 
90  //! Typedef for one result: score, category name
91  typedef std::pair<float, std::string> predresult;
92 
93  //! Processing function, results are stored internally in the underlying TensorFlow network object
94  /*! Expects an OpenCV byte RGB (CV_8UC3) or Gray (CV_8UC1) image, which may be further converted to float if that is
95  what the network wants as input. The image dims and number of channels must match the network's input dims and
96  channels. Returns the prediction time (neural net forward pass) in milliseconds. Throws std::logic_error if the
97  network is still loading and not ready. */
98  float predict(cv::Mat const & cvimg, std::vector<predresult> & results);
99 
100  //! Get input width, height, channels
101  /*! Throws std::logic_error if the network is still loading and not ready. */
102  void getInDims(int & w, int & h, int & c);
103 
104  // We leave these in the open in case one wants to access the probs, names, etc but just be careful with them
105  std::vector<std::string> labels;
106  size_t numlabels; // labels is padded to a multiple of 16; numlabels is the actual unpadded number of labels
107  std::unique_ptr<tflite::FlatBufferModel> model;
108  std::unique_ptr<tflite::Interpreter> interpreter;
109 
110  protected:
111  void onParamChange(tflow::netdir const & param, std::string const & newval);
112  void onParamChange(tflow::dataroot const & param, std::string const & newval);
113  void loadNet();
114  void readLabelsFile(std::string const & fname);
115 
116  template <class T>
117  void get_top_n(T * prediction, int prediction_size, std::vector<predresult> & top_results, bool input_floating);
118 
119  std::future<void> itsReadyFut;
120  std::atomic<bool> itsReady;
121  std::atomic<bool> itsNeedReload;
122 
123  struct JeVoisReporter : public tflite::ErrorReporter
124  {
125  int Report(char const * format, va_list args) override;
126  };
127 
129 };
size_t numlabels
Definition: TensorFlow.H:106
std::atomic< bool > itsNeedReload
Definition: TensorFlow.H:121
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
std::atomic< bool > itsReady
Definition: TensorFlow.H:120
JeVoisReporter itsErrorReporter
Definition: TensorFlow.H:128
std::unique_ptr< tflite::Interpreter > interpreter
Definition: TensorFlow.H:108
std::pair< float, std::string > predresult
Typedef for one result: score, category name.
Definition: TensorFlow.H:91
std::unique_ptr< tflite::FlatBufferModel > model
Definition: TensorFlow.H:107
std::future< void > itsReadyFut
Definition: TensorFlow.H:119
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:39
Identify an object using TensorFlow deep neural network.
Definition: TensorFlow.H:72
std::vector< std::string > labels
Definition: TensorFlow.H:105