JeVoisBase  1.9
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlow.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
22 #include <jevois/Types/Enum.H>
23 #include <jevois/Types/ObjReco.H>
24 #include <atomic>
25 
26 #include <tensorflow/contrib/lite/model.h>
27 #include <tensorflow/contrib/lite/interpreter.h>
28 
29 namespace tflow
30 {
31  static jevois::ParameterCategory const ParamCateg("TensorFlow Options");
32 
33  //! Parameter \relates TensorFlow
34  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(netdir, std::string, "Network to load. This should be the name of a "
35  "directory within JEVOIS:/share/tensorflow/ which should contain two files: "
36  "model.tflite and labels.txt",
37  "mobilenet_v1_224_android_quant_2017_11_08", ParamCateg);
38 
39  //! Parameter \relates TensorFlow
40  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. "
41  "If empty, use the module's path.",
42  JEVOIS_SHARE_PATH "/tensorflow", ParamCateg);
43 
44  //! Parameter \relates TensorFlow
45  JEVOIS_DECLARE_PARAMETER(top, unsigned int, "Max number of top-scoring predictions that score above thresh to return",
46  5, ParamCateg);
47 
48  //! Parameter \relates TensorFlow
49  JEVOIS_DECLARE_PARAMETER(thresh, float, "Threshold (in percent confidence) above which predictions will be reported",
50  20.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
51 
52  //! Parameter \relates TensorFlow
53  JEVOIS_DECLARE_PARAMETER(threads, int, "Number of parallel computation threads, or 0 for auto",
54  4, jevois::Range<int>(0, 1024), ParamCateg);
55 
56  //! Parameter \relates TensorFlow
57  JEVOIS_DECLARE_PARAMETER(scorescale, float, "Scaling factors applied to recognition scores, useful for InceptionV3",
58  1.0F, ParamCateg);
59 }
60 
61 //! Identify an object using TensorFlow deep neural network
62 /*! TensorFlow is a popular neural network framework. This component identifies the object in the given image crop. It
63  returns the top scoring candidates.
64 
65  See https://www.tensorflow.org
66 
67  TensorFlow is a great deep learning and deep neural network framework. TensorFlow Lite (used here) is optimized for
68  embedded systems like the small JeVois camera because it has a very small footprint and fewer dependencies than
69  other deep neural network frameworks like MXNet, Theano, Keras, PyTorch, etc. In addition, the port of TensorFlow to
70  JeVois includes acceleration using the ARM NEON multimedia instructions.
71 
72  \ingroup Components */
74  public jevois::Parameter<tflow::netdir, tflow::dataroot, tflow::top, tflow::thresh, tflow::threads,
75  tflow::scorescale>
76 {
77  public:
78  //! Constructor
79  TensorFlow(std::string const & instance);
80 
81  //! Initialize, configure and load the network in a thread
82  /*! Any call to predict() will simply throw until the network is loaded and ready */
83  void postInit() override;
84 
85  //! Virtual destructor for safe inheritance
86  virtual ~TensorFlow();
87 
88  //! Un-initialize and free resources
89  void postUninit() override;
90 
91  //! Processing function, results are stored internally in the underlying TensorFlow network object
92  /*! Expects an OpenCV byte RGB (CV_8UC3) or Gray (CV_8UC1) image, which may be further converted to float if that is
93  what the network wants as input. The image dims and number of channels must match the network's input dims and
94  channels. Returns the prediction time (neural net forward pass) in milliseconds. Throws std::logic_error if the
95  network is still loading and not ready. */
96  float predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results);
97 
98  //! Get input width, height, channels
99  /*! Throws std::logic_error if the network is still loading and not ready. */
100  void getInDims(int & w, int & h, int & c);
101 
102  // We leave these in the open in case one wants to access the probs, names, etc but just be careful with them
103  std::vector<std::string> labels;
104  size_t numlabels; // labels is padded to a multiple of 16; numlabels is the actual unpadded number of labels
105  std::unique_ptr<tflite::FlatBufferModel> model;
106  std::unique_ptr<tflite::Interpreter> interpreter;
107 
108  protected:
109  void onParamChange(tflow::netdir const & param, std::string const & newval);
110  void onParamChange(tflow::dataroot const & param, std::string const & newval);
111  void loadNet();
112  void readLabelsFile(std::string const & fname);
113 
114  template <class T>
115  void get_top_n(T * prediction, int prediction_size, std::vector<jevois::ObjReco> & top_results,
116  bool input_floating);
117 
118  std::future<void> itsReadyFut;
119  std::atomic<bool> itsReady;
120  std::atomic<bool> itsNeedReload;
121 
122  struct JeVoisReporter : public tflite::ErrorReporter
123  {
124  int Report(char const * format, va_list args) override;
125  };
126 
128 };
size_t numlabels
Definition: TensorFlow.H:104
std::atomic< bool > itsNeedReload
Definition: TensorFlow.H:120
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
std::atomic< bool > itsReady
Definition: TensorFlow.H:119
JeVoisReporter itsErrorReporter
Definition: TensorFlow.H:127
std::unique_ptr< tflite::Interpreter > interpreter
Definition: TensorFlow.H:106
std::unique_ptr< tflite::FlatBufferModel > model
Definition: TensorFlow.H:105
std::future< void > itsReadyFut
Definition: TensorFlow.H:118
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:40
Identify an object using TensorFlow deep neural network.
Definition: TensorFlow.H:73
std::vector< std::string > labels
Definition: TensorFlow.H:103