JeVoisBase  1.10
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlow.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 
19 // Much of this code inspired by the label_image example for TensorFlow Lite:
20 
21 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
22 
23 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
24 License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
25 
26 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
27 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
28 specific language governing permissions and limitations under the License. */
29 
31 #include <jevois/Core/Module.H>
32 #include <opencv2/imgproc/imgproc.hpp>
33 #include <queue>
34 #include <fstream>
35 #include <cstdio>
36 
37 #include <tensorflow/contrib/lite/kernels/register.h>
38 #include <tensorflow/contrib/lite/optional_debug_tools.h>
39 #include <tensorflow/contrib/lite/string_util.h>
40 
41 // ####################################################################################################
42 int TensorFlow::JeVoisReporter::Report(char const * format, va_list args)
43 {
44  char buf[1024];
45  int ret = vsnprintf(buf, 1024, format, args);
46  LERROR(buf);
47  return ret;
48 }
49 
50 // ####################################################################################################
51 TensorFlow::TensorFlow(std::string const & instance) :
52  jevois::Component(instance), itsReady(false), itsNeedReload(false)
53 { }
54 
55 // ####################################################################################################
57 { }
58 
59 // ####################################################################################################
60 void TensorFlow::onParamChange(tflow::netdir const & param, std::string const & newval)
61 { itsNeedReload.store(true); }
62 
63 // ####################################################################################################
64 void TensorFlow::onParamChange(tflow::dataroot const & param, std::string const & newval)
65 { itsNeedReload.store(true); }
66 
67 // ####################################################################################################
68 void TensorFlow::readLabelsFile(std::string const & fname)
69 {
70  std::ifstream file(fname);
71  if (!file) LFATAL("Could not open labels file " << fname);
72  labels.clear();
73 
74  std::string line;
75  while (std::getline(file, line)) labels.push_back(line);
76 
77  // Tensorflow requires the number of labels to be a multiple of 16:
78  numlabels = labels.size();
79  int const padding = 16;
80  while (labels.size() % padding) labels.emplace_back();
81 
82  LINFO("Loaded " << numlabels << " category names from " << fname);
83 }
84 
85 // ####################################################################################################
87 {
88  // Start a thread to load the desired network:
89  loadNet();
90 }
91 
92 // ####################################################################################################
93 template <class T>
94 void TensorFlow::get_top_n(T * prediction, int prediction_size, std::vector<jevois::ObjReco> & top_results,
95  bool input_floating)
96 {
97  int const topn = top::get();
98  float const th = thresh::get() * 0.01F;
99  top_results.clear();
100 
101  // Will contain top N results in ascending order.
102  std::priority_queue<std::pair<float, int>, std::vector<std::pair<float, int> >,
103  std::greater<std::pair<float, int> > > top_result_pq;
104 
105  float const scale = scorescale::get() * (input_floating ? 1.0 : 1.0F / 255.0F);
106 
107  for (int i = 0; i < prediction_size; ++i)
108  {
109  float value = prediction[i] * scale;
110 
111  // Only add it if it beats the threshold and has a chance at being in the top N.
112  if (value < th) continue;
113 
114  top_result_pq.push(std::pair<float, int>(value, i));
115 
116  // If at capacity, kick the smallest value out.
117  if (top_result_pq.size() > topn) top_result_pq.pop();
118  }
119 
120  // Copy to output vector and reverse into descending order.
121  while (!top_result_pq.empty())
122  {
123  auto const & tr = top_result_pq.top();
124  top_results.push_back( { tr.first * 100, std::string(labels[tr.second]) } );
125  top_result_pq.pop();
126  }
127  std::reverse(top_results.begin(), top_results.end());
128 }
129 
130 // ####################################################################################################
132 {
133  // Users will neet to wait until last load is done before they load again:
134  if (itsReadyFut.valid())
135  {
136  if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
137  try { itsReadyFut.get(); } catch (...) { }
138  else
139  throw std::logic_error("Loading already in progress. Attempt to load again rejected");
140  }
141 
142  // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
143  itsNeedReload.store(false);
144 
145  // We are not ready anymore:
146  itsReady.store(false);
147 
148  // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
149  // meantime. itsReady will flip to true when the load is complete.
150  itsReadyFut = std::async(std::launch::async, [&]() {
151  if (model)
152  {
153  LINFO("Closing model..");
154  model.reset();
155  interpreter.reset();
156  labels.clear();
157  numlabels = 0;
158  }
159 
160  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
161  std::string const modelfile = absolutePath(root + netdir::get() + "/model.tflite");
162  std::string const labelfile = absolutePath(root + netdir::get() + "/labels.txt");
163 
164  LINFO("Using model from " << modelfile);
165  LINFO("Using labels from " << labelfile);
166 
167  // Load the labels:
168  readLabelsFile(labelfile);
169 
170  // Create the model from flatbuffer file using mmap:
171  model = tflite::FlatBufferModel::BuildFromFile(modelfile.c_str(), &itsErrorReporter);
172  if (!model) LFATAL("Failed to mmap model " << modelfile);
173  LINFO("Loaded model " << modelfile);
174 
175  tflite::ops::builtin::BuiltinOpResolver resolver;
176  tflite::InterpreterBuilder(*model, resolver)(&interpreter);
177  if (!interpreter) LFATAL("Failed to construct interpreter");
178  //////interpreter->UseNNAPI(s->accel);
179 
180  LINFO("Tensors size: " << interpreter->tensors_size());
181  LINFO("Nodes size: " << interpreter->nodes_size());
182  LINFO("inputs: " << interpreter->inputs().size());
183  LINFO("input(0) name: " << interpreter->GetInputName(0));
184 
185  int t_size = interpreter->tensors_size();
186  for (int i = 0; i < t_size; ++i)
187  if (interpreter->tensor(i)->name)
188  LINFO(i << ": " << interpreter->tensor(i)->name << ", "
189  << interpreter->tensor(i)->bytes << ", "
190  << interpreter->tensor(i)->type << ", "
191  << interpreter->tensor(i)->params.scale << ", "
192  << interpreter->tensor(i)->params.zero_point);
193 
194  if (threads::get()) interpreter->SetNumThreads(threads::get());
195 
196  LINFO("input: " << interpreter->inputs()[0]);
197  LINFO("number of inputs: " << interpreter->inputs().size());
198  LINFO("number of outputs: " << interpreter->outputs().size());
199 
200  if (interpreter->AllocateTensors() != kTfLiteOk) LFATAL("Failed to allocate tensors");
201 
202  LINFO("TensorFlow network ready");
203 
204  // We are ready to rock:
205  itsReady.store(true);
206  });
207 }
208 
209 // ####################################################################################################
211 {
212  try { itsReadyFut.get(); } catch (...) { }
213 }
214 
215 // ####################################################################################################
216 float TensorFlow::predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results)
217 {
218  if (itsNeedReload.load()) loadNet();
219  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
220 
221  int const image_width = cvimg.cols;
222  int const image_height = cvimg.rows;
223  int const image_type = cvimg.type();
224 
225  // get input dimension from the input tensor metadata assuming one input only
226  int input = interpreter->inputs()[0];
227  TfLiteIntArray * dims = interpreter->tensor(input)->dims;
228  int const wanted_height = dims->data[1];
229  int const wanted_width = dims->data[2];
230  int const wanted_channels = dims->data[3];
231 
232  if (wanted_channels != 1 && wanted_channels != 3)
233  LFATAL("Network wants " << wanted_channels << " input channels, but only 1 or 3 are supported");
234  if (wanted_channels == 3 && image_type != CV_8UC3) LFATAL("Network wants RGB but input image is not CV_8UC3");
235  if (wanted_channels == 1 && image_type != CV_8UC1) LFATAL("Network wants Gray but input image is not CV_8UC1");
236 
237  if (image_width != wanted_width || image_height != wanted_height)
238  LFATAL("Wrong input size " << image_width << 'x' << image_height << " but network wants "
239  << wanted_width << 'x' << wanted_height);
240 
241  // Copy input image to input tensor, converting pixel type if needed:
242  switch (interpreter->tensor(input)->type)
243  {
244  case kTfLiteUInt8:
245  {
246  memcpy(interpreter->typed_tensor<uint8_t>(input), cvimg.data, cvimg.total() * cvimg.elemSize());
247  break;
248  }
249  case kTfLiteFloat32:
250  {
251  cv::Mat convimg;
252  if (wanted_channels == 1) cvimg.convertTo(convimg, CV_32FC1, 1.0F / 127.5F, -1.0F);
253  else cvimg.convertTo(convimg, CV_32FC3, 1.0F / 127.5F, -1.0F);
254  memcpy(interpreter->typed_tensor<float>(input), convimg.data, convimg.total() * convimg.elemSize());
255  break;
256  }
257  default:
258  LFATAL("only uint8 or float32 network input pixel types are supported");
259  }
260 
261  // Run the inference:
262  struct timeval start, stop;
263  gettimeofday(&start, 0);
264  if (interpreter->Invoke() != kTfLiteOk) LFATAL("Failed to invoke tflite");
265  gettimeofday(&stop, 0);
266  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
267 
268  const size_t num_results = 5;
269  const float threshold = 0.001f;
270 
271  std::vector<std::pair<float, int>> top_results;
272 
273  int output = interpreter->outputs()[0];
274  switch (interpreter->tensor(output)->type)
275  {
276  case kTfLiteFloat32:
277  get_top_n<float>(interpreter->typed_output_tensor<float>(0), numlabels, results, true);
278  break;
279 
280  case kTfLiteUInt8:
281  get_top_n<uint8_t>(interpreter->typed_output_tensor<uint8_t>(0), numlabels, results, false);
282  break;
283 
284  default:
285  LFATAL("cannot handle output type " << interpreter->tensor(input)->type << " yet");
286  }
287 
288  return predtime;
289 }
290 
291 // ####################################################################################################
292 void TensorFlow::getInDims(int & w, int & h, int & c)
293 {
294  if (itsNeedReload.load()) loadNet();
295  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
296 
297  // get input dimension from the input tensor metadata assuming one input only
298  int input = interpreter->inputs()[0];
299  TfLiteIntArray * dims = interpreter->tensor(input)->dims;
300  h = dims->data[1]; w = dims->data[2]; c = dims->data[3];
301 }
size_t numlabels
Definition: TensorFlow.H:104
void postInit() override
Initialize, configure and load the network in a thread.
Definition: TensorFlow.C:86
void onParamChange(tflow::netdir const &param, std::string const &newval)
Definition: TensorFlow.C:60
void get_top_n(T *prediction, int prediction_size, std::vector< jevois::ObjReco > &top_results, bool input_floating)
Definition: TensorFlow.C:94
std::string value
std::atomic< bool > itsNeedReload
Definition: TensorFlow.H:120
virtual ~TensorFlow()
Virtual destructor for safe inheritance.
Definition: TensorFlow.C:56
#define LERROR(msg)
void getInDims(int &w, int &h, int &c)
Get input width, height, channels.
Definition: TensorFlow.C:292
std::atomic< bool > itsReady
Definition: TensorFlow.H:119
JeVoisReporter itsErrorReporter
Definition: TensorFlow.H:127
std::unique_ptr< tflite::Interpreter > interpreter
Definition: TensorFlow.H:106
int Report(char const *format, va_list args) override
Definition: TensorFlow.C:42
void loadNet()
Definition: TensorFlow.C:131
std::unique_ptr< tflite::FlatBufferModel > model
Definition: TensorFlow.H:105
#define LFATAL(msg)
std::future< void > itsReadyFut
Definition: TensorFlow.H:118
float predict(cv::Mat const &cvimg, std::vector< jevois::ObjReco > &results)
Processing function, results are stored internally in the underlying TensorFlow network object...
Definition: TensorFlow.C:216
void readLabelsFile(std::string const &fname)
Definition: TensorFlow.C:68
friend friend class Component
#define LINFO(msg)
TensorFlow(std::string const &instance)
Constructor.
Definition: TensorFlow.C:51
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:40
void postUninit() override
Un-initialize and free resources.
Definition: TensorFlow.C:210
std::string absolutePath(std::string const &path="")
std::vector< std::string > labels
Definition: TensorFlow.H:103