JeVoisBase  1.17
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
TensorFlow.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 
19 // Much of this code inspired by the label_image example for TensorFlow Lite:
20 
21 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
22 
23 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
24 License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
25 
26 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
27 "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
28 specific language governing permissions and limitations under the License. */
29 
31 #include <jevois/Core/Module.H>
32 #include <opencv2/imgproc/imgproc.hpp>
33 #include <queue>
34 #include <fstream>
35 #include <cstdio>
36 
37 #include <tensorflow/lite/kernels/register.h>
38 #include <tensorflow/lite/optional_debug_tools.h>
39 #include <tensorflow/lite/string_util.h>
40 
41 // ####################################################################################################
42 int TensorFlow::JeVoisReporter::Report(char const * format, va_list args)
43 {
44  char buf[1024];
45  int ret = vsnprintf(buf, 1024, format, args);
46  LERROR(buf);
47  return ret;
48 }
49 
50 // ####################################################################################################
51 TensorFlow::TensorFlow(std::string const & instance) :
52  jevois::Component(instance), itsReady(false), itsNeedReload(false)
53 { }
54 
55 // ####################################################################################################
57 {
58  // Wait until any loading is complete:
60 }
61 
62 // ####################################################################################################
63 void TensorFlow::onParamChange(tflow::netdir const & param, std::string const & newval)
64 { itsNeedReload.store(true); }
65 
66 // ####################################################################################################
67 void TensorFlow::onParamChange(tflow::dataroot const & param, std::string const & newval)
68 { itsNeedReload.store(true); }
69 
70 // ####################################################################################################
71 void TensorFlow::readLabelsFile(std::string const & fname)
72 {
73  std::ifstream file(fname);
74  if (!file) LFATAL("Could not open labels file " << fname);
75  labels.clear();
76 
77  std::string line;
78  while (std::getline(file, line)) labels.push_back(line);
79 
80  // Tensorflow requires the number of labels to be a multiple of 16:
81  numlabels = labels.size();
82  int const padding = 16;
83  while (labels.size() % padding) labels.emplace_back();
84 
85  LINFO("Loaded " << numlabels << " category names from " << fname);
86 }
87 
88 // ####################################################################################################
90 {
91  // Start a thread to load the desired network:
92  loadNet();
93 }
94 
95 // ####################################################################################################
96 template <class T>
97 void TensorFlow::get_top_n(T * prediction, int prediction_size, std::vector<jevois::ObjReco> & top_results,
98  bool input_floating)
99 {
100  int const topn = top::get();
101  float const th = thresh::get() * 0.01F;
102  top_results.clear();
103 
104  // Will contain top N results in ascending order.
105  std::priority_queue<std::pair<float, int>, std::vector<std::pair<float, int> >,
106  std::greater<std::pair<float, int> > > top_result_pq;
107 
108  float const scale = scorescale::get() * (input_floating ? 1.0 : 1.0F / 255.0F);
109 
110  for (int i = 0; i < prediction_size; ++i)
111  {
112  float value = prediction[i] * scale;
113 
114  // Only add it if it beats the threshold and has a chance at being in the top N.
115  if (value < th) continue;
116 
117  top_result_pq.push(std::pair<float, int>(value, i));
118 
119  // If at capacity, kick the smallest value out.
120  if (top_result_pq.size() > topn) top_result_pq.pop();
121  }
122 
123  // Copy to output vector and reverse into descending order.
124  while (!top_result_pq.empty())
125  {
126  auto const & tr = top_result_pq.top();
127  top_results.push_back( { tr.first * 100, std::string(labels[tr.second]) } );
128  top_result_pq.pop();
129  }
130  std::reverse(top_results.begin(), top_results.end());
131 }
132 
133 // ####################################################################################################
135 {
136  // Users will neet to wait until last load is done before they load again:
137  if (itsReadyFut.valid())
138  {
139  if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
140  try { itsReadyFut.get(); } catch (...) { }
141  else
142  throw std::logic_error("Loading already in progress. Attempt to load again rejected");
143  }
144 
145  // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
146  itsNeedReload.store(false);
147 
148  // We are not ready anymore:
149  itsReady.store(false);
150 
151  // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
152  // meantime. itsReady will flip to true when the load is complete.
153  itsReadyFut = jevois::async([&]()
154  {
155  if (model)
156  {
157  LINFO("Closing model..");
158  model.reset();
159  interpreter.reset();
160  labels.clear();
161  numlabels = 0;
162  }
163 
164  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
165  std::string const modelfile = absolutePath(root + netdir::get() + "/model.tflite");
166  std::string const labelfile = absolutePath(root + netdir::get() + "/labels.txt");
167 
168  LINFO("Using model from " << modelfile);
169  LINFO("Using labels from " << labelfile);
170 
171  // Load the labels:
172  readLabelsFile(labelfile);
173 
174  // Create the model from flatbuffer file using mmap:
175  model = tflite::FlatBufferModel::BuildFromFile(modelfile.c_str(), &itsErrorReporter);
176  if (!model) LFATAL("Failed to mmap model " << modelfile);
177  LINFO("Loaded model " << modelfile);
178 
179  tflite::ops::builtin::BuiltinOpResolver resolver;
180  tflite::InterpreterBuilder(*model, resolver)(&interpreter);
181  if (!interpreter) LFATAL("Failed to construct interpreter");
182  //////interpreter->UseNNAPI(s->accel);
183 
184  LINFO("Tensors size: " << interpreter->tensors_size());
185  LINFO("Nodes size: " << interpreter->nodes_size());
186  LINFO("inputs: " << interpreter->inputs().size());
187  LINFO("input(0) name: " << interpreter->GetInputName(0));
188 
189  int t_size = interpreter->tensors_size();
190  for (int i = 0; i < t_size; ++i)
191  if (interpreter->tensor(i)->name)
192  LINFO(i << ": " << interpreter->tensor(i)->name << ", "
193  << interpreter->tensor(i)->bytes << ", "
194  << interpreter->tensor(i)->type << ", "
195  << interpreter->tensor(i)->params.scale << ", "
196  << interpreter->tensor(i)->params.zero_point);
197 
198  if (threads::get()) interpreter->SetNumThreads(threads::get());
199 
200  LINFO("input: " << interpreter->inputs()[0]);
201  LINFO("number of inputs: " << interpreter->inputs().size());
202  LINFO("number of outputs: " << interpreter->outputs().size());
203 
204  if (interpreter->AllocateTensors() != kTfLiteOk) LFATAL("Failed to allocate tensors");
205 
206  LINFO("TensorFlow network ready");
207 
208  // We are ready to rock:
209  itsReady.store(true);
210  });
211 }
212 
213 // ####################################################################################################
215 {
216  try { itsReadyFut.get(); } catch (...) { }
217 }
218 
219 // ####################################################################################################
220 float TensorFlow::predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results)
221 {
222  if (itsNeedReload.load()) loadNet();
223  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
224 
225  int const image_width = cvimg.cols;
226  int const image_height = cvimg.rows;
227  int const image_type = cvimg.type();
228 
229  // get input dimension from the input tensor metadata assuming one input only
230  int input = interpreter->inputs()[0];
231  TfLiteIntArray * dims = interpreter->tensor(input)->dims;
232  int const wanted_height = dims->data[1];
233  int const wanted_width = dims->data[2];
234  int const wanted_channels = dims->data[3];
235 
236  if (wanted_channels != 1 && wanted_channels != 3)
237  LFATAL("Network wants " << wanted_channels << " input channels, but only 1 or 3 are supported");
238  if (wanted_channels == 3 && image_type != CV_8UC3) LFATAL("Network wants RGB but input image is not CV_8UC3");
239  if (wanted_channels == 1 && image_type != CV_8UC1) LFATAL("Network wants Gray but input image is not CV_8UC1");
240 
241  if (image_width != wanted_width || image_height != wanted_height)
242  LFATAL("Wrong input size " << image_width << 'x' << image_height << " but network wants "
243  << wanted_width << 'x' << wanted_height);
244 
245  // Copy input image to input tensor, converting pixel type if needed:
246  switch (interpreter->tensor(input)->type)
247  {
248  case kTfLiteUInt8:
249  {
250  memcpy(interpreter->typed_tensor<uint8_t>(input), cvimg.data, cvimg.total() * cvimg.elemSize());
251  break;
252  }
253  case kTfLiteFloat32:
254  {
255  cv::Mat convimg;
256  if (wanted_channels == 1) cvimg.convertTo(convimg, CV_32FC1, 1.0F / 127.5F, -1.0F);
257  else cvimg.convertTo(convimg, CV_32FC3, 1.0F / 127.5F, -1.0F);
258  memcpy(interpreter->typed_tensor<float>(input), convimg.data, convimg.total() * convimg.elemSize());
259  break;
260  }
261  default:
262  LFATAL("only uint8 or float32 network input pixel types are supported");
263  }
264 
265  // Run the inference:
266  struct timeval start, stop;
267  gettimeofday(&start, 0);
268  if (interpreter->Invoke() != kTfLiteOk) LFATAL("Failed to invoke tflite");
269  gettimeofday(&stop, 0);
270  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
271 
272  const size_t num_results = 5;
273  const float threshold = 0.001f;
274 
275  std::vector<std::pair<float, int>> top_results;
276 
277  int output = interpreter->outputs()[0];
278  switch (interpreter->tensor(output)->type)
279  {
280  case kTfLiteFloat32:
281  get_top_n<float>(interpreter->typed_output_tensor<float>(0), numlabels, results, true);
282  break;
283 
284  case kTfLiteUInt8:
285  get_top_n<uint8_t>(interpreter->typed_output_tensor<uint8_t>(0), numlabels, results, false);
286  break;
287 
288  default:
289  LFATAL("cannot handle output type " << interpreter->tensor(input)->type << " yet");
290  }
291 
292  return predtime;
293 }
294 
295 // ####################################################################################################
296 void TensorFlow::getInDims(int & w, int & h, int & c)
297 {
298  if (itsNeedReload.load()) loadNet();
299  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
300 
301  // get input dimension from the input tensor metadata assuming one input only
302  int input = interpreter->inputs()[0];
303  TfLiteIntArray * dims = interpreter->tensor(input)->dims;
304  h = dims->data[1]; w = dims->data[2]; c = dims->data[3];
305 }
TensorFlow::~TensorFlow
virtual ~TensorFlow()
Virtual destructor for safe inheritance.
Definition: TensorFlow.C:56
JEVOIS_WAIT_GET_FUTURE
#define JEVOIS_WAIT_GET_FUTURE(f)
jevois::ParameterRegistry::Component
friend friend class Component
Module.H
TensorFlow::TensorFlow
TensorFlow(std::string const &instance)
Constructor.
Definition: TensorFlow.C:51
TensorFlow::loadNet
void loadNet()
Definition: TensorFlow.C:134
TensorFlow::model
std::unique_ptr< tflite::FlatBufferModel > model
Definition: TensorFlow.H:105
TensorFlow::postUninit
void postUninit() override
Un-initialize and free resources.
Definition: TensorFlow.C:214
value
std::string value
TensorFlow::labels
std::vector< std::string > labels
Definition: TensorFlow.H:103
TensorFlow::predict
float predict(cv::Mat const &cvimg, std::vector< jevois::ObjReco > &results)
Processing function, results are stored internally in the underlying TensorFlow network object.
Definition: TensorFlow.C:220
LERROR
#define LERROR(msg)
test-model.ret
ret
Definition: test-model.py:22
file
Data collection mode RAW means that the latest available raw data is returned each time hence timing may not be very accurate depending on how regularly grate into a FIFO and and accumulates resulting output data into the IMU s internal FIFO buffer at a fixed rate This parameter can only be set in a module s params cfg file
format
Type of detection output format
TensorFlow::numlabels
size_t numlabels
Definition: TensorFlow.H:104
TensorFlow::readLabelsFile
void readLabelsFile(std::string const &fname)
Definition: TensorFlow.C:71
TensorFlow::postInit
void postInit() override
Initialize, configure and load the network in a thread.
Definition: TensorFlow.C:89
TensorFlow::onParamChange
void onParamChange(tflow::netdir const &param, std::string const &newval) override
Definition: TensorFlow.C:63
jevois
F
float F
TensorFlow::JeVoisReporter::Report
int Report(char const *format, va_list args) override
Definition: TensorFlow.C:42
TensorFlow::itsNeedReload
std::atomic< bool > itsNeedReload
Definition: TensorFlow.H:120
dknet::dataroot
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:41
LFATAL
#define LFATAL(msg)
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
TensorFlow::itsReady
std::atomic< bool > itsReady
Definition: TensorFlow.H:119
TensorFlow::interpreter
std::unique_ptr< tflite::Interpreter > interpreter
Definition: TensorFlow.H:106
TensorFlow::getInDims
void getInDims(int &w, int &h, int &c)
Get input width, height, channels.
Definition: TensorFlow.C:296
TensorFlow::itsErrorReporter
JeVoisReporter itsErrorReporter
Definition: TensorFlow.H:127
h
int h
TensorFlow::get_top_n
void get_top_n(T *prediction, int prediction_size, std::vector< jevois::ObjReco > &top_results, bool input_floating)
Definition: TensorFlow.C:97
LINFO
#define LINFO(msg)
jevois::Component::absolutePath
std::string absolutePath(std::string const &path="")
TensorFlow::itsReadyFut
std::future< void > itsReadyFut
Definition: TensorFlow.H:118
TensorFlow.H