JeVoisBase  1.21
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
TensorFlow.C
Go to the documentation of this file.
1// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2//
3// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5//
6// This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8// Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10// License for more details. You should have received a copy of the GNU General Public License along with this program;
11// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12//
13// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16/*! \file */
17
18
19// Much of this code inspired by the label_image example for TensorFlow Lite:
20
21/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
22
23Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
24License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
25
26Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
27"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
28specific language governing permissions and limitations under the License. */
29
31#include <jevois/Core/Module.H>
32#include <opencv2/imgproc/imgproc.hpp>
33#include <queue>
34#include <fstream>
35#include <cstdio>
36
37#include <tensorflow/lite/kernels/register.h>
38#include <tensorflow/lite/optional_debug_tools.h>
39#include <tensorflow/lite/string_util.h>
40
41// ####################################################################################################
42int TensorFlow::JeVoisReporter::Report(char const * format, va_list args)
43{
44 char buf[1024];
45 int ret = vsnprintf(buf, 1024, format, args);
46 LERROR(buf);
47 return ret;
48}
49
50// ####################################################################################################
51TensorFlow::TensorFlow(std::string const & instance) :
52 jevois::Component(instance), itsReady(false), itsNeedReload(false)
53{ }
54
55// ####################################################################################################
57{
58 // Wait until any loading is complete:
60}
61
62// ####################################################################################################
63void TensorFlow::onParamChange(tflow::netdir const & param, std::string const & newval)
64{ itsNeedReload.store(true); }
65
66// ####################################################################################################
67void TensorFlow::onParamChange(tflow::dataroot const & param, std::string const & newval)
68{ itsNeedReload.store(true); }
69
70// ####################################################################################################
71void TensorFlow::readLabelsFile(std::string const & fname)
72{
73 std::ifstream file(fname);
74 if (!file) LFATAL("Could not open labels file " << fname);
75 labels.clear();
76
77 std::string line;
78 while (std::getline(file, line)) labels.push_back(line);
79
80 // Tensorflow requires the number of labels to be a multiple of 16:
81 numlabels = labels.size();
82 int const padding = 16;
83 while (labels.size() % padding) labels.emplace_back();
84
85 LINFO("Loaded " << numlabels << " category names from " << fname);
86}
87
88// ####################################################################################################
90{
91 // Start a thread to load the desired network:
92 loadNet();
93}
94
95// ####################################################################################################
96template <class T>
97void TensorFlow::get_top_n(T * prediction, int prediction_size, std::vector<jevois::ObjReco> & top_results,
98 bool input_floating)
99{
100 int const topn = top::get();
101 float const th = thresh::get() * 0.01F;
102 top_results.clear();
103
104 // Will contain top N results in ascending order.
105 std::priority_queue<std::pair<float, int>, std::vector<std::pair<float, int> >,
106 std::greater<std::pair<float, int> > > top_result_pq;
107
108 float const scale = scorescale::get() * (input_floating ? 1.0 : 1.0F / 255.0F);
109
110 for (int i = 0; i < prediction_size; ++i)
111 {
112 float value = prediction[i] * scale;
113
114 // Only add it if it beats the threshold and has a chance at being in the top N.
115 if (value < th) continue;
116
117 top_result_pq.push(std::pair<float, int>(value, i));
118
119 // If at capacity, kick the smallest value out.
120 if (top_result_pq.size() > topn) top_result_pq.pop();
121 }
122
123 // Copy to output vector and reverse into descending order.
124 while (!top_result_pq.empty())
125 {
126 auto const & tr = top_result_pq.top();
127 top_results.push_back( { tr.first * 100, std::string(labels[tr.second]) } );
128 top_result_pq.pop();
129 }
130 std::reverse(top_results.begin(), top_results.end());
131}
132
133// ####################################################################################################
135{
136 // Users will neet to wait until last load is done before they load again:
137 if (itsReadyFut.valid())
138 {
139 if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
140 try { itsReadyFut.get(); } catch (...) { }
141 else
142 throw std::logic_error("Loading already in progress. Attempt to load again rejected");
143 }
144
145 // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
146 itsNeedReload.store(false);
147
148 // We are not ready anymore:
149 itsReady.store(false);
150
151 // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
152 // meantime. itsReady will flip to true when the load is complete.
154 {
155 if (model)
156 {
157 LINFO("Closing model..");
158 model.reset();
159 interpreter.reset();
160 labels.clear();
161 numlabels = 0;
162 }
163
164 std::string root = dataroot::get(); if (root.empty() == false) root += '/';
165 std::string const modelfile = absolutePath(root + netdir::get() + "/model.tflite");
166 std::string const labelfile = absolutePath(root + netdir::get() + "/labels.txt");
167
168 LINFO("Using model from " << modelfile);
169 LINFO("Using labels from " << labelfile);
170
171 // Load the labels:
172 readLabelsFile(labelfile);
173
174 // Create the model from flatbuffer file using mmap:
175 model = tflite::FlatBufferModel::BuildFromFile(modelfile.c_str(), &itsErrorReporter);
176 if (!model) LFATAL("Failed to mmap model " << modelfile);
177 LINFO("Loaded model " << modelfile);
178
179 tflite::ops::builtin::BuiltinOpResolver resolver;
180 tflite::InterpreterBuilder(*model, resolver)(&interpreter);
181 if (!interpreter) LFATAL("Failed to construct interpreter");
182 //////interpreter->UseNNAPI(s->accel);
183
184 LINFO("Tensors size: " << interpreter->tensors_size());
185 LINFO("Nodes size: " << interpreter->nodes_size());
186 LINFO("inputs: " << interpreter->inputs().size());
187 LINFO("input(0) name: " << interpreter->GetInputName(0));
188
189 int t_size = interpreter->tensors_size();
190 for (int i = 0; i < t_size; ++i)
191 if (interpreter->tensor(i)->name)
192 LINFO(i << ": " << interpreter->tensor(i)->name << ", "
193 << interpreter->tensor(i)->bytes << ", "
194 << interpreter->tensor(i)->type << ", "
195 << interpreter->tensor(i)->params.scale << ", "
196 << interpreter->tensor(i)->params.zero_point);
197
198 if (threads::get()) interpreter->SetNumThreads(threads::get());
199
200 LINFO("input: " << interpreter->inputs()[0]);
201 LINFO("number of inputs: " << interpreter->inputs().size());
202 LINFO("number of outputs: " << interpreter->outputs().size());
203
204 if (interpreter->AllocateTensors() != kTfLiteOk) LFATAL("Failed to allocate tensors");
205
206 LINFO("TensorFlow network ready");
207
208 // We are ready to rock:
209 itsReady.store(true);
210 });
211}
212
213// ####################################################################################################
215{
216 try { itsReadyFut.get(); } catch (...) { }
217}
218
219// ####################################################################################################
220float TensorFlow::predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results)
221{
222 if (itsNeedReload.load()) loadNet();
223 if (itsReady.load() == false) throw std::logic_error("not ready yet...");
224
225 int const image_width = cvimg.cols;
226 int const image_height = cvimg.rows;
227 int const image_type = cvimg.type();
228
229 // get input dimension from the input tensor metadata assuming one input only
230 int input = interpreter->inputs()[0];
231 TfLiteIntArray * dims = interpreter->tensor(input)->dims;
232 int const wanted_height = dims->data[1];
233 int const wanted_width = dims->data[2];
234 int const wanted_channels = dims->data[3];
235
236 if (wanted_channels != 1 && wanted_channels != 3)
237 LFATAL("Network wants " << wanted_channels << " input channels, but only 1 or 3 are supported");
238 if (wanted_channels == 3 && image_type != CV_8UC3) LFATAL("Network wants RGB but input image is not CV_8UC3");
239 if (wanted_channels == 1 && image_type != CV_8UC1) LFATAL("Network wants Gray but input image is not CV_8UC1");
240
241 if (image_width != wanted_width || image_height != wanted_height)
242 LFATAL("Wrong input size " << image_width << 'x' << image_height << " but network wants "
243 << wanted_width << 'x' << wanted_height);
244
245 // Copy input image to input tensor, converting pixel type if needed:
246 switch (interpreter->tensor(input)->type)
247 {
248 case kTfLiteUInt8:
249 {
250 memcpy(interpreter->typed_tensor<uint8_t>(input), cvimg.data, cvimg.total() * cvimg.elemSize());
251 break;
252 }
253 case kTfLiteFloat32:
254 {
255 cv::Mat convimg;
256 if (wanted_channels == 1) cvimg.convertTo(convimg, CV_32FC1, 1.0F / 127.5F, -1.0F);
257 else cvimg.convertTo(convimg, CV_32FC3, 1.0F / 127.5F, -1.0F);
258 memcpy(interpreter->typed_tensor<float>(input), convimg.data, convimg.total() * convimg.elemSize());
259 break;
260 }
261 default:
262 LFATAL("only uint8 or float32 network input pixel types are supported");
263 }
264
265 // Run the inference:
266 struct timeval start, stop;
267 gettimeofday(&start, 0);
268 if (interpreter->Invoke() != kTfLiteOk) LFATAL("Failed to invoke tflite");
269 gettimeofday(&stop, 0);
270 float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
271
272 const size_t num_results = 5;
273 const float threshold = 0.001f;
274
275 std::vector<std::pair<float, int>> top_results;
276
277 int output = interpreter->outputs()[0];
278 switch (interpreter->tensor(output)->type)
279 {
280 case kTfLiteFloat32:
281 get_top_n<float>(interpreter->typed_output_tensor<float>(0), numlabels, results, true);
282 break;
283
284 case kTfLiteUInt8:
285 get_top_n<uint8_t>(interpreter->typed_output_tensor<uint8_t>(0), numlabels, results, false);
286 break;
287
288 default:
289 LFATAL("cannot handle output type " << interpreter->tensor(input)->type << " yet");
290 }
291
292 return predtime;
293}
294
295// ####################################################################################################
296void TensorFlow::getInDims(int & w, int & h, int & c)
297{
298 if (itsNeedReload.load()) loadNet();
299 if (itsReady.load() == false) throw std::logic_error("not ready yet...");
300
301 // get input dimension from the input tensor metadata assuming one input only
302 int input = interpreter->inputs()[0];
303 TfLiteIntArray * dims = interpreter->tensor(input)->dims;
304 h = dims->data[1]; w = dims->data[2]; c = dims->data[3];
305}
int h
#define JEVOIS_WAIT_GET_FUTURE(f)
std::atomic< bool > itsReady
Definition TensorFlow.H:119
virtual ~TensorFlow()
Virtual destructor for safe inheritance.
Definition TensorFlow.C:56
void readLabelsFile(std::string const &fname)
Definition TensorFlow.C:71
JeVoisReporter itsErrorReporter
Definition TensorFlow.H:127
float predict(cv::Mat const &cvimg, std::vector< jevois::ObjReco > &results)
Processing function, results are stored internally in the underlying TensorFlow network object.
Definition TensorFlow.C:220
TensorFlow(std::string const &instance)
Constructor.
Definition TensorFlow.C:51
void onParamChange(tflow::netdir const &param, std::string const &newval) override
Definition TensorFlow.C:63
std::unique_ptr< tflite::Interpreter > interpreter
Definition TensorFlow.H:106
void postInit() override
Initialize, configure and load the network in a thread.
Definition TensorFlow.C:89
size_t numlabels
Definition TensorFlow.H:104
std::future< void > itsReadyFut
Definition TensorFlow.H:118
std::atomic< bool > itsNeedReload
Definition TensorFlow.H:120
void get_top_n(T *prediction, int prediction_size, std::vector< jevois::ObjReco > &top_results, bool input_floating)
Definition TensorFlow.C:97
void postUninit() override
Un-initialize and free resources.
Definition TensorFlow.C:214
void getInDims(int &w, int &h, int &c)
Get input width, height, channels.
Definition TensorFlow.C:296
std::vector< std::string > labels
Definition TensorFlow.H:103
void loadNet()
Definition TensorFlow.C:134
std::unique_ptr< tflite::FlatBufferModel > model
Definition TensorFlow.H:105
std::filesystem::path absolutePath(std::filesystem::path const &path="")
friend friend class Component
#define LFATAL(msg)
#define LERROR(msg)
#define LINFO(msg)
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
int Report(char const *format, va_list args) override
Definition TensorFlow.C:42