JeVoisBase  1.16
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Darknet.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 #include <opencv2/imgproc/imgproc.hpp>
21 #include <darknet-nnpack/src/classifier.h>
22 #include <darknet-nnpack/src/option_list.h>
23 #include <darknet-nnpack/src/data.h>
24 #include <darknet-nnpack/src/network.h>
25 #include <stdlib.h> // for free()
26 
27 // ####################################################################################################
28 Darknet::Darknet(std::string const & instance, bool show_detail_params) :
29  jevois::Component(instance), net(nullptr), names(nullptr), classes(0), itsReady(false),
30  itsShowDetailParams(show_detail_params), itsNeedReload(false)
31 {
32  // Get NNPACK ready to rock:
33 #ifdef NNPACK
34  nnp_initialize();
35 #endif
36 
37  // Possibly hide some parameters:
38  if (itsShowDetailParams == false)
39  {
45  }
46 }
47 
48 // ####################################################################################################
50 {
51  // Wait until any loading is complete:
53 
54 #ifdef NNPACK
55  nnp_deinitialize();
56 #endif
57 }
58 
59 // ####################################################################################################
60 void Darknet::onParamChange(dknet::netw const & param, dknet::Net const & newval)
61 {
62  if (itsShowDetailParams == false)
63  {
69  }
70 
71  switch (newval)
72  {
73  case dknet::Net::Reference:
74  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
75  datacfg::set("cfg/imagenet1k.data");
76  cfgfile::set("cfg/darknet.cfg");
77  weightfile::set("weights/darknet.weights");
78  namefile::set("");
79  break;
80 
81  case dknet::Net::Tiny:
82  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
83  datacfg::set("cfg/imagenet1k.data");
84  cfgfile::set("cfg/tiny.cfg");
85  weightfile::set("weights/tiny.weights");
86  namefile::set("");
87  break;
88  }
89 
90  if (itsShowDetailParams == false)
91  {
97  }
98 }
99 
100 // ####################################################################################################
101 void Darknet::onParamChange(dknet::dataroot const & param, std::string const & newval) { itsNeedReload.store(true); }
102 void Darknet::onParamChange(dknet::datacfg const & param, std::string const & newval) { itsNeedReload.store(true); }
103 void Darknet::onParamChange(dknet::cfgfile const & param, std::string const & newval) { itsNeedReload.store(true); }
104 void Darknet::onParamChange(dknet::weightfile const & param, std::string const & newval) { itsNeedReload.store(true); }
105 void Darknet::onParamChange(dknet::namefile const & param, std::string const & newval) { itsNeedReload.store(true); }
106 
107 // ####################################################################################################
109 {
110  // Start a thread to load the desired network:
111  loadNet();
112 }
113 
114 // ####################################################################################################
116 {
117  // Users will neet to wait until last load is done before they load again:
118  if (itsReadyFut.valid())
119  {
120  if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
121  try { itsReadyFut.get(); } catch (...) { }
122  else
123  throw std::logic_error("Loading already in progress. Attempt to load again rejected");
124  }
125 
126  // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
127  itsNeedReload.store(false);
128 
129  // We are not ready anymore:
130  itsReady.store(false);
131 
132 #ifdef DARKNET_NNPACK
133  if (net && net->threadpool) pthreadpool_destroy(net->threadpool);
134 #endif
135  if (net) { free_network(*net); free(net); net = nullptr; }
136 
137  // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
138  // meantime. itsReady will flip to true when the load is complete.
139  itsReadyFut = jevois::async([&]() {
140  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
141 
142  // Note: darknet expects read/write pointers to the file names...
143  std::string const datacf = absolutePath(root + datacfg::get());
144  std::string const cfgfil = absolutePath(root + cfgfile::get());
145  std::string const weightfil = absolutePath(root + weightfile::get());
146 
147  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
148  std::string name_list = namefile::get();
149  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
150  else name_list = absolutePath(root + name_list);
151 
152  LINFO("Using data config from " << datacf);
153  LINFO("Using cfg from " << cfgfil);
154  LINFO("Using weights from " << weightfil);
155  LINFO("Using names from " << name_list);
156 
157  LINFO("Getting labels...");
158  names = get_labels(const_cast<char *>(name_list.c_str()));
159  LINFO("Parsing network and loading weights...");
160 
161  net = load_network(const_cast<char *>(cfgfil.c_str()), const_cast<char *>(weightfil.c_str()), 0);
162 
163  if (net == nullptr)
164  { free_list(options); LFATAL("Failed to load darknet network and/or weights -- ABORT"); }
165 
166  classes = option_find_int(options, "classes", 2);
167 
168  set_batch_network(net, 1);
169  srand(2222222);
170 
171 #ifdef DARKNET_NNPACK
172  net->threadpool = pthreadpool_create(threads::get());
173 #endif
174  LINFO("Darknet network ready");
175 
176  free_list(options);
177 
178  // We are ready to rock:
179  itsReady.store(true);
180  });
181 }
182 
183 // ####################################################################################################
185 {
186  try { itsReadyFut.get(); } catch (...) { }
187 
188  if (net)
189  {
190 #ifdef DARKNET_NNPACK
191  if (net->threadpool) pthreadpool_destroy(net->threadpool);
192 #endif
193  free_network(*net);
194  free(net);
195  net = nullptr;
196  }
197 
198  free_ptrs((void**)names, classes);
199 }
200 
201 // ####################################################################################################
202 float Darknet::predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results)
203 {
204  if (itsNeedReload.load()) loadNet();
205  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
206  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
207 
208  int const c = 3; // color channels
209  int const w = cvimg.cols;
210  int const h = cvimg.rows;
211  image im = make_image(w, h, c);
212  for (int k = 0; k < c; ++k)
213  for (int j = 0; j < h; ++j)
214  for (int i = 0; i < w; ++i)
215  {
216  int const dst_index = i + w*j + w*h*k;
217  int const src_index = k + c*i + c*w*j;
218  im.data[dst_index] = float(cvimg.data[src_index]) * (1.0F / 255.0F);
219  }
220 
221  float predtime = predict(im, results);
222 
223  free_image(im);
224 
225  return predtime;
226 }
227 
228 // ####################################################################################################
229 float Darknet::predict(image & im, std::vector<jevois::ObjReco> & results)
230 {
231  if (itsNeedReload.load()) loadNet();
232  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
233  int const topn = top::get();
234  float const th = thresh::get();
235  results.clear();
236 
237  // Resize the network if needed:
238  resize_network(net, im.w, im.h);
239 
240  // Run the predictions:
241  struct timeval start, stop;
242  gettimeofday(&start, 0);
243  float * predictions = network_predict(*net, im.data);
244  gettimeofday(&stop, 0);
245 
246  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
247 
248  if (net->hierarchy) hierarchy_predictions(predictions, net->outputs, net->hierarchy, 1);
249 
250  // Get the top scoring predictions and push them into results:
251  int indexes[topn];
252  top_k(predictions, net->outputs, topn, indexes);
253 
254  for (int i = 0; i < topn; ++i)
255  {
256  int const index = indexes[i];
257  float const score = float(predictions[index] * 100);
258  if (score >= th) results.push_back( { score, std::string(names[index]) } );
259  else break;
260  }
261 
262  return predtime;
263 }
264 
265 // ####################################################################################################
266 void Darknet::resizeInDims(int w, int h)
267 {
268  if (itsNeedReload.load()) loadNet();
269  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
270  resize_network(net, w, h);
271 }
272 
273 // ####################################################################################################
274 void Darknet::getInDims(int & w, int & h, int & c)
275 {
276  if (itsNeedReload.load()) loadNet();
277  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
278  w = net->w; h = net->h; c = net->c;
279 }
JEVOIS_WAIT_GET_FUTURE
#define JEVOIS_WAIT_GET_FUTURE(f)
Module.H
Darknet::loadNet
void loadNet()
Definition: Darknet.C:115
Darknet::itsReadyFut
std::future< void > itsReadyFut
Definition: Darknet.H:155
Darknet::Darknet
Darknet(std::string const &instance, bool show_detail_params=false)
Constructor.
Definition: Darknet.C:28
jevois
Darknet::~Darknet
virtual ~Darknet()
Virtual destructor for safe inheritance.
Definition: Darknet.C:49
Darknet::onParamChange
void onParamChange(dknet::netw const &param, dknet::Net const &newval) override
Definition: Darknet.C:60
Darknet::net
network * net
Definition: Darknet.H:142
dknet::weightfile
Network to load This meta parameter sets parameters weightfile
Definition: Darknet.H:41
Darknet::classes
int classes
Definition: Darknet.H:144
dknet::datacfg
Network to load This meta parameter sets parameters datacfg
Definition: Darknet.H:41
test-model.net
net
Definition: test-model.py:10
dknet::dataroot
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:41
LFATAL
#define LFATAL(msg)
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Darknet::names
char ** names
Definition: Darknet.H:143
unFreeze
void unFreeze()
dknet::cfgfile
Network to load This meta parameter sets parameters cfgfile
Definition: Darknet.H:41
Darknet::resizeInDims
void resizeInDims(int w, int h)
Resize the network's input image dims.
Definition: Darknet.C:266
dknet::Net
Net
Definition: Darknet.H:40
h
int h
freeze
void freeze()
Darknet::getInDims
void getInDims(int &w, int &h, int &c)
Get input width, height, channels.
Definition: Darknet.C:274
Darknet::itsNeedReload
std::atomic< bool > itsNeedReload
Definition: Darknet.H:158
LINFO
#define LINFO(msg)
jevois::Component::absolutePath
std::string absolutePath(std::string const &path="")
Darknet::predict
float predict(cv::Mat const &cvimg, std::vector< jevois::ObjReco > &results)
Processing function, results are stored internally in the underlying Darknet network object.
Definition: Darknet.C:202
Darknet::postUninit
void postUninit() override
Un-initialize and free resources.
Definition: Darknet.C:184
Darknet::postInit
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Darknet.C:108
Darknet::itsShowDetailParams
const bool itsShowDetailParams
Definition: Darknet.H:157
Darknet.H
Darknet::itsReady
std::atomic< bool > itsReady
Definition: Darknet.H:156