JeVoisBase  1.6
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Darknet.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 #include <opencv2/imgproc/imgproc.hpp>
21 
22 // ####################################################################################################
23 Darknet::Darknet(std::string const & instance, bool show_detail_params) :
24  jevois::Component(instance), net(nullptr), names(nullptr), classes(0), itsReady(false),
25  itsShowDetailParams(show_detail_params), itsNeedReload(false)
26 {
27  // Get NNPACK ready to rock:
28 #ifdef NNPACK
29  nnp_initialize();
30 #endif
31 
32  // Possibly hide some parameters:
33  if (itsShowDetailParams == false)
34  {
40  }
41 }
42 
43 // ####################################################################################################
45 {
46 #ifdef NNPACK
47  nnp_deinitialize();
48 #endif
49 }
50 
51 // ####################################################################################################
52 void Darknet::onParamChange(dknet::netw const & param, dknet::Net const & newval)
53 {
54  if (itsShowDetailParams == false)
55  {
61  }
62 
63  switch (newval)
64  {
65  case dknet::Net::Reference:
66  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
67  datacfg::set("cfg/imagenet1k.data");
68  cfgfile::set("cfg/darknet.cfg");
69  weightfile::set("weights/darknet.weights");
70  namefile::set("");
71  break;
72 
73  case dknet::Net::Tiny:
74  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
75  datacfg::set("cfg/imagenet1k.data");
76  cfgfile::set("cfg/tiny.cfg");
77  weightfile::set("weights/tiny.weights");
78  namefile::set("");
79  break;
80  /*
81  case dknet::Net::MobileNet:
82  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
83  datacfg::set("cfg/imagenet1k.data");
84  cfgfile::set("cfg/mobilenet.cfg");
85  weightfile::set("weights/mobilenet.weights");
86  namefile::set("");
87  break;
88 
89  case dknet::Net::CompMobileNet:
90  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
91  datacfg::set("cfg/imagenet1k.data");
92  cfgfile::set("cfg/compressmobilenet_0.1.cfg");
93  weightfile::set("weights/compressmobilenet_0.1.weights");
94  namefile::set("");
95  break;
96  */
97  }
98 
99  if (itsShowDetailParams == false)
100  {
102  datacfg::freeze();
103  cfgfile::freeze();
106  }
107 }
108 
109 // ####################################################################################################
110 void Darknet::onParamChange(dknet::dataroot const & param, std::string const & newval) { itsNeedReload.store(true); }
111 void Darknet::onParamChange(dknet::datacfg const & param, std::string const & newval) { itsNeedReload.store(true); }
112 void Darknet::onParamChange(dknet::cfgfile const & param, std::string const & newval) { itsNeedReload.store(true); }
113 void Darknet::onParamChange(dknet::weightfile const & param, std::string const & newval) { itsNeedReload.store(true); }
114 void Darknet::onParamChange(dknet::namefile const & param, std::string const & newval) { itsNeedReload.store(true); }
115 
116 // ####################################################################################################
118 {
119  // Start a thread to load the desired network:
120  loadNet();
121 }
122 
123 // ####################################################################################################
125 {
126  // Users will neet to wait until last load is done before they load again:
127  if (itsReadyFut.valid())
128  {
129  if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
130  try { itsReadyFut.get(); } catch (...) { }
131  else
132  throw std::logic_error("Loading already in progress. Attempt to load again rejected");
133  }
134 
135  // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
136  itsNeedReload.store(false);
137 
138  // We are not ready anymore:
139  itsReady.store(false);
140 
141 #ifdef DARKNET_NNPACK
142  if (net && net->threadpool) pthreadpool_destroy(net->threadpool);
143 #endif
144  if (net) { free_network(net); net = nullptr; }
145 
146  // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
147  // meantime. itsReady will flip to true when the load is complete.
148  itsReadyFut = std::async(std::launch::async, [&]() {
149  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
150 
151  // Note: darknet expects read/write pointers to the file names...
152  std::string const datacf = absolutePath(root + datacfg::get());
153  std::string const cfgfil = absolutePath(root + cfgfile::get());
154  std::string const weightfil = absolutePath(root + weightfile::get());
155 
156  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
157  std::string name_list = namefile::get();
158  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
159  else name_list = absolutePath(root + name_list);
160 
161  LINFO("Using data config from " << datacf);
162  LINFO("Using cfg from " << cfgfil);
163  LINFO("Using weights from " << weightfil);
164  LINFO("Using names from " << name_list);
165 
166  LINFO("Getting labels...");
167  names = get_labels(const_cast<char *>(name_list.c_str()));
168  LINFO("Parsing network and loading weights...");
169  net = load_network(const_cast<char *>(cfgfil.c_str()), const_cast<char *>(weightfil.c_str()), 0);
170  if (net == nullptr) LFATAL("Failed to load darknet network and/or weights -- ABORT");
171  classes = option_find_int(options, "classes", 2);
172 
173  set_batch_network(net, 1);
174  srand(2222222);
175 
176 #ifdef DARKNET_NNPACK
177  net->threadpool = pthreadpool_create(threads::get());
178 #endif
179  LINFO("Darknet network ready");
180 
181  free_list(options);
182 
183  // We are ready to rock:
184  itsReady.store(true);
185  });
186 }
187 
188 // ####################################################################################################
190 {
191  try { itsReadyFut.get(); } catch (...) { }
192 
193  if (net)
194  {
195 #ifdef DARKNET_NNPACK
196  if (net->threadpool) pthreadpool_destroy(net->threadpool);
197 #endif
198  free_network(net);
199  }
200 
201  free_ptrs((void**)names, classes);
202 }
203 
204 // ####################################################################################################
205 bool Darknet::ready() const
206 { return (itsReady.load() == true && itsNeedReload.load() == false); }
207 
208 // ####################################################################################################
209 float Darknet::predict(cv::Mat const & cvimg, std::vector<predresult> & results)
210 {
211  if (itsNeedReload.load()) loadNet();
212  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
213  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
214 
215  int const c = 3; // color channels
216  int const w = cvimg.cols;
217  int const h = cvimg.rows;
218  image im = make_image(w, h, c);
219  for (int k = 0; k < c; ++k)
220  for (int j = 0; j < h; ++j)
221  for (int i = 0; i < w; ++i)
222  {
223  int const dst_index = i + w*j + w*h*k;
224  int const src_index = k + c*i + c*w*j;
225  im.data[dst_index] = float(cvimg.data[src_index]) * (1.0F / 255.0F);
226  }
227 
228  float predtime = predict(im, results);
229 
230  free_image(im);
231 
232  return predtime;
233 }
234 
235 // ####################################################################################################
236 float Darknet::predict(image & im, std::vector<predresult> & results)
237 {
238  if (itsNeedReload.load()) loadNet();
239  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
240  int const topn = top::get();
241  float const th = thresh::get();
242  results.clear();
243 
244  // Resize the network if needed:
245  resize_network(net, im.w, im.h);
246 
247  // Run the predictions:
248  struct timeval start, stop;
249  gettimeofday(&start, 0);
250  float * predictions = network_predict(net, im.data);
251  gettimeofday(&stop, 0);
252 
253  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
254 
255  if (net->hierarchy) hierarchy_predictions(predictions, net->outputs, net->hierarchy, 1, 1);
256 
257  // Get the top scoring predictions and push them into results:
258  int indexes[topn];
259  top_k(predictions, net->outputs, topn, indexes);
260 
261  for (int i = 0; i < topn; ++i)
262  {
263  int const index = indexes[i];
264  float const score = float(predictions[index] * 100);
265  if (score >= th) results.push_back(std::make_pair(score, std::string(names[index])));
266  else break;
267  }
268 
269  return predtime;
270 }
271 
272 // ####################################################################################################
273 void Darknet::resizeInDims(int w, int h)
274 {
275  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
276  resize_network(net, w, h);
277 }
278 
279 // ####################################################################################################
280 void Darknet::getInDims(int & w, int & h, int & c) const
281 {
282  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
283  w = net->w; h = net->h; c = net->c;
284 }
int classes
Definition: Darknet.H:149
void unFreeze()
std::atomic< bool > itsReady
Definition: Darknet.H:161
bool const itsShowDetailParams
Definition: Darknet.H:162
void postUninit() override
Un-initialize and free resources.
Definition: Darknet.C:189
Darknet(std::string const &instance, bool show_detail_params=false)
Constructor.
Definition: Darknet.C:23
char ** names
Definition: Darknet.H:148
network * net
Definition: Darknet.H:147
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Darknet.C:117
Network to load This meta parameter sets parameters cfgfile
Definition: Darknet.H:39
Network to load This meta parameter sets parameters datacfg
Definition: Darknet.H:39
Net
Definition: Darknet.H:39
virtual ~Darknet()
Virtual destructor for safe inheritance.
Definition: Darknet.C:44
std::atomic< bool > itsNeedReload
Definition: Darknet.H:163
void freeze()
#define LFATAL(msg)
void loadNet()
Definition: Darknet.C:124
void getInDims(int &w, int &h, int &c) const
Get input width, height, channels.
Definition: Darknet.C:280
bool ready() const
Return true if th enetwork is ready (i.e., not loading)
Definition: Darknet.C:205
float predict(cv::Mat const &cvimg, std::vector< predresult > &results)
Processing function, results are stored internally in the underlying Darknet network object...
Definition: Darknet.C:209
Network to load This meta parameter sets parameters weightfile
Definition: Darknet.H:39
#define LINFO(msg)
std::future< void > itsReadyFut
Definition: Darknet.H:160
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:39
void onParamChange(dknet::netw const &param, dknet::Net const &newval)
Definition: Darknet.C:52
void resizeInDims(int w, int h)
Resize the network&#39;s input image dims.
Definition: Darknet.C:273
std::string absolutePath(std::string const &path="")