JeVoisBase  1.10
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Darknet.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 #include <opencv2/imgproc/imgproc.hpp>
21 
22 // ####################################################################################################
23 Darknet::Darknet(std::string const & instance, bool show_detail_params) :
24  jevois::Component(instance), net(nullptr), names(nullptr), classes(0), itsReady(false),
25  itsShowDetailParams(show_detail_params), itsNeedReload(false)
26 {
27  // Get NNPACK ready to rock:
28 #ifdef NNPACK
29  nnp_initialize();
30 #endif
31 
32  // Possibly hide some parameters:
33  if (itsShowDetailParams == false)
34  {
40  }
41 }
42 
43 // ####################################################################################################
45 {
46 #ifdef NNPACK
47  nnp_deinitialize();
48 #endif
49 }
50 
51 // ####################################################################################################
52 void Darknet::onParamChange(dknet::netw const & param, dknet::Net const & newval)
53 {
54  if (itsShowDetailParams == false)
55  {
61  }
62 
63  switch (newval)
64  {
65  case dknet::Net::Reference:
66  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
67  datacfg::set("cfg/imagenet1k.data");
68  cfgfile::set("cfg/darknet.cfg");
69  weightfile::set("weights/darknet.weights");
70  namefile::set("");
71  break;
72 
73  case dknet::Net::Tiny:
74  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
75  datacfg::set("cfg/imagenet1k.data");
76  cfgfile::set("cfg/tiny.cfg");
77  weightfile::set("weights/tiny.weights");
78  namefile::set("");
79  break;
80  }
81 
82  if (itsShowDetailParams == false)
83  {
89  }
90 }
91 
92 // ####################################################################################################
93 void Darknet::onParamChange(dknet::dataroot const & param, std::string const & newval) { itsNeedReload.store(true); }
94 void Darknet::onParamChange(dknet::datacfg const & param, std::string const & newval) { itsNeedReload.store(true); }
95 void Darknet::onParamChange(dknet::cfgfile const & param, std::string const & newval) { itsNeedReload.store(true); }
96 void Darknet::onParamChange(dknet::weightfile const & param, std::string const & newval) { itsNeedReload.store(true); }
97 void Darknet::onParamChange(dknet::namefile const & param, std::string const & newval) { itsNeedReload.store(true); }
98 
99 // ####################################################################################################
101 {
102  // Start a thread to load the desired network:
103  loadNet();
104 }
105 
106 // ####################################################################################################
108 {
109  // Users will neet to wait until last load is done before they load again:
110  if (itsReadyFut.valid())
111  {
112  if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
113  try { itsReadyFut.get(); } catch (...) { }
114  else
115  throw std::logic_error("Loading already in progress. Attempt to load again rejected");
116  }
117 
118  // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
119  itsNeedReload.store(false);
120 
121  // We are not ready anymore:
122  itsReady.store(false);
123 
124 #ifdef DARKNET_NNPACK
125  if (net && net->threadpool) pthreadpool_destroy(net->threadpool);
126 #endif
127  if (net) { free_network(net); net = nullptr; }
128 
129  // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
130  // meantime. itsReady will flip to true when the load is complete.
131  itsReadyFut = std::async(std::launch::async, [&]() {
132  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
133 
134  // Note: darknet expects read/write pointers to the file names...
135  std::string const datacf = absolutePath(root + datacfg::get());
136  std::string const cfgfil = absolutePath(root + cfgfile::get());
137  std::string const weightfil = absolutePath(root + weightfile::get());
138 
139  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
140  std::string name_list = namefile::get();
141  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
142  else name_list = absolutePath(root + name_list);
143 
144  LINFO("Using data config from " << datacf);
145  LINFO("Using cfg from " << cfgfil);
146  LINFO("Using weights from " << weightfil);
147  LINFO("Using names from " << name_list);
148 
149  LINFO("Getting labels...");
150  names = get_labels(const_cast<char *>(name_list.c_str()));
151  LINFO("Parsing network and loading weights...");
152 
153  net = load_network(const_cast<char *>(cfgfil.c_str()), const_cast<char *>(weightfil.c_str()), 0);
154 
155  if (net == nullptr)
156  { free_list(options); LFATAL("Failed to load darknet network and/or weights -- ABORT"); }
157 
158  classes = option_find_int(options, "classes", 2);
159 
160  set_batch_network(net, 1);
161  srand(2222222);
162 
163 #ifdef DARKNET_NNPACK
164  net->threadpool = pthreadpool_create(threads::get());
165 #endif
166  LINFO("Darknet network ready");
167 
168  free_list(options);
169 
170  // We are ready to rock:
171  itsReady.store(true);
172  });
173 }
174 
175 // ####################################################################################################
177 {
178  try { itsReadyFut.get(); } catch (...) { }
179 
180  if (net)
181  {
182 #ifdef DARKNET_NNPACK
183  if (net->threadpool) pthreadpool_destroy(net->threadpool);
184 #endif
185  free_network(net);
186  }
187 
188  free_ptrs((void**)names, classes);
189 }
190 
191 // ####################################################################################################
192 float Darknet::predict(cv::Mat const & cvimg, std::vector<jevois::ObjReco> & results)
193 {
194  if (itsNeedReload.load()) loadNet();
195  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
196  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
197 
198  int const c = 3; // color channels
199  int const w = cvimg.cols;
200  int const h = cvimg.rows;
201  image im = make_image(w, h, c);
202  for (int k = 0; k < c; ++k)
203  for (int j = 0; j < h; ++j)
204  for (int i = 0; i < w; ++i)
205  {
206  int const dst_index = i + w*j + w*h*k;
207  int const src_index = k + c*i + c*w*j;
208  im.data[dst_index] = float(cvimg.data[src_index]) * (1.0F / 255.0F);
209  }
210 
211  float predtime = predict(im, results);
212 
213  free_image(im);
214 
215  return predtime;
216 }
217 
218 // ####################################################################################################
219 float Darknet::predict(image & im, std::vector<jevois::ObjReco> & results)
220 {
221  if (itsNeedReload.load()) loadNet();
222  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
223  int const topn = top::get();
224  float const th = thresh::get();
225  results.clear();
226 
227  // Resize the network if needed:
228  resize_network(net, im.w, im.h);
229 
230  // Run the predictions:
231  struct timeval start, stop;
232  gettimeofday(&start, 0);
233  float * predictions = network_predict(net, im.data);
234  gettimeofday(&stop, 0);
235 
236  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
237 
238  if (net->hierarchy) hierarchy_predictions(predictions, net->outputs, net->hierarchy, 1, 1);
239 
240  // Get the top scoring predictions and push them into results:
241  int indexes[topn];
242  top_k(predictions, net->outputs, topn, indexes);
243 
244  for (int i = 0; i < topn; ++i)
245  {
246  int const index = indexes[i];
247  float const score = float(predictions[index] * 100);
248  if (score >= th) results.push_back( { score, std::string(names[index]) } );
249  else break;
250  }
251 
252  return predtime;
253 }
254 
255 // ####################################################################################################
256 void Darknet::resizeInDims(int w, int h)
257 {
258  if (itsNeedReload.load()) loadNet();
259  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
260  resize_network(net, w, h);
261 }
262 
263 // ####################################################################################################
264 void Darknet::getInDims(int & w, int & h, int & c)
265 {
266  if (itsNeedReload.load()) loadNet();
267  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
268  w = net->w; h = net->h; c = net->c;
269 }
int classes
Definition: Darknet.H:144
void unFreeze()
std::atomic< bool > itsReady
Definition: Darknet.H:156
bool const itsShowDetailParams
Definition: Darknet.H:157
void postUninit() override
Un-initialize and free resources.
Definition: Darknet.C:176
Darknet(std::string const &instance, bool show_detail_params=false)
Constructor.
Definition: Darknet.C:23
void getInDims(int &w, int &h, int &c)
Get input width, height, channels.
Definition: Darknet.C:264
char ** names
Definition: Darknet.H:143
network * net
Definition: Darknet.H:142
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Darknet.C:100
Network to load This meta parameter sets parameters cfgfile
Definition: Darknet.H:40
Network to load This meta parameter sets parameters datacfg
Definition: Darknet.H:40
Net
Definition: Darknet.H:40
virtual ~Darknet()
Virtual destructor for safe inheritance.
Definition: Darknet.C:44
std::atomic< bool > itsNeedReload
Definition: Darknet.H:158
void freeze()
#define LFATAL(msg)
void loadNet()
Definition: Darknet.C:107
float predict(cv::Mat const &cvimg, std::vector< jevois::ObjReco > &results)
Processing function, results are stored internally in the underlying Darknet network object...
Definition: Darknet.C:192
Network to load This meta parameter sets parameters weightfile
Definition: Darknet.H:40
#define LINFO(msg)
std::future< void > itsReadyFut
Definition: Darknet.H:155
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:40
void onParamChange(dknet::netw const &param, dknet::Net const &newval)
Definition: Darknet.C:52
void resizeInDims(int w, int h)
Resize the network&#39;s input image dims.
Definition: Darknet.C:256
std::string absolutePath(std::string const &path="")