JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Darknet.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 #include <opencv2/imgproc/imgproc.hpp>
21 
22 // ####################################################################################################
23 Darknet::Darknet(std::string const & instance, bool show_detail_params) :
24  jevois::Component(instance), itsReady(false), itsShowDetailParams(show_detail_params), itsNeedReload(false)
25 {
26 #ifdef DARKNET_NNPACK
27  net.threadpool = 0;
28 #endif
29 
30  // Get NNPACK ready to rock:
31 #ifdef NNPACK
32  nnp_initialize();
33 #endif
34 
35  // Possibly hide some parameters:
36  if (itsShowDetailParams == false)
37  {
43  }
44 }
45 
46 // ####################################################################################################
48 {
49 #ifdef NNPACK
50  nnp_deinitialize();
51 #endif
52 }
53 
54 // ####################################################################################################
55 void Darknet::onParamChange(dknet::netw const & param, dknet::Net const & newval)
56 {
57  if (itsShowDetailParams == false)
58  {
64  }
65 
66  switch (newval)
67  {
68  case dknet::Net::Reference:
69  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
70  datacfg::set("cfg/imagenet1k.data");
71  cfgfile::set("cfg/darknet.cfg");
72  weightfile::set("weights/darknet.weights");
73  namefile::set("");
74  break;
75 
76  case dknet::Net::Tiny:
77  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
78  datacfg::set("cfg/imagenet1k.data");
79  cfgfile::set("cfg/tiny.cfg");
80  weightfile::set("weights/tiny.weights");
81  namefile::set("");
82 
83  /* FIXME: need to implement depthwise convolutions first
84  case dknet::Net::MobileNet:
85  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
86  datacfg::set("cfg/imagenet1k.data");
87  cfgfile::set("cfg/mobilenet.cfg");
88  weightfile::set("weights/mobilenet.weights");
89  namefile::set("");
90  break;
91 
92  case dknet::Net::CompMobileNet:
93  dataroot::set(JEVOIS_SHARE_PATH "/darknet/single");
94  datacfg::set("cfg/imagenet1k.data");
95  cfgfile::set("cfg/compressmobilenet_0.1.cfg");
96  weightfile::set("weights/compressmobilenet_0.1.weights");
97  namefile::set("");
98  break;
99  */
100  }
101 
102  if (itsShowDetailParams == false)
103  {
105  datacfg::freeze();
106  cfgfile::freeze();
109  }
110 
111 }
112 
113 // ####################################################################################################
114 void Darknet::onParamChange(dknet::dataroot const & param, std::string const & newval) { itsNeedReload.store(true); }
115 void Darknet::onParamChange(dknet::datacfg const & param, std::string const & newval) { itsNeedReload.store(true); }
116 void Darknet::onParamChange(dknet::cfgfile const & param, std::string const & newval) { itsNeedReload.store(true); }
117 void Darknet::onParamChange(dknet::weightfile const & param, std::string const & newval) { itsNeedReload.store(true); }
118 void Darknet::onParamChange(dknet::namefile const & param, std::string const & newval) { itsNeedReload.store(true); }
119 
120 // ####################################################################################################
122 {
123  // Start a thread to load the desired network:
124  loadNet();
125 }
126 
127 // ####################################################################################################
129 {
130  // Users will neet to wait until last load is done before they load again:
131  if (itsReadyFut.valid())
132  {
133  if (itsReadyFut.wait_for(std::chrono::milliseconds(5)) == std::future_status::ready)
134  try { itsReadyFut.get(); } catch (...) { }
135  else
136  throw std::logic_error("Loading already in progress. Attempt to load again rejected");
137  }
138 
139  // Flip itsNeedReload to false so we do not get called several times for the same need to reload:
140  itsNeedReload.store(false);
141 
142  // We are not ready anymore:
143  itsReady.store(false);
144 
145 #ifdef DARKNET_NNPACK
146  if (net.threadpool) pthreadpool_destroy(net.threadpool);
147 #endif
148 
149  // Since loading big networks can take a while, do it in a thread so we can keep streaming video in the
150  // meantime. itsReady will flip to true when the load is complete.
151  itsReadyFut = std::async(std::launch::async, [&]() {
152  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
153 
154  // Note: darknet expects read/write pointers to the file names...
155  std::string const datacf = absolutePath(root + datacfg::get());
156  std::string const cfgfil = absolutePath(root + cfgfile::get());
157  std::string const weightfil = absolutePath(root + weightfile::get());
158 
159  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
160  std::string name_list = namefile::get();
161  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
162  else name_list = absolutePath(root + name_list);
163 
164  LINFO("Using data config from " << datacf);
165  LINFO("Using cfg from " << cfgfil);
166  LINFO("Using weights from " << weightfil);
167  LINFO("Using names from " << name_list);
168 
169  LINFO("Getting labels...");
170  names = get_labels(const_cast<char *>(name_list.c_str()));
171  LINFO("Parsing network...");
172  net = parse_network_cfg(const_cast<char *>(cfgfil.c_str()));
173  LINFO("Loading weights...");
174  load_weights(&net, const_cast<char *>(weightfil.c_str()));
175  classes = option_find_int(options, "classes", 2);
176 
177  set_batch_network(&net, 1);
178  srand(2222222);
179 
180 #ifdef DARKNET_NNPACK
181  net.threadpool = pthreadpool_create(threads::get());
182 #endif
183  LINFO("Darknet network ready");
184 
185  free_list(options);
186 
187  // We are ready to rock:
188  itsReady.store(true);
189  });
190 }
191 
192 // ####################################################################################################
194 {
195  try { itsReadyFut.get(); } catch (...) { }
196 
197  free_network(net);
198  free_ptrs((void**)names, classes);
199 
200 #ifdef DARKNET_NNPACK
201  pthreadpool_destroy(net.threadpool);
202 #endif
203 }
204 
205 // ####################################################################################################
206 bool Darknet::ready() const
207 { return (itsReady.load() == true && itsNeedReload.load() == false); }
208 
209 // ####################################################################################################
210 float Darknet::predict(cv::Mat const & cvimg, std::vector<predresult> & results)
211 {
212  if (itsNeedReload.load()) loadNet();
213  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
214  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
215 
216  int const c = 3; // color channels
217  int const w = cvimg.cols;
218  int const h = cvimg.rows;
219  image im = make_image(w, h, c);
220  for (int k = 0; k < c; ++k)
221  for (int j = 0; j < h; ++j)
222  for (int i = 0; i < w; ++i)
223  {
224  int const dst_index = i + w*j + w*h*k;
225  int const src_index = k + c*i + c*w*j;
226  im.data[dst_index] = float(cvimg.data[src_index]) * (1.0F / 255.0F);
227  }
228 
229  float predtime = predict(im, results);
230 
231  free_image(im);
232 
233  return predtime;
234 }
235 
236 // ####################################################################################################
237 float Darknet::predict(image & im, std::vector<predresult> & results)
238 {
239  if (itsNeedReload.load()) loadNet();
240  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
241  int const topn = top::get();
242  float const th = thresh::get();
243  results.clear();
244 
245  // Resize the network if needed:
246  resize_network(&net, im.w, im.h);
247 
248  // Run the predictions:
249  struct timeval start, stop;
250  gettimeofday(&start, 0);
251  float * predictions = network_predict(net, im.data);
252  gettimeofday(&stop, 0);
253 
254  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
255 
256  if (net.hierarchy) hierarchy_predictions(predictions, net.outputs, net.hierarchy, 1, 1);
257 
258  // Get the top scoring predictions and push them into results:
259  int indexes[topn];
260  top_k(predictions, net.outputs, topn, indexes);
261 
262  for (int i = 0; i < topn; ++i)
263  {
264  int const index = indexes[i];
265  float const score = float(predictions[index] * 100);
266  if (score >= th) results.push_back(std::make_pair(score, std::string(names[index])));
267  else break;
268  }
269 
270  return predtime;
271 }
272 
273 // ####################################################################################################
274 void Darknet::resizeInDims(int w, int h)
275 {
276  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
277  resize_network(&net, w, h);
278 }
279 
280 // ####################################################################################################
281 void Darknet::getInDims(int & w, int & h, int & c) const
282 {
283  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
284  w = net.w; h = net.h; c = net.c;
285 }
int classes
Definition: Darknet.H:149
void unFreeze()
std::atomic< bool > itsReady
Definition: Darknet.H:161
bool const itsShowDetailParams
Definition: Darknet.H:162
void postUninit() override
Un-initialize and free resources.
Definition: Darknet.C:193
Darknet(std::string const &instance, bool show_detail_params=false)
Constructor.
Definition: Darknet.C:23
network net
Definition: Darknet.H:147
char ** names
Definition: Darknet.H:148
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Darknet.C:121
Network to load This meta parameter sets parameters cfgfile
Definition: Darknet.H:39
Network to load This meta parameter sets parameters datacfg
Definition: Darknet.H:39
Net
Definition: Darknet.H:39
virtual ~Darknet()
Virtual destructor for safe inheritance.
Definition: Darknet.C:47
std::atomic< bool > itsNeedReload
Definition: Darknet.H:163
void freeze()
#define LFATAL(msg)
void loadNet()
Definition: Darknet.C:128
void getInDims(int &w, int &h, int &c) const
Get input width, height, channels.
Definition: Darknet.C:281
bool ready() const
Return true if th enetwork is ready (i.e., not loading)
Definition: Darknet.C:206
float predict(cv::Mat const &cvimg, std::vector< predresult > &results)
Processing function, results are stored internally in the underlying Darknet network object...
Definition: Darknet.C:210
Network to load This meta parameter sets parameters weightfile
Definition: Darknet.H:39
#define LINFO(msg)
std::future< void > itsReadyFut
Definition: Darknet.H:160
Network to load This meta parameter sets parameters dataroot
Definition: Darknet.H:39
void onParamChange(dknet::netw const &param, dknet::Net const &newval)
Definition: Darknet.C:55
void resizeInDims(int w, int h)
Resize the network&#39;s input image dims.
Definition: Darknet.C:274
std::string absolutePath(std::string const &path="")