JeVoisBase  1.11
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Yolo.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 
21 // ####################################################################################################
22 Yolo::Yolo(std::string const & instance) : jevois::Component(instance), net(nullptr), names(nullptr), nboxes(0),
23  dets(nullptr), classes(0), map(nullptr), itsReady(false)
24 {
25  // Get NNPACK ready to rock:
26 #ifdef NNPACK
27  nnp_initialize();
28 #endif
29 }
30 
31 // ####################################################################################################
33 {
34 #ifdef NNPACK
35  nnp_deinitialize();
36 #endif
37 }
38 
39 // ####################################################################################################
41 {
42  itsReadyFut = std::async(std::launch::async, [&]() {
43  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
44 
45  // Note: darknet expects read/write pointers to the file names...
46  std::string const datacf = absolutePath(root + datacfg::get());
47  std::string const cfgfil = absolutePath(root + cfgfile::get());
48  std::string const weightfil = absolutePath(root + weightfile::get());
49 
50  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
51  std::string name_list = namefile::get();
52  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
53  else name_list = absolutePath(root + name_list);
54 
55  LINFO("Using data config from " << datacf);
56  LINFO("Using cfg from " << cfgfil);
57  LINFO("Using weights from " << weightfil);
58  LINFO("Using names from " << name_list);
59 
60  LINFO("Getting labels...");
61  names = get_labels(const_cast<char *>(name_list.c_str()));
62 
63  char * mapf = option_find_str(options, "map", 0);
64  if (mapf) map = read_map(mapf);
65 
66  LINFO("Parsing network and loading weights...");
67 
68  net = load_network(const_cast<char *>(cfgfil.c_str()), const_cast<char *>(weightfil.c_str()), 0);
69 
70  if (net == nullptr)
71  {
72  free_list(options);
73  if (map) { free(map); map = nullptr; }
74  LFATAL("Failed to load YOLO network and/or weights -- ABORT");
75  }
76 
77  classes = option_find_int(options, "classes", 2);
78 
79  set_batch_network(net, 1);
80  srand(2222222);
81  LINFO("YOLO network ready");
82 
83 #ifdef DARKNET_NNPACK
84  net->threadpool = pthreadpool_create(threads::get());
85 #endif
86  free_list(options);
87  itsReady.store(true);
88  });
89 }
90 
91 // ####################################################################################################
93 {
94  if (itsReadyFut.valid()) try { itsReadyFut.get(); } catch (...) { }
95 
96  if (dets) { free_detections(dets, nboxes); dets = nullptr; nboxes = 0; }
97 
98  if (map) { free(map); map = nullptr; }
99 
100  if (net)
101  {
102 #ifdef DARKNET_NNPACK
103  if (net->threadpool) pthreadpool_destroy(net->threadpool);
104 #endif
105  free_network(net);
106  net = nullptr;
107  }
108 
109  free_ptrs((void**)names, classes);
110  names = nullptr; classes = 0;
111 }
112 
113 // ####################################################################################################
114 float Yolo::predict(cv::Mat const & cvimg)
115 {
116  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
117  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
118 
119  int const c = 3; // color channels
120  int const w = cvimg.cols;
121  int const h = cvimg.rows;
122 
123  image im = make_image(w, h, c);
124  for (int k = 0; k < c; ++k)
125  for (int j = 0; j < h; ++j)
126  for (int i = 0; i < w; ++i)
127  {
128  int dst_index = i + w*j + w*h*k;
129  int src_index = k + c*i + c*w*j;
130  im.data[dst_index] = float(cvimg.data[src_index]) / 255.0F;
131  }
132 
133  float predtime = predict(im);
134 
135  free_image(im);
136 
137  return predtime;
138 }
139 
140 // ####################################################################################################
141 float Yolo::predict(image & im)
142 {
143  image sized; bool need_free = false;
144  if (im.w == net->w && im.h == net->h) sized = im;
145  else { sized = letterbox_image(im, net->w, net->h); need_free = true; }
146 
147  struct timeval start, stop;
148 
149  gettimeofday(&start, 0);
150  network_predict(net, sized.data);
151  gettimeofday(&stop, 0);
152 
153  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
154 
155  if (need_free) free_image(sized);
156 
157  return predtime;
158 }
159 
160 // ####################################################################################################
161 void Yolo::computeBoxes(int inw, int inh)
162 {
163  layer & l = net->layers[net->n-1];
164 
165  if (dets) { free_detections(dets, nboxes); dets = nullptr; nboxes = 0; }
166 
167  dets = get_network_boxes(net, 1, 1, thresh::get() * 0.01F, hierthresh::get() * 0.01F, map, 0, &nboxes);
168 
169  float const nmsval = nms::get() * 0.01F;
170 
171  if (nmsval) do_nms_sort(dets, nboxes, l.classes, nmsval);
172 }
173 
174 // ####################################################################################################
175 void Yolo::drawDetections(jevois::RawImage & outimg, int inw, int inh, int xoff, int yoff)
176 {
177  float const thval = thresh::get();
178 
179  for (int i = 0; i < nboxes; ++i)
180  {
181  // For each detection, We need to get a list of labels and probabilities, sorted by score:
182  std::vector<jevois::ObjReco> data;
183 
184  for (int j = 0; j < classes; ++j)
185  {
186  float const p = dets[i].prob[j] * 100.0F;
187  if (p > thval) data.push_back( { p, std::string(names[j]) } );
188  }
189 
190  // End here if nothing above threshold:
191  if (data.empty()) continue;
192 
193  // Sort in descending order:
194  std::sort(data.begin(), data.end(), [](auto a, auto b) { return (b.score < a.score); });
195 
196  // Create our display label:
197  std::string labelstr;
198  for (auto const & d : data)
199  {
200  if (labelstr.empty() == false) labelstr += ", ";
201  labelstr += jevois::sformat("%s:%.1f", d.category.c_str(), d.score);
202  }
203 
204  box const & b = dets[i].bbox;
205 
206  int const left = std::max(xoff, int(xoff + (b.x - b.w / 2.0F) * inw + 0.499F));
207  int const bw = std::min(inw, int(b.w * inw + 0.499F));
208  int const top = std::max(yoff, int(yoff + (b.y - b.h / 2.0F) * inh + 0.499F));
209  int const bh = std::min(inh, int(b.h * inh + 0.499F));
210 
211  jevois::rawimage::drawRect(outimg, left, top, bw, bh, 2, jevois::yuyv::LightGreen);
212  jevois::rawimage::writeText(outimg, labelstr,
213  left + 6, top + 2, jevois::yuyv::LightGreen, jevois::rawimage::Font10x20);
214  }
215 }
216 
217 // ####################################################################################################
218 void Yolo::sendSerial(jevois::StdModule * mod, int inw, int inh)
219 {
220  float const thval = thresh::get();
221 
222  for (int i = 0; i < nboxes; ++i)
223  {
224  // For each detection, We need to get a list of labels and probabilities, sorted by score:
225  std::vector<jevois::ObjReco> data;
226 
227  for (int j = 0; j < classes; ++j)
228  {
229  float const p = dets[i].prob[j] * 100.0F;
230  if (p > thval) data.push_back({ p, std::string(names[j]) });
231  }
232 
233  // End here if nothing above threshold:
234  if (data.empty()) continue;
235 
236  // Sort in descending order:
237  std::sort(data.begin(), data.end(), [](auto a, auto b) { return (b.score < a.score); });
238 
239  box const & b = dets[i].bbox;
240 
241  int const left = (b.x - b.w / 2.0F) * inw;
242  int const bw = b.w * inw;
243  int const top = (b.y - b.h / 2.0F) * inh;
244  int const bh = b.h * inh;
245 
246  mod->sendSerialObjDetImg2D(inw, inh, left, top, bw, bh, data);
247  }
248 }
249 
250 // ####################################################################################################
251 void Yolo::resizeInDims(int w, int h)
252 {
253  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
254  resize_network(net, w, h);
255 }
256 
257 // ####################################################################################################
258 void Yolo::getInDims(int & w, int & h, int & c) const
259 {
260  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
261  w = net->w; h = net->h; c = net->c;
262 }
void computeBoxes(int inw, int inh)
Compute the boxes.
Definition: Yolo.C:161
std::atomic< bool > itsReady
Definition: Yolo.H:154
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
detection * dets
Definition: Yolo.H:148
void postUninit() override
Un-initialize and free resources.
Definition: Yolo.C:92
void resizeInDims(int w, int h)
Resize the network&#39;s input image dims.
Definition: Yolo.C:251
virtual ~Yolo()
Virtual destructor for safe inheritance.
Definition: Yolo.C:32
void getInDims(int &w, int &h, int &c) const
Get input width, height, channels.
Definition: Yolo.C:258
int nboxes
Definition: Yolo.H:147
char ** names
Definition: Yolo.H:146
int classes
Definition: Yolo.H:149
int * map
Definition: Yolo.H:150
Yolo(std::string const &instance)
Constructor.
Definition: Yolo.C:22
float predict(cv::Mat const &cvimg)
Processing function, results are stored internally in the underlying Darknet network object...
Definition: Yolo.C:114
void sendSerial(jevois::StdModule *mod, int inw, int inh)
Send serial messages about detections.
Definition: Yolo.C:218
void drawDetections(jevois::RawImage &outimg, int inw, int inh, int xoff, int yoff)
Draw the detections.
Definition: Yolo.C:175
#define LFATAL(msg)
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Yolo.C:40
std::future< void > itsReadyFut
Definition: Yolo.H:153
network * net
Definition: Yolo.H:145
#define LINFO(msg)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
void sendSerialObjDetImg2D(unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
std::string absolutePath(std::string const &path="")