JeVoisBase  1.16
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Yolo.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 #include <darknet-nnpack/src/classifier.h>
21 #include <darknet-nnpack/src/option_list.h>
22 #include <darknet-nnpack/src/data.h>
23 #include <darknet-nnpack/src/network.h>
24 #include <darknet-nnpack/src/utils.h>
25 #include <stdlib.h> // for free()
26 
27 // ####################################################################################################
28 Yolo::Yolo(std::string const & instance) : jevois::Component(instance), net(nullptr), names(nullptr), nboxes(0),
29  dets(nullptr), classes(0), map(nullptr), itsReady(false)
30 {
31  // Get NNPACK ready to rock:
32 #ifdef NNPACK
33  nnp_initialize();
34 #endif
35 }
36 
37 // ####################################################################################################
39 {
40 #ifdef NNPACK
41  nnp_deinitialize();
42 #endif
43 }
44 
45 // ####################################################################################################
47 {
48  itsReadyFut = jevois::async([&]() {
49  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
50 
51  // Note: darknet expects read/write pointers to the file names...
52  std::string const datacf = absolutePath(root + datacfg::get());
53  std::string const cfgfil = absolutePath(root + cfgfile::get());
54  std::string const weightfil = absolutePath(root + weightfile::get());
55 
56  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
57  std::string name_list = namefile::get();
58  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
59  else name_list = absolutePath(root + name_list);
60 
61  LINFO("Using data config from " << datacf);
62  LINFO("Using cfg from " << cfgfil);
63  LINFO("Using weights from " << weightfil);
64  LINFO("Using names from " << name_list);
65 
66  LINFO("Getting labels...");
67  names = get_labels(const_cast<char *>(name_list.c_str()));
68 
69  char * mapf = option_find_str(options, "map", 0);
70  if (mapf) map = read_map(mapf);
71 
72  LINFO("Parsing network and loading weights...");
73 
74  net = load_network(const_cast<char *>(cfgfil.c_str()), const_cast<char *>(weightfil.c_str()), 0);
75 
76  if (net == nullptr)
77  {
78  free_list(options);
79  if (map) { free(map); map = nullptr; }
80  LFATAL("Failed to load YOLO network and/or weights -- ABORT");
81  }
82 
83  classes = option_find_int(options, "classes", 2);
84 
85  set_batch_network(net, 1);
86  srand(2222222);
87  LINFO("YOLO network ready");
88 
89 #ifdef DARKNET_NNPACK
90  net->threadpool = pthreadpool_create(threads::get());
91 #endif
92  free_list(options);
93  itsReady.store(true);
94  });
95 }
96 
97 // ####################################################################################################
99 {
100  if (itsReadyFut.valid()) try { itsReadyFut.get(); } catch (...) { }
101 
102  if (dets) { free_detections(dets, nboxes); dets = nullptr; nboxes = 0; }
103 
104  if (map) { free(map); map = nullptr; }
105 
106  if (net)
107  {
108 #ifdef DARKNET_NNPACK
109  if (net->threadpool) pthreadpool_destroy(net->threadpool);
110 #endif
111  free_network(*net);
112  free(net);
113  net = nullptr;
114  }
115 
116  free_ptrs((void**)names, classes);
117  names = nullptr; classes = 0;
118 }
119 
120 // ####################################################################################################
121 float Yolo::predict(cv::Mat const & cvimg)
122 {
123  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
124  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
125 
126  int const c = 3; // color channels
127  int const w = cvimg.cols;
128  int const h = cvimg.rows;
129 
130  image im = make_image(w, h, c);
131  for (int k = 0; k < c; ++k)
132  for (int j = 0; j < h; ++j)
133  for (int i = 0; i < w; ++i)
134  {
135  int dst_index = i + w*j + w*h*k;
136  int src_index = k + c*i + c*w*j;
137  im.data[dst_index] = float(cvimg.data[src_index]) / 255.0F;
138  }
139 
140  float predtime = predict(im);
141 
142  free_image(im);
143 
144  return predtime;
145 }
146 
147 // ####################################################################################################
148 float Yolo::predict(image & im)
149 {
150  image sized; bool need_free = false;
151  if (im.w == net->w && im.h == net->h) sized = im;
152  else { sized = letterbox_image(im, net->w, net->h); need_free = true; }
153 
154  struct timeval start, stop;
155 
156  gettimeofday(&start, 0);
157  network_predict(*net, sized.data);
158  gettimeofday(&stop, 0);
159 
160  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
161 
162  if (need_free) free_image(sized);
163 
164  return predtime;
165 }
166 
167 // ####################################################################################################
168 void Yolo::computeBoxes(int inw, int inh)
169 {
170  layer & l = net->layers[net->n-1];
171 
172  if (dets) { free_detections(dets, nboxes); dets = nullptr; nboxes = 0; }
173 
174  dets = get_network_boxes(net, 1, 1, thresh::get() * 0.01F, hierthresh::get() * 0.01F, map, 0, &nboxes, 1);
175 
176  float const nmsval = nms::get() * 0.01F;
177 
178  if (nmsval) do_nms_sort(dets, nboxes, l.classes, nmsval);
179 }
180 
181 // ####################################################################################################
182 void Yolo::drawDetections(jevois::RawImage & outimg, int inw, int inh, int xoff, int yoff)
183 {
184  float const thval = thresh::get();
185 
186  for (int i = 0; i < nboxes; ++i)
187  {
188  // For each detection, We need to get a list of labels and probabilities, sorted by score:
189  std::vector<jevois::ObjReco> data;
190 
191  for (int j = 0; j < classes; ++j)
192  {
193  float const p = dets[i].prob[j] * 100.0F;
194  if (p > thval) data.push_back( { p, std::string(names[j]) } );
195  }
196 
197  // End here if nothing above threshold:
198  if (data.empty()) continue;
199 
200  // Sort in descending order:
201  std::sort(data.begin(), data.end(), [](auto a, auto b) { return (b.score < a.score); });
202 
203  // Create our display label:
204  std::string labelstr;
205  for (auto const & d : data)
206  {
207  if (labelstr.empty() == false) labelstr += ", ";
208  labelstr += jevois::sformat("%s:%.1f", d.category.c_str(), d.score);
209  }
210 
211  box const & b = dets[i].bbox;
212 
213  int const left = std::max(xoff, int(xoff + (b.x - b.w / 2.0F) * inw + 0.499F));
214  int const bw = std::min(inw, int(b.w * inw + 0.499F));
215  int const top = std::max(yoff, int(yoff + (b.y - b.h / 2.0F) * inh + 0.499F));
216  int const bh = std::min(inh, int(b.h * inh + 0.499F));
217 
218  jevois::rawimage::drawRect(outimg, left, top, bw, bh, 2, jevois::yuyv::LightGreen);
219  jevois::rawimage::writeText(outimg, labelstr,
220  left + 6, top + 2, jevois::yuyv::LightGreen, jevois::rawimage::Font10x20);
221  }
222 }
223 
224 // ####################################################################################################
225 void Yolo::sendSerial(jevois::StdModule * mod, int inw, int inh)
226 {
227  float const thval = thresh::get();
228 
229  for (int i = 0; i < nboxes; ++i)
230  {
231  // For each detection, We need to get a list of labels and probabilities, sorted by score:
232  std::vector<jevois::ObjReco> data;
233 
234  for (int j = 0; j < classes; ++j)
235  {
236  float const p = dets[i].prob[j] * 100.0F;
237  if (p > thval) data.push_back({ p, std::string(names[j]) });
238  }
239 
240  // End here if nothing above threshold:
241  if (data.empty()) continue;
242 
243  // Sort in descending order:
244  std::sort(data.begin(), data.end(), [](auto a, auto b) { return (b.score < a.score); });
245 
246  box const & b = dets[i].bbox;
247 
248  int const left = (b.x - b.w / 2.0F) * inw;
249  int const bw = b.w * inw;
250  int const top = (b.y - b.h / 2.0F) * inh;
251  int const bh = b.h * inh;
252 
253  mod->sendSerialObjDetImg2D(inw, inh, left, top, bw, bh, data);
254  }
255 }
256 
257 // ####################################################################################################
258 void Yolo::resizeInDims(int w, int h)
259 {
260  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
261  resize_network(net, w, h);
262 }
263 
264 // ####################################################################################################
265 void Yolo::getInDims(int & w, int & h, int & c) const
266 {
267  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
268  w = net->w; h = net->h; c = net->c;
269 }
Yolo::drawDetections
void drawDetections(jevois::RawImage &outimg, int inw, int inh, int xoff, int yoff)
Draw the detections.
Definition: Yolo.C:182
Yolo::map
int * map
Definition: Yolo.H:150
Yolo::classes
int classes
Definition: Yolo.H:149
Yolo::names
char ** names
Definition: Yolo.H:146
Module.H
jevois::sformat
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Yolo::itsReadyFut
std::future< void > itsReadyFut
Definition: Yolo.H:153
Yolo::Yolo
Yolo(std::string const &instance)
Constructor.
Definition: Yolo.C:28
Yolo::net
network * net
Definition: Yolo.H:145
Yolo::predict
float predict(cv::Mat const &cvimg)
Processing function, results are stored internally in the underlying Darknet network object.
Definition: Yolo.C:121
jevois::RawImage
Yolo::postInit
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Yolo.C:46
Yolo::computeBoxes
void computeBoxes(int inw, int inh)
Compute the boxes.
Definition: Yolo.C:168
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
jevois
F
float F
Yolo::itsReady
std::atomic< bool > itsReady
Definition: Yolo.H:154
jevois::StdModule::sendSerialObjDetImg2D
void sendSerialObjDetImg2D(unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
test-model.net
net
Definition: test-model.py:10
LFATAL
#define LFATAL(msg)
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Yolo::postUninit
void postUninit() override
Un-initialize and free resources.
Definition: Yolo.C:98
jevois::rawimage::drawRect
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
h
int h
Yolo::~Yolo
virtual ~Yolo()
Virtual destructor for safe inheritance.
Definition: Yolo.C:38
Yolo::getInDims
void getInDims(int &w, int &h, int &c) const
Get input width, height, channels.
Definition: Yolo.C:265
jevois::StdModule
jevois::rawimage::Font10x20
Font10x20
LINFO
#define LINFO(msg)
jevois::Component::absolutePath
std::string absolutePath(std::string const &path="")
Yolo::resizeInDims
void resizeInDims(int w, int h)
Resize the network's input image dims.
Definition: Yolo.C:258
Yolo::dets
detection * dets
Definition: Yolo.H:148
Yolo::sendSerial
void sendSerial(jevois::StdModule *mod, int inw, int inh)
Send serial messages about detections.
Definition: Yolo.C:225
Yolo.H
Yolo::nboxes
int nboxes
Definition: Yolo.H:147