JeVoisBase  1.6
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Yolo.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 
21 // ####################################################################################################
22 Yolo::Yolo(std::string const & instance) : jevois::Component(instance), net(nullptr), names(nullptr), boxes(nullptr),
23  probs(nullptr), classes(0), itsReady(false)
24 {
25  // Get NNPACK ready to rock:
26 #ifdef NNPACK
27  nnp_initialize();
28 #endif
29 }
30 
31 // ####################################################################################################
33 {
34 #ifdef NNPACK
35  nnp_deinitialize();
36 #endif
37 }
38 
39 // ####################################################################################################
41 {
42  itsReadyFut = std::async(std::launch::async, [&]() {
43  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
44 
45  // Note: darknet expects read/write pointers to the file names...
46  std::string const datacf = absolutePath(root + datacfg::get());
47  std::string const cfgfil = absolutePath(root + cfgfile::get());
48  std::string const weightfil = absolutePath(root + weightfile::get());
49 
50  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
51  std::string name_list = namefile::get();
52  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
53  else name_list = absolutePath(root + name_list);
54 
55  LINFO("Using data config from " << datacf);
56  LINFO("Using cfg from " << cfgfil);
57  LINFO("Using weights from " << weightfil);
58  LINFO("Using names from " << name_list);
59 
60  LINFO("Getting labels...");
61  names = get_labels(const_cast<char *>(name_list.c_str()));
62  LINFO("Parsing network and loading weights...");
63  net = load_network(const_cast<char *>(cfgfil.c_str()), const_cast<char *>(weightfil.c_str()), 0);
64  if (net == nullptr) LFATAL("Failed to load darknet network and/or weights -- ABORT");
65  classes = option_find_int(options, "classes", 2);
66 
67  set_batch_network(net, 1);
68  srand(2222222);
69  LINFO("YOLO network ready");
70 
71 #ifdef DARKNET_NNPACK
72  net->threadpool = pthreadpool_create(threads::get());
73 #endif
74  free_list(options);
75  itsReady.store(true);
76  });
77 }
78 
79 // ####################################################################################################
81 {
82  try { itsReadyFut.get(); } catch (...) { }
83 
84  if (boxes) { free(boxes); boxes = nullptr; }
85  if (probs) { layer & l = net->layers[net->n-1]; free_ptrs((void **)probs, l.w * l.h * l.n); probs = nullptr; }
86 
87  if (net)
88  {
89 #ifdef DARKNET_NNPACK
90  if (net->threadpool) pthreadpool_destroy(net->threadpool);
91 #endif
92  free_network(net);
93  }
94 
95  free_ptrs((void**)names, classes);
96 }
97 
98 // ####################################################################################################
99 float Yolo::predict(cv::Mat const & cvimg)
100 {
101  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
102  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
103 
104  int const c = 3; // color channels
105  int const w = cvimg.cols;
106  int const h = cvimg.rows;
107 
108  image im = make_image(w, h, c);
109  for (int k = 0; k < c; ++k)
110  for (int j = 0; j < h; ++j)
111  for (int i = 0; i < w; ++i)
112  {
113  int dst_index = i + w*j + w*h*k;
114  int src_index = k + c*i + c*w*j;
115  im.data[dst_index] = float(cvimg.data[src_index]) / 255.0F;
116  }
117 
118  float predtime = predict(im);
119 
120  free_image(im);
121 
122  return predtime;
123 }
124 
125 // ####################################################################################################
126 float Yolo::predict(image & im)
127 {
128  image sized; bool need_free = false;
129  if (im.w == net->w && im.h == net->h) sized = im; else { sized = letterbox_image(im, net->w, net->h); need_free = true; }
130 
131  struct timeval start, stop;
132 
133  gettimeofday(&start, 0);
134  network_predict(net, sized.data);
135  gettimeofday(&stop, 0);
136 
137  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
138 
139  if (need_free) free_image(sized);
140 
141  return predtime;
142 }
143 
144 // ####################################################################################################
145 void Yolo::computeBoxes(int inw, int inh)
146 {
147  layer & l = net->layers[net->n-1];
148 
149  if (boxes == nullptr)
150  boxes = (box *)calloc(l.w * l.h * l.n, sizeof(box));
151 
152  if (probs == nullptr)
153  {
154  probs = (float **)calloc(l.w * l.h * l.n, sizeof(float *));
155  for (int j = 0; j < l.w * l.h * l.n; ++j) probs[j] = (float *)calloc(l.classes + 1, sizeof(float));
156  }
157 
158  get_region_boxes(l, inw, inh, net->w, net->h, thresh::get()*0.01F, probs, boxes, 0, 0, 0, hierthresh::get()*0.01F, 1);
159 
160  float const nmsval = nms::get()*0.01F;
161  if (nmsval) do_nms_obj(boxes, probs, l.w * l.h * l.n, l.classes, nmsval);
162 }
163 
164 // ####################################################################################################
165 void Yolo::drawDetections(jevois::RawImage & outimg, int inw, int inh, int xoff, int yoff)
166 {
167  layer & l = net->layers[net->n-1];
168  int const num = l.w * l.h * l.n;
169 
170  float const thval = thresh::get();
171  float const hthval = hierthresh::get();
172 
173  for (int i = 0; i < num; ++i)
174  {
175  int const cls = max_index(probs[i], l.classes);
176  float const prob = probs[i][cls] * 100.0F;
177 
178  if (prob > thval)
179  {
180  box const & b = boxes[i];
181 
182  int const left = xoff + (b.x - b.w / 2.0F) * inw;
183  int const bw = b.w * inw;
184  int const top = yoff + (b.y - b.h / 2.0F) * inh;
185  int const bh = b.h * inh;
186 
187  jevois::rawimage::drawRect(outimg, left, top, bw, bh, 2, jevois::yuyv::LightGreen);
188  jevois::rawimage::writeText(outimg, jevois::sformat("%s: %.1f", names[cls], prob),
189  left, top - 22, jevois::yuyv::LightGreen, jevois::rawimage::Font10x20);
190  }
191  }
192 }
193 
194 // ####################################################################################################
195 void Yolo::sendSerial(jevois::StdModule * mod, int inw, int inh, unsigned long frame)
196 {
197  mod->sendSerial("DKY " + std::to_string(frame));
198 
199  layer & l = net->layers[net->n-1];
200  int const num = l.w * l.h * l.n;
201 
202  float const thval = thresh::get();
203  float const hthval = hierthresh::get();
204 
205  for (int i = 0; i < num; ++i)
206  {
207  int const cls = max_index(probs[i], l.classes);
208  float const prob = probs[i][cls] * 100.0F;
209 
210  if (prob > thval)
211  {
212  box const & b = boxes[i];
213 
214  int const left = (b.x - b.w / 2.0F) * inw;
215  int const bw = b.w * inw;
216  int const top = (b.y - b.h / 2.0F) * inh;
217  int const bh = b.h * inh;
218 
219  mod->sendSerialImg2D(inw, inh, left, top, bw, bh, names[cls], jevois::sformat("%.1f", prob));
220  }
221  }
222 }
223 
224 // ####################################################################################################
225 void Yolo::resizeInDims(int w, int h)
226 {
227  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
228  resize_network(net, w, h);
229 }
230 
231 // ####################################################################################################
232 void Yolo::getInDims(int & w, int & h, int & c) const
233 {
234  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
235  w = net->w; h = net->h; c = net->c;
236 }
void computeBoxes(int inw, int inh)
Compute the boxes.
Definition: Yolo.C:145
std::atomic< bool > itsReady
Definition: Yolo.H:154
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
void sendSerial(jevois::StdModule *mod, int inw, int inh, unsigned long frame)
Send serial messages about detections.
Definition: Yolo.C:195
void postUninit() override
Un-initialize and free resources.
Definition: Yolo.C:80
void resizeInDims(int w, int h)
Resize the network&#39;s input image dims.
Definition: Yolo.C:225
virtual ~Yolo()
Virtual destructor for safe inheritance.
Definition: Yolo.C:32
void getInDims(int &w, int &h, int &c) const
Get input width, height, channels.
Definition: Yolo.C:232
char ** names
Definition: Yolo.H:147
int classes
Definition: Yolo.H:150
void sendSerialImg2D(unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
float ** probs
Definition: Yolo.H:149
Yolo(std::string const &instance)
Constructor.
Definition: Yolo.C:22
float predict(cv::Mat const &cvimg)
Processing function, results are stored internally in the underlying Darknet network object...
Definition: Yolo.C:99
virtual void sendSerial(std::string const &str)
void drawDetections(jevois::RawImage &outimg, int inw, int inh, int xoff, int yoff)
Draw the detections.
Definition: Yolo.C:165
#define LFATAL(msg)
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Yolo.C:40
std::string to_string(T const &val)
std::future< void > itsReadyFut
Definition: Yolo.H:153
network * net
Definition: Yolo.H:146
#define LINFO(msg)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
box * boxes
Definition: Yolo.H:148
std::string absolutePath(std::string const &path="")