JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Yolo.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2017 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Core/Module.H>
20 
21 // ####################################################################################################
22 Yolo::Yolo(std::string const & instance) : jevois::Component(instance), itsReady(false)
23 {
24 #ifdef DARKNET_NNPACK
25  net.threadpool = 0;
26 #endif
27 
28  // Get NNPACK ready to rock:
29 #ifdef NNPACK
30  nnp_initialize();
31 #endif
32 }
33 
34 // ####################################################################################################
36 {
37 #ifdef NNPACK
38  nnp_deinitialize();
39 #endif
40 }
41 
42 // ####################################################################################################
44 {
45  itsReadyFut = std::async(std::launch::async, [&]() {
46  std::string root = dataroot::get(); if (root.empty() == false) root += '/';
47 
48  // Note: darknet expects read/write pointers to the file names...
49  std::string const datacf = absolutePath(root + datacfg::get());
50  std::string const cfgfil = absolutePath(root + cfgfile::get());
51  std::string const weightfil = absolutePath(root + weightfile::get());
52 
53  list * options = read_data_cfg(const_cast<char *>(datacf.c_str()));
54  std::string name_list = namefile::get();
55  if (name_list.empty()) name_list = absolutePath(root + option_find_str(options, "names", "data/names.list"));
56  else name_list = absolutePath(root + name_list);
57 
58  LINFO("Using data config from " << datacf);
59  LINFO("Using cfg from " << cfgfil);
60  LINFO("Using weights from " << weightfil);
61  LINFO("Using names from " << name_list);
62 
63  LINFO("Getting labels...");
64  names = get_labels(const_cast<char *>(name_list.c_str()));
65  LINFO("Parsing network...");
66  net = parse_network_cfg(const_cast<char *>(cfgfil.c_str()));
67  LINFO("Loading weights...");
68  load_weights(&net, const_cast<char *>(weightfil.c_str()));
69  classes = option_find_int(options, "classes", 2);
70 
71  set_batch_network(&net, 1);
72  srand(2222222);
73  LINFO("YOLO network ready");
74 
75 #ifdef DARKNET_NNPACK
76  net.threadpool = pthreadpool_create(threads::get());
77 #endif
78  free_list(options);
79  itsReady.store(true);
80  });
81 }
82 
83 // ####################################################################################################
85 {
86  try { itsReadyFut.get(); } catch (...) { }
87 
88 #ifdef DARKNET_NNPACK
89  pthreadpool_destroy(net.threadpool);
90 #endif
91 
92  if (boxes) { free(boxes); boxes = nullptr; }
93  if (probs) { layer & l = net.layers[net.n-1]; free_ptrs((void **)probs, l.w * l.h * l.n); probs = nullptr; }
94  free_ptrs((void**)names, classes);
95  free_network(net);
96 }
97 
98 // ####################################################################################################
99 float Yolo::predict(cv::Mat const & cvimg)
100 {
101  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
102  if (cvimg.type() != CV_8UC3) LFATAL("cvimg must have type CV_8UC3 and RGB pixels");
103 
104  int const c = 3; // color channels
105  int const w = cvimg.cols;
106  int const h = cvimg.rows;
107 
108  image im = make_image(w, h, c);
109  for (int k = 0; k < c; ++k)
110  for (int j = 0; j < h; ++j)
111  for (int i = 0; i < w; ++i)
112  {
113  int dst_index = i + w*j + w*h*k;
114  int src_index = k + c*i + c*w*j;
115  im.data[dst_index] = float(cvimg.data[src_index]) / 255.0F;
116  }
117 
118  float predtime = predict(im);
119 
120  free_image(im);
121 
122  return predtime;
123 }
124 
125 // ####################################################################################################
126 float Yolo::predict(image & im)
127 {
128  image sized; bool need_free = false;
129  if (im.w == net.w && im.h == net.h) sized = im; else { sized = letterbox_image(im, net.w, net.h); need_free = true; }
130 
131  struct timeval start, stop;
132 
133  gettimeofday(&start, 0);
134  network_predict(net, sized.data);
135  gettimeofday(&stop, 0);
136 
137  float predtime = (stop.tv_sec * 1000 + stop.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000);
138 
139  if (need_free) free_image(sized);
140 
141  return predtime;
142 }
143 
144 // ####################################################################################################
145 void Yolo::computeBoxes(int inw, int inh)
146 {
147  layer & l = net.layers[net.n-1];
148 
149  if (boxes == nullptr)
150  boxes = (box *)calloc(l.w * l.h * l.n, sizeof(box));
151 
152  if (probs == nullptr)
153  {
154  probs = (float **)calloc(l.w * l.h * l.n, sizeof(float *));
155  for (int j = 0; j < l.w * l.h * l.n; ++j) probs[j] = (float *)calloc(l.classes + 1, sizeof(float));
156  }
157 
158 #ifdef DARKNET_NNPACK
159  get_region_boxes(l, inw, inh, net.w, net.h, thresh::get()*0.01F, probs, boxes, 0, 0, hierthresh::get()*0.01F, 1);
160 #else
161  get_region_boxes(l, inw, inh, net.w, net.h, thresh::get()*0.01F, probs, boxes, 0, 0, 0, hierthresh::get()*0.01F, 1);
162 #endif
163 
164  float const nmsval = nms::get()*0.01F;
165  if (nmsval) do_nms_obj(boxes, probs, l.w * l.h * l.n, l.classes, nmsval);
166 }
167 
168 // ####################################################################################################
169 void Yolo::drawDetections(jevois::RawImage & outimg, int inw, int inh, int xoff, int yoff)
170 {
171  layer & l = net.layers[net.n-1];
172  int const num = l.w * l.h * l.n;
173 
174  float const thval = thresh::get();
175  float const hthval = hierthresh::get();
176 
177  for (int i = 0; i < num; ++i)
178  {
179  int const cls = max_index(probs[i], l.classes);
180  float const prob = probs[i][cls] * 100.0F;
181 
182  if (prob > thval)
183  {
184  box const & b = boxes[i];
185 
186  int const left = xoff + (b.x - b.w / 2.0F) * inw;
187  int const bw = b.w * inw;
188  int const top = yoff + (b.y - b.h / 2.0F) * inh;
189  int const bh = b.h * inh;
190 
191  jevois::rawimage::drawRect(outimg, left, top, bw, bh, 2, jevois::yuyv::LightGreen);
192  jevois::rawimage::writeText(outimg, jevois::sformat("%s: %.1f", names[cls], prob),
193  left, top - 22, jevois::yuyv::LightGreen, jevois::rawimage::Font10x20);
194  }
195  }
196 }
197 
198 // ####################################################################################################
199 void Yolo::sendSerial(jevois::StdModule * mod, int inw, int inh, unsigned long frame)
200 {
201  mod->sendSerial("DKY " + std::to_string(frame));
202 
203  layer & l = net.layers[net.n-1];
204  int const num = l.w * l.h * l.n;
205 
206  float const thval = thresh::get();
207  float const hthval = hierthresh::get();
208 
209  for (int i = 0; i < num; ++i)
210  {
211  int const cls = max_index(probs[i], l.classes);
212  float const prob = probs[i][cls] * 100.0F;
213 
214  if (prob > thval)
215  {
216  box const & b = boxes[i];
217 
218  int const left = (b.x - b.w / 2.0F) * inw;
219  int const bw = b.w * inw;
220  int const top = (b.y - b.h / 2.0F) * inh;
221  int const bh = b.h * inh;
222 
223  mod->sendSerialImg2D(inw, inh, left, top, bw, bh, names[cls], jevois::sformat("%.1f", prob));
224  }
225  }
226 }
227 
228 // ####################################################################################################
229 void Yolo::resizeInDims(int w, int h)
230 {
231  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
232  resize_network(&net, w, h);
233 }
234 
235 // ####################################################################################################
236 void Yolo::getInDims(int & w, int & h, int & c) const
237 {
238  if (itsReady.load() == false) throw std::logic_error("not ready yet...");
239  w = net.w; h = net.h; c = net.c;
240 }
void computeBoxes(int inw, int inh)
Compute the boxes.
Definition: Yolo.C:145
std::atomic< bool > itsReady
Definition: Yolo.H:154
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
void sendSerial(jevois::StdModule *mod, int inw, int inh, unsigned long frame)
Send serial messages about detections.
Definition: Yolo.C:199
void postUninit() override
Un-initialize and free resources.
Definition: Yolo.C:84
void resizeInDims(int w, int h)
Resize the network&#39;s input image dims.
Definition: Yolo.C:229
virtual ~Yolo()
Virtual destructor for safe inheritance.
Definition: Yolo.C:35
void getInDims(int &w, int &h, int &c) const
Get input width, height, channels.
Definition: Yolo.C:236
char ** names
Definition: Yolo.H:147
int classes
Definition: Yolo.H:150
void sendSerialImg2D(unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
float ** probs
Definition: Yolo.H:149
Yolo(std::string const &instance)
Constructor.
Definition: Yolo.C:22
float predict(cv::Mat const &cvimg)
Processing function, results are stored internally in the underlying Darknet network object...
Definition: Yolo.C:99
virtual void sendSerial(std::string const &str)
void drawDetections(jevois::RawImage &outimg, int inw, int inh, int xoff, int yoff)
Draw the detections.
Definition: Yolo.C:169
#define LFATAL(msg)
void postInit() override
Initialize, configure and load the network in a thread.
Definition: Yolo.C:43
std::string to_string(T const &val)
std::future< void > itsReadyFut
Definition: Yolo.H:153
#define LINFO(msg)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
network net
Definition: Yolo.H:146
box * boxes
Definition: Yolo.H:148
std::string absolutePath(std::string const &path="")