JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessorYuNet.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
20 #include <jevois/DNN/Utils.H>
21 #include <jevois/Util/Utils.H>
23 #include <jevois/Core/Engine.H>
24 #include <jevois/Core/Module.H>
25 #include <jevois/GPU/GUIhelper.H>
26 
27 #include <opencv2/dnn.hpp>
28 
29 // ####################################################################################################
30 // this code from https://github.com/khadas/OpenCV_NPU_Demo
31 namespace jevois { namespace dnn { namespace yunet {
32 
33  class PriorBox
34  {
35  public:
36  PriorBox(cv::Size const & input_shape, cv::Size const & output_shape)
37  {
38  // initialize
39  in_w = input_shape.width;
40  in_h = input_shape.height;
41  out_w = output_shape.width;
42  out_h = output_shape.height;
43 
44  cv::Size feature_map_2nd { int(int((in_w+1)/2)/2), int(int((in_h+1)/2)/2) };
45  cv::Size feature_map_3rd { int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) };
46  cv::Size feature_map_4th { int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) };
47  cv::Size feature_map_5th { int(feature_map_4th.width/2), int(feature_map_4th.height/2) };
48  cv::Size feature_map_6th { int(feature_map_5th.width/2), int(feature_map_5th.height/2) };
49 
50  // feature_map_sizes.push_back(feature_map_2nd);
51  feature_map_sizes.push_back(feature_map_3rd);
52  feature_map_sizes.push_back(feature_map_4th);
53  feature_map_sizes.push_back(feature_map_5th);
54  feature_map_sizes.push_back(feature_map_6th);
55 
56  // generate the priors:
57  for (size_t i = 0; i < feature_map_sizes.size(); ++i)
58  {
59  cv::Size feature_map_size = feature_map_sizes[i];
60  std::vector<float> min_size = min_sizes[i];
61 
62  for (int _h = 0; _h < feature_map_size.height; ++_h)
63  {
64  for (int _w = 0; _w < feature_map_size.width; ++_w)
65  {
66  for (size_t j = 0; j < min_size.size(); ++j)
67  {
68  float s_kx = min_size[j] / in_w;
69  float s_ky = min_size[j] / in_h;
70 
71  float cx = (_w + 0.5) * steps[i] / in_w;
72  float cy = (_h + 0.5) * steps[i] / in_h;
73 
74  Box anchor = { cx, cy, s_kx, s_ky };
75  priors.push_back(anchor);
76  }
77  }
78  }
79  }
80  }
81 
83  { }
84 
85  std::vector<Face> decode(cv::Mat const & loc, cv::Mat const & conf, cv::Mat const & iou,
86  float const ignore_score = 0.3)
87  {
88  std::vector<Face> dets; // num * [x1, y1, x2, y2, x_re, y_re, x_le, y_le, x_ml, y_ml, x_n, y_n, x_mr, y_ml]
89 
90  float const * loc_v = (float const *)(loc.data);
91  float const * conf_v = (float const *)(conf.data);
92  float const * iou_v = (float const *)(iou.data);
93  for (size_t i = 0; i < priors.size(); ++i)
94  {
95  // get score
96  float cls_score = conf_v[i*2+1];
97  float iou_score = iou_v[i];
98 
99  // clamp
100  if (iou_score < 0.f) iou_score = 0.f; else if (iou_score > 1.f) iou_score = 1.f;
101  float score = std::sqrt(cls_score * iou_score);
102 
103  // ignore low scores
104  if (score < ignore_score) { continue; }
105 
106  Face face;
107  face.score = score;
108 
109  // get bounding box
110  float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * out_w;
111  float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * out_h;
112  float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * out_w;
113  float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * out_h;
114  float x1 = cx - w / 2;
115  float y1 = cy - h / 2;
116  face.bbox_tlwh = { x1, y1, w, h };
117 
118  // get landmarks, loc->[right_eye, left_eye, mouth_left, nose, mouth_right]
119  float x_re = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * out_w;
120  float y_re = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * out_h;
121  float x_le = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * out_w;
122  float y_le = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * out_h;
123  float x_n = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * out_w;
124  float y_n = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * out_h;
125  float x_mr = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * out_w;
126  float y_mr = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * out_h;
127  float x_ml = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * out_w;
128  float y_ml = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * out_h;
129  face.landmarks = {
130  {x_re, y_re}, // right eye
131  {x_le, y_le}, // left eye
132  {x_n, y_n }, // nose
133  {x_mr, y_mr}, // mouth right
134  {x_ml, y_ml} // mouth left
135  };
136 
137  dets.push_back(face);
138  }
139  return dets;
140  }
141 
142  private:
143  std::vector<std::vector<float>> const min_sizes
144  {
145  {10.0f, 16.0f, 24.0f},
146  {32.0f, 48.0f},
147  {64.0f, 96.0f},
148  {128.0f, 192.0f, 256.0f}
149  };
150  std::vector<int> const steps { 8, 16, 32, 64 };
151  std::vector<float> const variance { 0.1, 0.2 };
152 
153  int in_w;
154  int in_h;
155  int out_w;
156  int out_h;
157 
158  std::vector<cv::Size> feature_map_sizes;
159  std::vector<Box> priors;
160  std::vector<Box> generate_priors();
161  };
162 
163  } // namespace yunet
164  } // namespace dnn
165 } // namespace jevois
166 
167 
168 // ####################################################################################################
170 { }
171 
172 // ####################################################################################################
173 void jevois::dnn::PostProcessorYuNet::freeze(bool JEVOIS_UNUSED_PARAM(doit))
174 { }
175 
176 // ####################################################################################################
177 void jevois::dnn::PostProcessorYuNet::process(std::vector<cv::Mat> const & outs, jevois::dnn::PreProcessor * preproc)
178 {
179  if (outs.size() != 3) LFATAL("Need exactly 3 outputs, received " << outs.size());
180  if (outs[0].rows < 16 || outs[0].cols != 2) LFATAL("loc size " << outs[0].size() << " instead of 2xN_Anchors");
181  if (outs[1].rows < 16 || outs[1].cols != 1) LFATAL("conf size " << outs[1].size() << " instead of 1xN_Anchors");
182  if (outs[2].rows < 16 || outs[2].cols != 14) LFATAL("iou size " << outs[2].size() << " instead of 14xN_Anchors");
183 
184  float const confThreshold = cthresh::get() * 0.01F;
185  float const boxThreshold = dthresh::get() * 0.01F;
186  float const nmsThreshold = nms::get() * 0.01F;
187  itsImageSize = preproc->imagesize();
188 
189  // To draw boxes, we will need to:
190  // - scale from [0..1]x[0..1] to blobw x blobh
191  // - scale and center from blobw x blobh to input image w x h, provided by PreProcessor::b2i()
192  // - when using the GUI, we further scale and translate to OpenGL display coordinates using GUIhelper::i2d()
193  // Here we assume that the first blob sets the input size.
194  cv::Size const bsiz = preproc->blobsize(0);
195 
196  // Initialize PriorBox on first inference:
197  if (!itsPriorBox) itsPriorBox.reset(new jevois::dnn::yunet::PriorBox(bsiz, bsiz));
198 
199  // Get the detections from the raw network outputs:
200  std::vector<jevois::dnn::yunet::Face> dets = itsPriorBox->decode(outs[2], outs[0], outs[1], boxThreshold);
201 
202  // NMS
203  itsDetections.clear();
204  if (dets.size() > 1)
205  {
206  std::vector<cv::Rect> face_boxes;
207  std::vector<float> face_scores;
208  for (auto const & d : dets) { face_boxes.push_back(d.bbox_tlwh); face_scores.push_back(d.score); }
209 
210  std::vector<int> keep_idx;
211  cv::dnn::NMSBoxes(face_boxes, face_scores, confThreshold, nmsThreshold, keep_idx, 1.f, top::get());
212  for (size_t i = 0; i < keep_idx.size(); i++) itsDetections.emplace_back(dets[keep_idx[i]]);
213  }
214 
215  // Now clamp boxes to be within blob, and adjust the boxes from blob size to input image size:
216  for (jevois::dnn::yunet::Face & f : itsDetections)
217  {
218  jevois::dnn::yunet::Box & b = f.bbox_tlwh;
219  jevois::dnn::clamp(b, bsiz.width, bsiz.height);
220 
221  cv::Point2f tl = b.tl(); preproc->b2i(tl.x, tl.y);
222  cv::Point2f br = b.br(); preproc->b2i(br.x, br.y);
223  b.x = tl.x; b.y = tl.y; b.width = br.x - tl.x; b.height = br.y - tl.y;
224 
225  preproc->b2i(f.landmarks.right_eye.x, f.landmarks.right_eye.y);
226  preproc->b2i(f.landmarks.left_eye.x, f.landmarks.left_eye.y);
227  preproc->b2i(f.landmarks.nose_tip.x, f.landmarks.nose_tip.y);
228  preproc->b2i(f.landmarks.mouth_right.x, f.landmarks.mouth_right.y);
229  preproc->b2i(f.landmarks.mouth_left.x, f.landmarks.mouth_left.y);
230  }
231 }
232 
233 // ####################################################################################################
235  jevois::OptGUIhelper * helper, bool overlay,
236  bool JEVOIS_UNUSED_PARAM(idle))
237 {
238  for (jevois::dnn::yunet::Face const & f : itsDetections)
239  {
240  jevois::dnn::yunet::Box const & b = f.bbox_tlwh;
241  std::string const scorestr = jevois::sformat("face: %.1f%%", f.score * 100.0F);
242 
243  // If desired, draw boxes in output image:
244  if (outimg && overlay)
245  {
246  unsigned int const col = jevois::yuyv::LightGreen;
247  unsigned int const rad = 5;
248 
249  // Draw the face box:
250  jevois::rawimage::drawRect(*outimg, b.x, b.y, b.width, b.height, 2, col);
251  jevois::rawimage::writeText(*outimg, scorestr, b.x + 6, b.y + 2, col, jevois::rawimage::Font10x20);
252 
253  // Draw the face landmarks:
254  jevois::rawimage::drawCircle(*outimg, f.landmarks.right_eye.x, f.landmarks.right_eye.y, rad*2, 2, col);
255  jevois::rawimage::drawCircle(*outimg, f.landmarks.left_eye.x, f.landmarks.left_eye.y, rad*2, 2, col);
256  jevois::rawimage::drawCircle(*outimg, f.landmarks.nose_tip.x, f.landmarks.nose_tip.y, rad, 2, col);
257  jevois::rawimage::drawCircle(*outimg, f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, rad, 2, col);
258  jevois::rawimage::drawCircle(*outimg, f.landmarks.mouth_left.x, f.landmarks.mouth_left.y, rad, 2, col);
259  jevois::rawimage::drawLine(*outimg, f.landmarks.mouth_left.x, f.landmarks.mouth_left.y,
260  f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, 2, col);
261  }
262 
263 #ifdef JEVOIS_PRO
264  // If desired, draw results on GUI:
265  if (helper)
266  {
267  ImU32 const col = 0x80ff80ff;
268  float const rad = 5.0f;
269 
270  // Draw the face box:
271  helper->drawRect(b.x, b.y, b.x + b.width, b.y + b.height, col, true);
272  helper->drawText(b.x + 3.0f, b.y + 3.0f, scorestr.c_str(), col);
273 
274  // Draw the face landmarks:
275  helper->drawCircle(f.landmarks.right_eye.x, f.landmarks.right_eye.y, rad*2, IM_COL32(255,0,0,255));
276  helper->drawCircle(f.landmarks.left_eye.x, f.landmarks.left_eye.y, rad*2, IM_COL32(0,0,255,255));
277  helper->drawCircle(f.landmarks.nose_tip.x, f.landmarks.nose_tip.y, rad, IM_COL32(0,255,0,255));
278  helper->drawCircle(f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, rad, IM_COL32(255,0,255,255));
279  helper->drawCircle(f.landmarks.mouth_left.x, f.landmarks.mouth_left.y, rad, IM_COL32(0,255,255,255));
280  helper->drawLine(f.landmarks.mouth_left.x, f.landmarks.mouth_left.y,
281  f.landmarks.mouth_right.x, f.landmarks.mouth_right.y, IM_COL32(255,255,255,255));
282  }
283 #else
284  (void)helper; // keep compiler happy
285 #endif
286 
287  // If desired, send results to serial port:
288  (void)mod;
289  //FIXME if (mod) mod->sendSerialObjDetImg2D(itsImageSize.width, itsImageSize.height, o);
290  }
291 }
jevois::dnn::PostProcessorYuNet::freeze
virtual void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PostProcessorYuNet.C:173
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::dnn::PostProcessorYuNet::~PostProcessorYuNet
virtual ~PostProcessorYuNet()
Destructor.
Definition: PostProcessorYuNet.C:169
jevois::dnn::PreProcessor::blobsize
cv::Size blobsize(size_t num) const
Access the width and height of a given blob, accounting for NCHW or NHWC.
Definition: PreProcessor.C:43
jevois::dnn::yunet::PriorBox::PriorBox
PriorBox(cv::Size const &input_shape, cv::Size const &output_shape)
Definition: PostProcessorYuNet.C:36
jevois::dnn::clamp
void clamp(cv::Rect &r, int width, int height)
Clamp a rectangle to within given image width and height.
Definition: Utils.C:288
PostProcessorYuNet.H
jevois::GUIhelper::drawCircle
void drawCircle(float x, float y, float r, ImU32 col=IM_COL32(128, 255, 128, 255), bool filled=true)
Draw circle over an image.
Definition: GUIhelper.C:497
Module.H
jevois::dnn::yunet::PriorBox::decode
std::vector< Face > decode(cv::Mat const &loc, cv::Mat const &conf, cv::Mat const &iou, float const ignore_score=0.3)
Definition: PostProcessorYuNet.C:85
RawImageOps.H
Box
cv::Rect2f Box
A face box for YuNet.
Definition: PostProcessorYuNet.H:30
jevois::dnn::yunet::PriorBox::~PriorBox
~PriorBox()
Definition: PostProcessorYuNet.C:82
jevois::sformat
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Create a string using printf style arguments.
Definition: Utils.C:419
Utils.H
jevois::dnn::yunet::PriorBox
Definition: PostProcessorYuNet.C:33
jevois::dnn::yunet::Face::landmarks
Landmarks_5 landmarks
Definition: PostProcessorYuNet.H:46
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::dnn::PostProcessorYuNet::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PostProcessorYuNet.C:234
jevois::rawimage::drawCircle
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
Draw a circle in a YUYV image.
Definition: RawImageOps.C:483
jevois::dnn::yunet::Face
A YuNet face detection.
Definition: PostProcessorYuNet.H:43
jevois::GUIhelper::drawRect
void drawRect(float x1, float y1, float x2, float y2, ImU32 col=IM_COL32(128, 255, 128, 255), bool filled=true)
Draw rectangular box over an image.
Definition: GUIhelper.C:448
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois::RawImage::width
unsigned int width
Image width in pixels.
Definition: RawImage.H:145
jevois::GUIhelper::drawLine
void drawLine(float x1, float y1, float x2, float y2, ImU32 col=IM_COL32(128, 255, 128, 255))
Draw line over an image.
Definition: GUIhelper.C:442
jevois::dnn::yunet::Face::score
float score
Definition: PostProcessorYuNet.H:47
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
Write some text in an image.
Definition: RawImageOps.C:689
jevois
Definition: Concepts.dox:1
Engine.H
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:102
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::GUIhelper::drawText
void drawText(float x, float y, char const *txt, ImU32 col=IM_COL32(128, 255, 128, 255))
Draw text over an image.
Definition: GUIhelper.C:510
PreProcessor.H
jevois::dnn::PostProcessorYuNet::process
void process(std::vector< cv::Mat > const &outs, PreProcessor *preproc) override
Process outputs and draw/send some results.
Definition: PostProcessorYuNet.C:177
jevois::rawimage::drawRect
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
Draw a rectangle in a YUYV image.
Definition: RawImageOps.C:607
Utils.H
jevois::dnn::PreProcessor::imagesize
const cv::Size & imagesize() const
Access the last processed image size.
Definition: PreProcessor.C:39
jevois::dnn::PreProcessor::b2i
void b2i(float &x, float &y, size_t blobnum=0)
Convert coordinates from blob back to original image.
Definition: PreProcessor.C:50
h
int h
Definition: GUIhelper.C:1968
jevois::dnn::yunet::Face::bbox_tlwh
Box bbox_tlwh
Definition: PostProcessorYuNet.H:45
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
jevois::rawimage::drawLine
void drawLine(RawImage &img, int x1, int y1, int x2, int y2, unsigned int thick, unsigned int col)
Draw a line in a YUYV image.
Definition: RawImageOps.C:564
jevois::rawimage::Font10x20
@ Font10x20
Definition: RawImageOps.H:159
GUIhelper.H