JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessNPUhelpers.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Debug/Log.H>
20 
21 #include <nn_detect_common.h>
22 
23 #include <stdlib.h>
24 #include <stdio.h>
25 #include <string.h>
26 #include <math.h>
27 #include <float.h>
28 
29 // ####################################################################################################
30 // Helper code from the detect_library of the NPU
31 namespace
32 {
33  inline float logistic_activate(float x)
34  { return 1./(1. + exp(-x)); }
35 
36  void flatten(float const * src, float * dst, int size, int layers, int batch, int forward)
37  {
38  for (int b = 0; b < batch; ++b)
39  for (int c = 0; c < layers; ++c)
40  for (int i = 0; i < size; ++i)
41  {
42  int i1 = b*layers*size + c*size + i;
43  int i2 = b*layers*size + i*layers + c;
44  if (forward) dst[i2] = src[i1];
45  else dst[i1] = src[i2];
46  }
47  }
48 } // anonymous namespace
49 
50 
51 // ####################################################################################################
52 void jevois::dnn::npu::yolo(cv::Mat const & out, std::vector<int> & classIds, std::vector<float> & confidences,
53  std::vector<cv::Rect> & boxes, size_t nclass, float const * biases, int const yolonum,
54  float confThreshold, cv::Size const & bsiz, int fudge)
55 {
56  if (nclass == 0) nclass = 1; // Assume 1 class if no list of classes was given
57  if (out.type() != CV_32F) LFATAL("Need FLOAT data");
58  cv::MatSize const & msiz = out.size;
59  if (msiz.dims() != 4 || msiz[0] != 1)
60  LFATAL("Incorrect tensor size: need 1xNxHxW where N=anchors*(4(coords)+1(box score)+nclass(object scores)), got "<<
62  // dim[1] is (coords = 4 + 1 for box score + classes) * n boxes:
63  // n = 5 for yoloface, yolov2
64  // n = 3 for yolo_v3
65 
66  int const w = msiz[3];
67  int const h = msiz[2];
68  int const coords = 4;
69  int bbsize = coords + 1 + nclass;
70  int const n = msiz[1] / bbsize;
71  if (msiz[1] % bbsize) LFATAL("Incorrect tensor size: need 1xNxHxW where N=anchors*(4(coords)+1(box score)"
72  "+nclass(object scores)), got "<< jevois::dnn::shapestr(out));
73  float const bfac = 1.0F / (8 << yolonum);
74  int const boff = n * 2 * yolonum;
75  int const whn = w * h * n;
76  int const total = whn * bbsize;
77  if (total != int(out.total())) LFATAL("Ooops");
78 
79  // Re-arrange the data order:
80  float predictions[total];
81  flatten((float const *)out.data, predictions, w*h, bbsize*n, 1, 1);
82 
83  // Apply logistic activation to box score and softmax to object scores:
84  for (int i = 0; i < total; i += bbsize)
85  {
86  predictions[i + coords] = logistic_activate(predictions[i + coords]);
87  jevois::dnn::softmax(predictions + i + coords + 1, nclass, 1, predictions + i + coords + 1);
88  }
89 
90  // Loop over all locations:
91  for (int i = 0; i < w * h; ++i)
92  {
93  int const row = i / w;
94  int const col = i % w;
95 
96  // Loop over all boxes per location:
97  for (int nn = 0; nn < n; ++nn)
98  {
99  int index = i*n + nn; // box number
100  int box_index = index * bbsize; // address of box in predictions[]
101  float scale = predictions[box_index + coords]; // box score
102  int class_index = box_index + coords + 1; // address of the nclass object scores for our box
103 
104  // Find max object score and its Id:
105  int maxid = -1; float maxscore = -1.0F;
106  for (size_t j = 0; j < nclass; ++j)
107  {
108  float prob = scale * predictions[class_index + j];
109  if (prob > confThreshold && prob > maxscore) { maxid = j; maxscore = prob; }
110  }
111 
112  // If at least one class was above threshold, keep that box:
113  if (maxid >= 0)
114  {
115  // Decode the box and scale it to input blob dims:
116  cv::Rect b( (col + logistic_activate(predictions[box_index + 0])) * bsiz.width / w + 0.499F, // x
117  (row + logistic_activate(predictions[box_index + 1])) * bsiz.height / h + 0.499F, // y
118  exp(predictions[box_index + 2]) * biases[2*nn + boff] * bfac * bsiz.width / w + 0.499F, // w
119  exp(predictions[box_index + 3]) * biases[2*nn+1 + boff] * bfac * bsiz.height / h + 0.499F); // h
120  b.x -= b.width / 2;
121  b.y -= b.height / 2;
122 
123  boxes.emplace_back(b);
124  classIds.emplace_back(maxid + fudge);
125  confidences.emplace_back(maxscore);
126  }
127  }
128  }
129 }
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:105
PostProcessNPUhelpers.H
Log.H
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
jevois::dnn::softmax
void softmax(float const *input, size_t n, float fac, float *output)
Apply softmax to a float vector.
Definition: Utils.C:443
jevois::dnn::npu::yolo
void yolo(cv::Mat const &out, std::vector< int > &classIds, std::vector< float > &confidences, std::vector< cv::Rect > &boxes, size_t nclass, float const *biases, int const yolonum, float confThreshold, cv::Size const &bsiz, int fudge)
Definition: PostProcessNPUhelpers.C:52
h
int h
Definition: GUIhelper.C:2150