JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessorSegment.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
20 #include <jevois/DNN/Utils.H>
22 #include <jevois/Core/Module.H>
23 #include <jevois/GPU/GUIhelper.H>
24 #include <jevois/Types/ObjReco.H>
25 
26 #include <opencv2/dnn.hpp>
27 
28 // ####################################################################################################
30 {
31 #ifdef JEVOIS_PRO
32  if (itsHelper) itsHelper->releaseImage("ppsr");
33 #endif
34 }
35 
36 // ####################################################################################################
38 {
39  segtype::freeze(doit);
40 }
41 
42 // ####################################################################################################
44 {
45  // Colormap from pascal VOC segmentation benchmark:
46  for (size_t i = 0; i < 256; ++i)
47  {
48  uint32_t & c = itsColor[i]; c = 0;
49  uint32_t ind = i;
50 
51  for (int shift = 7; shift >= 0; --shift)
52  {
53  for (int channel = 0; channel < 3; ++channel) c |= ((ind >> channel) & 1) << (shift + 8 * (3-channel));
54  ind >>= 3;
55  }
56  }
57 }
58 
59 // ####################################################################################################
60 template <typename T>
61 void jevois::dnn::PostProcessorSegment::process(cv::Mat const & results)
62 {
63  int const bgclass = bgid::get();
64  uint32_t const alph = alpha::get() << 24;
65  cv::MatSize const rs = results.size;
66  T const * r = reinterpret_cast<T const *>(results.data);
67 
68  switch (segtype::get())
69  {
70  // ----------------------------------------------------------------------------------------------------
71  case jevois::dnn::postprocessor::SegType::ClassesHWC:
72  {
73  // tensor should be 4D 1xHxWxC, where C is the number of classes. We pick the class index with max value and
74  // apply the colormap to it:
75  if (rs.dims() != 4 || rs[0] != 1)
76  LFATAL("Output tensor is " << jevois::dnn::shapestr(results) << ", but need 1xHxWxC for C classes");
77  int const numclass = rs[3]; int const siz = rs[1] * rs[2] * numclass;
78 
79  // Apply colormap, converting from RGB to RGBA:
80  itsOverlay = cv::Mat(rs[1], rs[2], CV_8UC4);
81  uint32_t * im = reinterpret_cast<uint32_t *>(itsOverlay.data);
82 
83  for (int i = 0; i < siz; i += numclass)
84  {
85  int maxc = -1; T maxval = 0;
86  for (int c = 0; c < numclass; ++c)
87  {
88  T v = *r++;
89  if (v > maxval) { maxval = v; maxc = c; }
90  }
91 
92  // Use full transparent for class bgclass or if out of bounds, otherwise colormap:
93  if (maxc < 0 || maxc > 255 || maxc == bgclass) *im++ = 0; else *im++ = itsColor[maxc] | alph;
94  }
95  }
96  break;
97 
98  // ----------------------------------------------------------------------------------------------------
99  case jevois::dnn::postprocessor::SegType::ClassesCHW:
100  {
101  // tensor should be 4D 1xCxHxW, where C is the number of classes. We pick the class index with max value and
102  // apply the colormap to it:
103  if (rs.dims() != 4 || rs[0] != 1)
104  LFATAL("Output tensor is " << jevois::dnn::shapestr(results) << ", but need 1xCxHxW for C classes");
105  int const numclass = rs[1]; int const hw = rs[2] * rs[3];
106 
107  // Apply colormap, converting from RGB to RGBA:
108  itsOverlay = cv::Mat(rs[2], rs[3], CV_8UC4);
109  uint32_t * im = reinterpret_cast<uint32_t *>(itsOverlay.data);
110 
111  for (int i = 0; i < hw; ++i)
112  {
113  int maxc = -1; T maxval = 0;
114  for (int c = 0; c < numclass; ++c)
115  {
116  T v = results.at<T>(i + c * hw);
117  if (v > maxval) { maxval = v; maxc = c; }
118  }
119 
120  // Use full transparent for class bgclass or if out of bounds, otherwise colormap:
121  if (maxc < 0 || maxc > 255 || maxc == bgclass) *im++ = 0; else *im++ = itsColor[maxc] | alph;
122  }
123  }
124  break;
125 
126  // ----------------------------------------------------------------------------------------------------
127  case jevois::dnn::postprocessor::SegType::ArgMax:
128  {
129  // tensor should be 2D HxW, 3D 1xHxW, or 4D 1xHxWx1 and contain class ID in each pixel:
130  if (rs.dims() != 2 && (rs.dims() != 3 || rs[0] != 1) && (rs.dims() != 4 || rs[0] != 1 || rs[3] != 1))
131  LFATAL("Output tensor is " << jevois::dnn::shapestr(results) << ", but need HxW, 1xHxW, or 1xHxWx1");
132  int const siz = rs[1] * rs[2];
133 
134  // Apply colormap, converting from RGB to RGBA:
135  itsOverlay = cv::Mat(rs[1], rs[2], CV_8UC4);
136  uint32_t * im = reinterpret_cast<uint32_t *>(itsOverlay.data);
137 
138  for (int i = 0; i < siz; ++i)
139  {
140  // Use full transparent for class bgclass or if out of bounds, otherwise colormap:
141  int32_t const id = *r++;
142  if (id < 0 || id > 255 || id == bgclass) *im++ = 0; else *im++ = itsColor[id] | alph;
143  }
144  }
145  break;
146  }
147 }
148 
149 // ####################################################################################################
150 void jevois::dnn::PostProcessorSegment::process(std::vector<cv::Mat> const & outs, jevois::dnn::PreProcessor * preproc)
151 {
152  if (outs.size() != 1) LFATAL("Need exactly one output blob, received " << outs.size());
153 
154  // Patch up the colormap if background class ID is 0:
155  if (bgid::get() != 0) itsColor[0] = 0xff0000; else itsColor[0] = 0;
156 
157  // Post-process:
158  cv::Mat const & results = outs[0];
159 
160  switch (results.type())
161  {
162  case CV_8UC1: process<uint8_t>(results); break;
163  case CV_32FC1: process<float>(results); break;
164  case CV_32SC1: process<int32_t>(results); break;
165 
166  default: LFATAL("Unsupported data type in tensor " << jevois::dnn::shapestr(results));
167  }
168 
169  // Compute overlay corner coords within the input image, for use in report():
170  preproc->getUnscaledCropRect(0, itsTLx, itsTLy, itsBRx, itsBRy);
171 }
172 
173 // ####################################################################################################
175  jevois::OptGUIhelper * helper, bool JEVOIS_UNUSED_PARAM(overlay),
176  bool JEVOIS_UNUSED_PARAM(idle))
177 {
178  // Remember our helper, will be used in destructor to free overlay OpenGL texture:
179  itsHelper = helper;
180 
181  // Outputs may not be ready yet:
182  if (itsOverlay.empty()) return;
183 
184  // If desired, draw boxes in output image:
185  if (outimg)
186  {
187  // todo: blend into YUYV
188  }
189 
190 #ifdef JEVOIS_PRO
191  // Draw the image on top of our input image, as a semi-transparent overlay. OpenGL will do scaling and blending:
192  if (helper)
193  {
194  // Convert box coords from input image to display:
195  ImVec2 tl = helper->i2d(itsTLx, itsTLy), br = helper->i2d(itsBRx, itsBRy);
196  int dtlx = tl.x, dtly = tl.y;
197  unsigned short dw = br.x - tl.x, dh = br.y - tl.y;
198 
199  // Draw overlay:
200  helper->drawImage("ppsr", itsOverlay, true, dtlx, dtly, dw, dh, false /* noalias */, true /* isoverlay */);
201  }
202 #else
203  (void)helper; // keep compiler happy
204 #endif
205 
206  if (mod)
207  {
208  // todo send results to serial
209  }
210 }
jevois::dnn::PostProcessorSegment::process
void process(std::vector< cv::Mat > const &outs, PreProcessor *preproc) override
Process outputs and draw/send some results.
Definition: PostProcessorSegment.C:150
jevois::GUIhelper::drawImage
void drawImage(char const *name, RawImage const &img, int &x, int &y, unsigned short &w, unsigned short &h, bool noalias=false, bool isoverlay=false)
Draw a RawImage, copying pixel data to an OpenGL texture.
Definition: GUIhelper.C:273
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
Module.H
jevois::dnn::PreProcessor::getUnscaledCropRect
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
Definition: PreProcessor.C:81
RawImageOps.H
jevois::dnn::PostProcessorSegment::~PostProcessorSegment
virtual ~PostProcessorSegment()
Destructor.
Definition: PostProcessorSegment.C:29
Utils.H
jevois::GUIhelper::releaseImage
void releaseImage(char const *name)
Release an image.
Definition: GUIhelper.C:574
PostProcessorSegment.H
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::GUIhelper::i2d
ImVec2 i2d(ImVec2 p, char const *name=nullptr)
Convert coordinates of a point from within a rendered image to on-screen.
Definition: GUIhelper.C:374
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:102
jevois::dnn::PostProcessorSegment::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PostProcessorSegment.C:37
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
PreProcessor.H
ObjReco.H
jevois::dnn::PostProcessorSegment::postInit
void postInit() override
Create colormap in postInit()
Definition: PostProcessorSegment.C:43
jevois::dnn::PostProcessorSegment::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PostProcessorSegment.C:174
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
GUIhelper.H