JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessorSegment.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
20 #include <jevois/DNN/Utils.H>
22 #include <jevois/Core/Module.H>
23 #include <jevois/GPU/GUIhelper.H>
24 #include <jevois/Types/ObjReco.H>
25 
26 #include <opencv2/dnn.hpp>
27 
28 // ####################################################################################################
30 {
31 #ifdef JEVOIS_PRO
32  if (itsHelper) itsHelper->releaseImage("ppsr");
33 #endif
34 }
35 
36 // ####################################################################################################
38 {
39  segtype::freeze(doit);
40 }
41 
42 // ####################################################################################################
44 {
45  // Colormap from pascal VOC segmentation benchmark:
46  for (size_t i = 0; i < 256; ++i)
47  {
48  uint32_t & c = itsColor[i]; c = 0;
49  uint32_t ind = i;
50 
51  for (int shift = 7; shift >= 0; --shift)
52  {
53  for (int channel = 0; channel < 3; ++channel) c |= ((ind >> channel) & 1) << (shift + 8 * (3-channel));
54  ind >>= 3;
55  }
56  }
57 }
58 
59 // ####################################################################################################
60 void jevois::dnn::PostProcessorSegment::process(std::vector<cv::Mat> const & outs, jevois::dnn::PreProcessor * preproc)
61 {
62  if (outs.empty()) return;
63 
64  switch (segtype::get())
65  {
66  // ----------------------------------------------------------------------------------------------------
67  case jevois::dnn::postprocessor::SegType::Classes:
68  {
69  // tensor should be 4D 1xHxWxN, where N is the number of classes. We pick the class index with max value and
70  // apply the colormap to it:
71  cv::Mat results = outs[0]; cv::MatSize rs = results.size;
72  if (rs.dims() != 4 || rs[0] != 1 || results.type() != CV_8UC1)
73  LFATAL("Output tensor is " << jevois::dnn::shapestr(results) << ", but need 4D UINT8 with 1 as first size");
74  int const numclass = rs[3]; int const siz = rs[1] * rs[2] * numclass;
75 
76  // Apply colormap, converting from RGB to RGBA. If the background class ID is 0, change its color in the colormap:
77  uint32_t const alph = alpha::get() << 24;
78  int const bgclass = bgid::get(); if (bgclass != 0) itsColor[0] = 0xff0000; else itsColor[0] = 0;
79 
80  itsOverlay = cv::Mat(rs[1], rs[2], CV_8UC4);
81  uint32_t * im = reinterpret_cast<uint32_t *>(itsOverlay.data);
82  uint8_t * r = reinterpret_cast<uint8_t *>(results.data);
83 
84  for (int i = 0; i < siz; i += numclass)
85  {
86  int maxc = -1; int maxval = -1;
87  for (int c = 0; c < numclass; ++c)
88  {
89  int v = *r++;
90  if (v > maxval) { maxval = v; maxc = c; }
91  }
92 
93  // Use full transparent for class bgclass or if out of bounds, otherwise colormap:
94  if (maxc < 0 || maxc > 255 || maxc == bgclass) *im++ = 0; else *im++ = itsColor[maxc] | alph;
95  }
96  }
97  break;
98 
99  // ----------------------------------------------------------------------------------------------------
100  case jevois::dnn::postprocessor::SegType::Classes2:
101  {
102  // tensor should be 4D 1xNxHxW, where N is the number of classes. We pick the class index with max value and
103  // apply the colormap to it:
104  cv::Mat results = outs[0]; cv::MatSize rs = results.size;
105  if (rs.dims() != 4 || rs[0] != 1 || results.type() != CV_32FC1)
106  LFATAL("Output tensor is " << jevois::dnn::shapestr(results) << ", but need 4D FLOAT32 with 1 as first size");
107  int const numclass = rs[1]; int const siz = rs[2] * rs[3];
108 
109  // Apply colormap, converting from RGB to RGBA. If the background class ID is 0, change its color in the colormap:
110  uint32_t const alph = alpha::get() << 24;
111  int const bgclass = bgid::get(); if (bgclass != 0) itsColor[0] = 0xff0000; else itsColor[0] = 0;
112 
113  itsOverlay = cv::Mat(rs[2], rs[3], CV_8UC4);
114  uint32_t * im = reinterpret_cast<uint32_t *>(itsOverlay.data);
115 
116  for (int i = 0; i < siz; ++i)
117  {
118  int maxc = -1; float maxval = -1.0F;
119  for (int c = 0; c < numclass; ++c)
120  {
121  float v = results.at<float>(i + c*siz);
122  if (v > maxval) { maxval = v; maxc = c; }
123  }
124 
125  // Use full transparent for class bgclass or if out of bounds, otherwise colormap:
126  if (maxc < 0 || maxc > 255 || maxc == bgclass) *im++ = 0; else *im++ = itsColor[maxc] | alph;
127  }
128  }
129  break;
130 
131  // ----------------------------------------------------------------------------------------------------
132  case jevois::dnn::postprocessor::SegType::ArgMax:
133  {
134  // tensor should be 3D 1xHxW INT32 and contain class ID in each pixel:
135  cv::Mat results = outs[0]; cv::MatSize rs = results.size;
136  if (rs.dims() != 3 || rs[0] != 1 || results.type() != CV_32SC1)
137  LFATAL("Output tensor is " << jevois::dnn::shapestr(results) << ", but need 3D INT32 with 1 as first size");
138  int const siz = rs[1] * rs[2];
139 
140  // Apply colormap, converting from RGB to RGBA. If the background class ID is 0, change its color in the colormap:
141  uint32_t const alph = alpha::get() << 24;
142  int const bgclass = bgid::get(); if (bgclass != 0) itsColor[0] = 0xff0000; else itsColor[0] = 0;
143 
144  itsOverlay = cv::Mat(rs[1], rs[2], CV_8UC4);
145  uint32_t * im = reinterpret_cast<uint32_t *>(itsOverlay.data);
146  int32_t const * r = reinterpret_cast<int32_t const *>(results.data);
147 
148  for (int i = 0; i < siz; ++i)
149  {
150  // Use full transparent for class bgclass or if out of bounds, otherwise colormap:
151  int32_t const id = *r++;
152  if (id < 0 || id > 255 || id == bgclass) *im++ = 0; else *im++ = itsColor[id] | alph;
153  }
154  }
155  break;
156  }
157 
158  // Compute overlay corner coords within the input image, for use in report():
159  preproc->getUnscaledCropRect(0, itsTLx, itsTLy, itsBRx, itsBRy);
160 }
161 
162 // ####################################################################################################
164  jevois::OptGUIhelper * helper, bool JEVOIS_UNUSED_PARAM(overlay),
165  bool JEVOIS_UNUSED_PARAM(idle))
166 {
167  // Remember our helper, will be used in destructor to free overlay OpenGL texture:
168  itsHelper = helper;
169 
170  // Outputs may not be ready yet:
171  if (itsOverlay.empty()) return;
172 
173  // If desired, draw boxes in output image:
174  if (outimg)
175  {
176  // todo: blend into YUYV
177  }
178 
179 #ifdef JEVOIS_PRO
180  // Draw the image on top of our input image, as a semi-transparent overlay. OpenGL will do scaling and blending:
181  if (helper)
182  {
183  // Convert box coords from input image to display:
184  ImVec2 tl = helper->i2d(itsTLx, itsTLy), br = helper->i2d(itsBRx, itsBRy);
185  int dtlx = tl.x, dtly = tl.y;
186  unsigned short dw = br.x - tl.x, dh = br.y - tl.y;
187 
188  // Draw overlay:
189  helper->drawImage("ppsr", itsOverlay, true, dtlx, dtly, dw, dh, false /* noalias */, true /* isoverlay */);
190  }
191 #else
192  (void)helper; // keep compiler happy
193 #endif
194 
195  if (mod)
196  {
197  // todo send results to serial
198  }
199 }
jevois::dnn::PostProcessorSegment::process
void process(std::vector< cv::Mat > const &outs, PreProcessor *preproc) override
Process outputs and draw/send some results.
Definition: PostProcessorSegment.C:60
jevois::GUIhelper::drawImage
void drawImage(char const *name, RawImage const &img, int &x, int &y, unsigned short &w, unsigned short &h, bool noalias=false, bool isoverlay=false)
Draw a RawImage, copying pixel data to an OpenGL texture.
Definition: GUIhelper.C:242
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
Module.H
jevois::dnn::PreProcessor::getUnscaledCropRect
cv::Rect getUnscaledCropRect(size_t blobnum=0)
Get unscaled crop rectangle in image coordinates.
Definition: PreProcessor.C:75
RawImageOps.H
jevois::dnn::PostProcessorSegment::~PostProcessorSegment
virtual ~PostProcessorSegment()
Destructor.
Definition: PostProcessorSegment.C:29
Utils.H
jevois::GUIhelper::releaseImage
void releaseImage(char const *name)
Release an image.
Definition: GUIhelper.C:543
PostProcessorSegment.H
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::GUIhelper::i2d
ImVec2 i2d(ImVec2 p, char const *name=nullptr)
Convert coordinates of a point from within a rendered image to on-screen.
Definition: GUIhelper.C:343
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:105
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:76
jevois::dnn::PostProcessorSegment::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PostProcessorSegment.C:37
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
PreProcessor.H
ObjReco.H
jevois::dnn::PostProcessorSegment::postInit
void postInit() override
Create colormap in postInit()
Definition: PostProcessorSegment.C:43
jevois::dnn::PostProcessorSegment::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PostProcessorSegment.C:163
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
GUIhelper.H