JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessorClassify.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/DNN/Utils.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Core/Engine.H>
23 #include <jevois/Core/Module.H>
24 #include <jevois/GPU/GUIhelper.H>
25 
26 // ####################################################################################################
28 { }
29 
30 // ####################################################################################################
32 {
33  classes::freeze(doit);
34 }
35 
36 // ####################################################################################################
37 void jevois::dnn::PostProcessorClassify::onParamChange(postprocessor::classes const & JEVOIS_UNUSED_PARAM(param),
38  std::string const & val)
39 {
40  if (val.empty()) return;
41 
42  // Get the dataroot of our network. We assume that there is a sub-component named "network" that is a sibling of us:
43  std::vector<std::string> dd = jevois::split(Component::descriptor(), ":");
44  dd.back() = "network"; dd.emplace_back("dataroot");
45  std::string const dataroot = engine()->getParamStringUnique(jevois::join(dd, ":"));
46 
47  itsLabels = jevois::dnn::readLabelsFile(jevois::absolutePath(dataroot, val));
48 }
49 
50 // ####################################################################################################
51 void jevois::dnn::PostProcessorClassify::process(std::vector<cv::Mat> const & outs,
52  jevois::dnn::PreProcessor * JEVOIS_UNUSED_PARAM(preproc))
53 {
54  if (outs.size() != 1) LFATAL("Expected exactly one output tensor, got " << outs.size());
55  cv::Mat const & out = outs[0]; uint32_t const sz = out.total();
56  if (out.type() != CV_32F) LFATAL("Need FLOAT32 tensor");
57  uint32_t topk = top::get(); if (topk > sz) topk = sz;
58  uint32_t const fudge = classoffset::get();
59  itsObjRec.clear();
60 
61  uint32_t MaxClass[topk]; float fMaxProb[topk];
62  if (softmax::get())
63  {
64  cv::Mat copy = out.clone();
65  jevois::dnn::softmax((float const *)out.data, sz, 1.0F, (float *)copy.data);
66  jevois::dnn::topK((float const *)copy.data, fMaxProb, MaxClass, sz, topk);
67  }
68  else jevois::dnn::topK((float const *)out.data, fMaxProb, MaxClass, sz, topk);
69 
70  // Collect the top-k results that are also above threshold:
71  float const t = thresh::get(); float const fac = 100.0F * scorescale::get();
72 
73  for (uint32_t i = 0; i < topk; ++i)
74  {
75  if (fMaxProb[i] * fac < t) break;
76  jevois::ObjReco o { fMaxProb[i] * fac, jevois::dnn::getLabel(itsLabels, MaxClass[i] + fudge) };
77  itsObjRec.push_back(o);
78  }
79 }
80 
81 // ####################################################################################################
83  jevois::OptGUIhelper * helper, bool overlay, bool idle)
84 {
85  uint32_t const topk = top::get();
86 
87  // If desired, write results to output image:
88  if (outimg && overlay)
89  {
90  int y = 16;
91  jevois::rawimage::writeText(*outimg, jevois::sformat("Top-%u above %.2F", topk, thresh::get()),
92  220, y, jevois::yuyv::White);
93  y += 15;
94 
95  for (jevois::ObjReco const & o : itsObjRec)
96  {
97  jevois::rawimage::writeText(*outimg, jevois::sformat("%s: %.2F", o.category.c_str(), o.score),
98  220, y, jevois::yuyv::White);
99  y += 11;
100  }
101  }
102 
103 #ifdef JEVOIS_PRO
104  // If desired, write results to GUI:
105  if (helper)
106  {
107  if (idle == false && ImGui::CollapsingHeader("Classification results", ImGuiTreeNodeFlags_DefaultOpen))
108  {
109  ImGui::Text("Top-%u classes above threshold %f", topk, thresh::get());
110  ImGui::Separator();
111  uint32_t done = 0;
112  for (jevois::ObjReco const & o : itsObjRec) { ImGui::Text("%s: %.2F", o.category.c_str(), o.score); ++done; }
113  while (done < topk) { ImGui::TextUnformatted("-"); ++done; }
114  }
115 
116  if (overlay)
117  {
118  uint32_t done = 0;
119  for (jevois::ObjReco const & o : itsObjRec)
120  { helper->itext(jevois::sformat("%s: %.2F", o.category.c_str(), o.score)); ++done; }
121  while (done < topk) { helper->itext("-"); ++done; }
122  }
123  }
124 #else
125  (void)idle; (void)helper; // keep compiler happy
126 #endif
127 
128  // If desired, send results to serial port:
129  if (mod) mod->sendSerialObjReco(itsObjRec);
130 }
PostProcessorClassify.H
jevois::Component::descriptor
std::string descriptor() const
Get our full descriptor (including all parents) as [Instancename]:[...]:[...].
Definition: Component.C:276
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
Module.H
jevois::GUIhelper::itext
void itext(char const *txt, ImU32 const &col=IM_COL32_BLACK_TRANS, int line=-1)
Draw some overlay text on top of an image.
Definition: GUIhelper.C:509
RawImageOps.H
jevois::sformat
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Create a string using printf style arguments.
Definition: Utils.C:401
jevois::dnn::PostProcessorClassify::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PostProcessorClassify.C:82
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:257
Utils.H
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
o
#define o
Definition: Font10x20.C:6
jevois::dnn::PostProcessorClassify::onParamChange
void onParamChange(postprocessor::classes const &param, std::string const &val) override
Definition: PostProcessorClassify.C:37
jevois::dnn::readLabelsFile
std::map< int, std::string > readLabelsFile(std::string const &fname)
Read a label file.
Definition: Utils.C:25
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
Write some text in an image.
Definition: RawImageOps.C:689
Engine.H
jevois::dnn::getLabel
std::string getLabel(std::map< int, std::string > const &labels, int id)
Get a label from an id.
Definition: Utils.C:68
jevois::ObjReco
A trivial struct to store object recognition results.
Definition: ObjReco.H:23
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:76
jevois::dnn::PostProcessorClassify::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PostProcessorClassify.C:31
jevois::join
std::string join(std::vector< std::string > const &strings, std::string const &delimiter)
Concatenate a vector of tokens into a string.
Definition: Utils.C:267
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
jevois::absolutePath
std::string absolutePath(std::string const &root, std::string const &path)
Compute an absolute path from two paths.
Definition: Utils.C:347
jevois::dnn::PostProcessorClassify::~PostProcessorClassify
virtual ~PostProcessorClassify()
Destructor.
Definition: PostProcessorClassify.C:27
jevois::dnn::softmax
void softmax(float const *input, size_t n, float fac, float *output)
Apply softmax to a float vector.
Definition: Utils.C:443
Utils.H
jevois::dnn::PostProcessorClassify::process
void process(std::vector< cv::Mat > const &outs, PreProcessor *preproc) override
Process outputs and draw/send some results.
Definition: PostProcessorClassify.C:51
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
GUIhelper.H
jevois::dnn::topK
void topK(float const *pfProb, float *pfMaxProb, uint32_t *pMaxClass, uint32_t outputCount, uint32_t topNum)
Get top-k entries and their indices.
Definition: Utils.C:85
jevois::StdModule::sendSerialObjReco
void sendSerialObjReco(std::vector< ObjReco > const &res)
Send a standardized object recognition message.
Definition: Module.C:540