JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
PostProcessorClassify.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/DNN/Utils.H>
20 #include <jevois/Util/Utils.H>
22 #include <jevois/Core/Engine.H>
23 #include <jevois/Core/Module.H>
24 #include <jevois/GPU/GUIhelper.H>
25 
26 // ####################################################################################################
28 { }
29 
30 // ####################################################################################################
32 {
33  classes::freeze(doit);
34 }
35 
36 // ####################################################################################################
37 void jevois::dnn::PostProcessorClassify::onParamChange(postprocessor::classes const & JEVOIS_UNUSED_PARAM(param),
38  std::string const & val)
39 {
40  if (val.empty()) return;
41 
42  // Get the dataroot of our network. We assume that there is a sub-component named "network" that is a sibling of us:
43  std::vector<std::string> dd = jevois::split(Component::descriptor(), ":");
44  dd.back() = "network"; dd.emplace_back("dataroot");
45  std::string const dataroot = engine()->getParamStringUnique(jevois::join(dd, ":"));
46 
47  itsLabels = jevois::dnn::readLabelsFile(jevois::absolutePath(dataroot, val));
48 }
49 
50 // ####################################################################################################
51 void jevois::dnn::PostProcessorClassify::process(std::vector<cv::Mat> const & outs,
52  jevois::dnn::PreProcessor * JEVOIS_UNUSED_PARAM(preproc))
53 {
54  if (outs.size() != 1 && itsFirstTime)
55  {
56  itsFirstTime = false;
57  LERROR("Expected 1 output tensor, got " << outs.size() << " - USING FIRST ONE");
58  }
59 
60  cv::Mat const & out = outs[0]; uint32_t const sz = out.total();
61  if (out.type() != CV_32F) LFATAL("Need FLOAT32 tensor");
62  uint32_t topk = top::get(); if (topk > sz) topk = sz;
63  uint32_t const fudge = classoffset::get();
64  itsObjRec.clear();
65 
66  uint32_t MaxClass[topk]; float fMaxProb[topk];
67  if (softmax::get())
68  {
69  float sm[out.total()];
70  jevois::dnn::softmax((float const *)out.data, sz, 1, 1.0F, sm, false);
71  jevois::dnn::topK(sm, fMaxProb, MaxClass, sz, topk);
72  }
73  else jevois::dnn::topK((float const *)out.data, fMaxProb, MaxClass, sz, topk);
74 
75  // Collect the top-k results that are also above threshold:
76  float const t = cthresh::get(); float const fac = 100.0F * scorescale::get();
77 
78  for (uint32_t i = 0; i < topk; ++i)
79  {
80  if (fMaxProb[i] * fac < t) break;
81  jevois::ObjReco o { fMaxProb[i] * fac, jevois::dnn::getLabel(itsLabels, MaxClass[i] + fudge) };
82  itsObjRec.push_back(o);
83  }
84 }
85 
86 // ####################################################################################################
88  jevois::OptGUIhelper * helper, bool overlay, bool idle)
89 {
90  uint32_t const topk = top::get();
91 
92  // If desired, write results to output image:
93  if (outimg && overlay)
94  {
95  int y = 16;
96  jevois::rawimage::writeText(*outimg, jevois::sformat("Top-%u above %.2F%%", topk, cthresh::get()),
97  220, y, jevois::yuyv::White);
98  y += 15;
99 
100  for (jevois::ObjReco const & o : itsObjRec)
101  {
102  jevois::rawimage::writeText(*outimg, jevois::sformat("%s: %.2F", o.category.c_str(), o.score),
103  220, y, jevois::yuyv::White);
104  y += 11;
105  }
106  }
107 
108 #ifdef JEVOIS_PRO
109  // If desired, write results to GUI:
110  if (helper)
111  {
112  if (idle == false && ImGui::CollapsingHeader("Classification results", ImGuiTreeNodeFlags_DefaultOpen))
113  {
114  ImGui::Text("Top-%u classes above threshold %.2f", topk, cthresh::get());
115  ImGui::Separator();
116  uint32_t done = 0;
117  for (jevois::ObjReco const & o : itsObjRec) { ImGui::Text("%s: %.2F", o.category.c_str(), o.score); ++done; }
118  while (done++ < topk) ImGui::TextUnformatted("-");
119  }
120 
121  if (overlay)
122  {
123  uint32_t done = 0;
124  for (jevois::ObjReco const & o : itsObjRec)
125  { helper->itext(jevois::sformat("%s: %.2F", o.category.c_str(), o.score)); ++done; }
126  while (done++ < topk) helper->itext("-");
127  }
128  }
129 #else
130  (void)idle; (void)helper; // keep compiler happy
131 #endif
132 
133  // If desired, send results to serial port:
134  if (mod) mod->sendSerialObjReco(itsObjRec);
135 }
PostProcessorClassify.H
jevois::Component::descriptor
std::string descriptor() const
Get our full descriptor (including all parents) as [Instancename]:[...]:[...].
Definition: Component.C:276
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
Module.H
jevois::GUIhelper::itext
void itext(char const *txt, ImU32 const &col=IM_COL32_BLACK_TRANS, int line=-1)
Draw some overlay text on top of an image.
Definition: GUIhelper.C:540
RawImageOps.H
jevois::sformat
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Create a string using printf style arguments.
Definition: Utils.C:419
jevois::dnn::PostProcessorClassify::report
void report(jevois::StdModule *mod, jevois::RawImage *outimg=nullptr, jevois::OptGUIhelper *helper=nullptr, bool overlay=true, bool idle=false) override
Report what happened in last process() to console/output video/GUI.
Definition: PostProcessorClassify.C:87
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:258
Utils.H
jevois::dnn::softmax
size_t softmax(float const *input, size_t const n, size_t const stride, float const fac, float *output, bool maxonly)
Apply softmax to a float vector.
Definition: Utils.C:589
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
o
#define o
Definition: Font10x20.C:6
jevois::dnn::PostProcessorClassify::onParamChange
void onParamChange(postprocessor::classes const &param, std::string const &val) override
Definition: PostProcessorClassify.C:37
jevois::dnn::readLabelsFile
std::map< int, std::string > readLabelsFile(std::string const &fname)
Read a label file.
Definition: Utils.C:24
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:211
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &root, std::filesystem::path const &path)
Compute an absolute path from two paths.
Definition: Utils.C:365
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
Write some text in an image.
Definition: RawImageOps.C:689
Engine.H
jevois::dnn::getLabel
std::string getLabel(std::map< int, std::string > const &labels, int id)
Get a label from an id.
Definition: Utils.C:67
jevois::ObjReco
A trivial struct to store object recognition results.
Definition: ObjReco.H:23
jevois::dnn::PreProcessor
Pre-Processor for neural network pipeline.
Definition: PreProcessor.H:102
jevois::dnn::PostProcessorClassify::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: PostProcessorClassify.C:31
jevois::join
std::string join(std::vector< std::string > const &strings, std::string const &delimiter)
Concatenate a vector of tokens into a string.
Definition: Utils.C:268
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::dnn::PostProcessorClassify::~PostProcessorClassify
virtual ~PostProcessorClassify()
Destructor.
Definition: PostProcessorClassify.C:27
Utils.H
jevois::dnn::PostProcessorClassify::process
void process(std::vector< cv::Mat > const &outs, PreProcessor *preproc) override
Process outputs and draw/send some results.
Definition: PostProcessorClassify.C:51
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
GUIhelper.H
jevois::dnn::topK
void topK(float const *pfProb, float *pfMaxProb, uint32_t *pMaxClass, uint32_t outputCount, uint32_t topNum)
Get top-k entries and their indices.
Definition: Utils.C:84
jevois::StdModule::sendSerialObjReco
void sendSerialObjReco(std::vector< ObjReco > const &res)
Send a standardized object recognition message.
Definition: Module.C:536