JeVois  1.20
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Pipeline.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/DNN/Pipeline.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
21 #include <jevois/Util/Async.H>
23 #include <jevois/Debug/SysInfo.H>
24 #include <jevois/DNN/Utils.H>
25 #include <jevois/Core/Engine.H>
26 
28 #include <jevois/DNN/NetworkONNX.H>
29 #include <jevois/DNN/NetworkNPU.H>
30 #include <jevois/DNN/NetworkTPU.H>
33 
36 
43 
44 #include <opencv2/core/utils/filesystem.hpp>
45 
46 #include <fstream>
47 
48 // ####################################################################################################
49 
50 // Simple class to hold a list of <name, value> pairs for our parameters, with updating the value of existing parameer
51 // names if they are set several times (e.g., first set as a global, then set again for a particular network). Note that
52 // here we do not check the validity of the parameters. This is delegated to Pipeline::setZooParam():
53 namespace
54 {
55  class ParHelper
56  {
57  public:
58  // ----------------------------------------------------------------------------------------------------
59  // Set a param from an entry in our yaml file
60  void set(cv::FileNode const & item, std::string const & zf, cv::FileNode const & node)
61  {
62  std::string k = item.name();
63  std::string v;
64  switch (item.type())
65  {
66  case cv::FileNode::INT: v = std::to_string((int)item); break;
67  case cv::FileNode::REAL: v = std::to_string((float)item); break;
68  case cv::FileNode::STRING: v = (std::string)item; break;
69  default:
70  if (&node == &item)
71  LFATAL("Invalid global zoo parameter " << k << " type " << item.type() << " in " << zf);
72  else
73  LFATAL("Invalid zoo parameter " << k << " type " << item.type() << " in " << zf << " node " << node.name());
74  }
75 
76  // Update value if param already exists, or add new key,value pair:
77  for (auto & p : params) if (p.first == k) { p.second = v; return; }
78  params.emplace_back(std::make_pair(k, v));
79  }
80 
81  // ----------------------------------------------------------------------------------------------------
82  // Get a value for entry subname under item if found, otherwise try our table of globals, otherwise empty
83  std::string pget(cv::FileNode & item, std::string const & subname)
84  {
85  std::string const v = (std::string)item[subname];
86  if (v.empty() == false) return v;
87  for (auto const & p : params) if (p.first == subname) return p.second;
88  return std::string();
89  }
90 
91  // ----------------------------------------------------------------------------------------------------
92  // Un-set a previously set global
93  void unset(std::string const & name)
94  {
95  for (auto itr = params.begin(); itr != params.end(); ++itr)
96  if (itr->first == name) { params.erase(itr); return; }
97  }
98 
99  // Ordering matters, so use a vector instead of map or unordered_map
100  std::vector<std::pair<std::string /* name */, std::string /* value */>> params;
101  };
102 }
103 
104 // ####################################################################################################
105 jevois::dnn::Pipeline::Pipeline(std::string const & instance) :
106  jevois::Component(instance), itsTpre("PreProc"), itsTnet("Network"), itsTpost("PstProc")
107 {
108  itsAccelerators["TPU"] = jevois::getNumInstalledTPUs();
109  itsAccelerators["VPU"] = jevois::getNumInstalledVPUs();
110  itsAccelerators["NPU"] = jevois::getNumInstalledNPUs();
111  itsAccelerators["SPU"] = jevois::getNumInstalledSPUs();
112  itsAccelerators["OpenCV"] = 1; // OpenCV always available
113  itsAccelerators["ORT"] = 1; // ONNX runtime always available
114  itsAccelerators["Python"] = 1; // Python always available
115 #ifdef JEVOIS_PLATFORM_PRO
116  itsAccelerators["VPUX"] = 1; // VPU emulation on CPU always available through OpenVino
117 #endif
118  itsAccelerators["NPUX"] = 1; // NPU over Tim-VX always available since compiled into OpenCV
119 
120  LINFO("Detected " <<
121  itsAccelerators["NPU"] << " JeVois-Pro NPUs, " <<
122  itsAccelerators["SPU"] << " Hailo8 SPUs, " <<
123  itsAccelerators["TPU"] << " Coral TPUs, " <<
124  itsAccelerators["VPU"] << " Myriad-X VPUs.");
125 }
126 
127 // ####################################################################################################
129 {
130  preproc::freeze(doit);
131  nettype::freeze(doit);
132  postproc::freeze(doit);
133 
134  if (itsPreProcessor) itsPreProcessor->freeze(doit);
135  if (itsNetwork) itsNetwork->freeze(doit);
136  if (itsPostProcessor) itsPostProcessor->freeze(doit);
137 }
138 
139 // ####################################################################################################
141 {
142  // Freeze all params that users should not modify at runtime:
143  freeze(true);
144 }
145 
146 // ####################################################################################################
148 {
149  // If we have a network running async, make sure we wait here until it is done:
150  asyncNetWait();
151 }
152 
153 // ####################################################################################################
155 {
156  // Make sure network is not running as we die:
157  asyncNetWait();
158 }
159 
160 // ####################################################################################################
161 void jevois::dnn::Pipeline::asyncNetWait()
162 {
163  // If we were currently doing async processing, wait until network is done:
164  if (itsNetFut.valid())
165  while (true)
166  {
167  if (itsNetFut.wait_for(std::chrono::seconds(5)) == std::future_status::timeout)
168  LERROR("Still waiting for network to finish running...");
169  else break;
170  }
171 
172  try { itsNetFut.get(); } catch (...) { }
173  itsOuts.clear();
174 }
175 
176 // ####################################################################################################
177 void jevois::dnn::Pipeline::onParamChange(pipeline::filter const & JEVOIS_UNUSED_PARAM(param),
178  jevois::dnn::pipeline::Filter const & JEVOIS_UNUSED_PARAM(val))
179 {
180  // Reload the zoo file so that the filter can be applied to create the parameter def of pipe, but first we need this
181  // parameter to indeed be updated. So here we just set a flag and the update will occur in process(), after we run the
182  // current model one last time:
183  itsZooChanged = true;
184 }
185 
186 // ####################################################################################################
187 void jevois::dnn::Pipeline::onParamChange(pipeline::zooroot const & JEVOIS_UNUSED_PARAM(param),
188  std::string const & JEVOIS_UNUSED_PARAM(val))
189 {
190  // Reload the zoo file, but first we need this parameter to indeed be updated. So here we just set a flag and the
191  // update will occur in process(), after we run the current model one last time:
192  itsZooChanged = true;
193 }
194 
195 // ####################################################################################################
196 void jevois::dnn::Pipeline::onParamChange(pipeline::benchmark const & JEVOIS_UNUSED_PARAM(param),
197  bool const & val)
198 {
199  if (val)
200  {
201  statsfile::set("benchmark.html");
202  statsfile::freeze(true);
203  }
204  else
205  {
206  statsfile::freeze(false);
207  statsfile::reset();
208  }
209 }
210 
211 // ####################################################################################################
212 void jevois::dnn::Pipeline::onParamChange(pipeline::zoo const & JEVOIS_UNUSED_PARAM(param),
213  std::string const & val)
214 {
215  // Just nuke everything:
216  itsPreProcessor.reset();
217  itsNetwork.reset();
218  itsPostProcessor.reset();
219  // Will get instantiated again when pipe param is set.
220 
221  // Load zoo file:
222  std::vector<std::string> pipes;
223  scanZoo(jevois::absolutePath(zooroot::get(), val), filter::strget(), pipes, "");
224  LINFO("Found a total of " << pipes.size() << " valid pipelines.");
225 
226  // Update the parameter def of pipe:
227  if (pipes.empty() == false)
228  {
229  jevois::ParameterDef<std::string> newdef("pipe", "Pipeline to use, determined by entries in the zoo file and "
230  "by the current filter",
231  pipes[0], pipes, jevois::dnn::pipeline::ParamCateg);
232  pipe::changeParameterDef(newdef);
233 
234  // Just changing the def does not change the param value, so change it now:
235  pipe::set(pipes[0]);
236  }
237 }
238 
239 // ####################################################################################################
240 void jevois::dnn::Pipeline::scanZoo(std::filesystem::path const & zoofile, std::string const & filt,
241  std::vector<std::string> & pipes, std::string const & indent)
242 {
243  LINFO(indent << "Scanning model zoo file " << zoofile << " with filter [" << filt << "]...");
244  int ntot = 0, ngood = 0;
245 
246  bool has_vpu = false;
247  auto itr = itsAccelerators.find("VPU");
248  if (itr != itsAccelerators.end() && itr->second > 0) has_vpu = true;
249 
250  // Scan the zoo file to update the parameter def of the pipe parameter:
251  cv::FileStorage fs(zoofile, cv::FileStorage::READ);
252  if (fs.isOpened() == false) LFATAL("Could not open zoo file " << zoofile);
253  cv::FileNode fn = fs.root();
254  ParHelper ph;
255 
256  for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
257  {
258  cv::FileNode item = *fit;
259 
260  // Process include: directives recursively:
261  if (item.name() == "include")
262  {
263  scanZoo(jevois::absolutePath(zooroot::get(), (std::string)item), filt, pipes, indent + " ");
264  }
265  // Process includedir: directives (only one level of directory is scanned):
266  else if (item.name() == "includedir")
267  {
268  std::filesystem::path const dir = jevois::absolutePath(zooroot::get(), (std::string)item);
269  for (auto const & dent : std::filesystem::recursive_directory_iterator(dir))
270  if (dent.is_regular_file())
271  {
272  std::filesystem::path const path = dent.path();
273  std::filesystem::path const ext = path.extension();
274  if (ext == ".yml" || ext == ".yaml") scanZoo(path, filt, pipes, indent + " ");
275  }
276  }
277  // Unset a previously set global?
278  else if (item.name() == "unset")
279  {
280  ph.unset((std::string)item);
281  }
282  // Set a global:
283  else if (! item.isMap())
284  {
285  ph.set(item, zoofile, item);
286  }
287  // Map type (model definition):
288  else
289  {
290  ++ntot;
291 
292  // As a prefix, we use OpenCV for OpenCV models on CPU/OpenCL backends, and VPU for InferenceEngine backend with
293  // Myriad target, VPUX for InferenceEngine/CPU (arm-compute OpenVino plugin, only works on platform), and NPUX for
294  // TimVX/NPU (NPU using TimVX OpenCV extension, uses NPU on platform or emulator on host):
295 
296  // Set then get nettype to account for globals:
297  std::string typ = ph.pget(item, "nettype");
298 
299  if (typ == "OpenCV")
300  {
301  std::string backend = ph.pget(item, "backend");
302  std::string target = ph.pget(item, "target");
303 
304  if (backend == "InferenceEngine")
305  {
306  if (target == "Myriad")
307  {
308  if (has_vpu) typ = "VPU"; // run VPU models on VPU if MyriadX accelerator is present
309 #ifdef JEVOIS_PLATFORM_PRO
310  else typ = "VPUX"; // emulate VPU models on CPU through ARM-Compute if MyriadX accelerator not present
311 #else
312  else continue; // VPU emulation does not work on host...
313 #endif
314  }
315  else if (target == "CPU") typ = "VPUX";
316  }
317  else if (backend == "TimVX" && target == "NPU") typ = "NPUX";
318  }
319 
320  // Do not consider a model if we do not have the accelerator for it:
321  bool has_accel = false;
322  itr = itsAccelerators.find(typ);
323  if (itr != itsAccelerators.end() && itr->second > 0) has_accel = true;
324 
325  // Add this pipe if it matches our filter and we have the accelerator for it:
326  if ((filt == "All" || typ == filt) && has_accel)
327  {
328  std::string const postproc = ph.pget(item, "postproc");
329  pipes.emplace_back(typ + ':' + postproc + ':' + item.name());
330  ++ngood;
331  }
332  }
333  }
334 
335  LINFO(indent << "Found " << ntot << " pipelines, " << ngood << " passed the filter.");
336 }
337 
338 // ####################################################################################################
339 void jevois::dnn::Pipeline::onParamChange(pipeline::pipe const & JEVOIS_UNUSED_PARAM(param), std::string const & val)
340 {
341  if (val.empty()) return;
342  itsPipeThrew = false;
343  freeze(false);
344 
345  // Clear any errors related to previous pipeline:
346  engine()->clearErrors();
347 
348  // Find the desired pipeline, and set it up:
349  std::string const z = jevois::absolutePath(zooroot::get(), zoo::get());
350  std::vector<std::string> tok = jevois::split(val, ":");
351  if (selectPipe(z, tok) == false)
352  LFATAL("Could not find pipeline entry [" << val << "] in zoo file " << z << " and its includes");
353 
354  freeze(true);
355 }
356 
357 // ####################################################################################################
358 bool jevois::dnn::Pipeline::selectPipe(std::string const & zoofile, std::vector<std::string> const & tok)
359 {
360  // We might have frozen processing to Sync if we ran a NetworkPython previously, so unfreeze here:
361  processing::freeze(false);
362  processing::set(jevois::dnn::pipeline::Processing::Async);
363 
364  // Check if we have a VPU, to use VPU vs VPUX:
365  bool has_vpu = false;
366  auto itr = itsAccelerators.find("VPU");
367  if (itr != itsAccelerators.end() && itr->second > 0) has_vpu = true;
368  bool vpu_emu = false;
369 
370  // Clear any old stats:
371  itsPreStats.clear(); itsNetStats.clear(); itsPstStats.clear();
372  itsStatsWarmup = true; // warmup before computing new stats
373 
374  // Open the zoo file:
375  cv::FileStorage fs(zoofile, cv::FileStorage::READ);
376  if (fs.isOpened() == false) LFATAL("Could not open zoo file " << zoofile);
377 
378  // Find the desired pipeline:
379  ParHelper ph;
380  cv::FileNode fn = fs.root(), node;
381 
382  for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
383  {
384  cv::FileNode item = *fit;
385 
386  // Process include: directives recursively, end the recursion if we found our pipe in there:
387  if (item.name() == "include")
388  {
389  if (selectPipe(jevois::absolutePath(zooroot::get(), (std::string)item), tok)) return true;
390  }
391 
392  // Process includedir: directives (only one level of directory is scanned), end recursion if we found our pipe:
393  else if (item.name() == "includedir")
394  {
395  std::filesystem::path const dir = jevois::absolutePath(zooroot::get(), (std::string)item);
396  for (auto const & dent : std::filesystem::recursive_directory_iterator(dir))
397  if (dent.is_regular_file())
398  {
399  std::filesystem::path const path = dent.path();
400  std::filesystem::path const ext = path.extension();
401  if (ext == ".yml" || ext == ".yaml") if (selectPipe(path, tok)) return true;
402  }
403  }
404 
405  // Unset a previously set global?
406  else if (item.name() == "unset")
407  {
408  ph.unset((std::string)item);
409  }
410  // Set a global:
411  else if (! item.isMap())
412  {
413  ph.set(item, zoofile, node);
414  }
415  // This is an entry for a pipeline with a bunch of params under it:
416  else
417  {
418  if (item.name() != tok.back()) continue;
419  if (tok.size() == 1) { node = item; break; }
420  if (tok.size() != 3) LFATAL("Malformed pipeline name: " << jevois::join(tok, ":"));
421 
422  // Skip if postproc is no match:
423  std::string postproc = ph.pget(item, "postproc");
424  if (postproc != tok[1] && postproc::strget() != tok[1]) continue;
425 
426  std::string nettype = ph.pget(item, "nettype");
427  std::string backend = ph.pget(item, "backend");
428  std::string target = ph.pget(item, "target");
429 
430  if (tok[0] == "VPU")
431  {
432  if (nettype == "OpenCV" && backend == "InferenceEngine" && target == "Myriad")
433  { node = item; break; }
434  }
435  else if (tok[0] == "VPUX")
436  {
437  if (nettype == "OpenCV" && backend == "InferenceEngine")
438  {
439  if (target == "Myriad" && has_vpu == false) { vpu_emu = true; node = item; break; }
440  else if (target == "CPU") { node = item; break; }
441  }
442  }
443  else if (tok[0] == "NPUX")
444  {
445  if (nettype == "OpenCV" && backend == "TimVX" && target == "NPU")
446  { node = item; break; }
447  }
448  else
449  {
450  if (nettype == tok[0])
451  { node = item; break; }
452  }
453  }
454  }
455 
456  // If the spec was not a match with any entries in the file, return false:
457  if (node.empty()) return false;
458 
459  // Found the pipe. First nuke our current pre/net/post:
460  asyncNetWait();
461  itsPreProcessor.reset(); removeSubComponent("preproc", false);
462  itsNetwork.reset(); removeSubComponent("network", false);
463  itsPostProcessor.reset(); removeSubComponent("postproc", false);
464 
465  // Then set all the global parameters of the current file:
466  //for (auto const & pp : ph.params)
467  // setZooParam(pp.first, pp.second, zoofile, fs.root());
468 
469  // Then iterate over all pipeline params and set them: first update our table, then set params from the whole table:
470  for (cv::FileNodeIterator fit = node.begin(); fit != node.end(); ++fit)
471  ph.set(*fit, zoofile, node);
472 
473  for (auto const & pp : ph.params)
474  {
475  if (vpu_emu && pp.first == "target") setZooParam(pp.first, "CPU", zoofile, node);
476  else setZooParam(pp.first, pp.second, zoofile, node);
477  }
478 
479  // Running a python net async segfaults instantly if we are also concurrently running pre or post processing in
480  // python, as python is not re-entrant... so force sync here:
481  if (dynamic_cast<jevois::dnn::NetworkPython *>(itsNetwork.get()) &&
482  (dynamic_cast<jevois::dnn::PreProcessorPython *>(itsPreProcessor.get()) ||
483  dynamic_cast<jevois::dnn::PostProcessorPython *>(itsPostProcessor.get())))
484  {
485  if (processing::get() != jevois::dnn::pipeline::Processing::Sync)
486  {
487  LERROR("Network of type Python cannot run Async if pre- or post- processor are also Python "
488  "-- FORCING Sync processing");
489  processing::set(jevois::dnn::pipeline::Processing::Sync);
490  }
491  processing::freeze(true);
492  }
493 
494  return true;
495 }
496 
497 // ####################################################################################################
498 void jevois::dnn::Pipeline::setZooParam(std::string const & k, std::string const & v,
499  std::string const & zf, cv::FileNode const & node)
500 {
501  // The zoo file may contain extra params, like download URL, etc. To ignore those while still catching invalid
502  // values on our parameters, we first check whether the parameter exists, and, if so, try to set it:
503  bool hasparam = false;
504  try { getParamStringUnique(k); hasparam = true; } catch (...) { }
505 
506  if (hasparam)
507  {
508  LINFO("Setting ["<<k<<"] to ["<<v<<']');
509 
510  try { setParamStringUnique(k, v); }
511  catch (std::exception const & e)
512  { LFATAL("While parsing [" << node.name() << "] in model zoo file " << zf << ": " << e.what()); }
513  catch (...)
514  { LFATAL("While parsing [" << node.name() << "] in model zoo file " << zf << ": unknown error"); }
515  }
516  else if (paramwarn::get())
517  engine()->reportError("WARNING: Unused parameter [" + k + "] in " + zf + " node [" + node.name() + "]");
518 }
519 
520 // ####################################################################################################
521 void jevois::dnn::Pipeline::onParamChange(pipeline::preproc const & JEVOIS_UNUSED_PARAM(param),
522  pipeline::PreProc const & val)
523 {
524  itsPreProcessor.reset(); removeSubComponent("preproc", false);
525 
526  switch (val)
527  {
528  case jevois::dnn::pipeline::PreProc::Blob:
529  itsPreProcessor = addSubComponent<jevois::dnn::PreProcessorBlob>("preproc");
530  break;
531 
532  case jevois::dnn::pipeline::PreProc::Python:
533  itsPreProcessor = addSubComponent<jevois::dnn::PreProcessorPython>("preproc");
534  break;
535 
536  case jevois::dnn::pipeline::PreProc::Custom:
537  // nothing here, user must call setCustomPreProcessor() later
538  break;
539  }
540 
541  if (itsPreProcessor) LINFO("Instantiated pre-processor of type " << itsPreProcessor->className());
542  else LINFO("No pre-processor");
543 }
544 
545 // ####################################################################################################
546 void jevois::dnn::Pipeline::setCustomPreProcessor(std::shared_ptr<jevois::dnn::PreProcessor> pp)
547 {
548  itsPreProcessor.reset(); removeSubComponent("preproc", false);
549  itsPreProcessor = pp;
550  preproc::set(jevois::dnn::pipeline::PreProc::Custom);
551 
552  if (itsPreProcessor) LINFO("Attached pre-processor of type " << itsPreProcessor->className());
553  else LINFO("No pre-processor");
554 }
555 
556 // ####################################################################################################
557 void jevois::dnn::Pipeline::onParamChange(pipeline::nettype const & JEVOIS_UNUSED_PARAM(param),
558  pipeline::NetType const & val)
559 {
560  asyncNetWait(); // If currently processing async net, wait until done
561 
562  itsNetwork.reset(); removeSubComponent("network", false);
563 
564  switch (val)
565  {
566  case jevois::dnn::pipeline::NetType::OpenCV:
567  itsNetwork = addSubComponent<jevois::dnn::NetworkOpenCV>("network");
568  break;
569 
570 #ifdef JEVOIS_PRO
571 
572  case jevois::dnn::pipeline::NetType::ORT:
573  itsNetwork = addSubComponent<jevois::dnn::NetworkONNX>("network");
574  break;
575 
576  case jevois::dnn::pipeline::NetType::NPU:
577 #ifdef JEVOIS_PLATFORM
578  itsNetwork = addSubComponent<jevois::dnn::NetworkNPU>("network");
579 #else // JEVOIS_PLATFORM
580  LFATAL("NPU network is only supported on JeVois-Pro Platform");
581 #endif
582  break;
583 
584  case jevois::dnn::pipeline::NetType::SPU:
585  itsNetwork = addSubComponent<jevois::dnn::NetworkHailo>("network");
586  break;
587 
588  case jevois::dnn::pipeline::NetType::TPU:
589  itsNetwork = addSubComponent<jevois::dnn::NetworkTPU>("network");
590  break;
591 #endif
592 
593  case jevois::dnn::pipeline::NetType::Python:
594  itsNetwork = addSubComponent<jevois::dnn::NetworkPython>("network");
595  break;
596 
597  case jevois::dnn::pipeline::NetType::Custom:
598  // Nothing here, user must call setCustomNetwork() later
599  break;
600  }
601 
602  if (itsNetwork) LINFO("Instantiated network of type " << itsNetwork->className());
603  else LINFO("No network");
604 
605  // We already display a "loading..." message while the network is loading, but some OpenCV networks take a long time
606  // to process their first frame after they are loaded (e.g., YuNet initializes all the anchors on first frame). So
607  // here we set some placeholder text that will appear after the network is loaded and is processing the first frame:
608  itsInputAttrs.clear();
609  itsNetInfo.clear();
610  itsNetInfo.emplace_back("* Input Tensors");
611  itsNetInfo.emplace_back("Initializing network...");
612  itsNetInfo.emplace_back("* Network");
613  itsNetInfo.emplace_back("Initializing network...");
614  itsNetInfo.emplace_back("* Output Tensors");
615  itsNetInfo.emplace_back("Initializing network...");
616  itsAsyncNetInfo = itsNetInfo;
617  itsAsyncNetworkTime = "Network: -";
618  itsAsyncNetworkSecs = 0.0;
619 }
620 
621 // ####################################################################################################
622 void jevois::dnn::Pipeline::setCustomNetwork(std::shared_ptr<jevois::dnn::Network> n)
623 {
624  asyncNetWait(); // If currently processing async net, wait until done
625  itsNetwork.reset(); removeSubComponent("network", false);
626  itsNetwork = n;
627  nettype::set(jevois::dnn::pipeline::NetType::Custom);
628 
629  if (itsNetwork) LINFO("Attached network of type " << itsNetwork->className());
630  else LINFO("No network");
631 
632  itsInputAttrs.clear();
633 }
634 
635 // ####################################################################################################
636 void jevois::dnn::Pipeline::onParamChange(pipeline::postproc const & JEVOIS_UNUSED_PARAM(param),
637  pipeline::PostProc const & val)
638 {
639  asyncNetWait(); // If currently processing async net, wait until done
640 
641  itsPostProcessor.reset(); removeSubComponent("postproc", false);
642 
643  switch (val)
644  {
645  case jevois::dnn::pipeline::PostProc::Classify:
646  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorClassify>("postproc");
647  break;
648  case jevois::dnn::pipeline::PostProc::Detect:
649  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorDetect>("postproc");
650  break;
651  case jevois::dnn::pipeline::PostProc::Segment:
652  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorSegment>("postproc");
653  break;
654  case jevois::dnn::pipeline::PostProc::YuNet:
655  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorYuNet>("postproc");
656  break;
657  case jevois::dnn::pipeline::PostProc::Python:
658  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorPython>("postproc");
659  break;
660  case jevois::dnn::pipeline::PostProc::Stub:
661  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorStub>("postproc");
662  case jevois::dnn::pipeline::PostProc::Custom:
663  // Nothing here, user must call setCustomPostProcessor() later
664  break;
665  }
666 
667  if (itsPostProcessor) LINFO("Instantiated post-processor of type " << itsPostProcessor->className());
668  else LINFO("No post-processor");
669 }
670 
671 // ####################################################################################################
672 void jevois::dnn::Pipeline::setCustomPostProcessor(std::shared_ptr<jevois::dnn::PostProcessor> pp)
673 {
674  asyncNetWait(); // If currently processing async net, wait until done
675  itsPostProcessor.reset(); removeSubComponent("postproc", false);
676  itsPostProcessor = pp;
677  postproc::set(jevois::dnn::pipeline::PostProc::Custom);
678 
679  if (itsPostProcessor) LINFO("Attached post-processor of type " << itsPostProcessor->className());
680  else LINFO("No post-processor");
681 }
682 
683 // ####################################################################################################
685 {
686  return itsPreProcessor && itsNetwork && itsNetwork->ready() && itsPostProcessor;
687 }
688 
689 // ####################################################################################################
690 bool jevois::dnn::Pipeline::checkAsyncNetComplete()
691 {
692  if (itsNetFut.valid() && itsNetFut.wait_for(std::chrono::milliseconds(2)) == std::future_status::ready)
693  {
694  itsOuts = itsNetFut.get();
695  itsNetInfo.clear();
696  std::swap(itsNetInfo, itsAsyncNetInfo);
697  itsProcTimes[1] = itsAsyncNetworkTime;
698  itsProcSecs[1] = itsAsyncNetworkSecs;
699  return true;
700  }
701  return false;
702 }
703 
704 // ####################################################################################################
706  jevois::OptGUIhelper * helper, bool idle)
707 {
708  // Reload the zoo file if filter has changed:
709  if (itsZooChanged) { itsZooChanged = false; zoo::set(zoo::get()); }
710 
711  // If the pipeline is throwing exception at any stage, do not do anything here, itsPipeThrew is cleared when selecting
712  // a new pipe:
713  if (itsPipeThrew) return;
714 
715  bool const ovl = overlay::get();
716  itsOutImgY = 5; // y text position when using outimg text drawings
717 
718 #ifdef JEVOIS_PRO
719  // Open an info window if using GUI and not idle:
720  if (helper && idle == false) ImGui::Begin((instanceName() + ':' + getParamStringUnique("pipe")).c_str());
721 #else
722  (void)helper; // avoid compiler warning
723 #endif
724 
725  // If we want an overlay, show network name on first line:
726  if (ovl)
727  {
728  if (outimg)
729  {
730  jevois::rawimage::writeText(*outimg, instanceName() + ':' + getParamStringUnique("pipe"),
731  5, itsOutImgY, jevois::yuyv::White);
732  itsOutImgY += 11;
733  }
734 
735 #ifdef JEVOIS_PRO
736  if (helper) helper->itext(instanceName() + ':' + getParamStringUnique("pipe"));
737 #endif
738  }
739 
740  // If network is not ready, inform user. Be careful that ready() may throw, eg, if bad network name was given and
741  // network could not be loaded:
742  try
743  {
744  if (ready() == false)
745  {
746  char const * msg = itsNetwork ? "Loading network..." : "No network selected...";
747 
748  if (outimg)
749  {
750  jevois::rawimage::writeText(*outimg, msg, 5, itsOutImgY, jevois::yuyv::White);
751  itsOutImgY += 11;
752  }
753 
754 #ifdef JEVOIS_PRO
755  if (helper)
756  {
757  if (idle == false) ImGui::TextUnformatted(msg);
758  if (ovl) helper->itext(msg);
759  }
760 #endif
761 
762  itsProcTimes = { "PreProc: -", "Network: -", "PstProc: -" };
763  itsProcSecs = { 0.0, 0.0, 0.0 };
764  }
765  else
766  {
767  // Network is ready, run processing, either single-thread (Sync) or threaded (Async):
768  switch (processing::get())
769  {
770  // --------------------------------------------------------------------------------
771  case jevois::dnn::pipeline::Processing::Sync:
772  {
773  asyncNetWait(); // If currently processing async net, wait until done
774 
775  // Pre-process:
776  itsTpre.start();
777  if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
778  itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
779  itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
780  itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
781 
782  // Network forward pass:
783  itsNetInfo.clear();
784  itsTnet.start();
785  itsOuts = itsNetwork->process(itsBlobs, itsNetInfo);
786  itsProcTimes[1] = itsTnet.stop(&itsProcSecs[1]);
787 
788  // Show network info:
789  showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
790 
791  // Post-Processing:
792  itsTpost.start();
793  itsPostProcessor->process(itsOuts, itsPreProcessor.get());
794  itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
795  itsPostProcessor->report(mod, outimg, helper, ovl, idle);
796  }
797  break;
798 
799  // --------------------------------------------------------------------------------
800  case jevois::dnn::pipeline::Processing::Async:
801  {
802  // We are going to run pre-processing and post-processing synchronously, and network in a thread. One small
803  // complication is that we are going to run post-processing on every frame so that the drawings do not
804  // flicker. We will keep post-processing the same results until new results replace them.
805 
806  // Are we running the network, and is it done? If so, get the outputs:
807  bool needpost = checkAsyncNetComplete();
808 
809  // If we are not running a network, start it:
810  if (itsNetFut.valid() == false)
811  {
812  // Pre-process in the current thread:
813  itsTpre.start();
814  if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
815  itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
816  itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
817 
818  // Network forward pass in a thread:
819  itsNetFut =
820  jevois::async([this]()
821  {
822  itsTnet.start();
823  std::vector<cv::Mat> outs = itsNetwork->process(itsBlobs, itsAsyncNetInfo);
824  itsAsyncNetworkTime = itsTnet.stop(&itsAsyncNetworkSecs);
825 
826  // OpenCV DNN seems to be re-using and overwriting the same output matrices,
827  // so we need to make a deep copy of the outputs if the network type is OpenCV:
828  if (dynamic_cast<jevois::dnn::NetworkOpenCV *>(itsNetwork.get()) == nullptr)
829  return outs;
830 
831  std::vector<cv::Mat> outscopy;
832  for (cv::Mat const & m : outs) outscopy.emplace_back(m.clone());
833  return outscopy;
834  });
835  }
836 
837  // Report pre-processing results on every frame:
838  itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
839 
840  // Show network info on every frame:
841  showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
842 
843  // Run post-processing if needed:
844  if (needpost && itsOuts.empty() == false)
845  {
846  itsTpost.start();
847  itsPostProcessor->process(itsOuts, itsPreProcessor.get());
848  itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
849  }
850 
851  // Report/draw post-processing results on every frame:
852  itsPostProcessor->report(mod, outimg, helper, ovl, idle);
853  }
854  break;
855  }
856 
857  // Update our rolling average of total processing time:
858  itsSecsSum += itsProcSecs[0] + itsProcSecs[1] + itsProcSecs[2];
859  if (++itsSecsSumNum == 20) { itsSecsAvg = itsSecsSum / itsSecsSumNum; itsSecsSum = 0.0; itsSecsSumNum = 0; }
860 
861  // If computing benchmarking stats, update them now:
862  if (statsfile::get().empty() == false && itsOuts.empty() == false)
863  {
864  static std::vector<std::string> pipelines;
865  static bool statswritten = false;
866  static size_t benchpipe = 0;
867 
868  if (benchmark::get())
869  {
870  if (pipelines.empty())
871  {
872  // User just turned on benchmark mode. List all pipes and start iterating over them:
873  // Valid values string format is List:[A|B|C] where A, B, C are replaced by the actual elements.
874  std::string pipes = pipe::def().validValuesString();
875  size_t const idx = pipes.find('[');
876  pipes = pipes.substr(idx + 1, pipes.length() - idx - 2); // risky code but we control the string's contents
877  pipelines = jevois::split(pipes, "\\|");
878  benchpipe = 0;
879  statswritten = false;
880  pipe::set(pipelines[benchpipe]);
881 #ifdef JEVOIS_PRO
882  if (helper)
883  {
884  helper->reportError("Starting DNN benchmark...");
885  helper->reportError("Benchmarking: " +pipelines[benchpipe]);
886  }
887 #endif
888  }
889  else
890  {
891  // Switch to the next pipeline after enough stats have been written:
892  if (statswritten)
893  {
894  ++benchpipe;
895  statswritten = false;
896  if (benchpipe >= pipelines.size())
897  {
898  pipelines.clear();
899  benchmark::set(false);
900 #ifdef JEVOIS_PRO
901  if (helper) helper->reportError("DNN benchmark complete.");
902 #endif
903  }
904  else
905  {
906  pipe::set(pipelines[benchpipe]);
907 #ifdef JEVOIS_PRO
908  if (helper) helper->reportError("Benchmarking: " +pipelines[benchpipe]);
909 #endif
910  }
911  }
912  }
913  }
914  else pipelines.clear();
915 
916  itsPreStats.push_back(itsProcSecs[0]);
917  itsNetStats.push_back(itsProcSecs[1]);
918  itsPstStats.push_back(itsProcSecs[2]);
919 
920  // Discard data for a few warmup frames after we start a new net:
921  if (itsStatsWarmup && itsPreStats.size() == 200)
922  { itsStatsWarmup = false; itsPreStats.clear(); itsNetStats.clear(); itsPstStats.clear(); }
923 
924  if (itsPreStats.size() == 500)
925  {
926  // Compute totals:
927  std::vector<double> tot;
928  for (size_t i = 0; i < itsPreStats.size(); ++i)
929  tot.emplace_back(itsPreStats[i] + itsNetStats[i] + itsPstStats[i]);
930 
931  // Append to stats file:
932  std::string const fn = jevois::absolutePath(JEVOIS_SHARE_PATH, statsfile::get());
933  std::ofstream ofs(fn, std::ios_base::app);
934  if (ofs.is_open())
935  {
936  ofs << "<tr><td class=jvpipe>" << pipe::get() << " </td>";
937 
938  std::vector<std::string> insizes;
939  for (cv::Mat const & m : itsBlobs)
940  insizes.emplace_back(jevois::replaceAll(jevois::dnn::shapestr(m), " ", "&nbsp;"));
941  ofs << "<td class=jvnetin>" << jevois::join(insizes, ", ") << "</td>";
942 
943  std::vector<std::string> outsizes;
944  for (cv::Mat const & m : itsOuts)
945  outsizes.emplace_back(jevois::replaceAll(jevois::dnn::shapestr(m), " ", "&nbsp;"));
946  ofs << "<td class=jvnetout>" << jevois::join(outsizes, ", ") << "</td>";
947 
948  ofs <<
949  "<td class=jvprestats>" << jevois::replaceAll(jevois::secs2str(itsPreStats), " ", "&nbsp;") << "</td>"
950  "<td class=jvnetstats>" << jevois::replaceAll(jevois::secs2str(itsNetStats), " ", "&nbsp;") << "</td>"
951  "<td class=jvpststats>" << jevois::replaceAll(jevois::secs2str(itsPstStats), " ", "&nbsp;") << "</td>"
952  "<td class=jvtotstats>" << jevois::replaceAll(jevois::secs2str(tot), " ", "&nbsp;") << "</td>";
953 
954  // Finally report average fps:
955  double avg = 0.0;
956  for (double t : tot) avg += t;
957  avg /= tot.size();
958  if (avg) avg = 1.0 / avg; // from s/frame to frames/s
959  ofs << "<td class=jvfps>" << std::fixed << std::showpoint << std::setprecision(1) <<
960  avg << "&nbsp;fps</td></tr>" << std::endl;
961 
962  // Ready for next round:
963  itsPreStats.clear();
964  itsNetStats.clear();
965  itsPstStats.clear();
966  LINFO("Network stats appended to " << fn);
967  statswritten = true;
968  }
969  }
970  }
971  }
972  }
973  catch (...)
974  {
975  itsPipeThrew = true;
976 
977 #ifdef JEVOIS_PRO
978  if (helper) helper->reportAndIgnoreException(instanceName());
979  else jevois::warnAndIgnoreException(instanceName());
980 #else
981  jevois::warnAndIgnoreException(instanceName());
982 #endif
983  }
984 
985 #ifdef JEVOIS_PRO
986  // Report processing times and close info window if we opened it:
987  if (helper)
988  {
989  std::string total;
990  if (idle == false || ovl) total = jevois::secs2str(itsSecsAvg);
991 
992  if (idle == false)
993  {
994  if (ImGui::CollapsingHeader("Processing Times", ImGuiTreeNodeFlags_DefaultOpen))
995  {
996  for (std::string const & s : itsProcTimes) ImGui::TextUnformatted(s.c_str());
997  ImGui::Text("OVERALL: %s/inference", total.c_str());
998  }
999  ImGui::End();
1000  }
1001 
1002  if (ovl)
1003  {
1004  for (std::string const & s : itsProcTimes) helper->itext(s);
1005  helper->itext("OVERALL: " + total + "/inference");
1006  }
1007  }
1008 #endif
1009 
1010  // Report processing times to outimg if present:
1011  if (outimg && ovl)
1012  {
1013  for (std::string const & s : itsProcTimes)
1014  {
1015  jevois::rawimage::writeText(*outimg, s, 5, itsOutImgY, jevois::yuyv::White);
1016  itsOutImgY += 11;
1017  }
1018  jevois::rawimage::writeText(*outimg, "OVERALL: " + jevois::secs2str(itsSecsAvg) + "/inference",
1019  5, itsOutImgY, jevois::yuyv::White);
1020  itsOutImgY += 11;
1021  }
1022 }
1023 
1024 // ####################################################################################################
1025 void jevois::dnn::Pipeline::showInfo(std::vector<std::string> const & info,
1026  jevois::StdModule * JEVOIS_UNUSED_PARAM(mod),
1027  jevois::RawImage * outimg,
1028  jevois::OptGUIhelper * helper, bool ovl, bool idle)
1029 {
1030  bool show = true;
1031 
1032  for (std::string const & s : info)
1033  {
1034  // On JeVois Pro, display info in the GUI:
1035 #ifdef JEVOIS_PRO
1036  if (helper && idle == false)
1037  {
1038  // Create collapsible header and get its collapsed status:
1039  if (jevois::stringStartsWith(s, "* "))
1040  show = ImGui::CollapsingHeader(s.c_str() + 2, ImGuiTreeNodeFlags_DefaultOpen);
1041  else if (show)
1042  {
1043  // If header not collapsed, show data:
1044  if (jevois::stringStartsWith(s, "- ")) ImGui::BulletText("%s", s.c_str() + 2);
1045  else ImGui::TextUnformatted(s.c_str());
1046  }
1047  }
1048 #else
1049  (void)idle; (void)show; (void)helper; // avoid warning
1050 #endif
1051 
1052  if (outimg && ovl)
1053  {
1054  jevois::rawimage::writeText(*outimg, s, 5, itsOutImgY, jevois::yuyv::White);
1055  itsOutImgY += 11;
1056  }
1057  }
1058 }
1059 
PostProcessorClassify.H
jevois::dnn::Pipeline::preUninit
void preUninit() override
Called before all sub-Components are uninit()ed.
Definition: Pipeline.C:147
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::GUIhelper::reportAndIgnoreException
void reportAndIgnoreException(std::string const &prefix="")
Report current exception in a modal dialog, then ignore it.
Definition: GUIhelper.C:2491
Pipeline.H
NetworkHailo.H
PostProcessorYuNet.H
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Async execution using a thread pool.
Async.H
PostProcessorDetect.H
jevois::GUIhelper::itext
void itext(char const *txt, ImU32 const &col=IM_COL32_BLACK_TRANS, int line=-1)
Draw some overlay text on top of an image.
Definition: GUIhelper.C:571
RawImageOps.H
jevois::dnn::Pipeline::setCustomNetwork
void setCustomNetwork(std::shared_ptr< Network > n)
Definition: Pipeline.C:622
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:270
jevois::Component
A component of a model hierarchy.
Definition: Component.H:181
jevois::dnn::NetworkOpenCV
Wrapper around an OpenCV DNN neural network.
Definition: NetworkOpenCV.H:31
jevois::dnn::Pipeline::Pipeline
Pipeline(std::string const &instance)
Constructor.
Definition: Pipeline.C:105
Utils.H
PostProcessorSegment.H
jevois::dnn::Pipeline::postInit
void postInit() override
Called after all sub-Components are init()ed.
Definition: Pipeline.C:140
jevois::dnn::Pipeline::ready
bool ready() const
Returns true when all three of preproc, net, and postproc are ready.
Definition: Pipeline.C:684
jevois::dnn::PreProcessorPython
Pre-Processor for neural network pipeline written in python.
Definition: PreProcessorPython.H:32
jevois::dnn::PostProcessorPython
Post-Processor for neural network pipeline.
Definition: PostProcessorPython.H:32
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::ParameterDef
A Parameter Definition.
Definition: ParameterDef.H:88
NetworkTPU.H
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:211
jevois::dnn::Pipeline::~Pipeline
virtual ~Pipeline()
Destructor.
Definition: Pipeline.C:154
jevois::dnn::Pipeline::setCustomPostProcessor
void setCustomPostProcessor(std::shared_ptr< PostProcessor > pp)
Definition: Pipeline.C:672
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:128
jevois::replaceAll
std::string replaceAll(std::string const &str, std::string const &from, std::string const &to)
Replace all instances of 'from' with 'to'.
Definition: Utils.C:361
PostProcessorStub.H
PreProcessorPython.H
jevois::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &root, std::filesystem::path const &path)
Compute an absolute path from two paths.
Definition: Utils.C:385
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
Write some text in an image.
Definition: RawImageOps.C:689
jevois::GUIhelper::reportError
void reportError(std::string const &err)
Report an error in an overlay window.
Definition: GUIhelper.C:2458
jevois
Definition: Concepts.dox:1
jevois::getNumInstalledNPUs
size_t getNumInstalledNPUs()
Get the number of JeVois-Pro NPUs present on this system.
Definition: SysInfo.C:127
Engine.H
Log.H
jevois::dnn::Pipeline::freeze
void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition: Pipeline.C:128
SysInfo.H
jevois::dnn::Pipeline::setCustomPreProcessor
void setCustomPreProcessor(std::shared_ptr< PreProcessor > pp)
Set a custom pre-processor.
Definition: Pipeline.C:546
jevois::join
std::string join(std::vector< std::string > const &strings, std::string const &delimiter)
Concatenate a vector of tokens into a string.
Definition: Utils.C:280
jevois::getNumInstalledTPUs
size_t getNumInstalledTPUs()
Get the number of Coral TPUs present on this system.
Definition: SysInfo.C:87
NetworkOpenCV.H
jevois::warnAndIgnoreException
std::string warnAndIgnoreException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition: Log.C:236
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::getNumInstalledSPUs
size_t getNumInstalledSPUs()
Get the number of Hailo8 SPUs present on this system.
Definition: SysInfo.C:138
jevois::stringStartsWith
bool stringStartsWith(std::string const &str, std::string const &prefix)
Return true if str starts with prefix (including if both strings are equal)
Definition: Utils.C:294
jevois::secs2str
std::string secs2str(double secs)
Report a duration given in seconds with variable units (ns, us, ms, or s), with precision of 2 decima...
Definition: Utils.C:478
jevois::getNumInstalledVPUs
size_t getNumInstalledVPUs()
Get the number of Myriad-X VPUs present on this system.
Definition: SysInfo.C:112
PostProcessorPython.H
NetworkONNX.H
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::Pipeline::process
void process(jevois::RawImage const &inimg, jevois::StdModule *mod, jevois::RawImage *outimg, jevois::OptGUIhelper *helper, bool idle=false)
Process an input image, send results to serial/image/gui.
Definition: Pipeline.C:705
Utils.H
PreProcessorBlob.H
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
NetworkPython.H
jevois::dnn::NetworkPython
Wrapper around an DNN neural network invoked through python.
Definition: NetworkPython.H:31
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
NetworkNPU.H