JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Pipeline.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/DNN/Pipeline.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
21 #include <jevois/Util/Async.H>
23 #include <jevois/Debug/SysInfo.H>
24 #include <jevois/DNN/Utils.H>
25 #include <jevois/Core/Engine.H>
26 
28 #include <jevois/DNN/NetworkNPU.H>
29 #include <jevois/DNN/NetworkTPU.H>
32 
35 
42 
43 #include <opencv2/core/utils/filesystem.hpp>
44 
45 #include <fstream>
46 
47 // ####################################################################################################
48 
49 // Simple class to hold a list of <name, value> pairs for our parameters, with updating the value of existing parameer
50 // names if they are set several times (e.g., first set as a global, then set again for a particular network). Note that
51 // here we do not check the validity of the parameters. This is delegated to Pipeline::setZooParam():
52 namespace
53 {
54  class ParHelper
55  {
56  public:
57  // ----------------------------------------------------------------------------------------------------
58  // Set a param from an entry in our yaml file
59  void set(cv::FileNode const & item, std::string const & zf, cv::FileNode const & node)
60  {
61  std::string k = item.name();
62  std::string v;
63  switch (item.type())
64  {
65  case cv::FileNode::INT: v = std::to_string((int)item); break;
66  case cv::FileNode::REAL: v = std::to_string((float)item); break;
67  case cv::FileNode::STRING: v = (std::string)item; break;
68  default:
69  if (&node == &item)
70  LFATAL("Invalid global zoo parameter " << k << " type " << item.type() << " in " << zf);
71  else
72  LFATAL("Invalid zoo parameter " << k << " type " << item.type() << " in " << zf << " node " << node.name());
73  }
74 
75  // Update value if param already exists, or add new key,value pair:
76  for (auto & p : params) if (p.first == k) { p.second = v; return; }
77  params.emplace_back(std::make_pair(k, v));
78  }
79 
80  // ----------------------------------------------------------------------------------------------------
81  // Get a value for entry subname under item if found, otherwise try our table of globals, otherwise empty
82  std::string pget(cv::FileNode & item, std::string const & subname)
83  {
84  std::string const v = (std::string)item[subname];
85  if (v.empty() == false) return v;
86  for (auto const & p : params) if (p.first == subname) return p.second;
87  return std::string();
88  }
89 
90  // ----------------------------------------------------------------------------------------------------
91  // Un-set a previously set global
92  void unset(std::string const & name)
93  {
94  for (auto itr = params.begin(); itr != params.end(); ++itr)
95  if (itr->first == name) { params.erase(itr); return; }
96  }
97 
98  // Ordering matters, so use a vector instead of map or unordered_map
99  std::vector<std::pair<std::string /* name */, std::string /* value */>> params;
100  };
101 }
102 
103 // ####################################################################################################
104 jevois::dnn::Pipeline::Pipeline(std::string const & instance) :
105  jevois::Component(instance), itsTpre("PreProc"), itsTnet("Network"), itsTpost("PstProc")
106 {
107  itsAccelerators["TPU"] = jevois::getNumInstalledTPUs();
108  itsAccelerators["VPU"] = jevois::getNumInstalledVPUs();
109  itsAccelerators["NPU"] = jevois::getNumInstalledNPUs();
110  itsAccelerators["SPU"] = jevois::getNumInstalledSPUs();
111  itsAccelerators["OpenCV"] = 1; // OpenCV always available
112  itsAccelerators["Python"] = 1; // Python always available
113 #ifdef JEVOIS_PLATFORM_PRO
114  itsAccelerators["VPUX"] = 1; // VPU emulation on CPU always available through OpenVino
115 #endif
116  itsAccelerators["NPUX"] = 1; // NPU over Tim-VX always available since compiled into OpenCV
117 
118  LINFO("Detected " <<
119  itsAccelerators["NPU"] << " JeVois-Pro NPUs, " <<
120  itsAccelerators["SPU"] << " Hailo8 SPUs, " <<
121  itsAccelerators["TPU"] << " Coral TPUs, " <<
122  itsAccelerators["VPU"] << " Myriad-X VPUs.");
123 }
124 
125 // ####################################################################################################
127 {
128  preproc::freeze(doit);
129  nettype::freeze(doit);
130  postproc::freeze(doit);
131 
132  if (itsPreProcessor) itsPreProcessor->freeze(doit);
133  if (itsNetwork) itsNetwork->freeze(doit);
134  if (itsPostProcessor) itsPostProcessor->freeze(doit);
135 }
136 
137 // ####################################################################################################
139 {
140  // Freeze all params that users should not modify at runtime:
141  freeze(true);
142 }
143 
144 // ####################################################################################################
146 {
147  // If we have a network running async, make sure we wait here until it is done:
148  asyncNetWait();
149 }
150 
151 // ####################################################################################################
153 {
154  // Make sure network is not running as we die:
155  asyncNetWait();
156 }
157 
158 // ####################################################################################################
159 void jevois::dnn::Pipeline::asyncNetWait()
160 {
161  // If we were currently doing async processing, wait until network is done:
162  if (itsNetFut.valid())
163  while (true)
164  {
165  if (itsNetFut.wait_for(std::chrono::seconds(5)) == std::future_status::timeout)
166  LERROR("Still waiting for network to finish running...");
167  else break;
168  }
169 
170  try { itsNetFut.get(); } catch (...) { }
171  itsOuts.clear();
172 }
173 
174 // ####################################################################################################
175 void jevois::dnn::Pipeline::onParamChange(pipeline::filter const & JEVOIS_UNUSED_PARAM(param),
176  jevois::dnn::pipeline::Filter const & JEVOIS_UNUSED_PARAM(val))
177 {
178  // Reload the zoo file so that the filter can be applied to create the parameter def of pipe, but first we need this
179  // parameter to indeed be updated. So here we just set a flag and the update will occur in process(), after we run the
180  // current model one last time:
181  itsZooChanged = true;
182 }
183 
184 // ####################################################################################################
185 void jevois::dnn::Pipeline::onParamChange(pipeline::zooroot const & JEVOIS_UNUSED_PARAM(param),
186  std::string const & JEVOIS_UNUSED_PARAM(val))
187 {
188  // Reload the zoo file, but first we need this parameter to indeed be updated. So here we just set a flag and the
189  // update will occur in process(), after we run the current model one last time:
190  itsZooChanged = true;
191 }
192 
193 // ####################################################################################################
194 void jevois::dnn::Pipeline::onParamChange(pipeline::zoo const & JEVOIS_UNUSED_PARAM(param),
195  std::string const & val)
196 {
197  // Just nuke everything:
198  itsPreProcessor.reset();
199  itsNetwork.reset();
200  itsPostProcessor.reset();
201  // Will get instantiated again when pipe param is set.
202 
203  // Load zoo file:
204  std::vector<std::string> pipes;
205  scanZoo(jevois::absolutePath(zooroot::get(), val), filter::strget(), pipes, "");
206  LINFO("Found a total of " << pipes.size() << " valid pipelines.");
207 
208  // Update the parameter def of pipe:
209  if (pipes.empty() == false)
210  {
211  jevois::ParameterDef<std::string> newdef("pipe", "Pipeline to use, determined by entries in the zoo file and "
212  "by the current filter",
213  pipes[0], pipes, jevois::dnn::pipeline::ParamCateg);
214  pipe::changeParameterDef(newdef);
215 
216  // Just changing the def does not change the param value, so change it now:
217  pipe::set(pipes[0]);
218  }
219 }
220 
221 // ####################################################################################################
222 void jevois::dnn::Pipeline::scanZoo(std::filesystem::path const & zoofile, std::string const & filt,
223  std::vector<std::string> & pipes, std::string const & indent)
224 {
225  LINFO(indent << "Scanning model zoo file " << zoofile << " with filter [" << filt << "]...");
226  int ntot = 0, ngood = 0;
227 
228  bool has_vpu = false;
229  auto itr = itsAccelerators.find("VPU");
230  if (itr != itsAccelerators.end() && itr->second > 0) has_vpu = true;
231 
232  // Scan the zoo file to update the parameter def of the pipe parameter:
233  cv::FileStorage fs(zoofile, cv::FileStorage::READ);
234  if (fs.isOpened() == false) LFATAL("Could not open zoo file " << zoofile);
235  cv::FileNode fn = fs.root();
236  ParHelper ph;
237 
238  for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
239  {
240  cv::FileNode item = *fit;
241 
242  // Process include: directives recursively:
243  if (item.name() == "include")
244  {
245  scanZoo(jevois::absolutePath(zooroot::get(), (std::string)item), filt, pipes, indent + " ");
246  }
247  // Process includedir: directives (only one level of directory is scanned):
248  else if (item.name() == "includedir")
249  {
250  std::filesystem::path const dir = jevois::absolutePath(zooroot::get(), (std::string)item);
251  for (auto const & dent : std::filesystem::recursive_directory_iterator(dir))
252  if (dent.is_regular_file())
253  {
254  std::filesystem::path const path = dent.path();
255  std::filesystem::path const ext = path.extension();
256  if (ext == ".yml" || ext == ".yaml") scanZoo(path, filt, pipes, indent + " ");
257  }
258  }
259  // Unset a previously set global?
260  else if (item.name() == "unset")
261  {
262  ph.unset((std::string)item);
263  }
264  // Set a global:
265  else if (! item.isMap())
266  {
267  ph.set(item, zoofile, item);
268  }
269  // Map type (model definition):
270  else
271  {
272  ++ntot;
273 
274  // As a prefix, we use OpenCV for OpenCV models on CPU/OpenCL backends, and VPU for InferenceEngine backend with
275  // Myriad target, VPUX for InferenceEngine/CPU (arm-compute OpenVino plugin, only works on platform), and NPUX for
276  // TimVX/NPU (NPU using TimVX OpenCV extension, uses NPU on platform or emulator on host):
277 
278  // Set then get nettype to account for globals:
279  std::string typ = ph.pget(item, "nettype");
280 
281  if (typ == "OpenCV")
282  {
283  std::string backend = ph.pget(item, "backend");
284  std::string target = ph.pget(item, "target");
285 
286  if (backend == "InferenceEngine")
287  {
288  if (target == "Myriad")
289  {
290  if (has_vpu) typ = "VPU"; // run VPU models on VPU if MyriadX accelerator is present
291 #ifdef JEVOIS_PLATFORM_PRO
292  else typ = "VPUX"; // emulate VPU models on CPU through ARM-Compute if MyriadX accelerator not present
293 #else
294  else continue; // VPU emulation does not work on host...
295 #endif
296  }
297  else if (target == "CPU") typ = "VPUX";
298  }
299  else if (backend == "TimVX" && target == "NPU") typ = "NPUX";
300  }
301 
302  // Do not consider a model if we do not have the accelerator for it:
303  bool has_accel = false;
304  itr = itsAccelerators.find(typ);
305  if (itr != itsAccelerators.end() && itr->second > 0) has_accel = true;
306 
307  // Add this pipe if it matches our filter and we have the accelerator for it:
308  if ((filt == "All" || typ == filt) && has_accel)
309  {
310  std::string const postproc = ph.pget(item, "postproc");
311  pipes.emplace_back(typ + ':' + postproc + ':' + item.name());
312  ++ngood;
313  }
314  }
315  }
316 
317  LINFO(indent << "Found " << ntot << " pipelines, " << ngood << " passed the filter.");
318 }
319 
320 // ####################################################################################################
321 void jevois::dnn::Pipeline::onParamChange(pipeline::pipe const & JEVOIS_UNUSED_PARAM(param), std::string const & val)
322 {
323  if (val.empty()) return;
324  itsPipeThrew = false;
325  freeze(false);
326 
327  // Clear any errors related to previous pipeline:
328  engine()->clearErrors();
329 
330  // Find the desired pipeline, and set it up:
331  std::string const z = jevois::absolutePath(zooroot::get(), zoo::get());
332  std::vector<std::string> tok = jevois::split(val, ":");
333  if (selectPipe(z, tok) == false)
334  LFATAL("Could not find pipeline entry [" << val << "] in zoo file " << z << " and its includes");
335 
336  freeze(true);
337 }
338 
339 // ####################################################################################################
340 bool jevois::dnn::Pipeline::selectPipe(std::string const & zoofile, std::vector<std::string> const & tok)
341 {
342  // We might have frozen processing to Sync if we ran a NetworkPython previously, so unfreeze here:
343  processing::freeze(false);
344  processing::set(jevois::dnn::pipeline::Processing::Async);
345 
346  // Check if we have a VPU, to use VPU vs VPUX:
347  bool has_vpu = false;
348  auto itr = itsAccelerators.find("VPU");
349  if (itr != itsAccelerators.end() && itr->second > 0) has_vpu = true;
350  bool vpu_emu = false;
351 
352  // Clear any old stats:
353  itsPreStats.clear(); itsNetStats.clear(); itsPstStats.clear();
354  itsStatsWarmup = true; // warmup before computing new stats
355 
356  // Open the zoo file:
357  cv::FileStorage fs(zoofile, cv::FileStorage::READ);
358  if (fs.isOpened() == false) LFATAL("Could not open zoo file " << zoofile);
359 
360  // Find the desired pipeline:
361  ParHelper ph;
362  cv::FileNode fn = fs.root(), node;
363 
364  for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
365  {
366  cv::FileNode item = *fit;
367 
368  // Process include: directives recursively, end the recursion if we found our pipe in there:
369  if (item.name() == "include")
370  {
371  if (selectPipe(jevois::absolutePath(zooroot::get(), (std::string)item), tok)) return true;
372  }
373 
374  // Process includedir: directives (only one level of directory is scanned), end recursion if we found our pipe:
375  else if (item.name() == "includedir")
376  {
377  std::filesystem::path const dir = jevois::absolutePath(zooroot::get(), (std::string)item);
378  for (auto const & dent : std::filesystem::recursive_directory_iterator(dir))
379  if (dent.is_regular_file())
380  {
381  std::filesystem::path const path = dent.path();
382  std::filesystem::path const ext = path.extension();
383  if (ext == ".yml" || ext == ".yaml") if (selectPipe(path, tok)) return true;
384  }
385  }
386 
387  // Unset a previously set global?
388  else if (item.name() == "unset")
389  {
390  ph.unset((std::string)item);
391  }
392  // Set a global:
393  else if (! item.isMap())
394  {
395  ph.set(item, zoofile, node);
396  }
397  // This is an entry for a pipeline with a bunch of params under it:
398  else
399  {
400  if (item.name() != tok.back()) continue;
401  if (tok.size() == 1) { node = item; break; }
402  if (tok.size() != 3) LFATAL("Malformed pipeline name: " << jevois::join(tok, ":"));
403 
404  // Skip if postproc is no match:
405  std::string postproc = ph.pget(item, "postproc");
406  if (postproc != tok[1] && postproc::strget() != tok[1]) continue;
407 
408  std::string nettype = ph.pget(item, "nettype");
409  std::string backend = ph.pget(item, "backend");
410  std::string target = ph.pget(item, "target");
411 
412  if (tok[0] == "VPU")
413  {
414  if (nettype == "OpenCV" && backend == "InferenceEngine" && target == "Myriad")
415  { node = item; break; }
416  }
417  else if (tok[0] == "VPUX")
418  {
419  if (nettype == "OpenCV" && backend == "InferenceEngine")
420  {
421  if (target == "Myriad" && has_vpu == false) { vpu_emu = true; node = item; break; }
422  else if (target == "CPU") { node = item; break; }
423  }
424  }
425  else if (tok[0] == "NPUX")
426  {
427  if (nettype == "OpenCV" && backend == "TimVX" && target == "NPU")
428  { node = item; break; }
429  }
430  else
431  {
432  if (nettype == tok[0])
433  { node = item; break; }
434  }
435  }
436  }
437 
438  // If the spec was not a match with any entries in the file, return false:
439  if (node.empty()) return false;
440 
441  // Found the pipe. First nuke our current pre/net/post:
442  asyncNetWait();
443  itsPreProcessor.reset(); removeSubComponent("preproc", false);
444  itsNetwork.reset(); removeSubComponent("network", false);
445  itsPostProcessor.reset(); removeSubComponent("postproc", false);
446 
447  // Then set all the global parameters of the current file:
448  //for (auto const & pp : ph.params)
449  // setZooParam(pp.first, pp.second, zoofile, fs.root());
450 
451  // Then iterate over all pipeline params and set them: first update our table, then set params from the whole table:
452  for (cv::FileNodeIterator fit = node.begin(); fit != node.end(); ++fit)
453  ph.set(*fit, zoofile, node);
454 
455  for (auto const & pp : ph.params)
456  {
457  if (vpu_emu && pp.first == "target") setZooParam(pp.first, "CPU", zoofile, node);
458  else setZooParam(pp.first, pp.second, zoofile, node);
459  }
460 
461  // Running a python net async segfaults instantly if we are also concurrently running pre or post processing in
462  // python, as python is not re-entrant... so force sync here:
463  if (dynamic_cast<jevois::dnn::NetworkPython *>(itsNetwork.get()) &&
464  (dynamic_cast<jevois::dnn::PreProcessorPython *>(itsPreProcessor.get()) ||
465  dynamic_cast<jevois::dnn::PostProcessorPython *>(itsPostProcessor.get())))
466  {
467  if (processing::get() != jevois::dnn::pipeline::Processing::Sync)
468  {
469  LERROR("Network of type Python cannot run Async if pre- or post- processor are also Python "
470  "-- FORCING Sync processing");
471  processing::set(jevois::dnn::pipeline::Processing::Sync);
472  }
473  processing::freeze(true);
474  }
475 
476  return true;
477 }
478 
479 // ####################################################################################################
480 void jevois::dnn::Pipeline::setZooParam(std::string const & k, std::string const & v,
481  std::string const & zf, cv::FileNode const & node)
482 {
483  // The zoo file may contain extra params, like download URL, etc. To ignore those while still catching invalid
484  // values on our parameters, we first check whether the parameter exists, and, if so, try to set it:
485  bool hasparam = false;
486  try { getParamStringUnique(k); hasparam = true; } catch (...) { }
487 
488  if (hasparam)
489  {
490  LINFO("Setting ["<<k<<"] to ["<<v<<']');
491 
492  try { setParamStringUnique(k, v); }
493  catch (std::exception const & e)
494  { LFATAL("While parsing [" << node.name() << "] in model zoo file " << zf << ": " << e.what()); }
495  catch (...)
496  { LFATAL("While parsing [" << node.name() << "] in model zoo file " << zf << ": unknown error"); }
497  }
498  else if (paramwarn::get())
499  engine()->reportError("WARNING: Unused parameter [" + k + "] in " + zf + " node [" + node.name() + "]");
500 }
501 
502 // ####################################################################################################
503 void jevois::dnn::Pipeline::onParamChange(pipeline::preproc const & JEVOIS_UNUSED_PARAM(param),
504  pipeline::PreProc const & val)
505 {
506  itsPreProcessor.reset(); removeSubComponent("preproc", false);
507 
508  switch (val)
509  {
510  case jevois::dnn::pipeline::PreProc::Blob:
511  itsPreProcessor = addSubComponent<jevois::dnn::PreProcessorBlob>("preproc");
512  break;
513 
514  case jevois::dnn::pipeline::PreProc::Python:
515  itsPreProcessor = addSubComponent<jevois::dnn::PreProcessorPython>("preproc");
516  break;
517 
518  case jevois::dnn::pipeline::PreProc::Custom:
519  // nothing here, user must call setCustomPreProcessor() later
520  break;
521  }
522 
523  if (itsPreProcessor) LINFO("Instantiated pre-processor of type " << itsPreProcessor->className());
524  else LINFO("No pre-processor");
525 }
526 
527 // ####################################################################################################
528 void jevois::dnn::Pipeline::setCustomPreProcessor(std::shared_ptr<jevois::dnn::PreProcessor> pp)
529 {
530  itsPreProcessor.reset(); removeSubComponent("preproc", false);
531  itsPreProcessor = pp;
532  preproc::set(jevois::dnn::pipeline::PreProc::Custom);
533 
534  if (itsPreProcessor) LINFO("Attached pre-processor of type " << itsPreProcessor->className());
535  else LINFO("No pre-processor");
536 }
537 
538 // ####################################################################################################
539 void jevois::dnn::Pipeline::onParamChange(pipeline::nettype const & JEVOIS_UNUSED_PARAM(param),
540  pipeline::NetType const & val)
541 {
542  asyncNetWait(); // If currently processing async net, wait until done
543 
544  itsNetwork.reset(); removeSubComponent("network", false);
545 
546  switch (val)
547  {
548  case jevois::dnn::pipeline::NetType::OpenCV:
549  itsNetwork = addSubComponent<jevois::dnn::NetworkOpenCV>("network");
550  break;
551 
552 #ifdef JEVOIS_PRO
553 
554  case jevois::dnn::pipeline::NetType::NPU:
555 #ifdef JEVOIS_PLATFORM
556  itsNetwork = addSubComponent<jevois::dnn::NetworkNPU>("network");
557 #else // JEVOIS_PLATFORM
558  LFATAL("NPU network is only supported on JeVois-Pro Platform");
559 #endif
560  break;
561 
562  case jevois::dnn::pipeline::NetType::SPU:
563  itsNetwork = addSubComponent<jevois::dnn::NetworkHailo>("network");
564  break;
565 
566  case jevois::dnn::pipeline::NetType::TPU:
567  itsNetwork = addSubComponent<jevois::dnn::NetworkTPU>("network");
568  break;
569 #endif
570 
571  case jevois::dnn::pipeline::NetType::Python:
572  itsNetwork = addSubComponent<jevois::dnn::NetworkPython>("network");
573  break;
574 
575  case jevois::dnn::pipeline::NetType::Custom:
576  // Nothing here, user must call setCustomNetwork() later
577  break;
578  }
579 
580  if (itsNetwork) LINFO("Instantiated network of type " << itsNetwork->className());
581  else LINFO("No network");
582 
583  // We already display a "loading..." message while the network is loading, but some OpenCV networks take a long time
584  // to process their first frame after they are loaded (e.g., YuNet initializes all the anchors on first frame). So
585  // here we set some placeholder text that will appear after the network is loaded and is processing the first frame:
586  itsInputAttrs.clear();
587  itsNetInfo.clear();
588  itsNetInfo.emplace_back("* Input Tensors");
589  itsNetInfo.emplace_back("Initializing network...");
590  itsNetInfo.emplace_back("* Network");
591  itsNetInfo.emplace_back("Initializing network...");
592  itsNetInfo.emplace_back("* Output Tensors");
593  itsNetInfo.emplace_back("Initializing network...");
594  itsAsyncNetInfo = itsNetInfo;
595  itsAsyncNetworkTime = "Network: -";
596  itsAsyncNetworkSecs = 0.0;
597 }
598 
599 // ####################################################################################################
600 void jevois::dnn::Pipeline::setCustomNetwork(std::shared_ptr<jevois::dnn::Network> n)
601 {
602  asyncNetWait(); // If currently processing async net, wait until done
603  itsNetwork.reset(); removeSubComponent("network", false);
604  itsNetwork = n;
605  nettype::set(jevois::dnn::pipeline::NetType::Custom);
606 
607  if (itsNetwork) LINFO("Attached network of type " << itsNetwork->className());
608  else LINFO("No network");
609 
610  itsInputAttrs.clear();
611 }
612 
613 // ####################################################################################################
614 void jevois::dnn::Pipeline::onParamChange(pipeline::postproc const & JEVOIS_UNUSED_PARAM(param),
615  pipeline::PostProc const & val)
616 {
617  asyncNetWait(); // If currently processing async net, wait until done
618 
619  itsPostProcessor.reset(); removeSubComponent("postproc", false);
620 
621  switch (val)
622  {
623  case jevois::dnn::pipeline::PostProc::Classify:
624  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorClassify>("postproc");
625  break;
626  case jevois::dnn::pipeline::PostProc::Detect:
627  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorDetect>("postproc");
628  break;
629  case jevois::dnn::pipeline::PostProc::Segment:
630  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorSegment>("postproc");
631  break;
632  case jevois::dnn::pipeline::PostProc::YuNet:
633  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorYuNet>("postproc");
634  break;
635  case jevois::dnn::pipeline::PostProc::Python:
636  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorPython>("postproc");
637  break;
638  case jevois::dnn::pipeline::PostProc::Stub:
639  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorStub>("postproc");
640  case jevois::dnn::pipeline::PostProc::Custom:
641  // Nothing here, user must call setCustomPostProcessor() later
642  break;
643  }
644 
645  if (itsPostProcessor) LINFO("Instantiated post-processor of type " << itsPostProcessor->className());
646  else LINFO("No post-processor");
647 }
648 
649 // ####################################################################################################
650 void jevois::dnn::Pipeline::setCustomPostProcessor(std::shared_ptr<jevois::dnn::PostProcessor> pp)
651 {
652  asyncNetWait(); // If currently processing async net, wait until done
653  itsPostProcessor.reset(); removeSubComponent("postproc", false);
654  itsPostProcessor = pp;
655  postproc::set(jevois::dnn::pipeline::PostProc::Custom);
656 
657  if (itsPostProcessor) LINFO("Attached post-processor of type " << itsPostProcessor->className());
658  else LINFO("No post-processor");
659 }
660 
661 // ####################################################################################################
663 {
664  return itsPreProcessor && itsNetwork && itsNetwork->ready() && itsPostProcessor;
665 }
666 
667 // ####################################################################################################
668 bool jevois::dnn::Pipeline::checkAsyncNetComplete()
669 {
670  if (itsNetFut.valid() && itsNetFut.wait_for(std::chrono::milliseconds(2)) == std::future_status::ready)
671  {
672  itsOuts = itsNetFut.get();
673  itsNetInfo.clear();
674  std::swap(itsNetInfo, itsAsyncNetInfo);
675  itsProcTimes[1] = itsAsyncNetworkTime;
676  itsProcSecs[1] = itsAsyncNetworkSecs;
677  return true;
678  }
679  return false;
680 }
681 
682 // ####################################################################################################
684  jevois::OptGUIhelper * helper, bool idle)
685 {
686  // Reload the zoo file if filter has changed:
687  if (itsZooChanged) { itsZooChanged = false; zoo::set(zoo::get()); }
688 
689  // If the pipeline is throwing exception at any stage, do not do anything here, itsPipeThrew is cleared when selecting
690  // a new pipe:
691  if (itsPipeThrew) return;
692 
693  bool const ovl = overlay::get();
694  itsOutImgY = 5; // y text position when using outimg text drawings
695 
696 #ifdef JEVOIS_PRO
697  // Open an info window if using GUI and not idle:
698  if (helper && idle == false) ImGui::Begin((instanceName() + ':' + getParamStringUnique("pipe")).c_str());
699 #else
700  (void)helper; // avoid compiler warning
701 #endif
702 
703  // If we want an overlay, show network name on first line:
704  if (ovl)
705  {
706  if (outimg)
707  {
708  jevois::rawimage::writeText(*outimg, instanceName() + ':' + getParamStringUnique("pipe"),
709  5, itsOutImgY, jevois::yuyv::White);
710  itsOutImgY += 11;
711  }
712 
713 #ifdef JEVOIS_PRO
714  if (helper) helper->itext(instanceName() + ':' + getParamStringUnique("pipe"));
715 #endif
716  }
717 
718  // If network is not ready, inform user. Be careful that ready() may throw, eg, if bad network name was given and
719  // network could not be loaded:
720  try
721  {
722  if (ready() == false)
723  {
724  char const * msg = itsNetwork ? "Loading network..." : "No network selected...";
725 
726  if (outimg)
727  {
728  jevois::rawimage::writeText(*outimg, msg, 5, itsOutImgY, jevois::yuyv::White);
729  itsOutImgY += 11;
730  }
731 
732 #ifdef JEVOIS_PRO
733  if (helper)
734  {
735  if (idle == false) ImGui::TextUnformatted(msg);
736  if (ovl) helper->itext(msg);
737  }
738 #endif
739 
740  itsProcTimes = { "PreProc: -", "Network: -", "PstProc: -" };
741  itsProcSecs = { 0.0, 0.0, 0.0 };
742  }
743  else
744  {
745  // Network is ready, run processing, either single-thread (Sync) or threaded (Async):
746  switch (processing::get())
747  {
748  // --------------------------------------------------------------------------------
749  case jevois::dnn::pipeline::Processing::Sync:
750  {
751  asyncNetWait(); // If currently processing async net, wait until done
752 
753  // Pre-process:
754  itsTpre.start();
755  if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
756  itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
757  itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
758  itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
759 
760  // Network forward pass:
761  itsNetInfo.clear();
762  itsTnet.start();
763  itsOuts = itsNetwork->process(itsBlobs, itsNetInfo);
764  itsProcTimes[1] = itsTnet.stop(&itsProcSecs[1]);
765 
766  // Show network info:
767  showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
768 
769  // Post-Processing:
770  itsTpost.start();
771  itsPostProcessor->process(itsOuts, itsPreProcessor.get());
772  itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
773  itsPostProcessor->report(mod, outimg, helper, ovl, idle);
774  }
775  break;
776 
777  // --------------------------------------------------------------------------------
778  case jevois::dnn::pipeline::Processing::Async:
779  {
780  // We are going to run pre-processing and post-processing synchronously, and network in a thread. One small
781  // complication is that we are going to run post-processing on every frame so that the drawings do not
782  // flicker. We will keep post-processing the same results until new results replace them.
783 
784  // Are we running the network, and is it done? If so, get the outputs:
785  bool needpost = checkAsyncNetComplete();
786 
787  // If we are not running a network, start it:
788  if (itsNetFut.valid() == false)
789  {
790  // Pre-process in the current thread:
791  itsTpre.start();
792  if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
793  itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
794  itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
795 
796  // Network forward pass in a thread:
797  itsNetFut =
798  jevois::async([this]()
799  {
800  itsTnet.start();
801  std::vector<cv::Mat> outs = itsNetwork->process(itsBlobs, itsAsyncNetInfo);
802  itsAsyncNetworkTime = itsTnet.stop(&itsAsyncNetworkSecs);
803 
804  // OpenCV DNN seems to be re-using and overwriting the same output matrices,
805  // so we need to make a deep copy of the outputs if the network type is OpenCV:
806  if (dynamic_cast<jevois::dnn::NetworkOpenCV *>(itsNetwork.get()) == nullptr)
807  return outs;
808 
809  std::vector<cv::Mat> outscopy;
810  for (cv::Mat const & m : outs) outscopy.emplace_back(m.clone());
811  return outscopy;
812  });
813  }
814 
815  // Report pre-processing results on every frame:
816  itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
817 
818  // Show network info on every frame:
819  showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
820 
821  // Run post-processing if needed:
822  if (needpost && itsOuts.empty() == false)
823  {
824  itsTpost.start();
825  itsPostProcessor->process(itsOuts, itsPreProcessor.get());
826  itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
827  }
828 
829  // Report/draw post-processing results on every frame:
830  itsPostProcessor->report(mod, outimg, helper, ovl, idle);
831  }
832  break;
833  }
834  }
835  }
836  catch (...)
837  {
838  itsPipeThrew = true;
839 
840 #ifdef JEVOIS_PRO
841  if (helper) helper->reportAndIgnoreException(instanceName());
842  else jevois::warnAndIgnoreException(instanceName());
843 #else
844  jevois::warnAndIgnoreException(instanceName());
845 #endif
846  }
847 
848  // Update our rolling average of total processing time:
849  itsSecsSum += itsProcSecs[0] + itsProcSecs[1] + itsProcSecs[2];
850  if (++itsSecsSumNum == 20) { itsSecsAvg = itsSecsSum / itsSecsSumNum; itsSecsSum = 0.0; itsSecsSumNum = 0; }
851 
852  // If computing benchmarking stats, update them now:
853  if (statsfile::get().empty() == false && ready())
854  {
855  itsPreStats.push_back(itsProcSecs[0]);
856  itsNetStats.push_back(itsProcSecs[1]);
857  itsPstStats.push_back(itsProcSecs[2]);
858 
859  // Discard data for a few warmup frames after we start a new net:
860  if (itsStatsWarmup && itsPreStats.size() == 30)
861  { itsStatsWarmup = false; itsPreStats.clear(); itsNetStats.clear(); itsPstStats.clear(); }
862 
863  if (itsPreStats.size() == 200)
864  {
865  // Compute totals:
866  std::vector<double> tot;
867  for (size_t i = 0; i < itsPreStats.size(); ++i)
868  tot.emplace_back(itsPreStats[i] + itsNetStats[i] + itsPstStats[i]);
869 
870  // Append to stats file:
871  std::string const fn = jevois::absolutePath(JEVOIS_SHARE_PATH, statsfile::get());
872  std::ofstream ofs(fn, std::ios_base::app);
873  if (ofs.is_open())
874  {
875  ofs << "<tr><td class=jvpipe>" << pipe::get() << " </td>";
876 
877  std::vector<std::string> insizes;
878  for (cv::Mat const & m : itsBlobs)
879  insizes.emplace_back(jevois::replaceAll(jevois::dnn::shapestr(m), " ", "&nbsp;"));
880  ofs << "<td class=jvnetin>" << jevois::join(insizes, ", ") << "</td>";
881 
882  std::vector<std::string> outsizes;
883  for (cv::Mat const & m : itsOuts)
884  outsizes.emplace_back(jevois::replaceAll(jevois::dnn::shapestr(m), " ", "&nbsp;"));
885  ofs << "<td class=jvnetout>" << jevois::join(outsizes, ", ") << "</td>";
886 
887  ofs <<
888  "<td class=jvprestats>" << jevois::replaceAll(jevois::secs2str(itsPreStats), " ", "&nbsp;") << "</td>"
889  "<td class=jvnetstats>" << jevois::replaceAll(jevois::secs2str(itsNetStats), " ", "&nbsp;") << "</td>"
890  "<td class=jvpststats>" << jevois::replaceAll(jevois::secs2str(itsPstStats), " ", "&nbsp;") << "</td>"
891  "<td class=jvtotstats>" << jevois::replaceAll(jevois::secs2str(tot), " ", "&nbsp;") << "</td>";
892 
893  // Finally report average fps:
894  double avg = 0.0;
895  for (double t : tot) avg += t;
896  avg /= tot.size();
897  if (avg) avg = 1.0 / avg; // from s/frame to frames/s
898  ofs << "<td class=jvfps>" << std::fixed << std::showpoint << std::setprecision(1) <<
899  avg << "&nbsp;fps</td></tr>" << std::endl;
900 
901  // Ready for next round:
902  itsPreStats.clear();
903  itsNetStats.clear();
904  itsPstStats.clear();
905  LINFO("Network stats appended to " << fn);
906 #ifdef JEVOIS_PRO
907  if (helper) helper->drawCircle(100, 100, 95);
908 #endif
909  }
910  }
911  }
912 
913 #ifdef JEVOIS_PRO
914  // Report processing times and close info window if we opened it:
915  if (helper)
916  {
917  std::string total;
918  if (idle == false || ovl) total = jevois::secs2str(itsSecsAvg);
919 
920  if (idle == false)
921  {
922  if (ImGui::CollapsingHeader("Processing Times", ImGuiTreeNodeFlags_DefaultOpen))
923  {
924  for (std::string const & s : itsProcTimes) ImGui::TextUnformatted(s.c_str());
925  ImGui::Text("OVERALL: %s/inference", total.c_str());
926  }
927  ImGui::End();
928  }
929 
930  if (ovl)
931  {
932  for (std::string const & s : itsProcTimes) helper->itext(s);
933  helper->itext("OVERALL: " + total + "/inference");
934  }
935  }
936 #endif
937 
938  // Report processing times to outimg if present:
939  if (outimg && ovl)
940  {
941  for (std::string const & s : itsProcTimes)
942  {
943  jevois::rawimage::writeText(*outimg, s, 5, itsOutImgY, jevois::yuyv::White);
944  itsOutImgY += 11;
945  }
946  jevois::rawimage::writeText(*outimg, "OVERALL: " + jevois::secs2str(itsSecsAvg) + "/inference",
947  5, itsOutImgY, jevois::yuyv::White);
948  itsOutImgY += 11;
949  }
950 }
951 
952 // ####################################################################################################
953 void jevois::dnn::Pipeline::showInfo(std::vector<std::string> const & info,
954  jevois::StdModule * JEVOIS_UNUSED_PARAM(mod),
955  jevois::RawImage * outimg,
956  jevois::OptGUIhelper * helper, bool ovl, bool idle)
957 {
958  bool show = true;
959 
960  for (std::string const & s : info)
961  {
962  // On JeVois Pro, display info in the GUI:
963 #ifdef JEVOIS_PRO
964  if (helper && idle == false)
965  {
966  // Create collapsible header and get its collapsed status:
967  if (jevois::stringStartsWith(s, "* "))
968  show = ImGui::CollapsingHeader(s.c_str() + 2, ImGuiTreeNodeFlags_DefaultOpen);
969  else if (show)
970  {
971  // If header not collapsed, show data:
972  if (jevois::stringStartsWith(s, "- ")) ImGui::BulletText("%s", s.c_str() + 2);
973  else ImGui::TextUnformatted(s.c_str());
974  }
975  }
976 #else
977  (void)idle; (void)show; (void)helper; // avoid warning
978 #endif
979 
980  if (outimg && ovl)
981  {
982  jevois::rawimage::writeText(*outimg, s, 5, itsOutImgY, jevois::yuyv::White);
983  itsOutImgY += 11;
984  }
985  }
986 }
987 
PostProcessorClassify.H
jevois::dnn::Pipeline::preUninit
void preUninit() override
Called before all sub-Components are uninit()ed.
Definition: Pipeline.C:145
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::GUIhelper::reportAndIgnoreException
void reportAndIgnoreException(std::string const &prefix="")
Report current exception in a modal dialog, then ignore it.
Definition: GUIhelper.C:2080
Pipeline.H
NetworkHailo.H
PostProcessorYuNet.H
jevois::GUIhelper::drawCircle
void drawCircle(float x, float y, float r, ImU32 col=IM_COL32(128, 255, 128, 255), bool filled=true)
Draw circle over an image.
Definition: GUIhelper.C:497
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Async execution using a thread pool.
Async.H
PostProcessorDetect.H
jevois::GUIhelper::itext
void itext(char const *txt, ImU32 const &col=IM_COL32_BLACK_TRANS, int line=-1)
Draw some overlay text on top of an image.
Definition: GUIhelper.C:540
RawImageOps.H
jevois::dnn::Pipeline::setCustomNetwork
void setCustomNetwork(std::shared_ptr< Network > n)
Definition: Pipeline.C:600
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:258
jevois::Component
A component of a model hierarchy.
Definition: Component.H:181
jevois::dnn::NetworkOpenCV
Wrapper around an OpenCV DNN neural network.
Definition: NetworkOpenCV.H:31
jevois::dnn::Pipeline::Pipeline
Pipeline(std::string const &instance)
Constructor.
Definition: Pipeline.C:104
Utils.H
PostProcessorSegment.H
jevois::dnn::Pipeline::postInit
void postInit() override
Called after all sub-Components are init()ed.
Definition: Pipeline.C:138
jevois::dnn::Pipeline::ready
bool ready() const
Returns true when all three of preproc, net, and postproc are ready.
Definition: Pipeline.C:662
jevois::dnn::PreProcessorPython
Pre-Processor for neural network pipeline written in python.
Definition: PreProcessorPython.H:32
jevois::dnn::PostProcessorPython
Post-Processor for neural network pipeline.
Definition: PostProcessorPython.H:32
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::ParameterDef
A Parameter Definition.
Definition: ParameterDef.H:88
NetworkTPU.H
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:211
jevois::dnn::Pipeline::~Pipeline
virtual ~Pipeline()
Destructor.
Definition: Pipeline.C:152
jevois::dnn::Pipeline::setCustomPostProcessor
void setCustomPostProcessor(std::shared_ptr< PostProcessor > pp)
Definition: Pipeline.C:650
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:122
jevois::replaceAll
std::string replaceAll(std::string const &str, std::string const &from, std::string const &to)
Replace all instances of 'from' with 'to'.
Definition: Utils.C:349
PostProcessorStub.H
PreProcessorPython.H
jevois::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &root, std::filesystem::path const &path)
Compute an absolute path from two paths.
Definition: Utils.C:365
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
Write some text in an image.
Definition: RawImageOps.C:689
jevois
Definition: Concepts.dox:1
jevois::getNumInstalledNPUs
size_t getNumInstalledNPUs()
Get the number of JeVois-Pro NPUs present on this system.
Definition: SysInfo.C:127
Engine.H
Log.H
jevois::dnn::Pipeline::freeze
void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition: Pipeline.C:126
SysInfo.H
jevois::dnn::Pipeline::setCustomPreProcessor
void setCustomPreProcessor(std::shared_ptr< PreProcessor > pp)
Set a custom pre-processor.
Definition: Pipeline.C:528
jevois::join
std::string join(std::vector< std::string > const &strings, std::string const &delimiter)
Concatenate a vector of tokens into a string.
Definition: Utils.C:268
jevois::getNumInstalledTPUs
size_t getNumInstalledTPUs()
Get the number of Coral TPUs present on this system.
Definition: SysInfo.C:87
NetworkOpenCV.H
jevois::warnAndIgnoreException
std::string warnAndIgnoreException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition: Log.C:236
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::getNumInstalledSPUs
size_t getNumInstalledSPUs()
Get the number of Hailo8 SPUs present on this system.
Definition: SysInfo.C:138
jevois::stringStartsWith
bool stringStartsWith(std::string const &str, std::string const &prefix)
Return true if str starts with prefix (including if both strings are equal)
Definition: Utils.C:282
jevois::secs2str
std::string secs2str(double secs)
Report a duration given in seconds with variable units (ns, us, ms, or s), with precision of 2 decima...
Definition: Utils.C:453
jevois::getNumInstalledVPUs
size_t getNumInstalledVPUs()
Get the number of Myriad-X VPUs present on this system.
Definition: SysInfo.C:112
PostProcessorPython.H
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::Pipeline::process
void process(jevois::RawImage const &inimg, jevois::StdModule *mod, jevois::RawImage *outimg, jevois::OptGUIhelper *helper, bool idle=false)
Process an input image, send results to serial/image/gui.
Definition: Pipeline.C:683
Utils.H
PreProcessorBlob.H
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:232
NetworkPython.H
jevois::dnn::NetworkPython
Wrapper around an DNN neural network invoked through python.
Definition: NetworkPython.H:31
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
NetworkNPU.H