JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Pipeline.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2021 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/DNN/Pipeline.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
21 #include <jevois/Util/Async.H>
23 #include <jevois/Debug/SysInfo.H>
24 
26 #include <jevois/DNN/NetworkNPU.H>
27 #include <jevois/DNN/NetworkTPU.H>
28 
30 
34 
35 #include <opencv2/core/utils/filesystem.hpp>
36 
37 #include <dirent.h>
38 
39 // ####################################################################################################
40 jevois::dnn::Pipeline::Pipeline(std::string const & instance) :
41  jevois::Component(instance), itsTpre("PreProc"), itsTnet("Network"), itsTpost("PstProc")
42 {
43  itsAccelerators["TPU"] = jevois::getNumInstalledTPUs();
44  itsAccelerators["VPU"] = jevois::getNumInstalledVPUs();
45  itsAccelerators["NPU"] = jevois::getNumInstalledNPUs();
46  itsAccelerators["OpenCV"] = 1;
47 
48  LINFO("Detected " << itsAccelerators["NPU"] << " JeVois-Pro NPUs, " << itsAccelerators["TPU"] <<
49  " Coral TPUs, " << itsAccelerators["VPU"] << " Myriad-X VPUs.");
50 }
51 
52 // ####################################################################################################
54 {
55  preproc::freeze(doit);
56  nettype::freeze(doit);
57  postproc::freeze(doit);
58 
59  if (itsPreProcessor) itsPreProcessor->freeze(doit);
60  if (itsNetwork) itsNetwork->freeze(doit);
61  if (itsPostProcessor) itsPostProcessor->freeze(doit);
62 }
63 
64 // ####################################################################################################
66 {
67  // Freeze all params that users should not modify at runtime:
68  freeze(true);
69 }
70 
71 // ####################################################################################################
73 {
74  // If we have a network running async, make sure we wait here until it is done:
75  asyncNetWait();
76 }
77 
78 // ####################################################################################################
80 {
81  // Make sure network is not running as we die:
82  asyncNetWait();
83 }
84 
85 // ####################################################################################################
86 void jevois::dnn::Pipeline::asyncNetWait()
87 {
88  // If we were currently doing async processing, wait until network is done:
89  if (itsNetFut.valid())
90  while (true)
91  {
92  if (itsNetFut.wait_for(std::chrono::seconds(5)) == std::future_status::timeout)
93  LERROR("Still waiting for network to finish running...");
94  else break;
95  }
96 
97  try { itsNetFut.get(); } catch (...) { }
98  itsOuts.clear();
99 }
100 
101 // ####################################################################################################
102 void jevois::dnn::Pipeline::onParamChange(pipeline::filter const & JEVOIS_UNUSED_PARAM(param),
103  jevois::dnn::pipeline::Filter const & JEVOIS_UNUSED_PARAM(val))
104 {
105  // Reload the zoo file so that the filter can be applied to create the parameter def of pipe, but first we need this
106  // parameter to indeed be updated. So here we just set a flag and the update will occur in process(), after we run the
107  // current model one last time:
108  itsZooChanged = true;
109 }
110 
111 // ####################################################################################################
112 void jevois::dnn::Pipeline::onParamChange(pipeline::zooroot const & JEVOIS_UNUSED_PARAM(param),
113  std::string const & JEVOIS_UNUSED_PARAM(val))
114 {
115  // Reload the zoo file, but first we need this parameter to indeed be updated. So here we just set a flag and the
116  // update will occur in process(), after we run the current model one last time:
117  itsZooChanged = true;
118 }
119 
120 // ####################################################################################################
121 void jevois::dnn::Pipeline::onParamChange(pipeline::zoo const & JEVOIS_UNUSED_PARAM(param),
122  std::string const & val)
123 {
124  // Just nuke everything:
125  itsPreProcessor.reset();
126  itsNetwork.reset();
127  itsPostProcessor.reset();
128  // Will get instantiated again when pipe param is set.
129 
130  // Load zoo file:
131  std::vector<std::string> pipes;
132  scanZoo(jevois::absolutePath(zooroot::get(), val), filter::strget(), pipes, "");
133  LINFO("Found a total of " << pipes.size() << " valid pipelines.");
134 
135  // Update the parameter def of pipe:
136  if (pipes.empty() == false)
137  {
138  jevois::ParameterDef<std::string> newdef("pipe", "Pipeline to use, determined by entries in the zoo file and "
139  "by the current filter",
140  pipes[0], pipes, jevois::dnn::pipeline::ParamCateg);
141  pipe::changeParameterDef(newdef);
142 
143  // Just changing the def does not change the param value, so change it now:
144  pipe::set(pipes[0]);
145  }
146 }
147 
148 // ####################################################################################################
149 void jevois::dnn::Pipeline::scanZoo(std::string const & zoofile, std::string const & filt,
150  std::vector<std::string> & pipes, std::string const & indent)
151 {
152  LINFO(indent << "Scanning model zoo file " << zoofile << " with filter [" << filt << "]...");
153  int ntot = 0, ngood = 0;
154 
155  // Scan the zoo file to update the parameter def of the pipe parameter:
156  cv::FileStorage fs(zoofile, cv::FileStorage::READ);
157  if (fs.isOpened() == false) LFATAL("Could not open zoo file " << zoofile);
158  cv::FileNode fn = fs.root();
159 
160  for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
161  {
162  cv::FileNode item = *fit;
163 
164  // Process include: directives recursively:
165  if (item.name() == "include")
166  scanZoo(jevois::absolutePath(zooroot::get(), (std::string)item), filt, pipes, indent + " ");
167 
168  // Process includedir: directives (only one level of directory is scanned):
169  if (item.name() == "includedir")
170  {
171  std::string const dir = jevois::absolutePath(zooroot::get(), (std::string)item);
172  DIR *dirp = opendir(dir.c_str());
173  if (dirp == nullptr) LFATAL("includedir: " << dir << " in zoo file " << zoofile << ": cannot read directory");
174  struct dirent * dent;
175  while ((dent = readdir(dirp)) != nullptr)
176  {
177  std::string const node = dent->d_name;
178  size_t const nl = node.length();
179  if ((nl > 4 && node.substr(nl-4) == ".yml") || (nl > 5 && node.substr(nl-5) == ".yaml"))
180  scanZoo(dir + "/" + node, filt, pipes, indent + " ");
181  }
182  closedir(dirp);
183  }
184 
185  // Process map types, ignore others:
186  if (item.isMap() == false) continue;
187 
188  ++ntot;
189 
190  // As a prefix, we use OpenCV for OpenCV models on CPU/OpenCL backends, and VPU for InferenceEngine backend with
191  // Myriad target:
192  std::string typ = (std::string)item["nettype"];
193  if (typ == "OpenCV" &&
194  (std::string)item["backend"] == "InferenceEngine" &&
195  (std::string)item["target"] == "Myriad")
196  typ = "VPU";
197 
198  // Do not consider a model if we do not have the accelerator for it:
199  bool has_accel = false;
200  auto itr = itsAccelerators.find(typ);
201  if (itr != itsAccelerators.end() && itr->second > 0) has_accel = true;
202 
203  // Add this pipe if it matches our filter and we have the accelerator for it:
204  if ((filt == "All" || typ == filt) && has_accel)
205  {
206  pipes.emplace_back(typ + ':' + (std::string)item["postproc"] + ':' + item.name());
207  ++ngood;
208  }
209  }
210  LINFO(indent << "Found " << ntot << " pipelines, " << ngood << " passed the filter.");
211 }
212 
213 // ####################################################################################################
214 void jevois::dnn::Pipeline::onParamChange(pipeline::pipe const & JEVOIS_UNUSED_PARAM(param), std::string const & val)
215 {
216  if (val.empty()) return;
217  itsPipeThrew = false;
218  freeze(false);
219 
220  // Find the desired pipeline, and set it up:
221  std::string const z = jevois::absolutePath(zooroot::get(), zoo::get());
222  std::vector<std::string> tok = jevois::split(val, ":");
223  if (selectPipe(z, tok) == false)
224  LFATAL("Could not find pipeline entry [" << val << "] in zoo file " << z << " and its includes");
225 
226  freeze(true);
227 }
228 
229 // ####################################################################################################
230 bool jevois::dnn::Pipeline::selectPipe(std::string const & zoofile, std::vector<std::string> const & tok)
231 {
232  // Open the zoo file:
233  cv::FileStorage fs(zoofile, cv::FileStorage::READ);
234  if (fs.isOpened() == false) LFATAL("Could not open zoo file " << zoofile);
235 
236  // Find the desired pipeline:
237  std::vector<cv::FileNodeIterator> globals;
238 
239  cv::FileNode fn = fs.root(), node;
240 
241  for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
242  {
243  cv::FileNode item = *fit;
244 
245  // Process include: directives recursively, end the recursion if we found our pipe in there:
246  if (item.name() == "include")
247  {
248  if (selectPipe(jevois::absolutePath(zooroot::get(), (std::string)item), tok)) return true;
249  }
250 
251  // Process includedir: directives (only one level of directory is scanned), end recursion if we found our pipe:
252  if (item.name() == "includedir")
253  {
254  std::string const dir = jevois::absolutePath(zooroot::get(), (std::string)item);
255  DIR *dirp = opendir(dir.c_str());
256  if (dirp == nullptr) LFATAL("includedir: " << dir << " in zoo file " << zoofile << ": cannot read directory");
257  struct dirent * dent;
258  while ((dent = readdir(dirp)) != nullptr)
259  {
260  std::string const node = dent->d_name;
261  size_t const nl = node.length();
262  if ((nl > 4 && node.substr(nl-4) == ".yml") || (nl > 5 && node.substr(nl-5) == ".yaml"))
263  {
264  if (selectPipe(dir + "/" + node, tok)) { closedir(dirp); return true; }
265  }
266  }
267  closedir(dirp);
268  }
269 
270  // This is an entry for a pipeline with a bunch of params under it:
271  else if (item.isMap())
272  {
273  if (item.name() != tok.back()) continue;
274  if (tok.size() == 1) { node = item; break; }
275  if (tok.size() != 3) LFATAL("Malformed pipeline name: " << jevois::join(tok, ":"));
276 
277  if (tok[0] == "VPU")
278  {
279  if ((std::string)item["nettype"] == "OpenCV" &&
280  ((std::string)item["postproc"] == tok[1] || postproc::strget() == tok[1]) &&
281  (std::string)item["backend"] == "InferenceEngine")
282  { node = item; break; }
283  }
284  else
285  {
286  if ((std::string)item["nettype"] == tok[0] &&
287  ((std::string)item["postproc"] == tok[1] || postproc::strget() == tok[1]))
288  { node = item; break; }
289  }
290  }
291  // Global parameter in the current file, keep it for later, if we find our pipe:
292  else globals.emplace_back(fit);
293  }
294 
295  // If the spec was not a match with any entries in the file, return false:
296  if (node.empty()) return false;
297 
298  // Found the pipe. First nuke our current pre/net/post:
299  asyncNetWait();
300  itsPreProcessor.reset(); try { removeSubComponent("preproc"); } catch (...) { }
301  itsNetwork.reset(); try { removeSubComponent("network"); } catch (...) { }
302  itsPostProcessor.reset(); try { removeSubComponent("postproc"); } catch (...) { }
303 
304  // Then set all the global parameters of the current file:
305  for (cv::FileNodeIterator fit : globals)
306  {
307  cv::FileNode item = *fit;
308  setZooParam(item, zoofile, fs.root());
309  }
310 
311  // Then iterate over all pipeline params and set them:
312  for (cv::FileNodeIterator fit = node.begin(); fit != node.end(); ++fit)
313  {
314  cv::FileNode item = *fit;
315  setZooParam(item, zoofile, node);
316  }
317  return true;
318 }
319 
320 // ####################################################################################################
321 void jevois::dnn::Pipeline::setZooParam(cv::FileNode & item, std::string const & zf, cv::FileNode const & node)
322 {
323  std::string k = item.name();
324  std::string v;
325  switch (item.type())
326  {
327  case cv::FileNode::INT: v = std::to_string((int)item); break;
328  case cv::FileNode::REAL: v = std::to_string((float)item); break;
329  case cv::FileNode::STRING: v = (std::string)item; break;
330  default: LFATAL("Invalid zoo parameter " << k << " type " << item.type() << " in " << zf << " node " << node.name());
331  }
332 
333  // The zoo file may contain extra params, like download URL, etc. To ignore those while still catching invalid
334  // values on our parameters, we first check whether the parameter exists, and, if so, try to set it:
335  bool hasparam = false;
336  try { getParamStringUnique(k); hasparam = true; } catch (...) { }
337 
338  if (hasparam)
339  {
340  LINFO("Setting ["<<k<<"] to ["<<v<<']');
341 
342  try { setParamStringUnique(k, v); }
343  catch (std::exception const & e)
344  { LFATAL("While parsing [" << node.name() << "] in model zoo file " << zf << ": " << e.what()); }
345  catch (...)
346  { LFATAL("While parsing [" << node.name() << "] in model zoo file " << zf << ": unknown error"); }
347  }
348 }
349 
350 // ####################################################################################################
351 void jevois::dnn::Pipeline::onParamChange(pipeline::preproc const & JEVOIS_UNUSED_PARAM(param),
352  pipeline::PreProc const & val)
353 {
354  itsPreProcessor.reset(); try { removeSubComponent("preproc"); } catch (...) { }
355 
356  switch (val)
357  {
358  case jevois::dnn::pipeline::PreProc::Blob:
359  itsPreProcessor = addSubComponent<jevois::dnn::PreProcessorBlob>("preproc");
360  break;
361 
362  case jevois::dnn::pipeline::PreProc::Custom:
363  // nothing here, user must call setCustomPreProcessor() later
364  break;
365  }
366 
367  if (itsPreProcessor) LINFO("Instantiated pre-processor of type " << itsPreProcessor->className());
368  else LINFO("No pre-processor");
369 }
370 
371 // ####################################################################################################
372 void jevois::dnn::Pipeline::setCustomPreProcessor(std::shared_ptr<jevois::dnn::PreProcessor> pp)
373 {
374  itsPreProcessor.reset(); try { removeSubComponent("preproc"); } catch (...) { }
375  itsPreProcessor = pp;
376  preproc::set(jevois::dnn::pipeline::PreProc::Custom);
377 
378  if (itsPreProcessor) LINFO("Attached pre-processor of type " << itsPreProcessor->className());
379  else LINFO("No pre-processor");
380 }
381 
382 // ####################################################################################################
383 void jevois::dnn::Pipeline::onParamChange(pipeline::nettype const & JEVOIS_UNUSED_PARAM(param),
384  pipeline::NetType const & val)
385 {
386  asyncNetWait(); // If currently processing async net, wait until done
387 
388  itsNetwork.reset(); try { removeSubComponent("network"); } catch (...) { }
389 
390  switch (val)
391  {
392  case jevois::dnn::pipeline::NetType::OpenCV:
393  itsNetwork = addSubComponent<jevois::dnn::NetworkOpenCV>("network");
394  break;
395 
396 #ifdef JEVOIS_PRO
397 
398  case jevois::dnn::pipeline::NetType::NPU:
399 #ifdef JEVOIS_PLATFORM
400  itsNetwork = addSubComponent<jevois::dnn::NetworkNPU>("network");
401 #else // JEVOIS_PLATFORM
402  LFATAL("NPU network is only supported on JeVois-Pro Platform");
403 #endif
404  break;
405 
406  case jevois::dnn::pipeline::NetType::TPU:
407  itsNetwork = addSubComponent<jevois::dnn::NetworkTPU>("network");
408  break;
409 #endif
410 
411  case jevois::dnn::pipeline::NetType::Custom:
412  // Nothing here, user must call setCustomNetwork() later
413  break;
414  }
415 
416  if (itsNetwork) LINFO("Instantiated network of type " << itsNetwork->className());
417  else LINFO("No network");
418 
419  itsInputAttrs.clear();
420 }
421 
422 // ####################################################################################################
423 void jevois::dnn::Pipeline::setCustomNetwork(std::shared_ptr<jevois::dnn::Network> n)
424 {
425  asyncNetWait(); // If currently processing async net, wait until done
426  itsNetwork.reset(); try { removeSubComponent("network"); } catch (...) { }
427  itsNetwork = n;
428  nettype::set(jevois::dnn::pipeline::NetType::Custom);
429 
430  if (itsNetwork) LINFO("Attached network of type " << itsNetwork->className());
431  else LINFO("No network");
432 
433  itsInputAttrs.clear();
434 }
435 
436 // ####################################################################################################
437 void jevois::dnn::Pipeline::onParamChange(pipeline::postproc const & JEVOIS_UNUSED_PARAM(param),
438  pipeline::PostProc const & val)
439 {
440  asyncNetWait(); // If currently processing async net, wait until done
441 
442  itsPostProcessor.reset(); try { removeSubComponent("postproc"); } catch (...) { }
443 
444  switch (val)
445  {
446  case jevois::dnn::pipeline::PostProc::Classify:
447  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorClassify>("postproc");
448  break;
449  case jevois::dnn::pipeline::PostProc::Detect:
450  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorDetect>("postproc");
451  break;
452  case jevois::dnn::pipeline::PostProc::Segment:
453  itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorSegment>("postproc");
454  break;
455  case jevois::dnn::pipeline::PostProc::Custom:
456  // Nothing here, user must call setCustomPostProcessor() later
457  break;
458  }
459 
460  if (itsPostProcessor) LINFO("Instantiated post-processor of type " << itsPostProcessor->className());
461  else LINFO("No post-processor");
462 }
463 
464 // ####################################################################################################
465 void jevois::dnn::Pipeline::setCustomPostProcessor(std::shared_ptr<jevois::dnn::PostProcessor> pp)
466 {
467  asyncNetWait(); // If currently processing async net, wait until done
468  itsPostProcessor.reset(); try { removeSubComponent("postproc"); } catch (...) { }
469  itsPostProcessor = pp;
470  postproc::set(jevois::dnn::pipeline::PostProc::Custom);
471 
472  if (itsPostProcessor) LINFO("Attached post-processor of type " << itsPostProcessor->className());
473  else LINFO("No post-processor");
474 }
475 
476 // ####################################################################################################
478 {
479  return itsPreProcessor && itsNetwork && itsNetwork->ready() && itsPostProcessor;
480 }
481 
482 // ####################################################################################################
483 bool jevois::dnn::Pipeline::checkAsyncNetComplete()
484 {
485  if (itsNetFut.valid() && itsNetFut.wait_for(std::chrono::milliseconds(2)) == std::future_status::ready)
486  {
487  itsOuts = itsNetFut.get();
488  itsNetInfo.clear();
489  std::swap(itsNetInfo, itsAsyncNetInfo);
490  itsProcTimes[1] = itsAsyncNetworkTime;
491  itsProcSecs[1] = itsAsyncNetworkSecs;
492  return true;
493  }
494  return false;
495 }
496 
497 // ####################################################################################################
499  jevois::OptGUIhelper * helper, bool idle)
500 {
501  // Reload the zoo file if filter has changed:
502  if (itsZooChanged) { itsZooChanged = false; zoo::set(zoo::get()); }
503 
504  // If the pipeline is throwing exception at any stage, do not do anything here, itsPipeThrew is cleared when selecting
505  // a new pipe:
506  if (itsPipeThrew) return;
507 
508  bool const ovl = overlay::get();
509  itsOutImgY = 5; // y text position when using outimg text drawings
510 
511 #ifdef JEVOIS_PRO
512  // Open an info window if using GUI and not idle:
513  if (helper && idle == false) ImGui::Begin((instanceName() + ':' + getParamStringUnique("pipe")).c_str());
514 #else
515  (void)helper; // avoid compiler warning
516 #endif
517 
518  // If we want an overlay, show network name on first line:
519  if (ovl)
520  {
521  if (outimg)
522  {
523  jevois::rawimage::writeText(*outimg, instanceName() + ':' + getParamStringUnique("pipe"),
524  5, itsOutImgY, jevois::yuyv::White);
525  itsOutImgY += 11;
526  }
527 
528 #ifdef JEVOIS_PRO
529  if (helper) helper->itext(instanceName() + ':' + getParamStringUnique("pipe"));
530 #endif
531  }
532 
533  // If network is not ready, inform user. Be careful that ready() may throw, eg, if bad network name was given and
534  // network could not be loaded:
535  try
536  {
537  if (ready() == false)
538  {
539  char const * msg = itsNetwork ? "Loading network..." : "No network selected...";
540 
541  if (outimg)
542  {
543  jevois::rawimage::writeText(*outimg, msg, 5, itsOutImgY, jevois::yuyv::White);
544  itsOutImgY += 11;
545  }
546 
547 #ifdef JEVOIS_PRO
548  if (helper)
549  {
550  if (idle == false) ImGui::TextUnformatted(msg);
551  if (ovl) helper->itext(msg);
552  }
553 #endif
554 
555  itsProcTimes = { "PreProc: -", "Network: -", "PstProc: -" };
556  itsProcSecs = { 0.0, 0.0, 0.0 };
557  }
558  else
559  {
560  // Network is ready, run processing, either single-thread (Sync) or threaded (Async):
561  switch (processing::get())
562  {
563  // --------------------------------------------------------------------------------
564  case jevois::dnn::pipeline::Processing::Sync:
565  {
566  asyncNetWait(); // If currently processing async net, wait until done
567 
568  // Pre-process:
569  itsTpre.start();
570  if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
571  itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
572  itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
573  itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
574 
575  // Network forward pass:
576  itsNetInfo.clear();
577  itsTnet.start();
578  itsOuts = itsNetwork->process(itsBlobs, itsNetInfo);
579  itsProcTimes[1] = itsTnet.stop(&itsProcSecs[1]);
580 
581  // Show network info:
582  showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
583 
584  // Post-Processing:
585  itsTpost.start();
586  itsPostProcessor->process(itsOuts, itsPreProcessor.get());
587  itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
588  itsPostProcessor->report(mod, outimg, helper, ovl, idle);
589  }
590  break;
591 
592  // --------------------------------------------------------------------------------
593  case jevois::dnn::pipeline::Processing::Async:
594  {
595  // We are going to run pre-processing and post-processing synchronously, and network in a thread. One small
596  // complication is that we are going to run post-processing on every frame so that the drawings do not
597  // flicker. We will keep post-processing the same results until new results replace them.
598 
599  // Are we running the network, and is it done? If so, get the outputs:
600  bool needpost = checkAsyncNetComplete();
601 
602  // If we are not running a network, start it:
603  if (itsNetFut.valid() == false)
604  {
605  // Pre-process in the current thread:
606  itsTpre.start();
607  if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
608  itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
609  itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
610 
611  // Network forward pass in a thread:
612  itsNetFut =
613  jevois::async([this]()
614  {
615  itsTnet.start();
616  std::vector<cv::Mat> outs = itsNetwork->process(itsBlobs, itsAsyncNetInfo);
617  itsAsyncNetworkTime = itsTnet.stop(&itsAsyncNetworkSecs);
618 
619  // OpenCV DNN seems to be re-using and overwriting the same output matrices,
620  // so we need to make a deep copy of the outputs if the network type is OpenCV:
621  if (dynamic_cast<jevois::dnn::NetworkOpenCV *>(itsNetwork.get()) == nullptr)
622  return outs;
623 
624  std::vector<cv::Mat> outscopy;
625  for (cv::Mat const & m : outs) outscopy.emplace_back(m.clone());
626  return outscopy;
627  });
628  }
629 
630  // Report pre-processing results on every frame:
631  itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
632 
633  // Show network info on every frame:
634  showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
635 
636  // Run post-processing if needed:
637  if (needpost && itsOuts.empty() == false)
638  {
639  itsTpost.start();
640  itsPostProcessor->process(itsOuts, itsPreProcessor.get());
641  itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
642  }
643 
644  // Report/draw post-processing results on every frame:
645  itsPostProcessor->report(mod, outimg, helper, ovl, idle);
646  }
647  break;
648  }
649  }
650  }
651  catch (...)
652  {
653  itsPipeThrew = true;
654 
655 #ifdef JEVOIS_PRO
656  if (helper) helper->reportAndIgnoreException(instanceName());
657  else jevois::warnAndIgnoreException(instanceName());
658 #else
659  jevois::warnAndIgnoreException(instanceName());
660 #endif
661  }
662 
663  // Update our rolling average of total processing time:
664  itsSecsSum += itsProcSecs[0] + itsProcSecs[1] + itsProcSecs[2];
665  ++itsSecsSumNum;
666  if (itsSecsSumNum == 20) { itsSecsAvg = itsSecsSum / itsSecsSumNum; itsSecsSum = 0.0; itsSecsSumNum = 0; }
667 
668 #ifdef JEVOIS_PRO
669  // Report processing times and close info window if we opened it:
670  if (helper)
671  {
672  std::string total;
673  if (idle == false || ovl) total = jevois::secs2str(itsSecsAvg);
674 
675  if (idle == false)
676  {
677  if (ImGui::CollapsingHeader("Processing Times", ImGuiTreeNodeFlags_DefaultOpen))
678  {
679  for (std::string const & s : itsProcTimes) ImGui::TextUnformatted(s.c_str());
680  ImGui::Text("OVERALL: %s/inference", total.c_str());
681  }
682  ImGui::End();
683  }
684 
685  if (ovl)
686  {
687  for (std::string const & s : itsProcTimes) helper->itext(s);
688  helper->itext("OVERALL: " + total + "/inference");
689  }
690  }
691 #endif
692 
693  // Report processing times to outimg if present:
694  if (outimg && ovl)
695  {
696  for (std::string const & s : itsProcTimes)
697  {
698  jevois::rawimage::writeText(*outimg, s, 5, itsOutImgY, jevois::yuyv::White);
699  itsOutImgY += 11;
700  }
701  jevois::rawimage::writeText(*outimg, "OVERALL: " + jevois::secs2str(itsSecsAvg) + "/inference",
702  5, itsOutImgY, jevois::yuyv::White);
703  itsOutImgY += 11;
704  }
705 }
706 
707 // ####################################################################################################
708 void jevois::dnn::Pipeline::showInfo(std::vector<std::string> const & info,
709  jevois::StdModule * JEVOIS_UNUSED_PARAM(mod),
710  jevois::RawImage * outimg,
711  jevois::OptGUIhelper * helper, bool ovl, bool idle)
712 {
713  if (ovl == false) return;
714 
715  bool show = true;
716 
717  for (std::string const & s : info)
718  {
719  // On JeVois Pro, display info in the GUI:
720 #ifdef JEVOIS_PRO
721  if (helper && idle == false)
722  {
723  // Create collapsible header and get its collapsed status:
724  if (jevois::stringStartsWith(s, "* "))
725  show = ImGui::CollapsingHeader(s.c_str() + 2, ImGuiTreeNodeFlags_DefaultOpen);
726  else if (show)
727  {
728  // If header not collapsed, show data:
729  if (jevois::stringStartsWith(s, "- ")) ImGui::BulletText("%s", s.c_str() + 2);
730  else ImGui::TextUnformatted(s.c_str());
731  }
732  }
733 #endif
734 
735  if (outimg)
736  {
737  jevois::rawimage::writeText(*outimg, s, 5, itsOutImgY, jevois::yuyv::White);
738  itsOutImgY += 11;
739  (void)idle; (void)ovl; (void)show; (void)helper; // avoid warning
740  }
741  }
742 }
743 
PostProcessorClassify.H
jevois::dnn::Pipeline::preUninit
void preUninit() override
Called before all sub-Components are uninit()ed.
Definition: Pipeline.C:72
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
jevois::GUIhelper::reportAndIgnoreException
void reportAndIgnoreException(std::string const &prefix="")
Report current exception in a modal dialog, then ignore it.
Definition: GUIhelper.C:2252
Pipeline.H
Async.H
PostProcessorDetect.H
jevois::GUIhelper::itext
void itext(char const *txt, ImU32 const &col=IM_COL32_BLACK_TRANS, int line=-1)
Draw some overlay text on top of an image.
Definition: GUIhelper.C:509
RawImageOps.H
jevois::dnn::Pipeline::setCustomNetwork
void setCustomNetwork(std::shared_ptr< Network > n)
Definition: Pipeline.C:423
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:257
jevois::Component
A component of a model hierarchy.
Definition: Component.H:180
jevois::dnn::NetworkOpenCV
Wrapper around an OpenCV DNN neural network.
Definition: NetworkOpenCV.H:31
jevois::dnn::Pipeline::Pipeline
Pipeline(std::string const &instance)
Constructor.
Definition: Pipeline.C:40
PostProcessorSegment.H
jevois::dnn::Pipeline::postInit
void postInit() override
Called after all sub-Components are init()ed.
Definition: Pipeline.C:65
jevois::dnn::Pipeline::ready
bool ready() const
Returns true when all three or preproc, net, and postproc are ready.
Definition: Pipeline.C:477
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::ParameterDef
A Parameter Definition.
Definition: ParameterDef.H:88
NetworkTPU.H
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:198
jevois::dnn::Pipeline::~Pipeline
virtual ~Pipeline()
Destructor.
Definition: Pipeline.C:79
jevois::dnn::Pipeline::setCustomPostProcessor
void setCustomPostProcessor(std::shared_ptr< PostProcessor > pp)
Definition: Pipeline.C:465
jevois::GUIhelper
Helper class to assist modules in creating graphical and GUI elements.
Definition: GUIhelper.H:108
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
Write some text in an image.
Definition: RawImageOps.C:689
jevois
Definition: Concepts.dox:1
jevois::getNumInstalledNPUs
size_t getNumInstalledNPUs()
Get the number of JeVois-Pro NPUs present on this system.
Definition: SysInfo.C:127
Log.H
jevois::dnn::Pipeline::freeze
void freeze(bool doit)
Freeze/unfreeze parameters that users should not change while running.
Definition: Pipeline.C:53
SysInfo.H
jevois::dnn::Pipeline::setCustomPreProcessor
void setCustomPreProcessor(std::shared_ptr< PreProcessor > pp)
Set a custom pre-processor.
Definition: Pipeline.C:372
jevois::join
std::string join(std::vector< std::string > const &strings, std::string const &delimiter)
Concatenate a vector of tokens into a string.
Definition: Utils.C:267
jevois::getNumInstalledTPUs
size_t getNumInstalledTPUs()
Get the number of Coral TPUs present on this system.
Definition: SysInfo.C:87
NetworkOpenCV.H
jevois::warnAndIgnoreException
std::string warnAndIgnoreException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition: Log.C:224
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
jevois::absolutePath
std::string absolutePath(std::string const &root, std::string const &path)
Compute an absolute path from two paths.
Definition: Utils.C:347
jevois::stringStartsWith
bool stringStartsWith(std::string const &str, std::string const &prefix)
Return true if str starts with prefix (including if both strings are equal)
Definition: Utils.C:281
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Async execution using a thread pool.
jevois::secs2str
std::string secs2str(double secs)
Report a duration given in seconds with variable units (ns, us, ms, or s), with precision of 2 decima...
Definition: Utils.C:435
jevois::getNumInstalledVPUs
size_t getNumInstalledVPUs()
Get the number of Myriad-X VPUs present on this system.
Definition: SysInfo.C:112
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::Pipeline::process
void process(jevois::RawImage const &inimg, jevois::StdModule *mod, jevois::RawImage *outimg, jevois::OptGUIhelper *helper, bool idle=false)
Process an input image, send results to serial/image/gui.
Definition: Pipeline.C:498
Utils.H
PreProcessorBlob.H
jevois::StdModule
Base class for a module that supports standardized serial messages.
Definition: Module.H:238
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:181
NetworkNPU.H