35 #include <opencv2/core/utils/filesystem.hpp>
41 jevois::
Component(instance), itsTpre(
"PreProc"), itsTnet(
"Network"), itsTpost(
"PstProc")
46 itsAccelerators[
"OpenCV"] = 1;
48 LINFO(
"Detected " << itsAccelerators[
"NPU"] <<
" JeVois-Pro NPUs, " << itsAccelerators[
"TPU"] <<
49 " Coral TPUs, " << itsAccelerators[
"VPU"] <<
" Myriad-X VPUs.");
55 preproc::freeze(doit);
56 nettype::freeze(doit);
57 postproc::freeze(doit);
59 if (itsPreProcessor) itsPreProcessor->freeze(doit);
60 if (itsNetwork) itsNetwork->freeze(doit);
61 if (itsPostProcessor) itsPostProcessor->freeze(doit);
86 void jevois::dnn::Pipeline::asyncNetWait()
89 if (itsNetFut.valid())
92 if (itsNetFut.wait_for(std::chrono::seconds(5)) == std::future_status::timeout)
93 LERROR(
"Still waiting for network to finish running...");
97 try { itsNetFut.get(); }
catch (...) { }
102 void jevois::dnn::Pipeline::onParamChange(pipeline::filter
const & JEVOIS_UNUSED_PARAM(param),
103 jevois::dnn::pipeline::Filter
const & JEVOIS_UNUSED_PARAM(val))
108 itsZooChanged =
true;
112 void jevois::dnn::Pipeline::onParamChange(pipeline::zooroot
const & JEVOIS_UNUSED_PARAM(param),
113 std::string
const & JEVOIS_UNUSED_PARAM(val))
117 itsZooChanged =
true;
121 void jevois::dnn::Pipeline::onParamChange(pipeline::zoo
const & JEVOIS_UNUSED_PARAM(param),
122 std::string
const & val)
125 itsPreProcessor.reset();
127 itsPostProcessor.reset();
131 std::vector<std::string> pipes;
133 LINFO(
"Found a total of " << pipes.size() <<
" valid pipelines.");
136 if (pipes.empty() ==
false)
139 "by the current filter",
140 pipes[0], pipes, jevois::dnn::pipeline::ParamCateg);
141 pipe::changeParameterDef(newdef);
149 void jevois::dnn::Pipeline::scanZoo(std::string
const & zoofile, std::string
const & filt,
150 std::vector<std::string> & pipes, std::string
const & indent)
152 LINFO(indent <<
"Scanning model zoo file " << zoofile <<
" with filter [" << filt <<
"]...");
153 int ntot = 0, ngood = 0;
156 cv::FileStorage fs(zoofile, cv::FileStorage::READ);
157 if (fs.isOpened() ==
false)
LFATAL(
"Could not open zoo file " << zoofile);
158 cv::FileNode fn = fs.root();
160 for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
162 cv::FileNode item = *fit;
165 if (item.name() ==
"include")
169 if (item.name() ==
"includedir")
172 DIR *dirp = opendir(dir.c_str());
173 if (dirp ==
nullptr)
LFATAL(
"includedir: " << dir <<
" in zoo file " << zoofile <<
": cannot read directory");
174 struct dirent * dent;
175 while ((dent = readdir(dirp)) !=
nullptr)
177 std::string
const node = dent->d_name;
178 size_t const nl = node.length();
179 if ((nl > 4 && node.substr(nl-4) ==
".yml") || (nl > 5 && node.substr(nl-5) ==
".yaml"))
180 scanZoo(dir +
"/" + node, filt, pipes, indent +
" ");
186 if (item.isMap() ==
false)
continue;
192 std::string typ = (std::string)item[
"nettype"];
193 if (typ ==
"OpenCV" &&
194 (std::string)item[
"backend"] ==
"InferenceEngine" &&
195 (std::string)item[
"target"] ==
"Myriad")
199 bool has_accel =
false;
200 auto itr = itsAccelerators.find(typ);
201 if (itr != itsAccelerators.end() && itr->second > 0) has_accel =
true;
204 if ((filt ==
"All" || typ == filt) && has_accel)
206 pipes.emplace_back(typ +
':' + (std::string)item[
"postproc"] +
':' + item.name());
210 LINFO(indent <<
"Found " << ntot <<
" pipelines, " << ngood <<
" passed the filter.");
214 void jevois::dnn::Pipeline::onParamChange(pipeline::pipe
const & JEVOIS_UNUSED_PARAM(param), std::string
const & val)
216 if (val.empty())
return;
217 itsPipeThrew =
false;
223 if (selectPipe(z, tok) ==
false)
224 LFATAL(
"Could not find pipeline entry [" << val <<
"] in zoo file " << z <<
" and its includes");
230 bool jevois::dnn::Pipeline::selectPipe(std::string
const & zoofile, std::vector<std::string>
const & tok)
233 cv::FileStorage fs(zoofile, cv::FileStorage::READ);
234 if (fs.isOpened() ==
false)
LFATAL(
"Could not open zoo file " << zoofile);
237 std::vector<cv::FileNodeIterator> globals;
239 cv::FileNode fn = fs.root(), node;
241 for (cv::FileNodeIterator fit = fn.begin(); fit != fn.end(); ++fit)
243 cv::FileNode item = *fit;
246 if (item.name() ==
"include")
252 if (item.name() ==
"includedir")
255 DIR *dirp = opendir(dir.c_str());
256 if (dirp ==
nullptr)
LFATAL(
"includedir: " << dir <<
" in zoo file " << zoofile <<
": cannot read directory");
257 struct dirent * dent;
258 while ((dent = readdir(dirp)) !=
nullptr)
260 std::string
const node = dent->d_name;
261 size_t const nl = node.length();
262 if ((nl > 4 && node.substr(nl-4) ==
".yml") || (nl > 5 && node.substr(nl-5) ==
".yaml"))
264 if (selectPipe(dir +
"/" + node, tok)) { closedir(dirp);
return true; }
271 else if (item.isMap())
273 if (item.name() != tok.back())
continue;
274 if (tok.size() == 1) { node = item;
break; }
279 if ((std::string)item[
"nettype"] ==
"OpenCV" &&
280 ((std::string)item[
"postproc"] == tok[1] || postproc::strget() == tok[1]) &&
281 (std::string)item[
"backend"] ==
"InferenceEngine")
282 { node = item;
break; }
286 if ((std::string)item[
"nettype"] == tok[0] &&
287 ((std::string)item[
"postproc"] == tok[1] || postproc::strget() == tok[1]))
288 { node = item;
break; }
292 else globals.emplace_back(fit);
296 if (node.empty())
return false;
300 itsPreProcessor.reset();
try { removeSubComponent(
"preproc"); }
catch (...) { }
301 itsNetwork.reset();
try { removeSubComponent(
"network"); }
catch (...) { }
302 itsPostProcessor.reset();
try { removeSubComponent(
"postproc"); }
catch (...) { }
305 for (cv::FileNodeIterator fit : globals)
307 cv::FileNode item = *fit;
308 setZooParam(item, zoofile, fs.root());
312 for (cv::FileNodeIterator fit = node.begin(); fit != node.end(); ++fit)
314 cv::FileNode item = *fit;
315 setZooParam(item, zoofile, node);
321 void jevois::dnn::Pipeline::setZooParam(cv::FileNode & item, std::string
const & zf, cv::FileNode
const & node)
323 std::string k = item.name();
329 case cv::FileNode::STRING: v = (std::string)item;
break;
330 default:
LFATAL(
"Invalid zoo parameter " << k <<
" type " << item.type() <<
" in " << zf <<
" node " << node.name());
335 bool hasparam =
false;
336 try { getParamStringUnique(k); hasparam =
true; }
catch (...) { }
340 LINFO(
"Setting ["<<k<<
"] to ["<<v<<
']');
342 try { setParamStringUnique(k, v); }
343 catch (std::exception
const & e)
344 {
LFATAL(
"While parsing [" << node.name() <<
"] in model zoo file " << zf <<
": " << e.what()); }
346 {
LFATAL(
"While parsing [" << node.name() <<
"] in model zoo file " << zf <<
": unknown error"); }
351 void jevois::dnn::Pipeline::onParamChange(pipeline::preproc
const & JEVOIS_UNUSED_PARAM(param),
352 pipeline::PreProc
const & val)
354 itsPreProcessor.reset();
try { removeSubComponent(
"preproc"); }
catch (...) { }
358 case jevois::dnn::pipeline::PreProc::Blob:
359 itsPreProcessor = addSubComponent<jevois::dnn::PreProcessorBlob>(
"preproc");
362 case jevois::dnn::pipeline::PreProc::Custom:
367 if (itsPreProcessor)
LINFO(
"Instantiated pre-processor of type " << itsPreProcessor->className());
368 else LINFO(
"No pre-processor");
374 itsPreProcessor.reset();
try { removeSubComponent(
"preproc"); }
catch (...) { }
375 itsPreProcessor = pp;
376 preproc::set(jevois::dnn::pipeline::PreProc::Custom);
378 if (itsPreProcessor)
LINFO(
"Attached pre-processor of type " << itsPreProcessor->className());
379 else LINFO(
"No pre-processor");
383 void jevois::dnn::Pipeline::onParamChange(pipeline::nettype
const & JEVOIS_UNUSED_PARAM(param),
384 pipeline::NetType
const & val)
388 itsNetwork.reset();
try { removeSubComponent(
"network"); }
catch (...) { }
392 case jevois::dnn::pipeline::NetType::OpenCV:
393 itsNetwork = addSubComponent<jevois::dnn::NetworkOpenCV>(
"network");
398 case jevois::dnn::pipeline::NetType::NPU:
399 #ifdef JEVOIS_PLATFORM
400 itsNetwork = addSubComponent<jevois::dnn::NetworkNPU>(
"network");
401 #else // JEVOIS_PLATFORM
402 LFATAL(
"NPU network is only supported on JeVois-Pro Platform");
406 case jevois::dnn::pipeline::NetType::TPU:
407 itsNetwork = addSubComponent<jevois::dnn::NetworkTPU>(
"network");
411 case jevois::dnn::pipeline::NetType::Custom:
416 if (itsNetwork)
LINFO(
"Instantiated network of type " << itsNetwork->className());
417 else LINFO(
"No network");
419 itsInputAttrs.clear();
426 itsNetwork.reset();
try { removeSubComponent(
"network"); }
catch (...) { }
428 nettype::set(jevois::dnn::pipeline::NetType::Custom);
430 if (itsNetwork)
LINFO(
"Attached network of type " << itsNetwork->className());
431 else LINFO(
"No network");
433 itsInputAttrs.clear();
437 void jevois::dnn::Pipeline::onParamChange(pipeline::postproc
const & JEVOIS_UNUSED_PARAM(param),
438 pipeline::PostProc
const & val)
442 itsPostProcessor.reset();
try { removeSubComponent(
"postproc"); }
catch (...) { }
446 case jevois::dnn::pipeline::PostProc::Classify:
447 itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorClassify>(
"postproc");
449 case jevois::dnn::pipeline::PostProc::Detect:
450 itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorDetect>(
"postproc");
452 case jevois::dnn::pipeline::PostProc::Segment:
453 itsPostProcessor = addSubComponent<jevois::dnn::PostProcessorSegment>(
"postproc");
455 case jevois::dnn::pipeline::PostProc::Custom:
460 if (itsPostProcessor)
LINFO(
"Instantiated post-processor of type " << itsPostProcessor->className());
461 else LINFO(
"No post-processor");
468 itsPostProcessor.reset();
try { removeSubComponent(
"postproc"); }
catch (...) { }
469 itsPostProcessor = pp;
470 postproc::set(jevois::dnn::pipeline::PostProc::Custom);
472 if (itsPostProcessor)
LINFO(
"Attached post-processor of type " << itsPostProcessor->className());
473 else LINFO(
"No post-processor");
479 return itsPreProcessor && itsNetwork && itsNetwork->ready() && itsPostProcessor;
483 bool jevois::dnn::Pipeline::checkAsyncNetComplete()
485 if (itsNetFut.valid() && itsNetFut.wait_for(std::chrono::milliseconds(2)) == std::future_status::ready)
487 itsOuts = itsNetFut.get();
489 std::swap(itsNetInfo, itsAsyncNetInfo);
490 itsProcTimes[1] = itsAsyncNetworkTime;
491 itsProcSecs[1] = itsAsyncNetworkSecs;
502 if (itsZooChanged) { itsZooChanged =
false; zoo::set(
zoo::get()); }
506 if (itsPipeThrew)
return;
513 if (helper && idle ==
false) ImGui::Begin((instanceName() +
':' + getParamStringUnique(
"pipe")).c_str());
524 5, itsOutImgY, jevois::yuyv::White);
529 if (helper) helper->
itext(instanceName() +
':' + getParamStringUnique(
"pipe"));
537 if (ready() ==
false)
539 char const * msg = itsNetwork ?
"Loading network..." :
"No network selected...";
550 if (idle ==
false) ImGui::TextUnformatted(msg);
551 if (ovl) helper->
itext(msg);
555 itsProcTimes = {
"PreProc: -",
"Network: -",
"PstProc: -" };
556 itsProcSecs = { 0.0, 0.0, 0.0 };
564 case jevois::dnn::pipeline::Processing::Sync:
570 if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
571 itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
572 itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
573 itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
578 itsOuts = itsNetwork->process(itsBlobs, itsNetInfo);
579 itsProcTimes[1] = itsTnet.stop(&itsProcSecs[1]);
582 showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
586 itsPostProcessor->process(itsOuts, itsPreProcessor.get());
587 itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
588 itsPostProcessor->report(mod, outimg, helper, ovl, idle);
593 case jevois::dnn::pipeline::Processing::Async:
600 bool needpost = checkAsyncNetComplete();
603 if (itsNetFut.valid() ==
false)
607 if (itsInputAttrs.empty()) itsInputAttrs = itsNetwork->inputShapes();
608 itsBlobs = itsPreProcessor->process(inimg, itsInputAttrs);
609 itsProcTimes[0] = itsTpre.stop(&itsProcSecs[0]);
616 std::vector<cv::Mat> outs = itsNetwork->process(itsBlobs, itsAsyncNetInfo);
617 itsAsyncNetworkTime = itsTnet.stop(&itsAsyncNetworkSecs);
624 std::vector<cv::Mat> outscopy;
625 for (cv::Mat
const & m : outs) outscopy.emplace_back(m.clone());
631 itsPreProcessor->sendreport(mod, outimg, helper, ovl, idle);
634 showInfo(itsNetInfo, mod, outimg, helper, ovl, idle);
637 if (needpost && itsOuts.empty() ==
false)
640 itsPostProcessor->process(itsOuts, itsPreProcessor.get());
641 itsProcTimes[2] = itsTpost.stop(&itsProcSecs[2]);
645 itsPostProcessor->report(mod, outimg, helper, ovl, idle);
664 itsSecsSum += itsProcSecs[0] + itsProcSecs[1] + itsProcSecs[2];
666 if (itsSecsSumNum == 20) { itsSecsAvg = itsSecsSum / itsSecsSumNum; itsSecsSum = 0.0; itsSecsSumNum = 0; }
677 if (ImGui::CollapsingHeader(
"Processing Times", ImGuiTreeNodeFlags_DefaultOpen))
679 for (std::string
const & s : itsProcTimes) ImGui::TextUnformatted(s.c_str());
680 ImGui::Text(
"OVERALL: %s/inference", total.c_str());
687 for (std::string
const & s : itsProcTimes) helper->
itext(s);
688 helper->
itext(
"OVERALL: " + total +
"/inference");
696 for (std::string
const & s : itsProcTimes)
702 5, itsOutImgY, jevois::yuyv::White);
708 void jevois::dnn::Pipeline::showInfo(std::vector<std::string>
const & info,
715 for (std::string
const & s : info)
719 if (helper && idle ==
false)
723 show = ImGui::CollapsingHeader(s.c_str() + 2, ImGuiTreeNodeFlags_DefaultOpen);
728 else ImGui::TextUnformatted(s.c_str());
732 (void)idle; (void)show; (void)helper;