JeVoisBase  1.16
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
DetectionDNN.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Debug/Timer.H>
21 #include <opencv2/core/core.hpp>
22 #include <opencv2/dnn.hpp>
23 #include <opencv2/imgproc.hpp>
24 #include <fstream>
25 
26 // icon from opencv
27 
28 static jevois::ParameterCategory const ParamCateg("Detection DNN Options");
29 
30 //! Enum \relates DetectionDNN
31 JEVOIS_DEFINE_ENUM_CLASS(Model, (Custom) (Face) (MobileNetSSDvoc) (MobileNetSSDcoco) (MobileNet2SSDcoco)
32  (TinyYOLOv3) (TinyYOLOv2) );
33 
34 //! Parameter \relates DetectionDNN
35 JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(model, Model, "Shortcut parameter to load a model. This sets parameters "
36  "classnames, configname, modelname, etc for the selected network. When "
37  "the selected model is Custom, those other parameters will be set instead "
38  "from the module's params.cfg config file.",
39  Model::Custom, Model_Values, ParamCateg);
40 
41 //! Parameter \relates DetectionDNN
42 JEVOIS_DECLARE_PARAMETER(classnames, std::string, "Path to a text file with names of classes to label detected objects",
43  JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.classes", ParamCateg);
44 
45 //! Parameter \relates DetectionDNN
46 JEVOIS_DECLARE_PARAMETER(configname, std::string, "Path to a text file that contains network configuration. "
47  "Can have extension .prototxt (Caffe), .pbtxt (TensorFlow), or .cfg (Darknet).",
48  JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.prototxt", ParamCateg);
49 
50 //! Parameter \relates DetectionDNN
51 JEVOIS_DECLARE_PARAMETER(modelname, std::string, "Path to a binary file of model contains trained weights. "
52  "Can have extension .caffemodel (Caffe), .pb (TensorFlow), .t7 or .net (Torch), "
53  "or .weights (Darknet).",
54  JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.caffemodel", ParamCateg);
55 
56 //! Parameter \relates DetectionDNN
57 JEVOIS_DECLARE_PARAMETER(netin, cv::Size, "Width and height (in pixels) of the neural network input layer, or [0 0] "
58  "to make it match camera frame size. NOTE: for YOLO v3 sizes must be multiples of 32.",
59  cv::Size(160, 120), ParamCateg);
60 
61 //! Parameter \relates DetectionDNN
62 JEVOIS_DECLARE_PARAMETER(thresh, float, "Detection threshold in percent confidence",
63  50.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
64 
65 //! Parameter \relates DetectionDNN
66 JEVOIS_DECLARE_PARAMETER(nms, float, "Non-maximum suppression intersection-over-union threshold in percent",
67  45.0F, jevois::Range<float>(0.0F, 100.0F), ParamCateg);
68 
69 //! Parameter \relates DetectionDNN
70 JEVOIS_DECLARE_PARAMETER(rgb, bool, "When true, model works with RGB input images instead BGR ones",
71  true, ParamCateg);
72 
73 //! Parameter \relates DetectionDNN
74 JEVOIS_DECLARE_PARAMETER(scale, float, "Value scaling factor applied to input pixels",
75  2.0F / 255.0F, ParamCateg);
76 
77 //! Parameter \relates DetectionDNN
78 JEVOIS_DECLARE_PARAMETER(mean, cv::Scalar, "Mean BGR value subtracted from input image",
79  cv::Scalar(127.5F, 127.5F, 127.5F), ParamCateg);
80 
81 #ifdef JEVOIS_PRO
82 //! Enum \relates DetectionDNN
83 JEVOIS_DEFINE_ENUM_CLASS(Target, (CPU) (OpenCL) (OpenCL_FP16) );
84 
85 //! Parameter \relates DetectionDNN
86 JEVOIS_DECLARE_PARAMETER(target, Target, "OpenCV compute target to use. Changes will take effect "
87  "next time you load a different model.",
88  Target::OpenCL, Target_Values, ParamCateg);
89 #endif
90 
91 //! Detect and recognize multiple objects in scenes using OpenCV Deep Neural Nets (DNN)
92 /*! This module runs an object detection deep neural network using the OpenCV DNN library. Detection networks analyze a
93  whole scene and produce a number of bounding boxes around detected objects, together with identity labels and
94  confidence scores for each detected box.
95 
96  This module runs the selected deep neural network and shows all detections obtained.
97 
98  Note that by default this module runs the OpenCV Face Detector DNN which can detect human faces.
99 
100  Included with the standard JeVois distribution are the following networks:
101 
102  - OpenCV Face Detector, Caffe model
103  - MobileNet + SSD trained on Pascal VOC (20 object classes), Caffe model
104  - MobileNet + SSD trained on Coco (80 object classes), TensorFlow model
105  - MobileNet v2 + SSD trained on Coco (80 object classes), TensorFlow model
106  - Darknet Tiny YOLO v3 trained on Coco (80 object classes), Darknet model
107  - Darknet Tiny YOLO v2 trained on Pascal VOC (20 object classes), Darknet model
108 
109  See the module's \b params.cfg file to switch network. Object categories are as follows:
110 
111  - The 80 COCO object categories are: person, bicycle, car, motorbike, aeroplane, bus, train, truck, boat, traffic,
112  fire, stop, parking, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra, giraffe, backpack, umbrella,
113  handbag, tie, suitcase, frisbee, skis, snowboard, sports, kite, baseball, baseball, skateboard, surfboard, tennis,
114  bottle, wine, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange, broccoli, carrot, hot, pizza, donut,
115  cake, chair, sofa, pottedplant, bed, diningtable, toilet, tvmonitor, laptop, mouse, remote, keyboard, cell,
116  microwave, oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy, hair, toothbrush.
117 
118  - The 20 Pascal-VOC object categories are: aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow,
119  diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor.
120 
121  Sometimes it will make mistakes! The performance of yolov3-tiny is about 33.1% correct (mean average precision) on
122  the COCO test set. The OpenCV Face Detector is quite fast and robust!
123 
124  Speed and network size
125  ----------------------
126 
127  The parameter \p netin allows you to rescale the neural network to the specified size. Beware that this will only
128  work if the network used is fully convolutional (as is the case with the default networks listed above). This not
129  only allows you to adjust processing speed (and, conversely, accuracy), but also to better match the network to the
130  input images (e.g., the default size for tiny-yolo is 416x416, and, thus, passing it a input image of size 640x480
131  will result in first scaling that input to 416x312, then letterboxing it by adding gray borders on top and bottom so
132  that the final input to the network is 416x416). This letterboxing can be completely avoided by just resizing the
133  network to 320x240.
134 
135  Here are expected processing speeds for the OpenCV Face Detector:
136  - when netin = [320 240], processes 320x240 inputs, about 650ms/image (1.5 frames/s)
137  - when netin = [160 120], processes 160x120 inputs, about 190ms/image (5.0 frames/s)
138 
139  Serial messages
140  ---------------
141 
142  When detections are found which are above threshold, one message will be sent for each detected
143  object (i.e., for each box that gets drawn when USB outputs are used), using a standardized 2D message:
144  + Serial message type: \b 2D
145  + `id`: the category of the recognized object, followed by ':' and the confidence score in percent
146  + `x`, `y`, or vertices: standardized 2D coordinates of object center or corners
147  + `w`, `h`: standardized object size
148  + `extra`: any number of additional category:score pairs which had an above-threshold score for that box
149 
150  See \ref UserSerialStyle for more on standardized serial messages, and \ref coordhelpers for more info on
151  standardized coordinates.
152 
153  This code is heavily inspired from:
154  https://github.com/opencv/opencv/blob/master/samples/dnn/object_detection.cpp
155 
156  @author Laurent Itti
157 
158  @displayname Detection DNN
159  @videomapping NONE 0 0 0.0 YUYV 640 480 15.0 JeVois DetectionDNN
160  @videomapping YUYV 640 498 15.0 YUYV 640 480 15.0 JeVois DetectionDNN
161  @email itti\@usc.edu
162  @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
163  @copyright Copyright (C) 2018 by Laurent Itti, iLab and the University of Southern California
164  @mainurl http://jevois.org
165  @supporturl http://jevois.org/doc
166  @otherurl http://iLab.usc.edu
167  @license GPL v3
168  @distribution Unrestricted
169  @restrictions None
170  \ingroup modules */
172  public jevois::Parameter<model, classnames, configname, modelname, netin,
173  thresh, nms, rgb, scale, mean
174 #ifdef JEVOIS_PRO
175  , target
176 #endif
177  >
178 {
179  public:
180  // ####################################################################################################
181  //! Constructor
182  // ####################################################################################################
183  DetectionDNN(std::string const & instance) : jevois::StdModule(instance)
184  { }
185 
186  // ####################################################################################################
187  //! Virtual destructor for safe inheritance
188  // ####################################################################################################
189  virtual ~DetectionDNN()
190  { }
191 
192  // ####################################################################################################
193  //! Parameter callback: set the selected model
194  // ####################################################################################################
195  void onParamChange(model const & param, Model const & val) override
196  {
197  // Un-freeze the dependent parameters:
201 
202  // Reset the other parameters:
203  netin::reset();
204  thresh::reset();
205  rgb::reset();
206  scale::reset();
207  nms::reset();
208  mean::reset();
209 
210  // Now set the parameters for the chosen model:
211  switch (val)
212  {
213  case Model::Custom:
214  break;
215 
216  case Model::Face:
217  classnames::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.classes");
218  modelname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.caffemodel");
219  configname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/opencv_face_detector.prototxt");
220  scale::set(1.0F);
221  mean::set(cv::Scalar(104.0F, 177.0F, 123.0F));
222  rgb::set(false);
223  break;
224 
225  case Model::MobileNetSSDvoc:
226  classnames::set(JEVOIS_SHARE_PATH "/darknet/yolo/data/voc.names");
227  modelname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/MobileNetSSD_deploy.caffemodel");
228  configname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/MobileNetSSD_deploy.prototxt");
229  break;
230 
231  case Model::MobileNetSSDcoco:
232  classnames::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/coco_tf.names");
233  modelname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/ssd_mobilenet_v1_coco_2017_11_17.pb");
234  configname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
235  nms::set(10.0F);
236  break;
237 
238  case Model::MobileNet2SSDcoco:
239  classnames::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/coco_tf.names");
240  modelname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/ssd_mobilenet_v2_coco_2018_03_29.pb");
241  configname::set(JEVOIS_SHARE_PATH "/opencv-dnn/detection/ssd_mobilenet_v2_coco_2018_03_29.pbtxt");
242  break;
243 
244  case Model::TinyYOLOv3:
245  classnames::set(JEVOIS_SHARE_PATH "/darknet/yolo/data/coco.names");
246  modelname::set(JEVOIS_SHARE_PATH "/darknet/yolo/weights/yolov3-tiny.weights");
247  configname::set(JEVOIS_SHARE_PATH "/darknet/yolo/cfg/yolov3-tiny.cfg");
248  break;
249 
250  case Model::TinyYOLOv2:
251  classnames::set(JEVOIS_SHARE_PATH "/darknet/yolo/data/voc.names");
252  modelname::set(JEVOIS_SHARE_PATH "/darknet/yolo/weights/yolov2-tiny-voc.weights");
253  configname::set(JEVOIS_SHARE_PATH "/darknet/yolo/cfg/yolov2-tiny-voc.cfg");
254  netin::set(cv::Size(320, 240));
255  }
256 
257  // Freeze the dependent parameters unless we are in custom mode:
258  if (val != Model::Custom)
259  {
263  }
264 
265  // Load/re-load the model:
266  load();
267  }
268 
269  // ####################################################################################################
270  //! Load and initialize a model
271  // ####################################################################################################
272  void load()
273  {
274  // Need to nuke the network first if it exists or we could run out of RAM:
275  if (itsNet.empty() == false) itsNet = cv::dnn::Net();
276 
277  // Load the class names:
278  itsClasses.clear();
279  std::ifstream ifs(classnames::get());
280  if (ifs.is_open() == false) LFATAL("Class names file " << classnames::get() << " not found");
281  std::string line;
282  while (std::getline(ifs, line)) itsClasses.push_back(line);
283 
284  // Create and load the network:
285  itsNet = cv::dnn::readNet(modelname::get(), configname::get());
286  itsNet.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
287 #ifdef JEVOIS_PRO
288  switch(target::get())
289  {
290  case Target::CPU: itsNet.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); break;
291  case Target::OpenCL: itsNet.setPreferableTarget(cv::dnn::DNN_TARGET_OPENCL); break;
292  case Target::OpenCL_FP16: itsNet.setPreferableTarget(cv::dnn::DNN_TARGET_OPENCL_FP16); break;
293  }
294 #else
295  itsNet.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
296 #endif
297  // Get names of the network's output layers:
298  itsOutLayers = itsNet.getUnconnectedOutLayers();
299  std::vector<cv::String> layersNames = itsNet.getLayerNames();
300  itsOutNames.resize(itsOutLayers.size());
301  for (size_t i = 0; i < itsOutLayers.size(); ++i) itsOutNames[i] = layersNames[itsOutLayers[i] - 1];
302  itsOutLayerType = itsNet.getLayer(itsOutLayers[0])->type;
303  }
304 
305  // ####################################################################################################
306  //! Post-processing to extract boxes from network outputs
307  // ####################################################################################################
308  void postprocess(cv::Mat const & frame, std::vector<cv::Mat> const & outs, jevois::RawImage * outframe = nullptr)
309  {
310  float const confThreshold = thresh::get() * 0.01F;
311  float const nmsThreshold = nms::get() * 0.01F;
312 
313  std::vector<int> classIds;
314  std::vector<float> confidences;
315  std::vector<cv::Rect> boxes;
316  if (itsNet.getLayer(0)->outputNameToIndex("im_info") != -1) // Faster-RCNN or R-FCN
317  {
318  // Network produces output blob with a shape 1x1xNx7 where N is a number of detections and an every detection is
319  // a vector of values [batchId, classId, confidence, left, top, right, bottom]
320  if (outs.size() != 1) LFATAL("Malformed output layers");
321  float* data = (float*)outs[0].data;
322  for (size_t i = 0; i < outs[0].total(); i += 7)
323  {
324  float confidence = data[i + 2];
325  if (confidence > confThreshold)
326  {
327  int left = (int)data[i + 3];
328  int top = (int)data[i + 4];
329  int right = (int)data[i + 5];
330  int bottom = (int)data[i + 6];
331  int width = right - left + 1;
332  int height = bottom - top + 1;
333  classIds.push_back((int)(data[i + 1]) - 1); // Skip 0th background class id.
334  boxes.push_back(cv::Rect(left, top, width, height));
335  confidences.push_back(confidence);
336  }
337  }
338  }
339  else if (itsOutLayerType == "DetectionOutput")
340  {
341  // Network produces output blob with a shape 1x1xNx7 where N is a number of detections and an every detection is
342  // a vector of values [batchId, classId, confidence, left, top, right, bottom]
343  if (outs.size() != 1) LFATAL("Malformed output layers");
344  float* data = (float*)outs[0].data;
345  for (size_t i = 0; i < outs[0].total(); i += 7)
346  {
347  float confidence = data[i + 2];
348  if (confidence > confThreshold)
349  {
350  int left = (int)(data[i + 3] * frame.cols);
351  int top = (int)(data[i + 4] * frame.rows);
352  int right = (int)(data[i + 5] * frame.cols);
353  int bottom = (int)(data[i + 6] * frame.rows);
354  int width = right - left + 1;
355  int height = bottom - top + 1;
356  classIds.push_back((int)(data[i + 1]) - 1); // Skip 0th background class id.
357  boxes.push_back(cv::Rect(left, top, width, height));
358  confidences.push_back(confidence);
359  }
360  }
361  }
362  else if (itsOutLayerType == "Region")
363  {
364  for (size_t i = 0; i < outs.size(); ++i)
365  {
366  // Network produces output blob with a shape NxC where N is a number of detected objects and C is a number of
367  // classes + 4 where the first 4 numbers are [center_x, center_y, width, height]
368  float* data = (float*)outs[i].data;
369  for (int j = 0; j < outs[i].rows; ++j, data += outs[i].cols)
370  {
371  cv::Mat scores = outs[i].row(j).colRange(5, outs[i].cols);
372  cv::Point classIdPoint;
373  double confidence;
374  minMaxLoc(scores, 0, &confidence, 0, &classIdPoint);
375  if (confidence > confThreshold)
376  {
377  int centerX = (int)(data[0] * frame.cols);
378  int centerY = (int)(data[1] * frame.rows);
379  int width = (int)(data[2] * frame.cols);
380  int height = (int)(data[3] * frame.rows);
381  int left = centerX - width / 2;
382  int top = centerY - height / 2;
383 
384  classIds.push_back(classIdPoint.x);
385  confidences.push_back((float)confidence);
386  boxes.push_back(cv::Rect(left, top, width, height));
387  }
388  }
389  }
390  }
391  else LFATAL("Unknown output layer type: " << itsOutLayerType);
392 
393  // Cleanup overlapping boxes:
394  std::vector<int> indices;
395  cv::dnn::NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
396 
397  // Send serial messages and draw boxes:
398  for (size_t i = 0; i < indices.size(); ++i)
399  {
400  int idx = indices[i];
401  cv::Rect const & box = boxes[idx];
402  std::vector<jevois::ObjReco> data;
403  float const conf = confidences[idx] * 100.0F;
404  std::string name;
405  if (classIds[idx] < itsClasses.size()) name = itsClasses[classIds[idx]]; else name = "Oooops";
406  data.push_back({ conf, name });
407 
408  std::string label = jevois::sformat("%s: %.2f", name.c_str(), conf);
409 
410  if (outframe)
411  {
412  jevois::rawimage::drawRect(*outframe, box.x, box.y, box.width, box.height, 2, jevois::yuyv::LightGreen);
413  jevois::rawimage::writeText(*outframe, label, box.x + 6, box.y + 2, jevois::yuyv::LightGreen,
415  }
416 
417  sendSerialObjDetImg2D(frame.cols, frame.rows, box.x, box.y, box.width, box.height, data);
418  }
419  }
420 
421  // ####################################################################################################
422  //! Processing function, no video output
423  // ####################################################################################################
424  virtual void process(jevois::InputFrame && inframe) override
425  {
426  // Wait for next available camera image:
427  jevois::RawImage const inimg = inframe.get();
428  unsigned int const w = inimg.width, h = inimg.height;
429 
430  // Convert input image to BGR for predictions:
431  cv::Mat cvimg = jevois::rawimage::convertToCvBGR(inimg);
432 
433  // Let camera know we are done processing the input image:
434  inframe.done();
435 
436  // Extract blob that will be sent to network:
437  cv::Mat blob;
438  cv::dnn::blobFromImage(cvimg, blob, scale::get(), netin::get(), mean::get(), rgb::get(), false);
439 
440  // Launch the predictions:
441  itsNet.setInput(blob);
442  std::vector<cv::Mat> outs; itsNet.forward(outs, itsOutNames);
443 
444  // Post-process the outputs and send serial messages:
445  postprocess(cvimg, outs);
446  }
447 
448  // ####################################################################################################
449  //! Processing function with video output to USB
450  // ####################################################################################################
451  virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
452  {
453  static jevois::Timer timer("processing", 10, LOG_DEBUG);
454 
455  // Wait for next available camera image:
456  jevois::RawImage const inimg = inframe.get();
457 
458  timer.start();
459 
460  // We only handle one specific pixel format, and any image size in this module:
461  unsigned int const w = inimg.width, h = inimg.height;
462  inimg.require("input", w, h, V4L2_PIX_FMT_YUYV);
463 
464  // While we process it, start a thread to wait for out frame and paste the input into it:
465  jevois::RawImage outimg;
466  auto paste_fut = jevois::async([&]() {
467  outimg = outframe.get();
468  outimg.require("output", w, h + 18, inimg.fmt);
469 
470  // Paste the current input image:
471  jevois::rawimage::paste(inimg, outimg, 0, 0);
472  jevois::rawimage::writeText(outimg, "JeVois ObjectDetection DNN", 3, 3, jevois::yuyv::White);
473  jevois::rawimage::drawFilledRect(outimg, 0, h, w, outimg.height - h, jevois::yuyv::Black);
474  });
475 
476  // Convert input image to BGR for predictions:
477  cv::Mat cvimg = jevois::rawimage::convertToCvBGR(inimg);
478 
479  // Extract blob that will be sent to network:
480  cv::Mat blob;
481  cv::dnn::blobFromImage(cvimg, blob, scale::get(), netin::get(), mean::get(), rgb::get(), false);
482 
483  // Let camera know we are done processing the input image:
484  inframe.done();
485 
486  // Launch the predictions:
487  itsNet.setInput(blob);
488  std::vector<cv::Mat> outs; itsNet.forward(outs, itsOutNames);
489 
490  // Wait for paste to finish up:
491  paste_fut.get();
492 
493  // Post-process the outputs, draw them, and send serial messages:
494  postprocess(cvimg, outs, &outimg);
495 
496  // Display efficiency information:
497  std::vector<double> layersTimes;
498  double freq = cv::getTickFrequency() / 1000;
499  double t = itsNet.getPerfProfile(layersTimes) / freq;
500  std::string label = model::strget() + jevois::sformat(" inference time: %.2f ms", t);
501  jevois::rawimage::writeText(outimg, label, 3, h + 3, jevois::yuyv::White);
502 
503  // Show processing fps:
504  std::string const & fpscpu = timer.stop();
505  jevois::rawimage::writeText(outimg, fpscpu, 3, h - 13, jevois::yuyv::White);
506 
507  // Send the output image with our processing results to the host over USB:
508  outframe.send();
509  }
510 
511  // ####################################################################################################
512  protected:
513  std::vector<std::string> itsClasses;
515  std::vector<cv::String> itsOutNames;
516  std::vector<int> itsOutLayers;
517  std::string itsOutLayerType;
518 };
519 
520 // Allow the module to be loaded as a shared object (.so) file:
DetectionDNN::itsClasses
std::vector< std::string > itsClasses
Definition: DetectionDNN.C:513
jevois::Range
jevois::OutputFrame
DetectionDNN::load
void load()
Load and initialize a model.
Definition: DetectionDNN.C:272
Timer.H
Module.H
jevois::sformat
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
reset
virtual void reset()=0
DetectionDNN::itsOutNames
std::vector< cv::String > itsOutNames
Definition: DetectionDNN.C:515
DetectionDNN::process
virtual void process(jevois::InputFrame &&inframe) override
Processing function, no video output.
Definition: DetectionDNN.C:424
jevois::RawImage
jevois::Timer::start
void start()
jevois::rawimage::convertToCvBGR
cv::Mat convertToCvBGR(RawImage const &src)
jevois::ParameterCategory
test-model.blob
blob
Definition: test-model.py:25
jevois::RawImage::require
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const
jevois::RawImage::width
unsigned int width
DetectionDNN::DetectionDNN
DetectionDNN(std::string const &instance)
Constructor.
Definition: DetectionDNN.C:183
jevois::rawimage::writeText
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
jevois
test-model.conf
conf
Definition: test-model.py:31
strget
virtual const std::string strget() const=0
DetectionDNN::process
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function with video output to USB.
Definition: DetectionDNN.C:451
F
float F
ARtoolkit::JEVOIS_DEFINE_ENUM_CLASS
JEVOIS_DEFINE_ENUM_CLASS(Dict,(AR_MATRIX_CODE_3x3)(AR_MATRIX_CODE_3x3_HAMMING63)(AR_MATRIX_CODE_3x3_PARITY65))
Enum for parameter.
DetectionDNN
Detect and recognize multiple objects in scenes using OpenCV Deep Neural Nets (DNN)
Definition: DetectionDNN.C:171
JEVOIS_REGISTER_MODULE
JEVOIS_REGISTER_MODULE(DetectionDNN)
jevois::Timer::stop
const std::string & stop(double *seconds)
jevois::StdModule::sendSerialObjDetImg2D
void sendSerialObjDetImg2D(unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
DetectionDNN::onParamChange
void onParamChange(model const &param, Model const &val) override
Parameter callback: set the selected model.
Definition: DetectionDNN.C:195
jevois::rawimage::drawFilledRect
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
jevois::StdModule::StdModule
StdModule(std::string const &instance)
LFATAL
#define LFATAL(msg)
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
RawImageOps.H
jevois::RawImage::height
unsigned int height
jevois::InputFrame
DetectionDNN::postprocess
void postprocess(cv::Mat const &frame, std::vector< cv::Mat > const &outs, jevois::RawImage *outframe=nullptr)
Post-processing to extract boxes from network outputs.
Definition: DetectionDNN.C:308
test-model.frame
frame
Definition: test-model.py:22
DetectionDNN::itsOutLayers
std::vector< int > itsOutLayers
Definition: DetectionDNN.C:516
unFreeze
void unFreeze()
jevois::rawimage::paste
void paste(RawImage const &src, RawImage &dest, int dx, int dy)
jevois::rawimage::drawRect
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
jevois::RawImage::fmt
unsigned int fmt
dknet::Net
Net
Definition: Darknet.H:40
DetectionDNN::itsNet
cv::dnn::Net itsNet
Definition: DetectionDNN.C:514
name
std::string name
h
int h
freeze
void freeze()
jevois::StdModule
DetectionDNN::~DetectionDNN
virtual ~DetectionDNN()
Virtual destructor for safe inheritance.
Definition: DetectionDNN.C:189
Darknet::JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(dataroot, std::string, "Root path for data, config, and weight files. " "If empty, use the module's path.", JEVOIS_SHARE_PATH "/darknet/single", ParamCateg)
Parameter.
ARtoolkit::JEVOIS_DECLARE_PARAMETER
JEVOIS_DECLARE_PARAMETER(camparams, std::string, "File stem of camera parameters, or empty. Camera resolution " "will be appended, as well as a .dat extension. For example, specifying 'camera_para' " "here and running the camera sensor at 320x240 will attempt to load " "camera_para320x240.dat from within the module's directory (if relative stem) or " "from the specified absolute location (if absolute stem).", JEVOIS_SHARE_PATH "/camera/camera_para", ParamCateg)
Parameter.
jevois::rawimage::Font10x20
Font10x20
DetectionDNN::itsOutLayerType
std::string itsOutLayerType
Definition: DetectionDNN.C:517
jevois::Timer