JeVois  1.9
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Module.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Module.H>
19 #include <jevois/Core/VideoInput.H>
21 #include <jevois/Core/Engine.H>
25 
26 #include <opencv2/imgproc/imgproc.hpp>
27 
28 #include <cmath>
29 #include <sstream>
30 #include <iomanip>
31 
32 // ####################################################################################################
33 jevois::InputFrame::InputFrame(std::shared_ptr<jevois::VideoInput> const & cam, bool turbo) :
34  itsCamera(cam), itsDidGet(false), itsDidDone(false), itsTurbo(turbo)
35 { }
36 
37 // ####################################################################################################
39 {
40  // If itsCamera is invalidated, we have been moved to another object, so do not do anything here:
41  if (itsCamera.get() == nullptr) return;
42 
43  // If we did not yet get(), just end now, camera will drop this frame:
44  if (itsDidGet == false) return;
45 
46  // If we did get() but not done(), signal done now:
47  if (itsDidDone == false) try { itsCamera->done(itsImage); } catch (...) { }
48 }
49 
50 // ####################################################################################################
51 jevois::RawImage const & jevois::InputFrame::get(bool casync) const
52 {
53  itsCamera->get(itsImage);
54  itsDidGet = true;
55  if (casync && itsTurbo) itsImage.buf->sync();
56  return itsImage;
57 }
58 
59 // ####################################################################################################
61 {
62  itsCamera->done(itsImage);
63  itsDidDone = true;
64 }
65 
66 // ####################################################################################################
67 cv::Mat jevois::InputFrame::getCvGRAY(bool casync) const
68 {
69  jevois::RawImage const & rawimg = get(casync);
70  cv::Mat cvimg = jevois::rawimage::convertToCvGray(rawimg);
71  done();
72  return cvimg;
73 }
74 
75 // ####################################################################################################
76 cv::Mat jevois::InputFrame::getCvBGR(bool casync) const
77 {
78  jevois::RawImage const & rawimg = get(casync);
79  cv::Mat cvimg = jevois::rawimage::convertToCvBGR(rawimg);
80  done();
81  return cvimg;
82 }
83 
84 // ####################################################################################################
85 cv::Mat jevois::InputFrame::getCvRGB(bool casync) const
86 {
87  jevois::RawImage const & rawimg = get(casync);
88  cv::Mat cvimg = jevois::rawimage::convertToCvRGB(rawimg);
89  done();
90  return cvimg;
91 }
92 
93 // ####################################################################################################
94 cv::Mat jevois::InputFrame::getCvRGBA(bool casync) const
95 {
96  jevois::RawImage const & rawimg = get(casync);
97  cv::Mat cvimg = jevois::rawimage::convertToCvRGBA(rawimg);
98  done();
99  return cvimg;
100 }
101 
102 // ####################################################################################################
103 // ####################################################################################################
104 jevois::OutputFrame::OutputFrame(std::shared_ptr<jevois::VideoOutput> const & gad, jevois::RawImage * excimg) :
105  itsGadget(gad), itsDidGet(false), itsDidSend(false), itsImagePtrForException(excimg)
106 { }
107 
108 // ####################################################################################################
110 {
111  // If itsGadget is invalidated, we have been moved to another object, so do not do anything here:
112  if (itsGadget.get() == nullptr) return;
113 
114  // If we did not get(), just end now:
115  if (itsDidGet == false) return;
116 
117  // If Engine gave us a non-zero image pointer for exceptions, and we did get(), pass down the buffer we did get, so
118  // that Engine can write exception text into it, unless it is too late (exception occurred after send()) or too early
119  // (before get()), in which case Engine will get its own buffer:
120  if (itsImagePtrForException)
121  {
122  if (itsDidSend == false) { *itsImagePtrForException = itsImage; }
123  // Engine will be responsible for the final send()
124  }
125  else
126  {
127  // If we did get() but not send(), send now (the image will likely contain garbage):
128  if (itsDidSend == false) try { itsGadget->send(itsImage); } catch (...) { }
129  }
130 }
131 
132 // ####################################################################################################
134 {
135  itsGadget->get(itsImage);
136  itsDidGet = true;
137  return itsImage;
138 }
139 
140 // ####################################################################################################
142 {
143  itsGadget->send(itsImage);
144  itsDidSend = true;
145  if (itsImagePtrForException) itsImagePtrForException->invalidate();
146 }
147 
148 // ####################################################################################################
149 void jevois::OutputFrame::sendCv(cv::Mat const & img, int quality) const
150 {
151  switch(img.type())
152  {
153  case CV_8UC3: sendScaledCvBGR(img, quality); break;
154  case CV_8UC1: sendScaledCvGRAY(img, quality); break;
155  case CV_8UC4: sendScaledCvRGBA(img, quality); break;
156  default: LFATAL("cv::Mat of type " << cvtypestr(img.type()) << " not supported.");
157  }
158 }
159 
160 // ####################################################################################################
161 void jevois::OutputFrame::sendCvGRAY(cv::Mat const & img, int quality) const
162 {
163  jevois::RawImage rawimg = get();
164  jevois::rawimage::convertCvGRAYtoRawImage(img, rawimg, quality);
165  send();
166 }
167 
168 // ####################################################################################################
169 void jevois::OutputFrame::sendCvBGR(cv::Mat const & img, int quality) const
170 {
171  jevois::RawImage rawimg = get();
172  jevois::rawimage::convertCvBGRtoRawImage(img, rawimg, quality);
173  send();
174 }
175 // ####################################################################################################
176 void jevois::OutputFrame::sendCvRGB(cv::Mat const & img, int quality) const
177 {
178  jevois::RawImage rawimg = get();
179  jevois::rawimage::convertCvRGBtoRawImage(img, rawimg, quality);
180  send();
181 }
182 
183 // ####################################################################################################
184 void jevois::OutputFrame::sendCvRGBA(cv::Mat const & img, int quality) const
185 {
186  jevois::RawImage rawimg = get();
187  jevois::rawimage::convertCvRGBAtoRawImage(img, rawimg, quality);
188  send();
189 }
190 
191 // ####################################################################################################
192 void jevois::OutputFrame::sendScaledCvGRAY(cv::Mat const & img, int quality) const
193 {
194  jevois::RawImage rawimg = get();
196  rawimg, quality);
197  send();
198 }
199 
200 // ####################################################################################################
201 void jevois::OutputFrame::sendScaledCvBGR(cv::Mat const & img, int quality) const
202 {
203  jevois::RawImage rawimg = get();
205  rawimg, quality);
206  send();
207 }
208 // ####################################################################################################
209 void jevois::OutputFrame::sendScaledCvRGB(cv::Mat const & img, int quality) const
210 {
211  jevois::RawImage rawimg = get();
213  rawimg, quality);
214  send();
215 }
216 
217 // ####################################################################################################
218 void jevois::OutputFrame::sendScaledCvRGBA(cv::Mat const & img, int quality) const
219 {
220  jevois::RawImage rawimg = get();
222  rawimg, quality);
223  send();
224 }
225 
226 // ####################################################################################################
227 // ####################################################################################################
228 jevois::Module::Module(std::string const & instance) :
229  jevois::Component(instance)
230 { }
231 
232 // ####################################################################################################
234 { }
235 
236 // ####################################################################################################
237 void jevois::Module::process(InputFrame && JEVOIS_UNUSED_PARAM(inframe), OutputFrame && JEVOIS_UNUSED_PARAM(outframe))
238 { LFATAL("Not implemented in this module"); }
239 
240 // ####################################################################################################
241 void jevois::Module::process(InputFrame && JEVOIS_UNUSED_PARAM(inframe))
242 { LFATAL("Not implemented in this module"); }
243 
244 // ####################################################################################################
245 void jevois::Module::sendSerial(std::string const & str)
246 {
247  jevois::Engine * e = dynamic_cast<jevois::Engine *>(itsParent);
248  if (e == nullptr) LFATAL("My parent is not Engine -- CANNOT SEND SERIAL");
249 
250  e->sendSerial(str);
251 }
252 
253 // ####################################################################################################
254 void jevois::Module::parseSerial(std::string const & str,
255  std::shared_ptr<jevois::UserInterface> JEVOIS_UNUSED_PARAM(s))
256 { throw std::runtime_error("Unsupported command [" + str + ']'); }
257 
258 // ####################################################################################################
259 void jevois::Module::supportedCommands(std::ostream & os)
260 { os << "None" << std::endl; }
261 
262 // ####################################################################################################
264 {
265  jevois::Engine * e = dynamic_cast<jevois::Engine *>(itsParent);
266  if (e == nullptr) LFATAL("My parent is not Engine -- CANNOT GET FRAME NUMBER");
267 
268  return e->frameNum();
269 }
270 
271 // ####################################################################################################
272 // ####################################################################################################
273 jevois::StdModule::StdModule(std::string const & instance) :
274  jevois::Module(instance)
275 { }
276 
277 // ####################################################################################################
279 { }
280 
281 // ####################################################################################################
282 std::string jevois::StdModule::getStamp() const
283 {
284  std::string ret;
285 
286  switch(serstamp::get())
287  {
288  case jevois::module::SerStamp::None:
289  break;
290 
291  case jevois::module::SerStamp::Frame:
292  ret = std::to_string(frameNum());
293  break;
294 
295  case jevois::module::SerStamp::Time:
296  {
297  std::time_t t = std::time(nullptr); char str[100];
298  std::strftime(str, sizeof(str), "%T", std::localtime(&t));
299  ret = std::string(str);
300  }
301  break;
302 
303  case jevois::module::SerStamp::FrameTime:
304  {
305  std::time_t t = std::time(nullptr); char str[100];
306  std::strftime(str, sizeof(str), "%T", std::localtime(&t));
307  ret = std::to_string(frameNum()) + '/' + std::string(str);
308  }
309  break;
310 
311  case jevois::module::SerStamp::FrameDateTime:
312  {
313  std::time_t t = std::time(nullptr); char str[100];
314  std::strftime(str, sizeof(str), "%F/%T", std::localtime(&t));
315  ret = std::to_string(frameNum()) + '/' + std::string(str);
316  }
317  break;
318  }
319 
320  if (ret.empty() == false) ret += ' ';
321  return ret;
322 }
323 
324 // ####################################################################################################
325 void jevois::StdModule::sendSerialImg1Dx(unsigned int camw, float x, float size, std::string const & id,
326  std::string const & extra)
327 {
328  // Normalize the coordinate and size using the given precision to do rounding:
329  float const eps = std::pow(10.0F, -float(serprec::get()));
330 
331  jevois::coords::imgToStdX(x, camw, eps);
332  float dummy = 0.0F; jevois::coords::imgToStdSize(size, dummy, camw, 100, eps);
333 
334  // Delegate:
335  sendSerialStd1Dx(x, size, id, extra);
336 }
337 
338 // ####################################################################################################
339 void jevois::StdModule::sendSerialStd1Dx(float x, float size, std::string const & id, std::string const & extra)
340 {
341  // Build the message depending on desired style:
342  std::ostringstream oss; oss << std::fixed << std::setprecision(serprec::get());
343 
344  // Prepend frame/date/time as possibly requested by parameter serstamp:
345  oss << getStamp();
346 
347  // Format the message depending on parameter serstyle:
348  switch (serstyle::get())
349  {
350  case jevois::module::SerStyle::Terse:
351  oss << "T1 " << x;
352  break;
353 
354  case jevois::module::SerStyle::Normal:
355  oss << "N1 ";
356  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
357  oss << x << ' ' << size;
358  break;
359 
360  case jevois::module::SerStyle::Detail:
361  case jevois::module::SerStyle::Fine:
362  oss << "D1 ";
363  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
364  oss << x - 0.5F * size << ' ' << x + 0.5F * size;
365  if (extra.empty() == false) oss << ' ' << extra;
366  break;
367  }
368 
369  // Send the message:
370  sendSerial(oss.str());
371 }
372 
373 // ####################################################################################################
374 void jevois::StdModule::sendSerialImg1Dy(unsigned int camh, float y, float size, std::string const & id,
375  std::string const & extra)
376 {
377  // Normalize the coordinate and size using the given precision to do rounding:
378  float const eps = std::pow(10.0F, -float(serprec::get()));
379  jevois::coords::imgToStdY(y, camh, eps);
380  float dummy = 0.0F; jevois::coords::imgToStdSize(dummy, size, 100, camh, eps);
381 
382  // Delegate:
383  sendSerialStd1Dy(y, size, id, extra);
384 }
385 
386 // ####################################################################################################
387 void jevois::StdModule::sendSerialStd1Dy(float y, float size, std::string const & id, std::string const & extra)
388 {
389  // Build the message depending on desired style:
390  std::ostringstream oss; oss << std::fixed << std::setprecision(serprec::get());
391 
392  // Prepend frame/date/time as possibly requested by parameter serstamp:
393  oss << getStamp();
394 
395  // Format the message depending on parameter serstyle:
396  switch (serstyle::get())
397  {
398  case jevois::module::SerStyle::Terse:
399  oss << "T1 " << y;
400  break;
401 
402  case jevois::module::SerStyle::Normal:
403  oss << "N1 ";
404  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
405  oss << y << ' ' << size;
406  break;
407 
408  case jevois::module::SerStyle::Detail:
409  case jevois::module::SerStyle::Fine:
410  oss << "D1 ";
411  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
412  oss << y - 0.5F * size << ' ' << y + 0.5F * size;
413  if (extra.empty() == false) oss << ' ' << extra;
414  break;
415  }
416 
417  // Send the message:
418  sendSerial(oss.str());
419 }
420 
421 // ####################################################################################################
422 void jevois::StdModule::sendSerialImg2D(unsigned int camw, unsigned int camh, float x, float y, float w, float h,
423  std::string const & id, std::string const & extra)
424 {
425  // Normalize the coordinates and sizes using the given precision to do rounding:
426  float const eps = std::pow(10.0F, -float(serprec::get()));
427 
428  jevois::coords::imgToStd(x, y, camw, camh, eps);
429  jevois::coords::imgToStdSize(w, h, camw, camh, eps);
430 
431  // Delegate:
432  sendSerialStd2D(x, y, w, h, id, extra);
433 }
434 // ####################################################################################################
435 void jevois::StdModule::sendSerialStd2D(float x, float y, float w, float h, std::string const & id,
436  std::string const & extra)
437 {
438  // Build the message depending on desired style:
439  std::ostringstream oss; oss << std::fixed << std::setprecision(serprec::get());
440 
441  // Prepend frame/date/time as possibly requested by parameter serstamp:
442  oss << getStamp();
443 
444  // Format the message depending on parameter serstyle:
445  switch (serstyle::get())
446  {
447  case jevois::module::SerStyle::Terse:
448  oss << "T2 " << x << ' ' << y;
449  break;
450 
451  case jevois::module::SerStyle::Normal:
452  oss << "N2 ";
453  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
454  oss << x << ' ' << y << ' ' << w << ' ' << h;
455  break;
456 
457  case jevois::module::SerStyle::Detail:
458  oss << "D2 ";
459  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
460  oss << x - 0.5F * w << ' ' << y - 0.5F * h << ' ';
461  oss << x + 0.5F * w << ' ' << y - 0.5F * h << ' ';
462  oss << x + 0.5F * w << ' ' << y + 0.5F * h << ' ';
463  oss << x + 0.5F * w << ' ' << y - 0.5F * h;
464  if (extra.empty() == false) oss << ' ' << extra;
465  break;
466 
467  case jevois::module::SerStyle::Fine:
468  oss << "F2 ";
469  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
470  oss << 4 << ' '; // number of vertices
471  oss << x - 0.5F * w << ' ' << y - 0.5F * h << ' ';
472  oss << x + 0.5F * w << ' ' << y - 0.5F * h << ' ';
473  oss << x + 0.5F * w << ' ' << y + 0.5F * h << ' ';
474  oss << x + 0.5F * w << ' ' << y - 0.5F * h;
475  if (extra.empty() == false) oss << ' ' << extra;
476  break;
477  }
478 
479  // Send the message:
480  sendSerial(oss.str());
481 }
482 
483 // ####################################################################################################
484 template <typename T>
485 void jevois::StdModule::sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector<cv::Point_<T> > points,
486  std::string const & id, std::string const & extra)
487 {
488  // Format the message depending on parameter serstyle:
489  switch (serstyle::get())
490  {
491  case jevois::module::SerStyle::Terse:
492  {
493  // Compute center of gravity:
494  float cx = 0.0F, cy = 0.0F;
495  for (cv::Point const & p : points) { cx += p.x; cy += p.y; }
496  if (points.size()) { cx /= points.size(); cy /= points.size(); }
497  sendSerialImg2D(camw, camh, cx, cy, 0.0F, 0.0F, id, extra);
498  }
499  break;
500 
501  case jevois::module::SerStyle::Normal:
502  {
503  // Compute upright bounding rectangle:
504  cv::Rect r = cv::boundingRect(points);
505  sendSerialImg2D(camw, camh, r.x + 0.5F * r.width, r.y + 0.5F * r.height, r.width, r.height, id, extra);
506  }
507  break;
508 
509  case jevois::module::SerStyle::Detail:
510  {
511  // Compute minimal rotated rectangle enclosing the points:
512  cv::RotatedRect r = cv::minAreaRect(points);
513 
514  // Build the message:
515  unsigned int const prec = serprec::get(); float const eps = std::pow(10.0F, -float(prec));
516  std::ostringstream oss; oss << std::fixed << std::setprecision(prec);
517 
518  // Prepend frame/date/time as possibly requested by parameter serstamp:
519  oss << getStamp();
520 
521  // Now the rest of the message:
522  oss << "D2 ";
523  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
524  cv::Point2f corners[4];
525  r.points(corners);
526 
527  oss << 4; // number of vertices
528 
529  for (int i = 0; i < 4; ++i)
530  {
531  float x = corners[i].x, y = corners[i].y;
532  jevois::coords::imgToStd(x, y, camw, camh, eps);
533  oss << ' ' << x << ' ' << y;
534  }
535  if (extra.empty() == false) oss << ' ' << extra;
536 
537  // Send the message:
538  sendSerial(oss.str());
539  }
540  break;
541 
542  case jevois::module::SerStyle::Fine:
543  {
544  // Build the message:
545  unsigned int const prec = serprec::get(); float const eps = std::pow(10.0F, -float(prec));
546  std::ostringstream oss; oss << std::fixed << std::setprecision(prec);
547 
548  // Prepend frame/date/time as possibly requested by parameter serstamp:
549  oss << getStamp();
550 
551  // Now the rest of the message:
552  oss << "F2 ";
553  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
554  oss << points.size(); // number of vertices
555 
556  for (cv::Point const & p : points)
557  {
558  float x = p.x, y = p.y;
559  jevois::coords::imgToStd(x, y, camw, camh, eps);
560  oss << ' ' << x << ' ' << y;
561  }
562  if (extra.empty() == false) oss << ' ' << extra;
563 
564  // Send the message:
565  sendSerial(oss.str());
566  }
567  break;
568  }
569 }
570 
571 // Compile in explicit template instantiations:
572 namespace jevois
573 {
574  template
575  void StdModule::sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector<cv::Point_<int> > points,
576  std::string const & id, std::string const & extra);
577  template
578  void StdModule::sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector<cv::Point_<float> > points,
579  std::string const & id, std::string const & extra);
580  template
581  void StdModule::sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector<cv::Point_<double> > points,
582  std::string const & id, std::string const & extra);
583 }
584 
585 // ####################################################################################################
586 void jevois::StdModule::sendSerialStd3D(float x, float y, float z, float w, float h, float d,
587  float q1, float q2, float q3, float q4,
588  std::string const & id, std::string const & extra)
589 {
590  // Build the message depending on desired style:
591  std::ostringstream oss; oss << std::fixed << std::setprecision(serprec::get());
592 
593  // Prepend frame/date/time as possibly requested by parameter serstamp:
594  oss << getStamp();
595 
596  // Format the message depending on parameter serstyle:
597  switch (serstyle::get())
598  {
599  case jevois::module::SerStyle::Terse:
600  oss << "T3 " << x << ' ' << y << ' ' << z;
601  break;
602 
603  case jevois::module::SerStyle::Normal:
604  oss << "N3 ";
605  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
606  oss << x << ' ' << y << ' ' << z << ' ' << w << ' ' << h << ' ' << d;
607  break;
608 
609  case jevois::module::SerStyle::Detail:
610  oss << "D3 ";
611  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
612  oss << x << ' ' << y << ' ' << z << ' ' << w << ' ' << h << ' ' << d << ' '
613  << q1 << ' ' << q2 << ' ' << q3 << ' ' << q4;
614  if (extra.empty() == false) oss << ' ' << extra;
615  break;
616 
617  case jevois::module::SerStyle::Fine:
618  oss << "F3 ";
619  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
620  oss << 8 << ' '; // number of vertices
621  oss << x - 0.5F * w << ' ' << y - 0.5F * h << ' ' << z - 0.5F * d << ' ';
622  oss << x + 0.5F * w << ' ' << y - 0.5F * h << ' ' << z - 0.5F * d << ' ';
623  oss << x + 0.5F * w << ' ' << y + 0.5F * h << ' ' << z - 0.5F * d << ' ';
624  oss << x + 0.5F * w << ' ' << y - 0.5F * h << ' ' << z - 0.5F * d << ' ';
625  oss << x - 0.5F * w << ' ' << y - 0.5F * h << ' ' << z + 0.5F * d << ' ';
626  oss << x + 0.5F * w << ' ' << y - 0.5F * h << ' ' << z + 0.5F * d << ' ';
627  oss << x + 0.5F * w << ' ' << y + 0.5F * h << ' ' << z + 0.5F * d << ' ';
628  oss << x + 0.5F * w << ' ' << y - 0.5F * h << ' ' << z + 0.5F * d << ' ';
629  if (extra.empty() == false) oss << ' ' << extra;
630  break;
631  }
632 
633  // Send the message:
634  sendSerial(oss.str());
635 }
636 
637 // ####################################################################################################
638 void jevois::StdModule::sendSerialStd3D(std::vector<cv::Point3f> points, std::string const & id,
639  std::string const & extra)
640 {
641  // Format the message depending on parameter serstyle:
642  switch (serstyle::get())
643  {
644  case jevois::module::SerStyle::Terse:
645  {
646  // Compute center of gravity:
647  cv::Point3f cg(0.0F, 0.0F, 0.0F);
648  for (cv::Point3f const & p : points) cg += p;
649  if (points.size()) { cg.x /= points.size(); cg.y /= points.size(); cg.z /= points.size(); }
650  sendSerialStd3D(cg.x, cg.y, cg.z, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, 0.0F, id, extra);
651  }
652  break;
653 
654  case jevois::module::SerStyle::Normal:
655  {
656  // Compute upright bounding parallelepiped:
657  cv::Point3f cg(0.0F, 0.0F, 0.0F), pmin(1e30F, 1e30F, 1e30F), pmax(-1e30F, -1e30F, -1e30F);
658  for (cv::Point3f const & p : points)
659  {
660  cg += p;
661  if (p.x < pmin.x) pmin.x = p.x;
662  if (p.y < pmin.y) pmin.y = p.y;
663  if (p.z < pmin.z) pmin.z = p.z;
664  if (p.x > pmax.x) pmax.x = p.x;
665  if (p.y > pmax.y) pmax.y = p.y;
666  if (p.z > pmax.z) pmax.z = p.z;
667  }
668  if (points.size()) { cg.x /= points.size(); cg.y /= points.size(); cg.z /= points.size(); }
669  sendSerialStd3D(cg.x, cg.y, cg.z, pmax.x - pmin.x, pmax.y - pmin.y, pmax.z - pmin.z, 0.0F, 0.0F, 0.0F, 0.0F,
670  id, extra);
671  }
672  break;
673 
674  case jevois::module::SerStyle::Detail:
675  {
676  // Compute upright bounding parallelepiped:
677  cv::Point3f cg(0.0F, 0.0F, 0.0F), pmin(1e30F, 1e30F, 1e30F), pmax(-1e30F, -1e30F, -1e30F);
678  for (cv::Point3f const & p : points)
679  {
680  cg += p;
681  if (p.x < pmin.x) pmin.x = p.x;
682  if (p.y < pmin.y) pmin.y = p.y;
683  if (p.z < pmin.z) pmin.z = p.z;
684  if (p.x > pmax.x) pmax.x = p.x;
685  if (p.y > pmax.y) pmax.y = p.y;
686  if (p.z > pmax.z) pmax.z = p.z;
687  }
688  if (points.size()) { cg.x /= points.size(); cg.y /= points.size(); cg.z /= points.size(); }
689  // FIXME what should we send for the quaternion?
690  sendSerialStd3D(cg.x, cg.y, cg.z, pmax.x - pmin.x, pmax.y - pmin.y, pmax.z - pmin.z, 0.0F, 0.0F, 0.0F, 1.0F,
691  id, extra);
692  }
693  break;
694 
695  case jevois::module::SerStyle::Fine:
696  {
697  // Build the message:
698  unsigned int const prec = serprec::get();
699  std::ostringstream oss; oss << std::fixed << std::setprecision(prec);
700 
701  // Prepend frame/date/time as possibly requested by parameter serstamp:
702  oss << getStamp();
703 
704  // Now the rest of the message:
705  oss << "F3 ";
706  if (id.empty()) oss << "unknown "; else oss << jevois::replaceWhitespace(id) << ' ';
707  oss << points.size(); // number of vertices
708 
709  for (cv::Point3f const & p : points) oss << ' ' << p.x << ' ' << p.y << ' ' << p.z;
710  if (extra.empty() == false) oss << ' ' << extra;
711 
712  // Send the message:
713  sendSerial(oss.str());
714  }
715  break;
716  }
717 }
718 
719 // ####################################################################################################
721 {
722  jevois::module::SerMark const m = sermark::get();
723  if (m == jevois::module::SerMark::None || m == jevois::module::SerMark::Stop) return;
724  sendSerial(getStamp() + "MARK START");
725 }
726 
727 // ####################################################################################################
729 {
730  jevois::module::SerMark const m = sermark::get();
731  if (m == jevois::module::SerMark::None || m == jevois::module::SerMark::Start) return;
732  sendSerial(getStamp() + "MARK STOP");
733 }
734 
735 // ####################################################################################################
736 void jevois::StdModule::sendSerialObjReco(std::vector<jevois::ObjReco> const & res)
737 {
738  if (res.empty()) return;
739 
740  // Build the message depending on desired style:
741  std::ostringstream oss; oss << std::fixed << std::setprecision(serprec::get());
742 
743  // Prepend frame/date/time as possibly requested by parameter serstamp:
744  oss << getStamp();
745 
746  // Format the message depending on parameter serstyle:
747  switch (serstyle::get())
748  {
749  case jevois::module::SerStyle::Terse:
750  oss << "TO " << jevois::replaceWhitespace(res[0].category);
751  break;
752 
753  case jevois::module::SerStyle::Normal:
754  oss << "NO " << jevois::replaceWhitespace(res[0].category) << ':' << res[0].score;
755  break;
756 
757  case jevois::module::SerStyle::Detail:
758  oss << "DO";
759  for (auto const & r : res) oss << ' ' << jevois::replaceWhitespace(r.category) << ':' << r.score;
760  break;
761 
762  case jevois::module::SerStyle::Fine:
763  oss << "FO";
764  for (auto const & r : res) oss << ' ' << jevois::replaceWhitespace(r.category) << ':' << r.score;
765  break;
766  }
767 
768  // Send the message:
769  sendSerial(oss.str());
770 }
771 
772 // ####################################################################################################
773 void jevois::StdModule::sendSerialObjDetImg2D(unsigned int camw, unsigned int camh, float x, float y, float w, float h,
774  std::vector<ObjReco> const & res)
775 {
776  if (res.empty()) return;
777 
778  std::string best, extra; std::string * ptr = &best;
779  std::string fmt = "%s:%." + std::to_string(serprec::get()) + "f";
780 
781  for (auto const & r : res)
782  {
783  switch (serstyle::get())
784  {
785  case jevois::module::SerStyle::Terse:
786  (*ptr) += jevois::replaceWhitespace(r.category);
787  break;
788 
789  default:
790  (*ptr) += jevois::sformat(fmt.c_str(), jevois::replaceWhitespace(r.category).c_str(), r.score);
791  }
792  if (ptr == &extra) (*ptr) += ' ';
793  ptr = &extra;
794  }
795 
796  // Remove last space:
797  if (extra.empty() == false) extra = extra.substr(0, extra.length() - 1);
798 
799  sendSerialImg2D(camw, camh, x, y, w, h, best, extra);
800 }
801 
cv::Mat convertToCvRGB(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV RGB byte...
Definition: RawImageOps.C:302
cv::Mat convertToCvRGBA(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV RGB-A byte...
Definition: RawImageOps.C:325
void sendSerialContour2D(unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
Send standardized 2D message for polygons in image coordinates.
Definition: Module.C:485
void imgToStdY(float &y, unsigned int const height, float const eps=0.1F)
Transform Y coordinate in-place from camera to standardized, using given image width and height...
Definition: Coordinates.C:43
Exception-safe wrapper around a raw camera input frame.
Definition: Module.H:57
void imgToStd(float &x, float &y, RawImage const &camimg, float const eps=0.1F)
Transform coordinates in-place from camera to standardized, using a RawImage to establish image size...
Definition: Coordinates.C:22
cv::Mat convertToCvBGR(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV BGR byte...
Definition: RawImageOps.C:279
unsigned int height
Image height in pixels.
Definition: RawImage.H:146
std::string cvtypestr(unsigned int cvtype)
Convert cv::Mat::type() code to to a string (e.g., CV_8UC1, CV_32SC3, etc)
Definition: Utils.C:52
void sendSerialStd1Dx(float x, float size=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 1D message for a standardized X coordinate.
Definition: Module.C:339
void send(RawImage const &img) override
Send an image out over USB to the host computer.
Definition: Gadget.C:852
cv::Mat getCvRGBA(bool casync=false) const
Shorthand to get the input image as a RGBA cv::Mat and release the raw buffer.
Definition: Module.C:94
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Create a string using printf style arguments.
Definition: Utils.C:226
void sendScaledCvRGB(cv::Mat const &img, int quality=75) const
Shorthand to send a RGB cv::Mat after converting it to the current output format. ...
Definition: Module.C:209
void sendScaledCvBGR(cv::Mat const &img, int quality=75) const
Shorthand to send a BGR cv::Mat after converting it to the current output format. ...
Definition: Module.C:201
void sendCvRGBA(cv::Mat const &img, int quality=75) const
Shorthand to send a RGBA cv::Mat after converting it to the current output format.
Definition: Module.C:184
void send() const
Send an image out over USB to the host computer.
Definition: Module.C:141
Module(std::string const &instance)
Constructor.
Definition: Module.C:228
void sendCvRGB(cv::Mat const &img, int quality=75) const
Shorthand to send a RGB cv::Mat after converting it to the current output format. ...
Definition: Module.C:176
void sendCv(cv::Mat const &img, int quality=75) const
Shorthand to send a cv::Mat after converting / scaling it to the current output format.
Definition: Module.C:149
RawImage const & get(bool casync=false) const
Get the next captured camera image.
Definition: Module.C:51
void sendCvGRAY(cv::Mat const &img, int quality=75) const
Shorthand to send a GRAY cv::Mat after converting it to the current output format.
Definition: Module.C:161
InputFrame(InputFrame &&other)=default
Move constructor.
virtual void supportedCommands(std::ostream &os)
Human-readable description of this Module&#39;s supported custom commands.
Definition: Module.C:259
void sendSerial(std::string const &str, bool islog=false)
Send a string to all serial ports.
Definition: Engine.C:867
std::string replaceWhitespace(std::string const &str, char rep='_')
Replace white space characters in a string with underscore (default) or another characters.
Definition: Utils.C:172
void sendScaledCvRGBA(cv::Mat const &img, int quality=75) const
Shorthand to send a RGBA cv::Mat after converting it to the current output format.
Definition: Module.C:218
size_t frameNum() const
Get frame number.
Definition: Engine.C:905
void sendCvBGR(cv::Mat const &img, int quality=75) const
Shorthand to send a BGR cv::Mat after converting it to the current output format. ...
Definition: Module.C:169
A component of a model hierarchy.
Definition: Component.H:177
virtual void process(InputFrame &&inframe, OutputFrame &&outframe)
Processing function, version that receives a frame from camera and sends a frame out over USB...
StdModule(std::string const &instance)
Constructor.
Definition: Module.C:273
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
cv::Mat getCvRGB(bool casync=false) const
Shorthand to get the input image as a RGB cv::Mat and release the raw buffer.
Definition: Module.C:85
virtual void done(RawImage &img)=0
Indicate that user processing is done with an image previously obtained via get() ...
void sendSerialStd2D(float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 2D message for standardized coordinates.
Definition: Module.C:435
void sendSerialImg2D(unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 2D message for image coordinates.
Definition: Module.C:422
void imgToStdSize(float &w, float &h, unsigned int const width, unsigned int const height, float const eps=0.1F)
Transform size in-place from camera to standardized, using given image width and height.
Definition: Coordinates.C:50
virtual void get(RawImage &img)=0
Get the next captured buffer.
cv::Mat convertToCvGray(RawImage const &src)
Convert RawImage to OpenCV doing color conversion from any RawImage source pixel to OpenCV gray byte...
Definition: RawImageOps.C:245
void convertCvBGRtoRawImage(cv::Mat const &src, RawImage &dst, int quality)
Convert a BGR cv::Mat to RawImage with already-allocated pixels and pixel type.
Definition: RawImageOps.C:1254
void imgToStdX(float &x, unsigned int const width, float const eps=0.1F)
Transform X coordinate in-place from camera to standardized, using given image width and height...
Definition: Coordinates.C:36
void sendSerialObjReco(std::vector< ObjReco > const &res)
Send a standardized object recognition message.
Definition: Module.C:736
virtual void sendSerial(std::string const &str)
Send a string over the &#39;serout&#39; serial port.
Definition: Module.C:245
void sendSerialImg1Dy(unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 1D message for an Y image coordinate.
Definition: Module.C:374
cv::Mat getCvGRAY(bool casync=false) const
Shorthand to get the input image as a GRAY cv::Mat and release the raw buffer.
Definition: Module.C:67
void sendSerialMarkStop()
Send a message MARK STOP to indicate the end of processing.
Definition: Module.C:728
void sendSerialStd3D(float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 3D message.
Definition: Module.C:586
void sendSerialImg1Dx(unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 1D message for an X image coordinate.
Definition: Module.C:325
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level. ...
Definition: Log.H:212
virtual ~StdModule()
Virtual destructor for safe inheritance.
Definition: Module.C:278
Prepend standardized serial messages with a frame time
Definition: Module.H:434
void convertCvRGBAtoRawImage(cv::Mat const &src, RawImage &dst, int quality)
Convert an RGBA cv::Mat to RawImage with already-allocated pixels and pixel type. ...
Definition: RawImageOps.C:1323
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:263
std::string to_string(T const &val)
Convert from type to string.
Definition: UtilsImpl.H:58
std::string getStamp() const
Get a string with the frame/date/time stamp in it, depending on serstamp parameter.
Definition: Module.C:282
void convertCvGRAYtoRawImage(cv::Mat const &src, RawImage &dst, int quality)
Convert a Gray cv::Mat to RawImage with already-allocated pixels and pixel type.
Definition: RawImageOps.C:1344
virtual void parseSerial(std::string const &str, std::shared_ptr< UserInterface > s)
Receive a string from a serial port which contains a user command.
Definition: Module.C:254
Virtual base class for a vision processing module.
Definition: Module.H:336
void sendScaledCvGRAY(cv::Mat const &img, int quality=75) const
Shorthand to send a GRAY cv::Mat after converting it to the current output format.
Definition: Module.C:192
void sendSerialStd1Dy(float y, float size=0.0F, std::string const &id="", std::string const &extra="")
Send standardized 1D message for a standardized Y coordinate.
Definition: Module.C:387
Exception-safe wrapper around a raw image to be sent over USB.
Definition: Module.H:145
~InputFrame()
Destructor, returns the buffers to the driver as needed.
Definition: Module.C:38
RawImage const & get() const
Get a pre-allocated image so that we can fill the pixel data and later send out over USB using send()...
Definition: Module.C:133
cv::Mat getCvBGR(bool casync=false) const
Shorthand to get the input image as a BGR cv::Mat and release the raw buffer.
Definition: Module.C:76
void done() const
Indicate that user processing is done with the image previously obtained via get() ...
Definition: Module.C:60
void sendSerialMarkStart()
Send a message MARK START to indicate the beginning of processing.
Definition: Module.C:720
cv::Mat rescaleCv(cv::Mat const &img, cv::Size const &newdims)
Rescale an OpenCV image, choosing the right kind of interpolation.
Definition: RawImageOps.C:1407
unsigned int width
Image width in pixels.
Definition: RawImage.H:145
~OutputFrame()
Destructor, returns the buffers to the driver as needed.
Definition: Module.C:109
virtual ~Module()
Virtual destructor for safe inheritance.
Definition: Module.C:233
void convertCvRGBtoRawImage(cv::Mat const &src, RawImage &dst, int quality)
Convert a RGB cv::Mat to RawImage with already-allocated pixels and pixel type.
Definition: RawImageOps.C:1276
void sendSerialObjDetImg2D(unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
Send a standardized object detection + recognition message.
Definition: Module.C:773
size_t frameNum() const
Get frame number from the Engine.
Definition: Module.C:263