JeVois  1.17
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
VideoMapping.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Util/Utils.H>
21 
22 #include <fstream>
23 #include <sstream>
24 #include <algorithm>
25 #include <cmath>
26 
27 #define PERROR(x) LERROR("In file " << JEVOIS_ENGINE_CONFIG_FILE << ':' << linenum << ": " << x)
28 
29 // ####################################################################################################
30 std::string jevois::VideoMapping::sopath() const
31 {
32  if (ispython) return JEVOIS_MODULE_PATH "/" + vendor + '/' + modulename + '/' + modulename + ".py";
33  else return JEVOIS_MODULE_PATH "/" + vendor + '/' + modulename + '/' + modulename + ".so";
34 }
35 
36 // ####################################################################################################
37 std::string jevois::VideoMapping::srcpath() const
38 {
39  if (ispython) return JEVOIS_MODULE_PATH "/" + vendor + '/' + modulename + '/' + modulename + ".py";
40  else return JEVOIS_MODULE_PATH "/" + vendor + '/' + modulename + '/' + modulename + ".C";
41 }
42 
43 // ####################################################################################################
44 unsigned int jevois::VideoMapping::osize() const
45 { return jevois::v4l2ImageSize(ofmt, ow, oh); }
46 
47 // ####################################################################################################
48 unsigned int jevois::VideoMapping::csize() const
49 { return jevois::v4l2ImageSize(cfmt, cw, ch); }
50 
51 // ####################################################################################################
52 unsigned int jevois::VideoMapping::c2size() const
53 { return jevois::v4l2ImageSize(c2fmt, c2w, c2h); }
54 
55 // ####################################################################################################
56 float jevois::VideoMapping::uvcToFps(unsigned int interval)
57 {
58  // Let's round it off to the nearest 1/100Hz:
59  return float(1000000000U / interval) * 0.01F;
60 }
61 
62 // ####################################################################################################
63 unsigned int jevois::VideoMapping::fpsToUvc(float fps)
64 {
65  return (unsigned int)(10000000.0F / fps + 0.499F);
66 }
67 
68 // ####################################################################################################
69 float jevois::VideoMapping::v4l2ToFps(struct v4l2_fract const & interval)
70 {
71  // Let's round it off to the nearest 1/100Hz:
72  return float(interval.denominator * 100U / interval.numerator) * 0.01F;
73 }
74 
75 // ####################################################################################################
76 struct v4l2_fract jevois::VideoMapping::fpsToV4l2(float fps)
77 {
78  return { 100U, (unsigned int)(fps * 100.0F) };
79 }
80 
81 // ####################################################################################################
82 std::string jevois::VideoMapping::ostr() const
83 {
84  std::ostringstream ss;
85  ss << jevois::fccstr(ofmt) << ' ' << ow << 'x' << oh << " @ " << ofps << "fps";
86  return ss.str();
87 }
88 
89 // ####################################################################################################
90 std::string jevois::VideoMapping::cstr() const
91 {
92  std::ostringstream ss;
93  ss << jevois::fccstr(cfmt) << ' ' << cw << 'x' << ch << " @ " << cfps << "fps";
94  return ss.str();
95 }
96 
97 // ####################################################################################################
98 std::string jevois::VideoMapping::c2str() const
99 {
100  std::ostringstream ss;
101  ss << jevois::fccstr(c2fmt) << ' ' << c2w << 'x' << c2h << " @ " << cfps << "fps";
102  return ss.str();
103 }
104 
105 // ####################################################################################################
106 std::string jevois::VideoMapping::cstrall() const
107 {
108  std::string ret = cstr();
109  if (crop == jevois::CropType::CropScale) ret += " + " + c2str();
110  return ret;
111 }
112 
113 // ####################################################################################################
114 std::string jevois::VideoMapping::str() const
115 {
116  std::ostringstream ss;
117 
118  ss << "OUT: " << ostr() << " CAM: " << cstr();
119  if (crop == jevois::CropType::CropScale) ss << " CAM2: " << c2str();
120  //ss << " (uvc " << uvcformat << '/' << uvcframe << '/' << jevois::VideoMapping::fpsToUvc(ofps) << ')';
121  ss << " MOD: " << vendor << ':' << modulename << ' ' << (ispython ? "Python" : "C++");
122  //ss << ' ' << this->sopath();
123  return ss.str();
124 }
125 
126 // ####################################################################################################
127 std::string jevois::VideoMapping::menustr() const
128 {
129  std::ostringstream ss;
130 
131  ss << modulename << (ispython ? " (Py)" : " (C++)");
132  ss << " CAM: " << cstr();
133  if (crop == jevois::CropType::CropScale) ss << " + " << c2str();
134  if (ofmt != 0 && ofmt != JEVOISPRO_FMT_GUI) ss << ", OUT: " << ostr() << ' ';
135  return ss.str();
136 }
137 
138 // ####################################################################################################
139 bool jevois::VideoMapping::hasSameSpecsAs(VideoMapping const & other) const
140 {
141  return (ofmt == other.ofmt && ow == other.ow && oh == other.oh && std::abs(ofps - other.ofps) < 0.01F &&
142  cfmt == other.cfmt && cw == other.cw && ch == other.ch && std::abs(cfps - other.cfps) < 0.01F &&
143  crop == other.crop &&
144  (crop != jevois::CropType::CropScale ||
145  (c2fmt == other.c2fmt && c2w == other.c2w && c2h == other.c2h && std::abs(cfps - other.cfps) < 0.01F)));
146 }
147 
148 // ####################################################################################################
149 bool jevois::VideoMapping::isSameAs(VideoMapping const & other) const
150 {
151  return (hasSameSpecsAs(other) && wdr == other.wdr && vendor == other.vendor && modulename == other.modulename &&
152  ispython == other.ispython);
153 }
154 
155 // ####################################################################################################
156 std::ostream & jevois::operator<<(std::ostream & out, jevois::VideoMapping const & m)
157 {
158  out << jevois::fccstr(m.ofmt) << ' ' << m.ow << ' ' << m.oh << ' ' << m.ofps << ' ';
159 
160  if (m.wdr != jevois::WDRtype::Linear)
161  out << m.wdr << ':';
162 
163  switch (m.crop)
164  {
165  case jevois::CropType::Scale:
166  break;
168  out << m.crop << ':'; break;
169  case jevois::CropType::CropScale:
170  out << m.crop << '=' << jevois::fccstr(m.c2fmt) << '@' << m.c2w << 'x' << m.c2h << ':'; break;
171  }
172 
173  out << jevois::fccstr(m.cfmt) << ' ' << m.cw << ' ' << m.ch << ' ' << m.cfps << ' '
174  << m.vendor << ' ' << m.modulename;
175  return out;
176 }
177 
178 namespace
179 {
180  // Return either the absolute value in str, or c +/- stoi(str)
181  int parse_relative_dim(std::string const & str, int c)
182  {
183  if (str.empty()) throw std::range_error("Invalid empty output width");
184  if (str[0] == '+') return c + std::stoi(str.substr(1));
185  else if (str[0] == '-') return c - std::stoi(str.substr(1));
186  return std::stoi(str);
187  }
188 
189  void parse_cam_format(std::string const & str, unsigned int & fmt, jevois::WDRtype & wdr, jevois::CropType & crop,
190  unsigned int & c2fmt, unsigned int & c2w, unsigned int & c2h)
191  {
192  // Set the defaults in case no qualifier is given:
193  wdr = jevois::WDRtype::Linear;
194  crop = jevois::CropType::Scale;
195 
196  // Parse:
197  auto tok = jevois::split(str, ":");
198  if (tok.empty()) throw std::range_error("Empty camera format is not allowed");
199  fmt = jevois::strfcc(tok.back()); tok.pop_back();
200  for (std::string & t : tok)
201  {
202  // WDR, crop type can be specified in any order. So try to get each and see if we succeed:
203  try { wdr = jevois::from_string<jevois::WDRtype>(t); continue; } catch (...) { }
204 
205  // If not WDR, then it should be Crop|Scale|CropScale=FCC@WxH
206  auto ttok = jevois::split(t, "[=@x]");
207  if (ttok.empty()) throw std::range_error("Invalid empty camera format modifier: " + t);
208 
209  try
210  {
211  crop = jevois::from_string<jevois::CropType>(ttok[0]);
212 
213  switch (crop)
214  {
215  case jevois::CropType::Crop: if (ttok.size() == 1) continue; break;
216  case jevois::CropType::Scale: if (ttok.size() == 1) continue; break;
217  case jevois::CropType::CropScale:
218  if (ttok.size() == 4)
219  {
220  c2fmt = jevois::strfcc(ttok[1]);
221  c2w = std::stoi(ttok[2]);
222  c2h = std::stoi(ttok[3]);
223  continue;
224  }
225  }
226  } catch (...) { }
227 
228  throw std::range_error("Invalid camera format modifier [" + t +
229  "] - must be Linear|DOL or Crop|Scale|CropScale=FCC@WxH");
230  }
231  }
232 }
233 
234 // ####################################################################################################
235 std::istream & jevois::operator>>(std::istream & in, jevois::VideoMapping & m)
236 {
237  std::string of, cf, ows, ohs;
238  in >> of >> ows >> ohs >> m.ofps >> cf >> m.cw >> m.ch >> m.cfps >> m.vendor >> m.modulename;
239 
240  // Output width and height can be either absolute or relative to camera width and height; for relative values, they
241  // must start with either a + or - symbol:
242  m.ow = parse_relative_dim(ows, m.cw);
243  m.oh = parse_relative_dim(ohs, m.ch);
244 
245  m.ofmt = jevois::strfcc(of);
246 
247  // Parse any wdr, crop, or stream modulators on camera format, and the format itself:
248  parse_cam_format(cf, m.cfmt, m.wdr, m.crop, m.c2fmt, m.c2w, m.c2h);
249 
250  m.setModuleType(); // set python vs C++, check that file is here, and throw otherwise
251 
252  return in;
253 }
254 
255 // ####################################################################################################
256 std::vector<jevois::VideoMapping> jevois::loadVideoMappings(jevois::CameraSensor s, size_t & defidx, bool checkso,
257  bool hasgui)
258 {
259  std::ifstream ifs(JEVOIS_ENGINE_CONFIG_FILE);
260  if (ifs.is_open() == false) LFATAL("Could not open [" << JEVOIS_ENGINE_CONFIG_FILE << ']');
261  return jevois::videoMappingsFromStream(s, ifs, defidx, checkso, hasgui);
262 }
263 
264 // ####################################################################################################
265 std::vector<jevois::VideoMapping> jevois::videoMappingsFromStream(jevois::CameraSensor s, std::istream & is,
266  size_t & defidx, bool checkso, bool hasgui)
267 {
268  size_t linenum = 1;
269  std::vector<jevois::VideoMapping> mappings;
270  jevois::VideoMapping defmapping = { };
271 
272  for (std::string line; std::getline(is, line); ++linenum)
273  {
274  std::vector<std::string> tok = jevois::split(line);
275  if (tok.empty()) continue; // skip blank lines
276  if (tok.size() == 1 && tok[0].empty()) continue; // skip blank lines
277  if (tok[0][0] == '#') continue; // skip comments
278  if (tok.size() < 10) { PERROR("Found " << tok.size() << " tokens instead of >= 10 -- SKIPPING"); continue; }
279 
280  jevois::VideoMapping m;
281  try
282  {
283  m.ofmt = jevois::strfcc(tok[0]);
284  m.ofps = std::stof(tok[3]);
285 
286  parse_cam_format(tok[4], m.cfmt, m.wdr, m.crop, m.c2fmt, m.c2w, m.c2h);
287  m.cw = std::stoi(tok[5]);
288  m.ch = std::stoi(tok[6]);
289  m.cfps = std::stof(tok[7]);
290 
291  m.ow = parse_relative_dim(tok[1], m.cw);
292  m.oh = parse_relative_dim(tok[2], m.ch);
293  }
294  catch (std::exception const & e) { PERROR("Skipping entry because of parsing error: " << e.what()); continue; }
295  catch (...) { PERROR("Skipping entry because of parsing errors"); continue; }
296 
297  m.vendor = tok[8];
298  m.modulename = tok[9];
299 
300  // Determine C++ vs python, silently skip this module if none of those and checkso was given:
301  try { m.setModuleType(); }
302  catch (...)
303  {
304  if (checkso)
305  { PERROR("No .so|.py found for " << m.vendor << '/' << m.modulename << " -- SKIPPING."); continue; }
306  }
307 
308  // Skip if the sensor cannot support this mapping:
309  if (m.sensorOk(s) == false)
310  { PERROR("Camera video format [" << m.cstr() << "] not supported by sensor -- SKIPPING."); continue; }
311 
312  // Skip gui modes if we do not have a gui:
313  if (hasgui == false && m.ofmt == JEVOISPRO_FMT_GUI)
314  { PERROR("Graphical user interface not available or disabled -- SKIPPING"); continue; }
315 
316 #ifndef JEVOIS_PRO
317  // Skip if not jevois-pro and trying to use GUI output:
318  if (m.ofmt == JEVOISPRO_FMT_GUI)
319  { PERROR("GUI output only supported on JeVois-Pro -- SKIPPING"); continue; }
320 
321 #ifndef JEVOIS_PLATFORM
322  // Skip if not jevois-pro platform and trying to do dual-frame hardware scaling through the ISP:
323  if (m.crop == jevois::CropType::CropScale || m.crop == jevois::CropType::Crop)
324  { PERROR("Crop or Crop+Scale camera input only supported on JeVois-Pro platform -- SKIPPING"); continue; }
325 #endif // JEVOIS_PLATFORM
326 #endif // JEVOIS_PRO
327 
328  // Handle optional star for default mapping. We tolerate several and pick the first one:
329  if (tok.size() > 10)
330  {
331  if (tok[10] == "*")
332  {
333  if (defmapping.cfmt == 0) defmapping = m;
334  if (tok.size() > 11 && tok[11][0] != '#') PERROR("Extra garbage after 11th token ignored");
335  }
336  else if (tok[10][0] != '#') PERROR("Extra garbage after 10th token ignored");
337  }
338 
339  mappings.push_back(m);
340  //LINFO("Successfully parsed mapping: " << m.str());
341  }
342 
343  // Sort the array:
344  std::sort(mappings.begin(), mappings.end(),
345  [=](jevois::VideoMapping const & a, jevois::VideoMapping const & b)
346  {
347  // Return true if a should be ordered before b:
348  if (a.ofmt < b.ofmt) return true;
349  if (a.ofmt == b.ofmt) {
350  if (a.ow > b.ow) return true;
351  if (a.ow == b.ow) {
352  if (a.oh > b.oh) return true;
353  if (a.oh == b.oh) {
354  if (a.ofps > b.ofps) return true;
355  if (std::abs(a.ofps - b.ofps) < 0.01F) {
356  // The two output modes are identical. Warn unless the output format is NONE or JVUI. We will
357  // adjust the framerates later to distinguish the offenders:
358  if (a.ofmt != 0 && a.ofmt != JEVOISPRO_FMT_GUI)
359  PERROR("WARNING: Two modes have identical output format: " << a.ostr());
360 
361  // All right, all USB stuff being equal, just sort according to the camera format:
362  if (a.cfmt < b.cfmt) return true;
363  if (a.cfmt == b.cfmt) {
364  if (a.cw > b.cw) return true;
365  if (a.cw == b.cw) {
366  if (a.ch > b.ch) return true;
367  if (a.ch == b.ch) {
368  if (a.cfps > b.cfps) return true;
369  // it's ok to have duplicates here since either those are NONE USB modes or JVUI that are
370  // selected manually, or we will adjust below
371  }
372  }
373  }
374  }
375  }
376  }
377  }
378  return false;
379  });
380 
381  // If we are not checking for .so, run the shortcut version where we also do not check for no USB mode, duplicates,
382  // default index, etc. This is used, by, e.g., the jevois-add-videomapping program:
383  if (checkso == false) { defidx = 0; return mappings; }
384 
385  // We need at least one mapping to work, and we need at least one with UVC output too keep hosts happy:
386  if (mappings.empty() || mappings.back().ofmt == 0 || mappings.back().ofmt == JEVOISPRO_FMT_GUI)
387  {
388  PERROR("No valid video mapping with UVC output found -- INSERTING A DEFAULT ONE");
389  jevois::VideoMapping m;
390  m.ofmt = V4L2_PIX_FMT_YUYV; m.ow = 640; m.oh = 480; m.ofps = 30.0F;
391  m.cfmt = V4L2_PIX_FMT_YUYV; m.cw = 640; m.ch = 480; m.cfps = 30.0F;
392  m.vendor = "JeVois"; m.modulename = "PassThrough"; m.ispython = false;
393 
394  // We are guaranteed that this will not create a duplicate output mapping since either mappings was empty or it had
395  // no USB-out modes:
396  mappings.push_back(m);
397  }
398 
399  // If we had duplicate output formats, discard full exact duplicates (including same module), and otherwise adjust
400  // framerates slightly: In the sorting above, we ordered by decreasing ofps (all else being equal). Here we are going
401  // to decrease ofps on the second mapping when we hit a match. We need to beware that this should propagate down to
402  // subsequent matching mappings while preserving the ordering:
403  auto a = mappings.begin(), b = a + 1;
404  while (b != mappings.end())
405  {
406  // Discard exact duplicates, adjust frame rates for matching specs but different modules:
407  if (a->isSameAs(*b)) { b = mappings.erase(b); continue; }
408  else if (b->ofmt != 0 && b->ofmt != JEVOISPRO_FMT_GUI && a->ofmt == b->ofmt && a->ow == b->ow && a->oh == b->oh)
409  {
410  if (std::abs(a->ofps - b->ofps) < 0.01F) b->ofps -= 1.0F; // equal fps, decrease b.ofps by 1fps
411  else if (b->ofps > a->ofps) b->ofps = a->ofps - 1.0F; // got out of order because of a previous decrease
412  }
413  a = b; ++b;
414  }
415 
416  // Find back our default mapping index in the sorted array:
417  if (defmapping.cfmt == 0)
418  {
419  LERROR("No default video mapping provided, using first one with UVC output");
420  for (size_t i = 0; i < mappings.size(); ++i) if (mappings[i].ofmt) { defidx = i; break; }
421  }
422  else
423  {
424  // Default was set, find its index after sorting:
425  for (size_t i = 0; i < mappings.size(); ++i)
426  if (mappings[i].ofmt == defmapping.ofmt && mappings[i].ow == defmapping.ow &&
427  mappings[i].oh == defmapping.oh && mappings[i].ofps == defmapping.ofps)
428  { defidx = i; break; }
429  }
430 
431  // Now that everything is sorted, compute our UVC format and frame indices, those are 1-based, and frame is reset each
432  // time format changes. Note that all the intervals will be passed as a list to the USB host for a given format and
433  // frame combination, so all the mappings that have identical pixel format and frame size here receive the same
434  // uvcformat and uvcframe numbers. Note that we skip over all the NONE ofmt modes here:
435  unsigned int ofmt = ~0U, ow = ~0U, oh = ~0U, iformat = 0, iframe = 0;
436  for (jevois::VideoMapping & m : mappings)
437  {
438  if (m.ofmt == 0 || m.ofmt == JEVOISPRO_FMT_GUI) { m.uvcformat = 0; m.uvcframe = 0; LDEBUG(m.str()); continue; }
439  if (m.ofmt != ofmt) { ofmt = m.ofmt; ow = ~0U; oh = ~0U; ++iformat; iframe = 0; } // Switch to the next format
440  if (m.ow != ow || m.oh != oh) { ow = m.ow; oh = m.oh; ++iframe; } // Switch to the next frame size
441  m.uvcformat = iformat; m.uvcframe = iframe;
442  LDEBUG(m.str());
443  }
444 
445  return mappings;
446 }
447 
448 // ####################################################################################################
449 bool jevois::VideoMapping::match(unsigned int oformat, unsigned int owidth, unsigned int oheight,
450  float oframespersec) const
451 {
452  if (ofmt == oformat && ow == owidth && oh == oheight && (std::abs(ofps - oframespersec) < 0.1F)) return true;
453  return false;
454 }
455 
456 // ####################################################################################################
457 bool jevois::VideoMapping::sensorOk(jevois::CameraSensor s)
458 {
459  return jevois::sensorSupportsFormat(s, cfmt, cw, ch, cfps);
460 }
461 
462 // ####################################################################################################
463 void jevois::VideoMapping::setModuleType()
464 {
465  // First assume that it is a C++ compiled module and check for the .so file:
466  ispython = false;
467  std::string sopa = sopath();
468  std::ifstream testifs(sopa);
469  if (testifs.is_open() == false)
470  {
471  // Could not find the .so, maybe it is a python module:
472  ispython = true; sopa = sopath();
473  std::ifstream testifs2(sopa);
474  if (testifs2.is_open() == false) throw std::runtime_error("Could not open module file " + sopa + "|.so");
475  }
476 }
JEVOISPRO_FMT_GUI
#define JEVOISPRO_FMT_GUI
JeVois-Pro zero-copy display of camera input frame (to be used as output mode in VideoMapping)
Definition: Utils.H:30
LDEBUG
#define LDEBUG(msg)
Convenience macro for users to print out console or syslog messages, DEBUG level.
Definition: Log.H:160
jevois::split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
Split string into vector of tokens using a regex to specify what to split on; default regex splits by...
Definition: Utils.C:257
PERROR
#define PERROR(x)
Definition: VideoMapping.C:27
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:198
jevois
Definition: Concepts.dox:1
F
float F
Definition: GUIhelper.C:2157
Log.H
jevois::sensorSupportsFormat
bool sensorSupportsFormat(CameraSensor s, unsigned int fmt, unsigned int w, unsigned int h, float fps)
Check whether a given resolution and frame rate is supported by a sensor.
jevois::fccstr
std::string fccstr(unsigned int fcc)
Convert a V4L2 four-cc code (V4L2_PIX_FMT_...) to a 4-char string.
Definition: Utils.C:44
jevois::Crop
() Scale() Crop(CropScale)) struct VideoMapping
Simple struct to hold video mapping definitions for the processing Engine.
Definition: VideoMapping.H:43
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
VideoMapping.H
Utils.H
jevois::v4l2ImageSize
unsigned int v4l2ImageSize(unsigned int fcc, unsigned int width, unsigned int height)
Return the image size in bytes for a given V4L2_PIX_FMT_..., width, height.
Definition: Utils.C:161
jevois::strfcc
unsigned int strfcc(std::string const &str)
Convert a JeVois video format string to V4L2 four-cc code (V4L2_PIX_FMT_...)
Definition: Utils.C:110