JeVois  1.20
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
CameraSensor.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2018 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/CameraSensor.H>
19 #include <jevois/Core/Camera.H>
20 #include <jevois/Core/VideoMapping.H>
21 #include <jevois/Util/Utils.H>
22 
23 // ####################################################################################################
24 bool jevois::sensorSupportsFormat(jevois::CameraSensor s, jevois::VideoMapping const & m)
25 {
26  switch (s)
27  {
28  // ----------------------------------------------------------------------------------------------------
29  case jevois::CameraSensor::any:
30  return true;
31 
32 #ifdef JEVOIS_PRO
33  // ----------------------------------------------------------------------------------------------------
34  case jevois::CameraSensor::imx290:
35  // This sensor supports: BAYER, GREY, YUYV, RGB24, ARGB32, and more. Native size is 1920x1080 at up to
36  // 120fps. Sensor also supports 1280x720 and cropping from 1920x1080, but the A311D ISP has issues with these (frame
37  // collision as soon as capture height is not 1080). So we always capture at 1920x1080 and use the ISP to
38  // crop/rescale to any other resolution. Any size (multiple of 4) is supported through cropping & rescaling.
39 
40  /* Supported formats as reported by the Amlogic camera ISP kernel driver:
41 
42  Supported format 0 is [32-bit A/XRGB 8-8-8-8] fcc 0x34424752 [RGB4]
43  Supported format 1 is [24-bit RGB 8-8-8] fcc 0x33424752 [RGB3]
44  Supported format 2 is [Y/CbCr 4:2:0] fcc 0x3231564e [NV12]
45  Supported format 3 is [16-bit A/XYUV 4-4-4-4] fcc 0x34343459 [Y444]
46  Supported format 4 is [YUYV 4:2:2] fcc 0x56595559 [YUYV]
47  Supported format 5 is [UYVY 4:2:2] fcc 0x59565955 [UYVY]
48  Supported format 6 is [8-bit Greyscale] fcc 0x59455247 [GREY]
49  Supported format 7 is [16-bit Bayer BGBG/GRGR (Exp.)] fcc 0x32525942 [BYR2] */
50  switch (m.cfmt)
51  {
52  // Here is what we support:
53  case V4L2_PIX_FMT_SBGGR16:
54  case V4L2_PIX_FMT_RGB32:
55  case V4L2_PIX_FMT_RGB24:
56  case V4L2_PIX_FMT_YUYV:
57  case V4L2_PIX_FMT_GREY:
58  //case V4L2_PIX_FMT_NV12: not supported yet, uses 3 planes: 12-bit (8 for Y, 8 for UV half size) Y/CbCr 4:2:0
59  //case V4L2_PIX_FMT_YUV444: not supported: 16-bit xxxxyyyy uuuuvvvv
60  //case V4L2_PIX_FMT_UYVY: not supported to avoid confusions with YUYV
61  break;
62  // Everything else is unsupported:
63  default: return false;
64  }
65 
66  // Any size up to 1080P is supported. Beware that some image processing algorithms may require image width to be a
67  // multiple of 8 or 16. Here, we do not impose this constraint as some neural nets do not have it (some even use odd
68  // input image sizes). The sensor supports up to 120fps:
69  if (m.cw <= 1920 && m.ch <= 1080 && m.cfps <= 120.0) return true;
70 
71  return false;
72 
73  // ----------------------------------------------------------------------------------------------------
74  case jevois::CameraSensor::os08a10:
75  // This sensor supports: BAYER, GREY, YUYV, RGB24, ARGB32, and more. Native size is 3840x2160 at up to 60fps. Sensor
76  // also supports 1920x1080, 1280x720 and cropping, but the A311D ISP may have issues with these (frame collision
77  // problem as with the imx290 when not at native resolution). Any size (multiple of 4) is supported through cropping
78  // & rescaling.
79 
80  /* Supported formats as reported by the Amlogic camera ISP kernel driver:
81 
82  Supported format 0 is [32-bit A/XRGB 8-8-8-8] fcc 0x34424752 [RGB4]
83  Supported format 1 is [24-bit RGB 8-8-8] fcc 0x33424752 [RGB3]
84  Supported format 2 is [Y/CbCr 4:2:0] fcc 0x3231564e [NV12]
85  Supported format 3 is [16-bit A/XYUV 4-4-4-4] fcc 0x34343459 [Y444]
86  Supported format 4 is [YUYV 4:2:2] fcc 0x56595559 [YUYV]
87  Supported format 5 is [UYVY 4:2:2] fcc 0x59565955 [UYVY]
88  Supported format 6 is [8-bit Greyscale] fcc 0x59455247 [GREY]
89  Supported format 7 is [16-bit Bayer BGBG/GRGR (Exp.)] fcc 0x32525942 [BYR2] */
90  switch (m.cfmt)
91  {
92  // Here is what we support:
93  case V4L2_PIX_FMT_SBGGR16:
94  case V4L2_PIX_FMT_RGB32:
95  case V4L2_PIX_FMT_RGB24:
96  case V4L2_PIX_FMT_YUYV:
97  case V4L2_PIX_FMT_GREY:
98  //case V4L2_PIX_FMT_NV12: not supported yet, uses 3 planes: 12-bit (8 for Y, 8 for UV half size) Y/CbCr 4:2:0
99  //case V4L2_PIX_FMT_YUV444: not supported: 16-bit xxxxyyyy uuuuvvvv
100  //case V4L2_PIX_FMT_UYVY: not supported to avoid confusions with YUYV
101  break;
102  // Everything else is unsupported:
103  default: return false;
104  }
105 
106  // Any size up to 4k is supported. Beware that some image processing algorithms may require image width to be a
107  // multiple of 8 or 16. Here, we do not impose this constraint as some neural nets do not have it (some even use odd
108  // input image sizes). The sensor supports up to 60fps:
109  if (m.cw <= 3840 && m.ch <= 2160 && m.cfps <= 60.0) return true;
110 
111  return false;
112 
113  // ----------------------------------------------------------------------------------------------------
114  case jevois::CameraSensor::ar0234:
115  // This sensor supports: BAYER, GREY, YUYV, RGB24, ARGB32, and more. Native size is 1920x1200 at up to
116  // 120fps. Sensor also supports 1920x1080, 1280x720 and cropping, but the A311D ISP may have issues with these
117  // (frame collision problem as with the imx290 when not at native resolution). Any size (multiple of 4) is supported
118  // through cropping & rescaling.
119 
120  /* Supported formats as reported by the Amlogic camera ISP kernel driver:
121 
122  Supported format 0 is [32-bit A/XRGB 8-8-8-8] fcc 0x34424752 [RGB4]
123  Supported format 1 is [24-bit RGB 8-8-8] fcc 0x33424752 [RGB3]
124  Supported format 2 is [Y/CbCr 4:2:0] fcc 0x3231564e [NV12]
125  Supported format 3 is [16-bit A/XYUV 4-4-4-4] fcc 0x34343459 [Y444]
126  Supported format 4 is [YUYV 4:2:2] fcc 0x56595559 [YUYV]
127  Supported format 5 is [UYVY 4:2:2] fcc 0x59565955 [UYVY]
128  Supported format 6 is [8-bit Greyscale] fcc 0x59455247 [GREY]
129  Supported format 7 is [16-bit Bayer BGBG/GRGR (Exp.)] fcc 0x32525942 [BYR2] */ // FIXME this sensor is GRBG
130  switch (m.cfmt)
131  {
132  // Here is what we support:
133  case V4L2_PIX_FMT_SGRBG16:
134  case V4L2_PIX_FMT_RGB32:
135  case V4L2_PIX_FMT_RGB24:
136  case V4L2_PIX_FMT_YUYV:
137  case V4L2_PIX_FMT_GREY:
138  //case V4L2_PIX_FMT_NV12: not supported yet, uses 3 planes: 12-bit (8 for Y, 8 for UV half size) Y/CbCr 4:2:0
139  //case V4L2_PIX_FMT_YUV444: not supported: 16-bit xxxxyyyy uuuuvvvv
140  //case V4L2_PIX_FMT_UYVY: not supported to avoid confusions with YUYV
141  break;
142  // Everything else is unsupported:
143  default: return false;
144  }
145 
146  // Any size up to 1920x1200 is supported. Beware that some image processing algorithms may require image width to be
147  // a multiple of 8 or 16. Here, we do not impose this constraint as some neural nets do not have it (some even use
148  // odd input image sizes). The sensor supports up to 120fps:
149  if (m.cw <= 1920 && m.ch <= 1200 && m.cfps <= 120.0) return true;
150 
151  return false;
152 
153 #else // JEVOIS_PRO
154  // ----------------------------------------------------------------------------------------------------
155  case jevois::CameraSensor::ov9650:
156 
157  if (m.wdr != jevois::WDRtype::Linear) return false; // Wide dynamic range not supported by this sensor
158  if (m.crop == jevois::CropType::CropScale) return false; // Dual-stream capture not supported by this sensor
159 
160  // This sensor supports: YUYV, BAYER, RGB565
161  // SXGA (1280 x 1024): up to 15 fps
162  // VGA ( 640 x 480): up to 30 fps
163  // CIF ( 352 x 288): up to 60 fps
164  // QVGA ( 320 x 240): up to 60 fps
165  // QCIF ( 176 x 144): up to 120 fps
166  // QQVGA ( 160 x 120): up to 60 fps
167  // QQCIF ( 88 x 72): up to 120 fps
168 
169  if (m.cfmt != V4L2_PIX_FMT_YUYV && m.cfmt != V4L2_PIX_FMT_SRGGB8 && m.cfmt != V4L2_PIX_FMT_RGB565) return false;
170 
171  if (m.cw == 1280 && m.ch == 1024) { if (m.cfps <= 15.0F) return true; else return false; }
172  if (m.cw == 640 && m.ch == 480) { if (m.cfps <= 30.0F) return true; else return false; }
173  if (m.cw == 352 && m.ch == 288) { if (m.cfps <= 60.0F) return true; else return false; }
174  if (m.cw == 320 && m.ch == 240) { if (m.cfps <= 60.0F) return true; else return false; }
175  if (m.cw == 176 && m.ch == 144) { if (m.cfps <= 120.0F) return true; else return false; }
176  if (m.cw == 160 && m.ch == 120) { if (m.cfps <= 60.0F) return true; else return false; }
177  if (m.cw == 88 && m.ch == 72) { if (m.cfps <= 120.0F) return true; else return false; }
178 
179  return false;
180 
181  // ----------------------------------------------------------------------------------------------------
182  case jevois::CameraSensor::ov2640:
183 
184  if (m.wdr != jevois::WDRtype::Linear) return false; // Wide dynamic range not supported by this sensor
185  if (m.crop == jevois::CropType::CropScale) return false; // Dual-stream capture not supported by this sensor
186 
187  // This sensor supports: YUYV, BAYER, RGB565
188  // UXGA (1600 x 1200): up to 15 fps
189  // SXGA (1280 x 1024): up to 15 fps
190  // 720p (1280 x 720): up to 15 fps
191  // XGA (1024 x 768): up to 15 fps
192  // SVGA ( 800 x 600): up to 40 fps
193  // VGA ( 640 x 480): up to 40 fps
194  // CIF ( 352 x 288): up to 60 fps
195  // QVGA ( 320 x 240): up to 60 fps
196  // QCIF ( 176 x 144): up to 60 fps
197  // QQVGA ( 160 x 120): up to 60 fps
198  // QQCIF ( 88 x 72): up to 60 fps
199 
200  if (m.cfmt != V4L2_PIX_FMT_YUYV && m.cfmt != V4L2_PIX_FMT_SRGGB8 && m.cfmt != V4L2_PIX_FMT_RGB565) return false;
201 
202  if (m.cw == 1600 && m.ch == 1200) { if (m.cfps <= 15.0F) return true; else return false; }
203  if (m.cw == 1280 && m.ch == 1024) { if (m.cfps <= 15.0F) return true; else return false; }
204  if (m.cw == 1280 && m.ch == 720) { if (m.cfps <= 15.0F) return true; else return false; }
205  if (m.cw == 1024 && m.ch == 768) { if (m.cfps <= 15.0F) return true; else return false; }
206  if (m.cw == 800 && m.ch == 600) { if (m.cfps <= 40.0F) return true; else return false; }
207  if (m.cw == 640 && m.ch == 480) { if (m.cfps <= 40.0F) return true; else return false; }
208  if (m.cw == 352 && m.ch == 288) { if (m.cfps <= 60.0F) return true; else return false; }
209  if (m.cw == 320 && m.ch == 240) { if (m.cfps <= 60.0F) return true; else return false; }
210  if (m.cw == 176 && m.ch == 144) { if (m.cfps <= 60.0F) return true; else return false; }
211  if (m.cw == 160 && m.ch == 120) { if (m.cfps <= 60.0F) return true; else return false; }
212  if (m.cw == 88 && m.ch == 72) { if (m.cfps <= 60.0F) return true; else return false; }
213 
214  return false;
215 
216  // ----------------------------------------------------------------------------------------------------
217  case jevois::CameraSensor::ov7725:
218 
219  if (m.wdr != jevois::WDRtype::Linear) return false; // Wide dynamic range not supported by this sensor
220  if (m.crop == jevois::CropType::CropScale) return false; // Dual-stream capture not supported by this sensor
221 
222  // This sensor supports: YUYV, BAYER, RGB565
223  // VGA ( 640 x 480): up to 60 fps
224  // CIF ( 352 x 288): up to 60 fps
225  // QVGA ( 320 x 240): up to 60 fps
226  // QCIF ( 176 x 144): up to 60 fps
227  // QQVGA ( 160 x 120): up to 60 fps
228  // QQCIF ( 88 x 72): up to 60 fps
229 
230  if (m.cfmt != V4L2_PIX_FMT_YUYV && m.cfmt != V4L2_PIX_FMT_SRGGB8 && m.cfmt != V4L2_PIX_FMT_RGB565) return false;
231 
232  if (m.cw == 640 && m.ch == 480) { if (m.cfps <= 60.0F) return true; else return false; }
233  if (m.cw == 352 && m.ch == 288) { if (m.cfps <= 60.0F) return true; else return false; }
234  if (m.cw == 320 && m.ch == 240) { if (m.cfps <= 60.0F) return true; else return false; }
235  if (m.cw == 176 && m.ch == 144) { if (m.cfps <= 60.0F) return true; else return false; }
236  if (m.cw == 160 && m.ch == 120) { if (m.cfps <= 60.0F) return true; else return false; }
237  if (m.cw == 88 && m.ch == 72) { if (m.cfps <= 60.0F) return true; else return false; }
238 
239  return false;
240 
241  // ----------------------------------------------------------------------------------------------------
242  case jevois::CameraSensor::ar0135:
243 
244  if (m.wdr != jevois::WDRtype::Linear) return false; // Wide dynamic range not supported by this sensor
245  if (m.crop == jevois::CropType::CropScale) return false; // Dual-stream capture not supported by this sensor
246 
247  // This sensor supports: BAYER or MONO
248  // SXGA (1280 x 960): up to 54 fps
249  // 720p (1280 x 720): up to 60 fps
250  // VGA ( 640 x 480): up to 54 fps (binned version of SXGA)
251  // 360p ( 640 x 360): up to 60 fps
252  // QVGA ( 320 x 240): up to 54 fps (central crop of binned version of SXGA)
253  // 180p ( 320 x 180): up to 60 fps
254  // QQVGA ( 160 x 120): up to 54 fps (central crop of binned version of SXGA)
255  // 90p ( 160 x 90): up to 60 fps
256 
257  // We support native Bayer or Mono, and YUYV through Bayer/mono to YUYV conversion in the Camera class:
258  if (m.cfmt != V4L2_PIX_FMT_SRGGB8 && m.cfmt != V4L2_PIX_FMT_GREY && m.cfmt != V4L2_PIX_FMT_YUYV) return false;
259 
260  if (m.cw == 1280 && m.ch == 960) { if (m.cfps <= 54.0F) return true; else return false; }
261  if (m.cw == 1280 && m.ch == 720) { if (m.cfps <= 60.0F) return true; else return false; }
262 
263  if (m.cw == 640 && m.ch == 480) { if (m.cfps <= 54.0F) return true; else return false; }
264  if (m.cw == 640 && m.ch == 360) { if (m.cfps <= 60.0F) return true; else return false; }
265 
266  if (m.cw == 320 && m.ch == 240) { if (m.cfps <= 60.0F) return true; else return false; }
267  if (m.cw == 320 && m.ch == 180) { if (m.cfps <= 60.0F) return true; else return false; }
268 
269  if (m.cw == 160 && m.ch == 120) { if (m.cfps <= 60.0F) return true; else return false; }
270  if (m.cw == 160 && m.ch == 90) { if (m.cfps <= 60.0F) return true; else return false; }
271 
272  return false;
273 #endif // JEVOIS_PRO
274  }
275  return false;
276 }
277 
278 // ####################################################################################################
279 bool jevois::sensorHasIMU(CameraSensor s)
280 {
281  switch (s)
282  {
283  // These sensors have an ICM20948 IMU:
284 #ifdef JEVOIS_PRO
285  case jevois::CameraSensor::imx290:
286  case jevois::CameraSensor::os08a10:
287  case jevois::CameraSensor::ar0234:
288 #else
289  case jevois::CameraSensor::ar0135:
290 #endif
291  return true;
292 
293  // All other sensors do not have an IMU:
294  default:
295  return false;
296  }
297 }
298 
299 // ####################################################################################################
300 void jevois::sensorPrepareSetFormat(jevois::CameraSensor s, jevois::VideoMapping const & m,
301  unsigned int & capw, unsigned int & caph, int & preset)
302 {
303  if (jevois::sensorSupportsFormat(s, m) == false) throw std::runtime_error("Requested format not supported by sensor");
304 
305  // Keep this code in sync with the kernel drivers:
306  switch(s)
307  {
308 #ifdef JEVOIS_PRO
309  case jevois::CameraSensor::imx290:
310  // Capture always 1080p. Presets:
311  // 0: 1080p30
312  // 1: 1080p60
313  // 2: 1080p120
314  capw = 1920; caph = 1080;
315  if (m.cfps > 60.0F) preset = 2;
316  else if (m.cfps > 30.0F) preset = 1;
317  else preset = 0;
318 
319  break;
320 
321  case jevois::CameraSensor::os08a10:
322  // Capture 2160p or 1080p. Presets:
323  // 0: 2160p30
324  // 1: 2160p60
325  // 2: 1080p30
326  // 3: 1080p60
327  // 4: 1080p30 WDR (DOL_VC)
328  if (m.cw > 1920 || m.ch > 1080)
329  {
330  capw = 3840; caph = 2160;
331  if (m.cfps > 30.0F) preset = 1;
332  else preset = 0;
333  }
334  else
335  {
336  capw = 1920; caph = 1080;
337  if (m.cfps > 30.0F) preset = 3;
338  else if (m.wdr != jevois::WDRtype::Linear) preset = 4;
339  else preset = 2;
340  }
341  break;
342 
343  case jevois::CameraSensor::ar0234:
344  // Capture 2160p or 1080p. Presets:
345  // 0: 1080p30
346  // 1: 1080p60
347  // 2: 1080p120
348  // 3: 1920x1200p30
349  // 4: 1920x1200p60
350  // 5: 1920x1200p120
351  if (m.ch > 1080)
352  {
353  capw = 1920; caph = 1200;
354  if (m.cfps > 60.0F) preset = 5;
355  else if (m.cfps > 30.0F) preset = 4;
356  else preset = 3;
357  }
358  else
359  {
360  capw = 1920; caph = 1080;
361  if (m.cfps > 60.0F) preset = 2;
362  else if (m.cfps > 30.0F) preset = 1;
363  else preset = 0;
364  }
365  break;
366 
367 #endif // JEVOIS_PRO
368 
369  default:
370  // Just capture as specified, sensorSupportsFormat() already removed invalid modes:
371  capw = m.cw; caph = m.ch; preset = -1;
372  }
373 }
jevois::sensorSupportsFormat
bool sensorSupportsFormat(CameraSensor s, VideoMapping const &m)
Check whether a given resolution and frame rate is supported by a sensor.
jevois::sensorHasIMU
bool sensorHasIMU(CameraSensor s)
Check whether sensor has an IMU (inertial measurement unit)
Definition: CameraSensor.C:279
jevois::sensorPrepareSetFormat
void sensorPrepareSetFormat(CameraSensor s, VideoMapping const &m, unsigned int &capw, unsigned int &caph, int &preset)
Load a sensor preset (JeVois-Pro only) and return native sensor grab dims (when cropping and/or scali...
VideoMapping.H
Camera.H
CameraSensor.H
Utils.H