JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Gadget.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #include <jevois/Core/Gadget.H>
19 #include <jevois/Debug/Log.H>
20 #include <jevois/Core/VideoInput.H>
21 #include <jevois/Util/Utils.H>
22 #include <jevois/Util/Async.H>
24 #include <jevois/Core/Engine.H>
25 
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <fcntl.h>
29 #include <unistd.h>
30 #include <sys/time.h> // for gettimeofday()
31 
32 namespace
33 {
34  inline void debugCtrlReq(struct usb_ctrlrequest const & ctrl)
35  {
36  (void)ctrl; // avoid compiler warning about unused param if LDEBUG is turned off
37 
38  LDEBUG(std::showbase << std::hex << "bRequestType " << ctrl.bRequestType << " bRequest " << ctrl.bRequest
39  << " wValue " << ctrl.wValue << " wIndex " << ctrl.wIndex << " wLength " << ctrl.wLength);
40  }
41 
42  inline void debugStreamingCtrl(std::string const & msg, struct uvc_streaming_control const & ctrl)
43  {
44  (void)ctrl; // avoid compiler warning about unused param if LDEBUG is turned off
45  (void)msg; // avoid compiler warning about unused param if LDEBUG is turned off
46  LDEBUG(msg << ": " << std::showbase << std::hex << "bmHint=" << ctrl.bmHint << ", bFormatIndex=" <<
47  ctrl.bFormatIndex << ", bFrameIndex=" << ctrl.bFrameIndex << ", dwFrameInterval=" << ctrl.dwFrameInterval <<
48  ", wKeyFrameRate=" << ctrl.wKeyFrameRate << ", wPFrameRate=" << ctrl.wPFrameRate <<
49  ", wCompQuality=" << ctrl.wCompQuality << ", wCompWindowSize=" << ctrl.wCompWindowSize <<
50  ", wDelay=" << ctrl.wDelay << ", dwMaxVideoFrameSize=" << ctrl.dwMaxVideoFrameSize <<
51  ", dwMaxPayloadTransferSize=" << ctrl.dwMaxPayloadTransferSize << ", dwClockFrequency=" <<
52  ctrl.dwClockFrequency << ", bmFramingInfo=" << ctrl.bmFramingInfo << ", bPreferedVersion=" <<
53  ctrl.bPreferedVersion << ", bMinVersion=" << ctrl.bMinVersion << ", bMaxVersion=" << ctrl.bMaxVersion);
54  }
55 
56  unsigned int uvcToV4Lcontrol(unsigned int entity, unsigned int cs)
57  {
58  switch (entity)
59  {
60  case 1: // Our camera unit
61  // -- #define UVC_CT_SCANNING_MODE_CONTROL 0x01
62  // OK #define UVC_CT_AE_MODE_CONTROL 0x02
63  // OK #define UVC_CT_AE_PRIORITY_CONTROL 0x03
64  // OK #define UVC_CT_EXPOSURE_TIME_ABSOLUTE_CONTROL 0x04
65  // -- #define UVC_CT_EXPOSURE_TIME_RELATIVE_CONTROL 0x05
66  // -- #define UVC_CT_FOCUS_ABSOLUTE_CONTROL 0x06
67  // -- #define UVC_CT_FOCUS_RELATIVE_CONTROL 0x07
68  // -- #define UVC_CT_FOCUS_AUTO_CONTROL 0x08
69  // -- #define UVC_CT_IRIS_ABSOLUTE_CONTROL 0x09
70  // -- #define UVC_CT_IRIS_RELATIVE_CONTROL 0x0a
71  // -- #define UVC_CT_ZOOM_ABSOLUTE_CONTROL 0x0b
72  // -- #define UVC_CT_ZOOM_RELATIVE_CONTROL 0x0c
73  // -- #define UVC_CT_PANTILT_ABSOLUTE_CONTROL 0x0d
74  // -- #define UVC_CT_PANTILT_RELATIVE_CONTROL 0x0e
75  // -- #define UVC_CT_ROLL_ABSOLUTE_CONTROL 0x0f
76  // -- #define UVC_CT_ROLL_RELATIVE_CONTROL 0x10
77  // -- #define UVC_CT_PRIVACY_CONTROL 0x11
78  // note: UVC 1.5 has a few more...
79  //
80  // Windows 10 insists on doing a GET_DEF on this 10-byte control even though we never said we support it:
81  // -- #define UVC_CT_REGION_OF_INTEREST_CONTROL 0x14
82  // This is now handled by sending a default blank reply to all unsupported controls.
83  switch (cs)
84  {
85  case UVC_CT_AE_MODE_CONTROL: return V4L2_CID_EXPOSURE_AUTO;
86  case UVC_CT_AE_PRIORITY_CONTROL: return V4L2_CID_EXPOSURE_AUTO_PRIORITY;
87  case UVC_CT_EXPOSURE_TIME_ABSOLUTE_CONTROL: return V4L2_CID_EXPOSURE_ABSOLUTE;
88  }
89  break;
90 
91  case 2: // Our processing unit
92  // From uvcvideo.h and the UVC specs, here are the available processing unit controls:
93  // A.9.5. Processing Unit Control Selectors
94 
95  // Note one trick here: UVC_PU_WHITE_BALANCE_COMPONENT_CONTROL contains both the V4L2_CID_RED_BALANCE and
96  // V4L2_CID_BLUE_BALANCE; we handle that in the ioctl processing section, here we just return V4L2_CID_RED_BALANCE
97 
98  // OK #define UVC_PU_BACKLIGHT_COMPENSATION_CONTROL 0x01
99  // OK #define UVC_PU_BRIGHTNESS_CONTROL 0x02
100  // OK #define UVC_PU_CONTRAST_CONTROL 0x03
101  // OK #define UVC_PU_GAIN_CONTROL 0x04
102  // OK #define UVC_PU_POWER_LINE_FREQUENCY_CONTROL 0x05
103  // OK #define UVC_PU_HUE_CONTROL 0x06
104  // OK #define UVC_PU_SATURATION_CONTROL 0x07
105  // OK #define UVC_PU_SHARPNESS_CONTROL 0x08
106  // -- #define UVC_PU_GAMMA_CONTROL 0x09
107  // -- #define UVC_PU_WHITE_BALANCE_TEMPERATURE_CONTROL 0x0a
108  // -- #define UVC_PU_WHITE_BALANCE_TEMPERATURE_AUTO_CONTROL 0x0b
109  // OK #define UVC_PU_WHITE_BALANCE_COMPONENT_CONTROL 0x0c
110  // OK #define UVC_PU_WHITE_BALANCE_COMPONENT_AUTO_CONTROL 0x0d
111  // TODO #define UVC_PU_DIGITAL_MULTIPLIER_CONTROL 0x0e
112  // TODO #define UVC_PU_DIGITAL_MULTIPLIER_LIMIT_CONTROL 0x0f
113  // TODO #define UVC_PU_HUE_AUTO_CONTROL 0x10
114  // TODO #define UVC_PU_ANALOG_VIDEO_STANDARD_CONTROL 0x11
115  // TODO #define UVC_PU_ANALOG_LOCK_STATUS_CONTROL 0x12
116  switch (cs)
117  {
118  case UVC_PU_BACKLIGHT_COMPENSATION_CONTROL: return V4L2_CID_BACKLIGHT_COMPENSATION;
119  case UVC_PU_BRIGHTNESS_CONTROL: return V4L2_CID_BRIGHTNESS;
120  case UVC_PU_CONTRAST_CONTROL: return V4L2_CID_CONTRAST;
121  case UVC_PU_GAIN_CONTROL: return V4L2_CID_GAIN;
122  case UVC_PU_POWER_LINE_FREQUENCY_CONTROL: return V4L2_CID_POWER_LINE_FREQUENCY;
123  case UVC_PU_HUE_CONTROL: return V4L2_CID_HUE;
124  case UVC_PU_SATURATION_CONTROL: return V4L2_CID_SATURATION;
125  case UVC_PU_SHARPNESS_CONTROL: return V4L2_CID_SHARPNESS;
126  case UVC_PU_WHITE_BALANCE_COMPONENT_CONTROL: return V4L2_CID_RED_BALANCE;
127  case UVC_PU_WHITE_BALANCE_COMPONENT_AUTO_CONTROL: return V4L2_CID_AUTO_WHITE_BALANCE;
128  //case UVC_PU_GAMMA_CONTROL: return V4L2_CID_GAMMA;
129  }
130  break;
131  }
132 
133  LFATAL("Request to access unsupported control " << cs << " on entity " << entity);
134  }
135 
136 } // namespace
137 
138 // ##############################################################################################################
139 jevois::Gadget::Gadget(std::string const & devname, jevois::VideoInput * camera, jevois::Engine * engine,
140  size_t const nbufs, bool multicam) :
141  itsFd(-1), itsMulticam(multicam), itsNbufs(nbufs), itsBuffers(nullptr), itsCamera(camera), itsEngine(engine),
142  itsRunning(false), itsFormat(), itsFps(0.0F), itsStreaming(false), itsErrorCode(0), itsControl(0), itsEntity(0)
143 {
144  JEVOIS_TRACE(1);
145 
146  if (itsCamera == nullptr) LFATAL("Gadget requires a valid camera to work");
147 
148  jevois::VideoMapping const & m = itsEngine->getDefaultVideoMapping();
149  fillStreamingControl(&itsProbe, m);
150  fillStreamingControl(&itsCommit, m);
151 
152  // Get our run() thread going and wait until it is cranking, it will flip itsRunning to true as it starts:
153  itsRunFuture = jevois::async_little(std::bind(&jevois::Gadget::run, this));
154  while (itsRunning.load() == false) std::this_thread::sleep_for(std::chrono::milliseconds(5));
155 
156  // Open the device:
157  itsFd = open(devname.c_str(), O_RDWR | O_NONBLOCK);
158  if (itsFd == -1) PLFATAL("Gadget device open failed for " << devname);
159 
160  // Get ready to handle UVC events:
161  struct v4l2_event_subscription sub = { };
162 
163  sub.type = UVC_EVENT_SETUP;
164  XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
165 
166  sub.type = UVC_EVENT_DATA;
167  XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
168 
169  sub.type = UVC_EVENT_STREAMON;
170  XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
171 
172  sub.type = UVC_EVENT_STREAMOFF;
173  XIOCTL(itsFd, VIDIOC_SUBSCRIBE_EVENT, &sub);
174 
175  // Find out what the driver can do:
176  struct v4l2_capability cap = { };
177  XIOCTL(itsFd, VIDIOC_QUERYCAP, &cap);
178 
179  LINFO('[' << itsFd << "] UVC gadget " << devname << " card " << cap.card << " bus " << cap.bus_info);
180  if ((cap.capabilities & V4L2_CAP_VIDEO_OUTPUT) == 0) LFATAL(devname << " is not a video output device");
181  if ((cap.capabilities & V4L2_CAP_STREAMING) == 0) LFATAL(devname << " does not support streaming");
182 }
183 
184 // ##############################################################################################################
186 {
187  JEVOIS_TRACE(1);
188 
189  streamOff();
190 
191  // Tell run() thread to finish up:
192  itsRunning.store(false);
193 
194  // Will block until the run() thread completes:
195  if (itsRunFuture.valid()) try { itsRunFuture.get(); } catch (...) { jevois::warnAndIgnoreException(); }
196 
197  if (close(itsFd) == -1) PLERROR("Error closing UVC gadget -- IGNORED");
198 }
199 
200 // ##############################################################################################################
201 void jevois::Gadget::setFormat(jevois::VideoMapping const & m)
202 {
203  JEVOIS_TRACE(2);
204 
205  JEVOIS_TIMED_LOCK(itsMtx);
206 
207  // Set the format:
208  memset(&itsFormat, 0, sizeof(struct v4l2_format));
209 
210  itsFormat.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
211  itsFormat.fmt.pix.width = m.ow;
212  itsFormat.fmt.pix.height = m.oh;
213  itsFormat.fmt.pix.pixelformat = m.ofmt;
214  itsFormat.fmt.pix.field = V4L2_FIELD_NONE;
215  itsFormat.fmt.pix.sizeimage = m.osize();
216  itsFps = m.ofps;
217 
218  // Do not do anything if ofmt is NONE:
219  if (m.ofmt == 0) { LINFO("USB Gadget set video format to NONE"); return; }
220 
221  // First try to set our own format, will throw if phony:
222  XIOCTL(itsFd, VIDIOC_S_FMT, &itsFormat);
223 
224  // Note that the format does not include fps, this is done with VIDIOC_S_PARM:
225  try
226  {
227  // The gadget driver may not support this ioctl...
228  struct v4l2_streamparm sparm = { };
229  sparm.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
230  sparm.parm.output.outputmode = 2; // V4L2_MODE_VIDEO not defined in our headers? its value is 2.
231  sparm.parm.output.timeperframe = jevois::VideoMapping::fpsToV4l2(m.ofps);
232  XIOCTL_QUIET(itsFd, VIDIOC_S_PARM, &sparm);
233  } catch (...) { }
234 
235  LINFO("USB Gadget set video format to " << itsFormat.fmt.pix.width << 'x' << itsFormat.fmt.pix.height << ' ' <<
236  jevois::fccstr(itsFormat.fmt.pix.pixelformat));
237 }
238 
239 // ##############################################################################################################
240 void jevois::Gadget::run()
241 {
242  JEVOIS_TRACE(1);
243 
244  fd_set wfds; // For UVC video streaming
245  fd_set efds; // For UVC events
246  struct timeval tv;
247 
248  // Switch to running state:
249  itsRunning.store(true);
250 
251  // We may have to wait until the device is opened:
252  while (itsFd == -1) std::this_thread::sleep_for(std::chrono::milliseconds(1));
253 
254  // Wait for event from the gadget kernel driver and process them:
255  while (itsRunning.load())
256  {
257  // Wait until we either receive an event or we are ready to send the next buffer over:
258  FD_ZERO(&wfds); FD_ZERO(&efds); FD_SET(itsFd, &wfds); FD_SET(itsFd, &efds);
259  tv.tv_sec = 0; tv.tv_usec = 10000;
260 
261  int ret = select(itsFd + 1, nullptr, &wfds, &efds, &tv);
262 
263  if (ret == -1) { PLERROR("Select error"); if (errno == EINTR) continue; else break; }
264  else if (ret > 0) // We have some events, handle them right away:
265  {
266  // Note: we may have more than one event, so here we try processEvents() several times to be sure:
267  if (FD_ISSET(itsFd, &efds))
268  {
269  // First event, we will report error if any:
270  try { processEvents(); } catch (...) { jevois::warnAndIgnoreException(); }
271 
272  // Let's try to dequeue one more, in most cases it should throw:
273  while (true) try { processEvents(); } catch (...) { break; }
274  }
275 
276  if (FD_ISSET(itsFd, &wfds)) try { processVideo(); } catch (...) { jevois::warnAndIgnoreException(); }
277  }
278 
279  // We timed out
280 
281  // Sometimes we miss events in the main loop, likely because more events come while we are unlocked in the USB UDC
282  // driver and processing here. So let's try to dequeue one more, in most cases it should throw:
283  while (true) try { processEvents(); } catch (...) { break; }
284 
285  // While the driver is not busy in select(), queue at most one buffer that is ready to send off:
286  try
287  {
288  JEVOIS_TIMED_LOCK(itsMtx);
289  if (itsDoneImgs.size())
290  {
291  LDEBUG("Queuing image " << itsDoneImgs.front() << " for sending over USB");
292 
293  // We need to prepare a legit v4l2_buffer, including bytesused:
294  struct v4l2_buffer buf = { };
295 
296  buf.type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
297  buf.memory = V4L2_MEMORY_MMAP;
298  buf.index = itsDoneImgs.front();
299  buf.length = itsBuffers->get(buf.index)->length();
300 
301  if (itsFormat.fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
302  buf.bytesused = itsBuffers->get(buf.index)->bytesUsed();
303  else
304  buf.bytesused = buf.length;
305 
306  buf.field = V4L2_FIELD_NONE;
307  buf.flags = 0;
308  gettimeofday(&buf.timestamp, nullptr);
309 
310  // Queue it up so it can be sent to the host:
311  itsBuffers->qbuf(buf);
312 
313  // This one is done:
314  itsDoneImgs.pop_front();
315  }
316  } catch (...) { jevois::warnAndIgnoreException(); std::this_thread::sleep_for(std::chrono::milliseconds(10)); }
317  }
318 
319  // Switch out of running state in case we did interrupt the loop here by a break statement:
320  itsRunning.store(false);
321 }
322 
323 // ##############################################################################################################
324 void jevois::Gadget::processEvents()
325 {
326  JEVOIS_TRACE(3);
327 
328  // Get the event from the driver:
329  struct v4l2_event v4l2ev = { };
330  XIOCTL_QUIET(itsFd, VIDIOC_DQEVENT, &v4l2ev);
331  struct uvc_event * uvcev = reinterpret_cast<struct uvc_event *>(&v4l2ev.u.data);
332 
333  // Prepare our response, if any will be sent:
334  struct uvc_request_data resp = { };
335  resp.length = -EL2HLT;
336 
337  // Act according to the event type:
338  try
339  {
340  switch (v4l2ev.type)
341  {
342  case UVC_EVENT_CONNECT: return;
343  case UVC_EVENT_DISCONNECT: LDEBUG("EVENT DISCONNECT"); itsEngine->streamOff(); return;
344  case UVC_EVENT_SETUP: LDEBUG("EVENT SETUP"); processEventSetup(uvcev->req, resp); return;
345  case UVC_EVENT_DATA: LDEBUG("EVENT DATA"); processEventData(uvcev->data); return;
346  case UVC_EVENT_STREAMON: LDEBUG("EVENT STREAMON"); itsEngine->streamOn(); return;
347  case UVC_EVENT_STREAMOFF: LDEBUG("EVENT STREAMOFF"); itsEngine->streamOff(); return;
348  }
349  } catch (...) { }
350 }
351 
352 // ##############################################################################################################
353 void jevois::Gadget::processVideo()
354 {
355  JEVOIS_TRACE(3);
356 
357  jevois::RawImage img;
358  JEVOIS_TIMED_LOCK(itsMtx);
359 
360  // If we are not streaming anymore, abort:
361  if (itsStreaming.load() == false) LFATAL("Aborted while not streaming");
362 
363  // Dequeue a buffer from the gadget driver, this is an image that has been sent to the host and hence the buffer is
364  // now available to be filled up with image data and later queued again to the gadget driver:
365  struct v4l2_buffer buf;
366  itsBuffers->dqbuf(buf);
367 
368  // Create a RawImage from that buffer:
369  img.width = itsFormat.fmt.pix.width;
370  img.height = itsFormat.fmt.pix.height;
371  img.fmt = itsFormat.fmt.pix.pixelformat;
372  img.fps = itsFps;
373  img.buf = itsBuffers->get(buf.index);
374  img.bufindex = buf.index;
375 
376  // Push the RawImage to outside consumers:
377  itsImageQueue.push_back(img);
378  LDEBUG("Empty image " << img.bufindex << " ready for filling in by application code");
379 }
380 
381 // ##############################################################################################################
382 void jevois::Gadget::processEventSetup(struct usb_ctrlrequest const & ctrl, struct uvc_request_data & resp)
383 {
384  JEVOIS_TRACE(3);
385 
386  itsControl = 0; itsEntity = 0;
387 
388  debugCtrlReq(ctrl);
389 
390  switch (ctrl.bRequestType & USB_TYPE_MASK)
391  {
392  case USB_TYPE_STANDARD: processEventStandard(ctrl, resp); break;
393  case USB_TYPE_CLASS: processEventClass(ctrl, resp); break;
394  default: LERROR("Unsupported setup event type " << std::showbase << std::hex <<
395  (ctrl.bRequestType & USB_TYPE_MASK) << " -- IGNORED");
396  }
397  if (ctrl.bRequestType != 0x21) XIOCTL(itsFd, UVCIOC_SEND_RESPONSE, &resp);
398 }
399 
400 // ##############################################################################################################
401 void jevois::Gadget::processEventStandard(struct usb_ctrlrequest const & ctrl,
402  struct uvc_request_data & JEVOIS_UNUSED_PARAM(resp))
403 {
404  JEVOIS_TRACE(3);
405 
406  LDEBUG("UVC standard setup event ignored:");
407  debugCtrlReq(ctrl);
408 }
409 
410 // ##############################################################################################################
411 void jevois::Gadget::processEventClass(struct usb_ctrlrequest const & ctrl, struct uvc_request_data & resp)
412 {
413  JEVOIS_TRACE(3);
414 
415  if ((ctrl.bRequestType & USB_RECIP_MASK) != USB_RECIP_INTERFACE) return;
416 
417  switch (ctrl.wIndex & 0xff)
418  {
419  case UVC_INTF_CONTROL:
420  processEventControl(ctrl.bRequest, ctrl.wValue >> 8, ctrl.wIndex >> 8, ctrl.wLength, resp);
421  break;
422 
423  case UVC_INTF_STREAMING:
424  processEventStreaming(ctrl.bRequest, ctrl.wValue >> 8, resp);
425  break;
426 
427  default:
428  LERROR("Unsupported setup event class " << std::showbase << std::hex << (ctrl.wIndex & 0xff) << " -- IGNORED");
429  }
430 }
431 
432 // ##############################################################################################################
433 void jevois::Gadget::processEventControl(uint8_t req, uint8_t cs, uint8_t entity_id, uint8_t len,
434  struct uvc_request_data & resp)
435 {
436  JEVOIS_TRACE(3);
437 
438  // Local function we run on successful processing of an event: we just reset our internal error code
439 #define success() { itsErrorCode = 0; }
440 
441  // Local function we run on failed processing of an event: stall the request, set our internal error code
442 #define failure(code) { resp.length = -EL2HLT; itsErrorCode = code; }
443 
444  // Shortcurt to successfully send a 1-byte response:
445 #define byteresponse(val) { resp.data[0] = val; resp.length = 1; itsErrorCode = 0; }
446 
447  // Shortcurt to successfully send a 2-byte response:
448 #define wordresponse(val) { resp.data[0] = val & 0xff; resp.data[1] = (val >> 8) & 0xff; \
449  resp.length = 2; itsErrorCode = 0; }
450 
451  // Shortcurt to successfully send a 4-byte response:
452 #define intresponse(val) { resp.data[0] = val & 0xff; resp.data[1] = (val >> 8) & 0xff; \
453  resp.data[2] = (val >> 16) & 0xff; resp.data[3] = (val >> 24) & 0xff; \
454  resp.length = 4; itsErrorCode = 0; }
455 
456  // Shortcurt to successfully send a blank N-byte response:
457 #define arrayblankresponse(len) { memset(resp.data, 0, len); resp.length = len; itsErrorCode = 0; }
458 
459  // If anything throws here, we will return failure:
460  try
461  {
462  // First handle any request that is directed to entity 0:
463  if (entity_id == 0)
464  {
465  switch (cs)
466  {
467  case UVC_VC_REQUEST_ERROR_CODE_CONTROL: byteresponse(itsErrorCode); return; // Send error code last prepared
468  default: failure(0x06); return;
469  }
470  }
471 
472  // Process according to the entity that this event is directed to and the control that is requested:
473  if (req == UVC_SET_CUR)
474  {
475  // We need to wait for the data phase, so for now just remember the control and return success:
476  itsEntity = entity_id; itsControl = cs;
477  resp.data[0] = 0x0; resp.length = len; success();
478  LDEBUG("SET_CUR ent " << itsEntity <<" ctrl "<< itsControl <<" len "<< len);
479  }
480  else if (req == UVC_GET_INFO)
481  {
482  // FIXME: controls could also be disabled, autoupdate, or asynchronous:
483  byteresponse(UVC_CONTROL_CAP_GET | UVC_CONTROL_CAP_SET);
484  }
485  else if (req == UVC_GET_CUR)
486  {
487  // Fetch the current value from the camera. Note: both Windows and Android insist on querying some controls, like
488  // IRIS and GAMMA, which we did not declare as supported in the kernel driver. We need to handle those requests
489  // and to send some phony data:
490  struct v4l2_control ctrl = { };
491  try { ctrl.id = uvcToV4Lcontrol(entity_id, cs); itsCamera->getControl(ctrl); } catch (...) { ctrl.id = 0; }
492 
493  // We need a special handling of white balance here:
494  if (ctrl.id == V4L2_CID_RED_BALANCE)
495  {
496  unsigned int redval = (ctrl.value & 0xffff) << 16; // red is at offset 2 in PU_WHITE_BALANCE_COMPONENT_CONTROL
497 
498  // Also get the blue balance value:
499  ctrl.id = V4L2_CID_BLUE_BALANCE;
500  itsCamera->getControl(ctrl);
501 
502  // Combine both red and blue values:
503  ctrl.value = (ctrl.value & 0xffff) | redval;
504  }
505  // We also need to remap auto exposure values:
506  else if (ctrl.id == V4L2_CID_EXPOSURE_AUTO)
507  {
508  if (ctrl.value == V4L2_EXPOSURE_MANUAL) ctrl.value = 0x01; // manual mode, set UVC bit D0
509  else if (ctrl.value == V4L2_EXPOSURE_AUTO) ctrl.value = 0x02; // auto mode, set UVC bit D1
510  else ctrl.value = 0x03;
511  // Note, there are 2 more bits under CT_AE_MODE_CONTROL
512  }
513  // Handle the unknown controls:
514  else if (ctrl.id == 0) ctrl.value = 0;
515 
516  switch (len)
517  {
518  case 1: byteresponse(ctrl.value); break;
519  case 2: wordresponse(ctrl.value); break;
520  case 4: intresponse(ctrl.value); break;
521  default: LERROR("Unsupported control with length " << len << " -- SENDING BLANK RESPONSE");
522  arrayblankresponse(len); break;
523  }
524  }
525  else
526  {
527  // It's a GET_DEF/RES/MIN/MAX let's first get the data from the camera: Note: both Windows and Android insist on
528  // querying some controls, like IRIS and GAMMA, which we did not declare as supported in the kernel driver. We
529  // need to handle those requests and to send some phony data:
530  struct v4l2_queryctrl qc = { };
531  try { qc.id = uvcToV4Lcontrol(entity_id, cs); itsCamera->queryControl(qc); } catch (...) { qc.id = 0; }
532 
533  // We need a special handling of white balance here:
534  if (qc.id == V4L2_CID_RED_BALANCE)
535  {
536  // Also get the blue balance values:
537  struct v4l2_queryctrl qc2 = { };
538  qc2.id = V4L2_CID_BLUE_BALANCE;
539  itsCamera->queryControl(qc2);
540 
541  // Combine red and blue values into qc:
542  qc.default_value = (qc.default_value << 16) | qc2.default_value;
543  qc.step = (qc.step << 16) | qc2.step;
544  qc.minimum = (qc.minimum << 16) | qc2.minimum;
545  qc.maximum = (qc.maximum << 16) | qc2.maximum;
546  }
547  // We also need to remap auto exposure values:
548  else if (qc.id == V4L2_CID_EXPOSURE_AUTO)
549  {
550  // Tricky: in the 'step' field, we are supposed to provide a bitmap of the modes that are supported, see UVC
551  // specs. D0=manual, D1=auto, D2=shutter priority, D3=aperture priority. Min and max are ignored for this
552  // control, default is handled.
553  qc.minimum = 0; qc.step = 3; qc.maximum = 3; qc.default_value = 1;
554  }
555  // Also handle the unknown controls here:
556  else if (qc.id == 0)
557  { qc.minimum = 0; qc.step = 1; qc.maximum = 1; qc.default_value = 0; }
558 
559  int val = 0;
560  switch (req)
561  {
562  case UVC_GET_DEF: val = qc.default_value; break;
563  case UVC_GET_RES: val = qc.step; break;
564  case UVC_GET_MIN: val = qc.minimum; break;
565  case UVC_GET_MAX: val = qc.maximum; break;
566  default: failure(0x07); return;
567  }
568 
569  switch (len)
570  {
571  case 1: byteresponse(val); break;
572  case 2: wordresponse(val); break;
573  case 4: intresponse(val); break;
574  default: LERROR("Unsupported control with length " << len << " -- SENDING BLANK RESPONSE");
575  arrayblankresponse(len); break;
576  }
577  }
578  }
579  catch (...)
580  {
581  LERROR("FAILED entity " << entity_id << " cs " << cs << " len " << len);
582  failure(0x06);
583  }
584 }
585 
586 // ##############################################################################################################
587 void jevois::Gadget::fillStreamingControl(struct uvc_streaming_control * ctrl, jevois::VideoMapping const & m)
588 {
589  JEVOIS_TRACE(3);
590 
591  memset(ctrl, 0, sizeof(struct uvc_streaming_control));
592 
593  ctrl->bFormatIndex = m.uvcformat;
594  ctrl->bFrameIndex = m.uvcframe;
595  ctrl->dwFrameInterval = jevois::VideoMapping::fpsToUvc(m.ofps);
596  ctrl->dwMaxVideoFrameSize = m.osize();
597  ctrl->dwMaxPayloadTransferSize = itsMulticam ? 1024 : 3072;
598  ctrl->bmFramingInfo = 3;
599  ctrl->bPreferedVersion = 1;
600  ctrl->bMaxVersion = 1;
601 }
602 
603 // ##############################################################################################################
604 void jevois::Gadget::processEventStreaming(uint8_t req, uint8_t cs, struct uvc_request_data & resp)
605 {
606  JEVOIS_TRACE(3);
607 
608  int const datalen = 26; // uvc 1.0 as reported by our kernel driver
609  if (cs != UVC_VS_PROBE_CONTROL && cs != UVC_VS_COMMIT_CONTROL) return;
610 
611  struct uvc_streaming_control * ctrl = reinterpret_cast<struct uvc_streaming_control *>(&resp.data);
612  struct uvc_streaming_control * target = (cs == UVC_VS_PROBE_CONTROL) ? &itsProbe : &itsCommit;
613  resp.length = datalen;
614 
615  switch (req)
616  {
617  case UVC_SET_CUR: itsControl = cs; resp.length = datalen; break; // will finish up in data stage
618 
619  case UVC_GET_CUR:
620  case UVC_GET_MIN: // we have nothing to negotiate
621  case UVC_GET_MAX: // we have nothing to negotiate
622  memcpy(ctrl, target, datalen);
623  break;
624 
625  case UVC_GET_DEF:
626  {
627  // If requested format index, frame index, or interval is bogus (including zero), initialize to our default mapping,
628  // otherwise pass down the selected mapping:
629  size_t idx = itsEngine->getDefaultVideoMappingIdx();
630  try { idx = itsEngine->getVideoMappingIdx(ctrl->bFormatIndex, ctrl->bFrameIndex, ctrl->dwFrameInterval); }
631  catch (...) { }
632  fillStreamingControl(target, itsEngine->getVideoMapping(idx));
633  memcpy(ctrl, target, datalen);
634  }
635  break;
636 
637  case UVC_GET_RES: memset(ctrl, 0, datalen); break;
638 
639  case UVC_GET_LEN: resp.data[0] = 0x00; resp.data[1] = datalen; resp.length = 2; break;
640 
641  case UVC_GET_INFO: resp.data[0] = 0x03; resp.length = 1; break;
642  }
643 }
644 
645 // ##############################################################################################################
646 void jevois::Gadget::processEventData(struct uvc_request_data & data)
647 {
648  JEVOIS_TRACE(3);
649 
650  struct uvc_streaming_control * target;
651 
652  // If entity is 1 or 2, this is to set a control:
653  if (itsEntity == 2 || itsEntity == 1) { processEventControlData(data); return; }
654 
655  switch (itsControl)
656  {
657  case UVC_VS_PROBE_CONTROL: target = &itsProbe; break;
658  case UVC_VS_COMMIT_CONTROL: target = &itsCommit; break;
659  default: processEventControlData(data); return;
660  }
661 
662  // Find the selected format and frame info and fill-in the control data:
663  struct uvc_streaming_control * ctrl = reinterpret_cast<struct uvc_streaming_control *>(&data.data);
664 
665  size_t idx = itsEngine->getVideoMappingIdx(ctrl->bFormatIndex, ctrl->bFrameIndex, ctrl->dwFrameInterval);
666 
667  fillStreamingControl(target, itsEngine->getVideoMapping(idx));
668 
669  LDEBUG("Host requested " << ctrl->bFormatIndex << '/' << ctrl->bFrameIndex << '/' << ctrl->dwFrameInterval <<
670  ", " << ((itsControl == UVC_VS_COMMIT_CONTROL) ? "setting " : "returning ") <<
671  itsEngine->getVideoMapping(idx).str());
672 
673  // Set the format if we are doing a commit control:
674  if (itsControl == UVC_VS_COMMIT_CONTROL) itsEngine->setFormat(idx);
675 }
676 
677 // ##############################################################################################################
678 void jevois::Gadget::processEventControlData(struct uvc_request_data & data)
679 {
680  JEVOIS_TRACE(3);
681 
682  struct v4l2_control ctrl;
683 
684  // Get the control ID for V4L or throw if unsupported:
685  ctrl.id = uvcToV4Lcontrol(itsEntity, itsControl);
686 
687  // Copy the data we received into the control's value:
688  switch (data.length)
689  {
690  case 1: ctrl.value = static_cast<int>(data.data[0]); break;
691  case 2: ctrl.value = static_cast<int>(__s16(data.data[0] | (static_cast<short>(data.data[1]) << 8))); break;
692  case 4: ctrl.value = data.data[0] | (data.data[1] << 8) | (data.data[2] << 16) | (data.data[3] << 24); break;
693  default: LFATAL("Ooops data len is " << data.length);
694  }
695 
696  // Tell the camera to set the control. We do not have enough time here to do it as otherwise our USB transaction will
697  // time out while we transfer a bunch of bytes to the camera over the 400kHz serial control link, so we just push it
698  // to a queue and our run() thread will do the work. First, handle special cases:
699  switch (ctrl.id)
700  {
701  case V4L2_CID_RED_BALANCE:
702  {
703  // We need to set both the red and the blue:
704  int blue = ctrl.value & 0xffff;
705  ctrl.value >>= 16; itsCamera->setControl(ctrl);
706  ctrl.id = V4L2_CID_BLUE_BALANCE; ctrl.value = blue; itsCamera->setControl(ctrl);
707  }
708  break;
709 
710  case V4L2_CID_EXPOSURE_AUTO:
711  if (ctrl.value & 0x01) ctrl.value = V4L2_EXPOSURE_MANUAL; // UVC bit D0 set for manual mode
712  else if (ctrl.value & 0x02) ctrl.value = V4L2_EXPOSURE_AUTO; // auto mode
713  // Note, there are 2 more bits under CT_AE_MODE_CONTROL
714  itsCamera->setControl(ctrl);
715  break;
716 
717  default: itsCamera->setControl(ctrl);
718  }
719 }
720 
721 // ##############################################################################################################
723 {
724  JEVOIS_TRACE(2);
725 
726  LDEBUG("Turning on UVC stream");
727 
728  JEVOIS_TIMED_LOCK(itsMtx);
729 
730  if (itsStreaming.load() || itsBuffers) { LERROR("Stream is already on -- IGNORED"); return; }
731  if (itsFormat.fmt.pix.pixelformat == 0) { LINFO("Gadget output format is NONE"); return; }
732 
733  // If number of buffers is zero, adjust it depending on frame size:
734  unsigned int nbuf = itsNbufs;
735  if (nbuf == 0)
736  {
737  unsigned int framesize = jevois::v4l2ImageSize(itsFormat.fmt.pix.pixelformat, itsFormat.fmt.pix.width,
738  itsFormat.fmt.pix.height);
739 
740  // Aim for about 4 mbyte when using small images, and no more than 4 buffers in any case:
741  nbuf = (4U * 1024U * 1024U) / framesize;
742  if (nbuf > 4) nbuf = 4;
743  }
744 
745  // Force number of buffers to a sane value:
746  if (nbuf < 3) nbuf = 3; else if (nbuf > 16) nbuf = 16;
747 
748  // Allocate our buffers for the currently selected resolution, format, etc:
749  itsBuffers = new jevois::VideoBuffers("gadget", itsFd, V4L2_BUF_TYPE_VIDEO_OUTPUT, nbuf);
750  LINFO(itsBuffers->size() << " buffers of " << itsBuffers->get(0)->length() << " bytes allocated");
751 
752  // Fill itsImageQueue with blank frames that can be given off to application code:
753  for (size_t i = 0; i < nbuf; ++i)
754  {
755  jevois::RawImage img;
756  img.width = itsFormat.fmt.pix.width;
757  img.height = itsFormat.fmt.pix.height;
758  img.fmt = itsFormat.fmt.pix.pixelformat;
759  img.fps = itsFps;
760  img.buf = itsBuffers->get(i);
761  img.bufindex = i;
762 
763  // Push the RawImage to outside consumers:
764  itsImageQueue.push_back(img);
765  LDEBUG("Empty image " << img.bufindex << " ready for filling in by application code");
766  }
767 
768  // Start streaming over the USB link:
769  int type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
770  XIOCTL(itsFd, VIDIOC_STREAMON, &type);
771  LDEBUG("Device stream on");
772 
773  itsStreaming.store(true);
774  LDEBUG("Stream is on");
775 }
776 
777 // ##############################################################################################################
779 {
780  JEVOIS_TRACE(2);
781 
782  itsStreaming.store(false);
783 }
784 
785 // ##############################################################################################################
787 {
788  JEVOIS_TRACE(2);
789 
790  // Note: we allow for several streamOff() without complaining, this happens, e.g., when destroying a Gadget that is
791  // not currently streaming.
792 
793  LDEBUG("Turning off gadget stream");
794 
795  // Abort stream in case it was not already done, which will introduce some sleeping in our run() thread, thereby
796  // helping us acquire our needed double lock:
797  abortStream();
798 
799  JEVOIS_TIMED_LOCK(itsMtx);
800 
801  // Stop streaming over the USB link:
802  int type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
803  try { XIOCTL_QUIET(itsFd, VIDIOC_STREAMOFF, &type); } catch (...) { }
804 
805  // Nuke all our buffers:
806  if (itsBuffers) { delete itsBuffers; itsBuffers = nullptr; }
807  itsImageQueue.clear();
808  itsDoneImgs.clear();
809 
810  LDEBUG("Gadget stream is off");
811 }
812 
813 // ##############################################################################################################
815 {
816  JEVOIS_TRACE(4);
817  int retry = 2000;
818 
819  while (--retry >= 0)
820  {
821  if (itsStreaming.load() == false)
822  { LDEBUG("Not streaming"); throw std::runtime_error("Gadget get() rejected while not streaming"); }
823 
824  if (itsMtx.try_lock_for(std::chrono::milliseconds(100)))
825  {
826  if (itsStreaming.load() == false)
827  {
828  LDEBUG("Not streaming");
829  itsMtx.unlock();
830  throw std::runtime_error("Gadget get() rejected while not streaming");
831  }
832 
833  if (itsImageQueue.size())
834  {
835  img = itsImageQueue.front();
836  itsImageQueue.pop_front();
837  itsMtx.unlock();
838  LDEBUG("Empty image " << img.bufindex << " handed over to application code for filling");
839  return;
840  }
841 
842  // No image in the queue, unlock and wait for one:
843  itsMtx.unlock();
844  LDEBUG("Waiting for blank UVC image...");
845  std::this_thread::sleep_for(std::chrono::milliseconds(5));
846  }
847  else
848  {
849  LDEBUG("Waiting for lock");
850  std::this_thread::sleep_for(std::chrono::milliseconds(5));
851  }
852  }
853  LFATAL("Giving up waiting for blank UVC image");
854 }
855 
856 // ##############################################################################################################
858 {
859  JEVOIS_TRACE(4);
860  int retry = 2000;
861 
862  while (--retry >= 0)
863  {
864  if (itsStreaming.load() == false)
865  { LDEBUG("Not streaming"); throw std::runtime_error("Gadget send() rejected while not streaming"); }
866 
867  if (itsMtx.try_lock_for(std::chrono::milliseconds(100)))
868  {
869  if (itsStreaming.load() == false)
870  {
871  LDEBUG("Not streaming");
872  itsMtx.unlock();
873  throw std::runtime_error("Gadget send() rejected while not streaming");
874  }
875 
876  // Check that the format matches, this may not be the case if we changed format while the buffer was out for
877  // processing. IF so, we just drop this image since it cannot be sent to the host anymore:
878  if (img.width != itsFormat.fmt.pix.width ||
879  img.height != itsFormat.fmt.pix.height ||
880  img.fmt != itsFormat.fmt.pix.pixelformat)
881  {
882  LDEBUG("Dropping image to send out as format just changed");
883  itsMtx.unlock();
884  return;
885  }
886 
887  // We cannot just qbuf() here as our run() thread is likely in select() and the driver will bomb the qbuf as
888  // resource unavailable. So we just enqueue the buffer index and the run() thread will handle the qbuf later:
889  itsDoneImgs.push_back(img.bufindex);
890  itsMtx.unlock();
891  LDEBUG("Filled image " << img.bufindex << " received from application code");
892  return;
893  }
894  else
895  {
896  LDEBUG("Waiting for lock");
897  std::this_thread::sleep_for(std::chrono::milliseconds(5));
898  }
899  }
900  LFATAL("Giving up waiting for lock");
901 }
902 
UVC_EVENT_DISCONNECT
#define UVC_EVENT_DISCONNECT
Definition: uvc.h:27
intresponse
#define intresponse(val)
Async.H
LDEBUG
#define LDEBUG(msg)
Convenience macro for users to print out console or syslog messages, DEBUG level.
Definition: Log.H:160
uvc_request_data::data
__u8 data[UVC_MAX_REQUEST_SIZE - sizeof(__s32)]
Definition: uvc.h:39
JEVOIS_TIMED_LOCK
#define JEVOIS_TIMED_LOCK(mtx)
Helper macro to create a timed_lock_guard object.
Definition: Log.H:304
jevois::VideoInput
Base class for video input, which will get derived into Camera and MovieInput.
Definition: VideoInput.H:31
UVC_INTF_STREAMING
#define UVC_INTF_STREAMING
Definition: uvc.h:54
uvc_event::req
struct usb_ctrlrequest req
Definition: uvc.h:46
uvc_request_data::length
__s32 length
Definition: uvc.h:38
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::RawImage::bufindex
size_t bufindex
The index of the data buffer in the kernel driver.
Definition: RawImage.H:150
jevois::RawImage::fps
float fps
Programmed frames/s as given by current video mapping, may not be actual.
Definition: RawImage.H:148
UVCIOC_SEND_RESPONSE
#define UVCIOC_SEND_RESPONSE
Definition: uvc.h:51
jevois::Gadget::setFormat
void setFormat(jevois::VideoMapping const &m) override
Set the video format and frame rate.
Definition: Gadget.C:201
UVC_EVENT_STREAMOFF
#define UVC_EVENT_STREAMOFF
Definition: uvc.h:29
JEVOIS_TRACE
#define JEVOIS_TRACE(level)
Trace object.
Definition: Log.H:272
wordresponse
#define wordresponse(val)
uvc_event::data
struct uvc_request_data data
Definition: uvc.h:47
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:198
jevois::Engine::getDefaultVideoMapping
const VideoMapping & getDefaultVideoMapping() const
Allow access to the default video mapping.
Definition: Engine.C:1277
jevois::async_little
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async_little(Function &&f, Args &&... args)
Async execution using a thread pool.
VideoInput.H
jevois::Gadget::streamOff
void streamOff() override
Stop streaming.
Definition: Gadget.C:786
jevois::RawImage::width
unsigned int width
Image width in pixels.
Definition: RawImage.H:145
UVC_EVENT_CONNECT
#define UVC_EVENT_CONNECT
Definition: uvc.h:26
PLFATAL
#define PLFATAL(msg)
Like LDEBUG but appends errno and strerror(errno), to be used when some system call fails.
Definition: Log.H:226
F
float F
Definition: GUIhelper.C:2150
Engine.H
Log.H
success
#define success()
arrayblankresponse
#define arrayblankresponse(len)
jevois::Gadget::abortStream
void abortStream() override
Abort streaming.
Definition: Gadget.C:778
jevois::fccstr
std::string fccstr(unsigned int fcc)
Convert a V4L2 four-cc code (V4L2_PIX_FMT_...) to a 4-char string.
Definition: Utils.C:44
jevois::Engine
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:381
jevois::warnAndIgnoreException
std::string warnAndIgnoreException(std::string const &prefix="")
Convenience function to catch an exception, issue some LERROR (depending on type),...
Definition: Log.C:224
byteresponse
#define byteresponse(val)
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
UVC_EVENT_SETUP
#define UVC_EVENT_SETUP
Definition: uvc.h:30
VideoBuffers.H
jevois::RawImage::height
unsigned int height
Image height in pixels.
Definition: RawImage.H:146
jevois::Gadget::streamOn
void streamOn() override
Start streaming.
Definition: Gadget.C:722
UVC_EVENT_STREAMON
#define UVC_EVENT_STREAMON
Definition: uvc.h:28
jevois::Gadget::~Gadget
virtual ~Gadget()
Close the device and free all resources.
Definition: Gadget.C:185
uvc_event
Definition: uvc.h:42
jevois::RawImage::fmt
unsigned int fmt
Pixel format as a V4L2_PIX_FMT_XXX.
Definition: RawImage.H:147
Utils.H
jevois::v4l2ImageSize
unsigned int v4l2ImageSize(unsigned int fcc, unsigned int width, unsigned int height)
Return the image size in bytes for a given V4L2_PIX_FMT_..., width, height.
Definition: Utils.C:161
jevois::Gadget::send
void send(RawImage const &img) override
Send an image out over USB to the host computer.
Definition: Gadget.C:857
PLERROR
#define PLERROR(msg)
Like LERROR but appends errno and strerror(errno), to be used when some system call fails.
Definition: Log.H:206
UVC_INTF_CONTROL
#define UVC_INTF_CONTROL
Definition: uvc.h:53
jevois::RawImage::buf
std::shared_ptr< VideoBuf > buf
The pixel data buffer.
Definition: RawImage.H:149
jevois::Gadget::Gadget
Gadget(std::string const &devname, VideoInput *camera, Engine *engine, size_t const nbufs=0, bool multicam=false)
Construct and open the device.
Definition: Gadget.C:139
UVC_EVENT_DATA
#define UVC_EVENT_DATA
Definition: uvc.h:31
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:181
jevois::VideoBuffers
Collection of buffers for V4L2 video frames (Camera or Gadget) with hooks to the MMAP'd areas.
Definition: VideoBuffers.H:42
jevois::Gadget::get
void get(RawImage &img) override
Get a pre-allocated image so that we can fill the pixel data and later send out over USB using send()
Definition: Gadget.C:814
uvc_request_data
Definition: uvc.h:36
Gadget.H
failure
#define failure(code)