JeVois  1.8
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See and for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - - -
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
18 #pragma once
22 #include <jevois/Types/Enum.H>
23 #include <jevois/Image/RawImage.H>
25 #include <memory>
26 #include <mutex>
27 #include <vector>
28 #include <list>
29 #include <atomic>
32 // On the platform (JeVois hardware), we use a gadget driver by default to send output frames over USB, one hardware
33 // serial driver, and one serial-over-USB driver:
35 //! On platform hardware, device for the camera sensor
36 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
38 //! On platform hardware, device for the USB gadget driver (which sends video frames over USB to a host computer)
39 #define JEVOIS_GADGET_DEFAULT "/dev/video1"
41 //! On platform hardware, device for the 4-pin hardware serial port
42 #define JEVOIS_SERIAL_DEFAULT "/dev/ttyS0"
44 //! On platform hardware, device for serial-over-USB port
45 #define JEVOIS_USBSERIAL_DEFAULT "/dev/ttyGS0"
47 #else
48 // On the host, we have no gadget (which will trigger displaying output frames to a window) and we use the terminal in
49 // which jevois-daemon was started for serial commands:
51 //! On generic computer hardware, device for the camera sensor
52 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
54 //! On generic computer hardware, device for the USB gadget driver should always be empty
57 //! On generic computer hardware, device for serial port should always be stdio to use an StdioInterface
58 #define JEVOIS_SERIAL_DEFAULT "stdio"
60 //! On generic computer hardware, device for the serial-over-USB port should always be empty
63 #endif
65 namespace jevois
66 {
67  class VideoInput;
68  class VideoOutput;
69  class Module;
70  class DynamicLoader;
71  class UserInterface;
73  namespace engine
74  {
75  static ParameterCategory const ParamCateg("Engine Options");
77  //! Parameter \relates jevois::Engine
78  JEVOIS_DECLARE_PARAMETER(cameradev, std::string, "Camera device name (if starting with /dev/v...), or movie "
79  "file name (e.g., movie.mpg) or image sequence (e.g., im%02d.jpg, to read frames "
80  "im00.jpg, im01.jpg, etc).",
83  //! Parameter \relates jevois::Engine
84  JEVOIS_DECLARE_PARAMETER(cameranbuf, unsigned int, "Number of video input (camera) buffers, or 0 for automatic.",
85  0, ParamCateg);
87  //! Parameter \relates jevois::Engine
88  JEVOIS_DECLARE_PARAMETER(gadgetdev, std::string, "Gadget device name. This is used on platform hardware only. "
89  "On host hardware, a display window will be used unless gadgetdev is None (useful "
90  "for benchmarking) or is a file stem for a movie file that does not start with /dev/ "
91  "(and which should contain a printf-style directive for a single int argument, "
92  "the movie number).",
95  //! Parameter \relates jevois::Engine
96  JEVOIS_DECLARE_PARAMETER(gadgetnbuf, unsigned int, "Number of video output (USB video) buffers, or 0 for auto",
97  0, ParamCateg);
99  //! Parameter \relates jevois::Engine
100  JEVOIS_DECLARE_PARAMETER(videomapping, int, "Index of Video Mapping to use, or -1 to use the default mapping",
101  -1, ParamCateg);
103  //! Parameter \relates jevois::Engine
104  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(serialdev, std::string, "Hardware (4-pin connector) serial device name, "
105  "or 'stdio' to use the console, or empty for no hardware serial port",
108  //! Parameter \relates jevois::Engine
109  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(usbserialdev, std::string, "Over-the-USB serial device name, or empty",
112  //! Parameter \relates jevois::Engine
113  JEVOIS_DECLARE_PARAMETER(camreg, bool, "Enable raw access to camera registers through setcamreg and getcamreg",
114  false, ParamCateg);
116  //! Parameter \relates jevois::Engine
117  JEVOIS_DECLARE_PARAMETER(camturbo, bool, "Enable camera turbo mode by relaxing the need for DMA-coherent video "
118  "buffer memory. This can accelerate severalfolds access to the captured image data, but "
119  "it may also yield stripe artifacts with some modules, such as PassThrough. The stripes "
120  "are pieces of incorrect data in the cache. You should experiment with each particular "
121  "module. Turbo mode is not recommended for any production-grade application.",
122  false, ParamCateg);
124  //! Enum for Parameter \relates jevois::Engine
125  JEVOIS_DEFINE_ENUM_CLASS(SerPort, (None) (All) (Hard) (USB) );
127  //! Parameter \relates jevois::Engine
128  JEVOIS_DECLARE_PARAMETER(serlog, SerPort, "Show log and debug messages on selected serial port(s)",
129  SerPort::None, SerPort_Values, ParamCateg);
131  //! Parameter \relates jevois::Engine
132  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(videoerrors, bool, "Show any machine vision module errors (exceptions) "
133  "in the video stream. Only takes effect if streaming video to USB.",
134  true, ParamCateg);
136  //! Parameter \relates jevois::Engine
137  JEVOIS_DECLARE_PARAMETER(serout, SerPort, "Send module serial messages to selected serial port(s)",
138  SerPort::None, SerPort_Values, ParamCateg);
140  //! Enum for Parameter \relates jevois::Engine
141  JEVOIS_DEFINE_ENUM_CLASS(CPUmode, (PowerSave) (Conservative) (OnDemand) (Interactive) (Performance) );
143  //! Parameter \relates jevois::Engine
144  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumode, CPUmode, "CPU frequency modulation mode",
145  CPUmode::Performance, CPUmode_Values, ParamCateg);
147  // keep this in sync with sunxi-cpufreq.c
148  //! Parameter \relates jevois::Engine
149  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumax, unsigned int, "CPU maximum frequency in MHz",
150  1344, { 120, 240, 312, 408, 480, 504, 600, 648, 720, 816, 912, 1008,
151  1044, 1056, 1080, 1104, 1116, 1152, 1200, 1224, 1248, 1296, 1344 },
152  ParamCateg);
154  //! Parameter \relates jevois::Engine
155  JEVOIS_DECLARE_PARAMETER(multicam, bool, "Allow up to 3 JeVois cameras on one USB bus. Enabling this "
156  "reduces the amount of USB bandwidth used by each JeVois camera, from 3kb "
157  "per USB isochronous microframe to 1kb. All 3 JeVois cameras must have this "
158  "option enabled, and the JeVois linux kernel module should also have "
159  "been loaded with multicam on.",
160  false, ParamCateg);
162  //! Parameter \relates jevois::Engine
163  JEVOIS_DECLARE_PARAMETER(quietcmd, bool, "When true, do not issue a message 'OK' after every correct command "
164  "received at the command-line interface. Recommended for advanced users only.",
165  false, ParamCateg);
167  //! Parameter \relates jevois::Engine
168  JEVOIS_DECLARE_PARAMETER(python, bool, "When true, enable support for modules written in Python. Otherwise, "
169  "attempting to load a python module will throw an exception. Disabling python saves "
170  "a lot of memory and may be useful when using C++ modules that run large deep neural "
171  "networks.",
172  true, ParamCateg);
174  //! Parameter \relates jevois::Engine
175  JEVOIS_DECLARE_PARAMETER(serlimit, size_t, "Maximum number of serial messages that can be sent by a module "
176  "using sendSerial(), for each video frame, or 0 for no limit. Any message sent by "
177  "the module beyond the first serlimit ones will be dropped. This is useful to avoid "
178  "overloading the serial link, for example in case one is running a ArUco detector and "
179  "a large number of ArUco tags are present in the field of view of JeVois.",
180  0, ParamCateg);
181  }
183  //! JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB
184  /*! The Engine is orchestrating the execution of vision processing software. It is a Manager, i.e., it is the root of
185  a hierarchy of Component objects and it handles access to their Parameter settings and their construction, init(),
186  unInit(), and destruction. The component hierarchy consists of Engine at the root, then one Module which is
187  selected by the user at runtime, e.g., by selecting a given video format on video camera software running on a
188  host computer connected to the JeVois hardware. The Module may then contain an arbitrarily complex hierarchy of
189  Component objects with Parameter settings in them. Module derives from Component and thus may also have its own
190  Parameter settings.
192  Engine contains the following basic elements:
194  - A VideoInput, instantiated as either a Camera for live video streaming or a MovieInput for processing of
195  pre-recorded video files or sequences of images (useful during algorithm development, to test and optimize on
196  reproducible inputs);
198  - A VideoOutput, instantiated either as a USB Gadget driver when running on the JeVois hardware platform, or as a
199  VideoDisplay when running on a computer that has a graphics display, or as a MovieOutput to save output video
200  frames to disk, or as a VideoOutputNone if desired for benchmarking of vision algorithms while discounting any
201  work related to transmitting output frames.
203  - A DynamicLoader which handles loading the chosen vision processing Module at runtime depending on user
204  selections;
206  - Any number of UserInterface objects, instantiated as either a hardware Serial port (for the 4-pin JST 1.0mm
207  connector on the platform hardware), a serial-over-USB Serial port (visible on the host computer to which the
208  JeVois hardware is connected by USB), or an StdioInterface (used to accept commands and print results directly
209  in the terminal where the JeVois Engine was started, particularly useful when running on a generic computer as
210  opposed to the platform hardware). When running on platform hardware, usually two UserInterface objects are
211  created (one hardware Serial, one serial-over-USB Serial), while, when running on a generic computer, usually
212  only one UserInterface is created (of type StdioInterface to accept commands directly in the terminal in which
213  the jevois-daemon was started);
215  - The list of VideoMapping definitions imported from your videomappings.cfg file. These definitions specify which
216  video output modes are available over USB and their corresponding Camera settings and which Module to use, as
217  well as which modes are available that do not have any sreaming video output over USB (e.g., when connecting the
218  hardware platform to an Arduino only).
220  The main loop of Engine runs until the user decides to quit, and basically goes through the following steps:
222  - Create an InputFrame object which is an exception-safe wrapper around the next available Camera frame. The frame
223  may not have been captured yet. The InputFrame can be understood as a mechanism to gain access to that frame in
224  the future, when it has become available (i.e., has been captured by the camera). This is very similar to the
225  std::future framework of C++11.
227  - When the current VideoMapping specifies that we will be streaming video frames out over USB, also create an
228  OutputFrame object which is an exception-safe wrapper around the next available Gadget frame. This is also just
229  a mechanism for gaining access to the next blank video buffer that is available from the USB driver and that we
230  should fill with interesting pixel data before sending it over USB to a host computer.
232  - Call the currently-loaded Module's process() function, either as process(InputFrame, OutputFrame) when the
233  current VideoMapping specifies that some video output is to be sent over USB, or as process(InputFrame) when the
234  current VideoMapping specifies no video output. Any exception thrown by the Module's process() function will be
235  caught, reported, and ignored. The process() function would typically request the next available camera image
236  through the InputFrame wrapper (this request may block until the frame has been captured by the camera sensor
237  hardware), process that image, request the next available output image through the OutputFrame wrapper (when
238  VideoMapping specifies that there is USB video output), and paint some results into that output image, which
239  will then be sent to the host coputer over USB, for display by some webcam program or for further processing by
240  some custom vision software running on that computer. In addition, the currently loaded Module may issue
241  messages over the UserInterface ports (e.g., indicating the location at which an object was found, to let an
242  Arduino know about it).
244  - Read any new commands issued by users over the UserInterface ports and execute the appropriate commands.
246  - Handle user requests to change VideoMapping, when they select a different video mode in their webcam software
247  running on the host computer connected to the JeVois hardware. Such requests may trigger unloading of the
248  current Module and loading a new one, and changing camera pixel format, image size, etc. These changes are
249  guaranteed to occur when the Module's process() function is not running, i.e., Module programmers do not have to
250  worry about possible changes in image dimensions or pixel formats during execution of their process() function.
252  - Pass any user requests received over USB or UserInterface to adjust camera parameters to the actual Camera
253  hardware driver (e.g., when users change contrast in their webcam program, that request is sent to the Engine
254  over USB, and the Engine then forwards it to the Camera hardware driver).
256  \ingroup core */
257  class Engine : public Manager,
258  public Parameter<engine::cameradev, engine::cameranbuf, engine::gadgetdev, engine::gadgetnbuf,
259  engine::videomapping, engine::serialdev, engine::usbserialdev, engine::camreg,
260  engine::camturbo, engine::serlog, engine::videoerrors, engine::serout,
261  engine::cpumode, engine::cpumax, engine::multicam, engine::quietcmd,
262  engine::python, engine::serlimit>
263  {
264  public:
265  //! Constructor
266  Engine(std::string const & instance);
268  //! Constructor with command-line parsing
269  Engine(int argc, char const* argv[], std::string const & instance);
271  //! Destructor
272  ~Engine();
274  //! Find the VideoMapping that has the given output specs, or throw if not found
275  VideoMapping const & findVideoMapping(unsigned int oformat, unsigned int owidth, unsigned int oheight,
276  float oframespersec) const;
278  //! Get the current video mapping
279  /*! Note that the current mapping may not have an entry in our list of mappings obtained from videomappings.cfg,
280  if the current one was set on the fly by the setmapping2 CLI command. */
281  VideoMapping const & getCurrentVideoMapping() const;
283  //! Return the number of video mappings
284  size_t numVideoMappings() const;
286  //! Allow access to our video mappings which are parsed from file at construction
287  VideoMapping const & getVideoMapping(size_t idx) const;
289  //! Get the video mapping index for a given UVC iformat, iframe and interval
290  size_t getVideoMappingIdx(unsigned int iformat, unsigned int iframe, unsigned int interval) const;
292  //! Allow access to the default video mapping
293  VideoMapping const & getDefaultVideoMapping() const;
295  //! Allow access to the default video mapping index
296  size_t getDefaultVideoMappingIdx() const;
298  //! Callback for when the user selects a new output video format
299  /*! Here, we stop streaming, nuke any current processing module, set the camera format, set the gadget output
300  format, load the new processing module, and start streaming again. The given VideoMapping will typically be
301  obtained using findVideoMapping() from output specs received over the USB link. */
302  void setFormat(size_t idx);
304  //! Start streaming on video from camera, processing, and USB
305  void streamOn();
307  //! Stop streaming on video from camera, processing, and USB
308  void streamOff();
310  //! Main loop: grab, process, send over USB. Should be called by main application thread
311  void mainLoop();
313  //! Send a string to all serial ports
314  /*! \note When islog is true, this is assumes to be a log message, and it will be sent to the port(s) specified by
315  parameter serlog. Otherwise, the message will be sent to the ports specified by parameter serout. Note how the
316  number of messages that can be sent for each video frame may be limited by parameter \p serlimit; only up to
317  \serlimit messages will be sent for a given video frame. This is useful to avoid overloading the serial link,
318  for example in cases one is running a ArUco detector and a large number of ArUco tags are present in the field
319  of view of JeVois. */
320  void sendSerial(std::string const & str, bool islog = false);
322  //! Get frame number
323  /*! The Engine maintains a master frame counter that is incremented on each call to a Module's process(), whether
324  or not the call succeeds. The counter is not incremented when a module has not been loaded (e.g., failed to
325  load). The counter is reset to zero each time a new module is loaded. */
326  size_t frameNum() const;
328  protected:
329  //! Run a script from file
330  /*! The filename should be absolute. The file should have any of the commands supported by Engine, one per
331  line. Filename should be relative to the current module's path. */
332  void runScriptFromFile(std::string const & filename, std::shared_ptr<UserInterface> ser,
333  bool throw_no_file);
335  //! Parameter callback
336  void onParamChange(engine::cameradev const & param, std::string const & newval);
338  //! Parameter callback
339  void onParamChange(engine::gadgetdev const & param, std::string const & newval);
341  //! Parameter callback
342  void onParamChange(engine::serialdev const & param, std::string const & newval);
344  //! Parameter callback
345  void onParamChange(engine::usbserialdev const & param, std::string const & newval);
347  //! Parameter callback
348  void onParamChange(engine::cpumode const & param, engine::CPUmode const & newval);
350  //! Parameter callback
351  void onParamChange(engine::cpumax const & param, unsigned int const & newval);
353  //! Parameter callback
354  void onParamChange(engine::videoerrors const & param, bool const & newval);
356  size_t itsDefaultMappingIdx; //!< Index of default mapping
357  std::vector<VideoMapping> const itsMappings; //!< All our mappings from videomappings.cfg
358  VideoMapping itsCurrentMapping; //!< Current video mapping, may not match any in itsMappings if setmapping2 used
360  std::shared_ptr<VideoInput> itsCamera; //!< Our camera
361  std::shared_ptr<VideoOutput> itsGadget; //!< Our gadget
363  std::unique_ptr<DynamicLoader> itsLoader; //!< Our module loader
364  std::shared_ptr<Module> itsModule; //!< Our current module
366  std::atomic<bool> itsRunning; //!< True when we are running
367  std::atomic<bool> itsStreaming; //!< True when we are streaming video
368  std::atomic<bool> itsStopMainLoop; //!< Flag used to stop the main loop
370  mutable std::timed_mutex itsMtx; //!< Mutex to protect our internals
372  void preInit() override; //!< Override of Manager::preInit()
373  void postInit() override; //!< Override of Manager::postInit()
375  //! Parse a user command received over serial port
376  /*! Throw upon receiving an incorrect command (eg, bad parameter value), return true if success, return false if
377  command was not recognized and should be tried by Module. pfx is an optional prefix which will be added to all
378  produced messages or errors. */
379  bool parseCommand(std::string const & str, std::shared_ptr<UserInterface> s, std::string const & pfx = "");
381  private:
382  std::list<std::shared_ptr<UserInterface> > itsSerials;
384  void setFormatInternal(size_t idx); // itsMtx should be locked by caller
385  void setFormatInternal(jevois::VideoMapping const & m, bool reload = false); // itsMtx should be locked by caller
387  // Return help string for a camera control or throw
388  std::string camCtrlHelp(struct v4l2_queryctrl & qc, std::set<int> & doneids);
390  // Return machine-oriented string for a camera control or throw
391  std::string camCtrlInfo(struct v4l2_queryctrl & qc, std::set<int> & doneids);
393  // Send info about built-in engine commands
394  void cmdInfo(std::shared_ptr<UserInterface> s, bool showAll, std::string const & pfx = "");
396  // Send info about module commands
397  void modCmdInfo(std::shared_ptr<UserInterface> s, std::string const & pfx = "");
399  // Get short name from V4L2 ID, long name is a backup in case we don't find the control in our list
400  std::string camctrlname(int id, char const * longname) const;
402  // Get V4L2 ID from short name
403  int camctrlid(std::string const & shortname);
405  bool itsTurbo;
406  bool itsManualStreamon; // allow manual streamon when outputing video to None or file
407  std::atomic<bool> itsVideoErrors; // fast cached value for engine::videoerrors
408  jevois::RawImage itsVideoErrorImage;
409  std::string itsModuleConstructionError; // Non-empty error message if module constructor threw
412  // Things related to mass storage gadget to export our /jevois partition as a virtual USB flash drive:
413  void checkMassStorage(); // thread to check mass storage gadget status
414  std::future<void> itsCheckMassStorageFut;
415  std::atomic<bool> itsCheckingMassStorage;
416  std::atomic<bool> itsMassStorageMode;
417  void startMassStorageMode();
418  void stopMassStorageMode();
419  void reboot();
420 #endif
422  std::atomic<size_t> itsFrame; // Master frame counter, can be used by modules
423  std::atomic<size_t> itsNumSerialSent; // Number of serial messages sent this frame; see serlimit
424  };
426 } // namespace jevois
size_t itsDefaultMappingIdx
Index of default mapping.
Definition: Engine.H:356
std::shared_ptr< VideoInput > itsCamera
Our camera.
Definition: Engine.H:360
Generic variadic template class template definition for Component Parameters.
On generic computer hardware, device for the USB gadget driver should always be empty.
Definition: Engine.H:55
std::unique_ptr< DynamicLoader > itsLoader
Our module loader.
Definition: Engine.H:363
std::atomic< bool > itsStreaming
True when we are streaming video.
Definition: Engine.H:367
Manager of a hierarchy of Component objects.
Definition: Manager.H:73
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
std::shared_ptr< VideoOutput > itsGadget
Our gadget.
Definition: Engine.H:361
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
VideoMapping itsCurrentMapping
Current video mapping, may not match any in itsMappings if setmapping2 used.
Definition: Engine.H:358
Simple struct to hold video mapping definitions for the processing Engine.
Definition: VideoMapping.H:41
std::atomic< bool > itsStopMainLoop
Flag used to stop the main loop.
Definition: Engine.H:368
On generic computer hardware, device for serial port should always be stdio to use an StdioInterface...
Definition: Engine.H:58
std::vector< VideoMapping > const itsMappings
All our mappings from videomappings.cfg.
Definition: Engine.H:357
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:257
std::timed_mutex itsMtx
Mutex to protect our internals.
Definition: Engine.H:370
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Parameter callback.
On generic computer hardware, device for the camera sensor.
Definition: Engine.H:52
std::atomic< bool > itsRunning
True when we are running.
Definition: Engine.H:366
On generic computer hardware, device for the serial-over-USB port should always be empty...
Definition: Engine.H:61
std::shared_ptr< Module > itsModule
Our current module.
Definition: Engine.H:364