JeVois  1.8
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See and for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - - -
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
18 #pragma once
22 #include <jevois/Types/Enum.H>
23 #include <jevois/Image/RawImage.H>
25 #include <memory>
26 #include <mutex>
27 #include <vector>
28 #include <list>
29 #include <atomic>
32 // On the platform (JeVois hardware), we use a gadget driver by default to send output frames over USB, one hardware
33 // serial driver, and one serial-over-USB driver:
35 //! On platform hardware, device for the camera sensor
36 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
38 //! On platform hardware, device for the USB gadget driver (which sends video frames over USB to a host computer)
39 #define JEVOIS_GADGET_DEFAULT "/dev/video1"
41 //! On platform hardware, device for the 4-pin hardware serial port
42 #define JEVOIS_SERIAL_DEFAULT "/dev/ttyS0"
44 //! On platform hardware, device for serial-over-USB port
45 #define JEVOIS_USBSERIAL_DEFAULT "/dev/ttyGS0"
47 #else
48 // On the host, we have no gadget (which will trigger displaying output frames to a window) and we use the terminal in
49 // which jevois-daemon was started for serial commands:
51 //! On generic computer hardware, device for the camera sensor
52 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
54 //! On generic computer hardware, device for the USB gadget driver should always be empty
57 //! On generic computer hardware, device for serial port should always be stdio to use an StdioInterface
58 #define JEVOIS_SERIAL_DEFAULT "stdio"
60 //! On generic computer hardware, device for the serial-over-USB port should always be empty
63 #endif
65 namespace jevois
66 {
67  class VideoInput;
68  class VideoOutput;
69  class Module;
70  class DynamicLoader;
71  class UserInterface;
73  namespace engine
74  {
75  static ParameterCategory const ParamCateg("Engine Options");
77  //! Parameter \relates jevois::Engine
78  JEVOIS_DECLARE_PARAMETER(cameradev, std::string, "Camera device name (if starting with /dev/v...), or movie "
79  "file name (e.g., movie.mpg) or image sequence (e.g., im%02d.jpg, to read frames "
80  "im00.jpg, im01.jpg, etc).",
83  //! Parameter \relates jevois::Engine
84  JEVOIS_DECLARE_PARAMETER(cameranbuf, unsigned int, "Number of video input (camera) buffers, or 0 for automatic.",
85  0, ParamCateg);
87  //! Parameter \relates jevois::Engine
88  JEVOIS_DECLARE_PARAMETER(gadgetdev, std::string, "Gadget device name. This is used on platform hardware only. "
89  "On host hardware, a display window will be used unless gadgetdev is None (useful "
90  "for benchmarking) or is a file stem for a movie file that does not start with /dev/ "
91  "(and which should contain a printf-style directive for a single int argument, "
92  "the movie number).",
95  //! Parameter \relates jevois::Engine
96  JEVOIS_DECLARE_PARAMETER(gadgetnbuf, unsigned int, "Number of video output (USB video) buffers, or 0 for auto",
97  0, ParamCateg);
99  //! Parameter \relates jevois::Engine
100  JEVOIS_DECLARE_PARAMETER(videomapping, int, "Index of Video Mapping to use, or -1 to use the default mapping",
101  -1, ParamCateg);
103  //! Parameter \relates jevois::Engine
104  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(serialdev, std::string, "Hardware (4-pin connector) serial device name, "
105  "or 'stdio' to use the console, or empty for no hardware serial port",
108  //! Parameter \relates jevois::Engine
109  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(usbserialdev, std::string, "Over-the-USB serial device name, or empty",
112  //! Parameter \relates jevois::Engine
113  JEVOIS_DECLARE_PARAMETER(camreg, bool, "Enable raw access to camera registers through setcamreg and getcamreg",
114  false, ParamCateg);
116  //! Parameter \relates jevois::Engine
117  JEVOIS_DECLARE_PARAMETER(camturbo, bool, "Enable camera turbo mode by relaxing the need for DMA-coherent video "
118  "buffer memory. This can accelerate severalfolds access to the captured image data, but "
119  "it may also yield stripe artifacts with some modules, such as PassThrough. The stripes "
120  "are pieces of incorrect data in the cache. You should experiment with each particular "
121  "module. Turbo mode is not recommended for any production-grade application.",
122  false, ParamCateg);
124  //! Enum for Parameter \relates jevois::Engine
125  JEVOIS_DEFINE_ENUM_CLASS(SerPort, (None) (All) (Hard) (USB) );
127  //! Parameter \relates jevois::Engine
128  JEVOIS_DECLARE_PARAMETER(serlog, SerPort, "Show log and debug messages on selected serial port(s)",
129  SerPort::None, SerPort_Values, ParamCateg);
131  //! Parameter \relates jevois::Engine
132  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(videoerrors, bool, "Show any machine vision module errors (exceptions) "
133  "in the video stream. Only takes effect if streaming video to USB.",
134  true, ParamCateg);
136  //! Parameter \relates jevois::Engine
137  JEVOIS_DECLARE_PARAMETER(serout, SerPort, "Send module serial messages to selected serial port(s)",
138  SerPort::None, SerPort_Values, ParamCateg);
140  //! Enum for Parameter \relates jevois::Engine
141  JEVOIS_DEFINE_ENUM_CLASS(CPUmode, (PowerSave) (Conservative) (OnDemand) (Interactive) (Performance) );
143  //! Parameter \relates jevois::Engine
144  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumode, CPUmode, "CPU frequency modulation mode",
145  CPUmode::Performance, CPUmode_Values, ParamCateg);
147  // keep this in sync with sunxi-cpufreq.c
148  //! Parameter \relates jevois::Engine
149  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumax, unsigned int, "CPU maximum frequency in MHz",
150  1344, { 120, 240, 312, 408, 480, 504, 600, 648, 720, 816, 912, 1008,
151  1044, 1056, 1080, 1104, 1116, 1152, 1200, 1224, 1248, 1296, 1344 },
152  ParamCateg);
154  //! Parameter \relates jevois::Engine
155  JEVOIS_DECLARE_PARAMETER(multicam, bool, "Allow up to 3 JeVois cameras on one USB bus. Enabling this "
156  "reduces the amount of USB bandwidth used by each JeVois camera, from 3kb "
157  "per USB isochronous microframe to 1kb. All 3 JeVois cameras must have this "
158  "option enabled, and the JeVois linux kernel module should also have "
159  "been loaded with multicam on.",
160  false, ParamCateg);
162  //! Parameter \relates jevois::Engine
163  JEVOIS_DECLARE_PARAMETER(quietcmd, bool, "When true, do not issue a message 'OK' after every correct command "
164  "received at the command-line interface. Recommended for advanced users only.",
165  false, ParamCateg);
167  //! Parameter \relates jevois::Engine
168  JEVOIS_DECLARE_PARAMETER(python, bool, "When true, enable support for modules written in Python. Otherwise, "
169  "attempting to load a python module will throw an exception. Disabling python saves "
170  "a lot of memory and may be useful when using C++ modules that run large deep neural "
171  "networks.",
172  true, ParamCateg);
173  }
175  //! JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB
176  /*! The Engine is orchestrating the execution of vision processing software. It is a Manager, i.e., it is the root of
177  a hierarchy of Component objects and it handles access to their Parameter settings and their construction, init(),
178  unInit(), and destruction. The component hierarchy consists of Engine at the root, then one Module which is
179  selected by the user at runtime, e.g., by selecting a given video format on video camera software running on a
180  host computer connected to the JeVois hardware. The Module may then contain an arbitrarily complex hierarchy of
181  Component objects with Parameter settings in them. Module derives from Component and thus may also have its own
182  Parameter settings.
184  Engine contains the following basic elements:
186  - A VideoInput, instantiated as either a Camera for live video streaming or a MovieInput for processing of
187  pre-recorded video files or sequences of images (useful during algorithm development, to test and optimize on
188  reproducible inputs);
190  - A VideoOutput, instantiated either as a USB Gadget driver when running on the JeVois hardware platform, or as a
191  VideoDisplay when running on a computer that has a graphics display, or as a MovieOutput to save output video
192  frames to disk, or as a VideoOutputNone if desired for benchmarking of vision algorithms while discounting any
193  work related to transmitting output frames.
195  - A DynamicLoader which handles loading the chosen vision processing Module at runtime depending on user
196  selections;
198  - Any number of UserInterface objects, instantiated as either a hardware Serial port (for the 4-pin JST 1.0mm
199  connector on the platform hardware), a serial-over-USB Serial port (visible on the host computer to which the
200  JeVois hardware is connected by USB), or an StdioInterface (used to accept commands and print results directly
201  in the terminal where the JeVois Engine was started, particularly useful when running on a generic computer as
202  opposed to the platform hardware). When running on platform hardware, usually two UserInterface objects are
203  created (one hardware Serial, one serial-over-USB Serial), while, when running on a generic computer, usually
204  only one UserInterface is created (of type StdioInterface to accept commands directly in the terminal in which
205  the jevois-daemon was started);
207  - The list of VideoMapping definitions imported from your videomappings.cfg file. These definitions specify which
208  video output modes are available over USB and their corresponding Camera settings and which Module to use, as
209  well as which modes are available that do not have any sreaming video output over USB (e.g., when connecting the
210  hardware platform to an Arduino only).
212  The main loop of Engine runs until the user decides to quit, and basically goes through the following steps:
214  - Create an InputFrame object which is an exception-safe wrapper around the next available Camera frame. The frame
215  may not have been captured yet. The InputFrame can be understood as a mechanism to gain access to that frame in
216  the future, when it has become available (i.e., has been captured by the camera). This is very similar to the
217  std::future framework of C++11.
219  - When the current VideoMapping specifies that we will be streaming video frames out over USB, also create an
220  OutputFrame object which is an exception-safe wrapper around the next available Gadget frame. This is also just
221  a mechanism for gaining access to the next blank video buffer that is available from the USB driver and that we
222  should fill with interesting pixel data before sending it over USB to a host computer.
224  - Call the currently-loaded Module's process() function, either as process(InputFrame, OutputFrame) when the
225  current VideoMapping specifies that some video output is to be sent over USB, or as process(InputFrame) when the
226  current VideoMapping specifies no video output. Any exception thrown by the Module's process() function will be
227  caught, reported, and ignored. The process() function would typically request the next available camera image
228  through the InputFrame wrapper (this request may block until the frame has been captured by the camera sensor
229  hardware), process that image, request the next available output image through the OutputFrame wrapper (when
230  VideoMapping specifies that there is USB video output), and paint some results into that output image, which
231  will then be sent to the host coputer over USB, for display by some webcam program or for further processing by
232  some custom vision software running on that computer. In addition, the currently loaded Module may issue
233  messages over the UserInterface ports (e.g., indicating the location at which an object was found, to let an
234  Arduino know about it).
236  - Read any new commands issued by users over the UserInterface ports and execute the appropriate commands.
238  - Handle user requests to change VideoMapping, when they select a different video mode in their webcam software
239  running on the host computer connected to the JeVois hardware. Such requests may trigger unloading of the
240  current Module and loading a new one, and changing camera pixel format, image size, etc. These changes are
241  guaranteed to occur when the Module's process() function is not running, i.e., Module programmers do not have to
242  worry about possible changes in image dimensions or pixel formats during execution of their process() function.
244  - Pass any user requests received over USB or UserInterface to adjust camera parameters to the actual Camera
245  hardware driver (e.g., when users change contrast in their webcam program, that request is sent to the Engine
246  over USB, and the Engine then forwards it to the Camera hardware driver).
248  \ingroup core */
249  class Engine : public Manager,
250  public Parameter<engine::cameradev, engine::cameranbuf, engine::gadgetdev, engine::gadgetnbuf,
251  engine::videomapping, engine::serialdev, engine::usbserialdev, engine::camreg,
252  engine::camturbo, engine::serlog, engine::videoerrors, engine::serout,
253  engine::cpumode, engine::cpumax, engine::multicam, engine::quietcmd,
254  engine::python>
255  {
256  public:
257  //! Constructor
258  Engine(std::string const & instance);
260  //! Constructor with command-line parsing
261  Engine(int argc, char const* argv[], std::string const & instance);
263  //! Destructor
264  ~Engine();
266  //! Find the VideoMapping that has the given output specs, or throw if not found
267  VideoMapping const & findVideoMapping(unsigned int oformat, unsigned int owidth, unsigned int oheight,
268  float oframespersec) const;
270  //! Get the current video mapping
271  /*! Note that the current mapping may not have an entry in our list of mappings obtained from videomappings.cfg,
272  if the current one was set on the fly by the setmapping2 CLI command. */
273  VideoMapping const & getCurrentVideoMapping() const;
275  //! Return the number of video mappings
276  size_t numVideoMappings() const;
278  //! Allow access to our video mappings which are parsed from file at construction
279  VideoMapping const & getVideoMapping(size_t idx) const;
281  //! Get the video mapping index for a given UVC iformat, iframe and interval
282  size_t getVideoMappingIdx(unsigned int iformat, unsigned int iframe, unsigned int interval) const;
284  //! Allow access to the default video mapping
285  VideoMapping const & getDefaultVideoMapping() const;
287  //! Allow access to the default video mapping index
288  size_t getDefaultVideoMappingIdx() const;
290  //! Callback for when the user selects a new output video format
291  /*! Here, we stop streaming, nuke any current processing module, set the camera format, set the gadget output
292  format, load the new processing module, and start streaming again. The given VideoMapping will typically be
293  obtained using findVideoMapping() from output specs received over the USB link. */
294  void setFormat(size_t idx);
296  //! Start streaming on video from camera, processing, and USB
297  void streamOn();
299  //! Stop streaming on video from camera, processing, and USB
300  void streamOff();
302  //! Main loop: grab, process, send over USB. Should be called by main application thread
303  void mainLoop();
305  //! Send a string to all serial ports
306  /*! \note When islog is true, this is assumes to be a log message, and it will be sent to the port(s) specified by
307  parameter serlog. Otherwise, the message will be sent to the ports specified by parameter serout. */
308  void sendSerial(std::string const & str, bool islog = false);
310  protected:
311  //! Run a script from file
312  /*! The filename should be absolute. The file should have any of the commands supported by Engine, one per
313  line. Filename should be relative to the current module's path. */
314  void runScriptFromFile(std::string const & filename, std::shared_ptr<UserInterface> ser,
315  bool throw_no_file);
317  //! Parameter callback
318  void onParamChange(engine::cameradev const & param, std::string const & newval);
320  //! Parameter callback
321  void onParamChange(engine::gadgetdev const & param, std::string const & newval);
323  //! Parameter callback
324  void onParamChange(engine::serialdev const & param, std::string const & newval);
326  //! Parameter callback
327  void onParamChange(engine::usbserialdev const & param, std::string const & newval);
329  //! Parameter callback
330  void onParamChange(engine::cpumode const & param, engine::CPUmode const & newval);
332  //! Parameter callback
333  void onParamChange(engine::cpumax const & param, unsigned int const & newval);
335  //! Parameter callback
336  void onParamChange(engine::videoerrors const & param, bool const & newval);
338  size_t itsDefaultMappingIdx; //!< Index of default mapping
339  std::vector<VideoMapping> const itsMappings; //!< All our mappings from videomappings.cfg
340  VideoMapping itsCurrentMapping; //!< Current video mapping, may not match any in itsMappings if setmapping2 used
342  std::shared_ptr<VideoInput> itsCamera; //!< Our camera
343  std::shared_ptr<VideoOutput> itsGadget; //!< Our gadget
345  std::unique_ptr<DynamicLoader> itsLoader; //!< Our module loader
346  std::shared_ptr<Module> itsModule; //!< Our current module
348  std::atomic<bool> itsRunning; //!< True when we are running
349  std::atomic<bool> itsStreaming; //!< True when we are streaming video
350  std::atomic<bool> itsStopMainLoop; //!< Flag used to stop the main loop
352  mutable std::timed_mutex itsMtx; //!< Mutex to protect our internals
354  void preInit() override; //!< Override of Manager::preInit()
355  void postInit() override; //!< Override of Manager::postInit()
357  //! Parse a user command received over serial port
358  /*! Throw upon receiving an incorrect command (eg, bad parameter value), return true if success, return false if
359  command was not recognized and should be tried by Module. */
360  bool parseCommand(std::string const & str, std::shared_ptr<UserInterface> s);
362  private:
363  std::list<std::shared_ptr<UserInterface> > itsSerials;
365  void setFormatInternal(size_t idx); // itsMtx should be locked by caller
366  void setFormatInternal(jevois::VideoMapping const & m); // itsMtx should be locked by caller
368  // Return help string for a camera control or throw
369  std::string camCtrlHelp(struct v4l2_queryctrl & qc, std::set<int> & doneids);
371  // Get short name from V4L2 ID, long name is a backup in case we don't find the control in our list
372  std::string camctrlname(int id, char const * longname) const;
374  // Get V4L2 ID from short name
375  int camctrlid(std::string const & shortname);
377  bool itsTurbo;
378  bool itsManualStreamon; // allow manual streamon when outputing video to None or file
379  std::atomic<bool> itsVideoErrors; // fast cached value for engine::videoerrors
380  jevois::RawImage itsVideoErrorImage;
381  std::string itsModuleConstructionError; // Non-empty error message if module constructor threw
384  // Things related to mass storage gadget to export our /jevois partition as a virtual USB flash drive:
385  void checkMassStorage(); // thread to check mass storage gadget status
386  std::future<void> itsCheckMassStorageFut;
387  std::atomic<bool> itsCheckingMassStorage;
388  std::atomic<bool> itsMassStorageMode;
389  void startMassStorageMode();
390  void stopMassStorageMode();
391  void reboot();
392 #endif
393  };
395 } // namespace jevois
size_t itsDefaultMappingIdx
Index of default mapping.
Definition: Engine.H:338
std::shared_ptr< VideoInput > itsCamera
Our camera.
Definition: Engine.H:342
Generic variadic template class template definition for Component Parameters.
On generic computer hardware, device for the USB gadget driver should always be empty.
Definition: Engine.H:55
std::unique_ptr< DynamicLoader > itsLoader
Our module loader.
Definition: Engine.H:345
std::atomic< bool > itsStreaming
True when we are streaming video.
Definition: Engine.H:349
Manager of a hierarchy of Component objects.
Definition: Manager.H:73
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
std::shared_ptr< VideoOutput > itsGadget
Our gadget.
Definition: Engine.H:343
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
VideoMapping itsCurrentMapping
Current video mapping, may not match any in itsMappings if setmapping2 used.
Definition: Engine.H:340
Simple struct to hold video mapping definitions for the processing Engine.
Definition: VideoMapping.H:41
std::atomic< bool > itsStopMainLoop
Flag used to stop the main loop.
Definition: Engine.H:350
On generic computer hardware, device for serial port should always be stdio to use an StdioInterface...
Definition: Engine.H:58
std::vector< VideoMapping > const itsMappings
All our mappings from videomappings.cfg.
Definition: Engine.H:339
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:249
std::timed_mutex itsMtx
Mutex to protect our internals.
Definition: Engine.H:352
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Parameter callback.
On generic computer hardware, device for the camera sensor.
Definition: Engine.H:52
std::atomic< bool > itsRunning
True when we are running.
Definition: Engine.H:348
On generic computer hardware, device for the serial-over-USB port should always be empty...
Definition: Engine.H:61
std::shared_ptr< Module > itsModule
Our current module.
Definition: Engine.H:346