JeVois  1.8
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Engine.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
22 #include <jevois/Types/Enum.H>
23 #include <jevois/Image/RawImage.H>
24 
25 #include <memory>
26 #include <mutex>
27 #include <vector>
28 #include <list>
29 #include <atomic>
30 
31 #ifdef JEVOIS_PLATFORM
32 // On the platform (JeVois hardware), we use a gadget driver by default to send output frames over USB, one hardware
33 // serial driver, and one serial-over-USB driver:
34 
35 //! On platform hardware, device for the camera sensor
36 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
37 
38 //! On platform hardware, device for the USB gadget driver (which sends video frames over USB to a host computer)
39 #define JEVOIS_GADGET_DEFAULT "/dev/video1"
40 
41 //! On platform hardware, device for the 4-pin hardware serial port
42 #define JEVOIS_SERIAL_DEFAULT "/dev/ttyS0"
43 
44 //! On platform hardware, device for serial-over-USB port
45 #define JEVOIS_USBSERIAL_DEFAULT "/dev/ttyGS0"
46 
47 #else
48 // On the host, we have no gadget (which will trigger displaying output frames to a window) and we use the terminal in
49 // which jevois-daemon was started for serial commands:
50 
51 //! On generic computer hardware, device for the camera sensor
52 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
53 
54 //! On generic computer hardware, device for the USB gadget driver should always be empty
55 #define JEVOIS_GADGET_DEFAULT ""
56 
57 //! On generic computer hardware, device for serial port should always be stdio to use an StdioInterface
58 #define JEVOIS_SERIAL_DEFAULT "stdio"
59 
60 //! On generic computer hardware, device for the serial-over-USB port should always be empty
61 #define JEVOIS_USBSERIAL_DEFAULT ""
62 
63 #endif
64 
65 namespace jevois
66 {
67  class VideoInput;
68  class VideoOutput;
69  class Module;
70  class DynamicLoader;
71  class UserInterface;
72 
73  namespace engine
74  {
75  static ParameterCategory const ParamCateg("Engine Options");
76 
77  //! Parameter \relates jevois::Engine
78  JEVOIS_DECLARE_PARAMETER(cameradev, std::string, "Camera device name (if starting with /dev/v...), or movie "
79  "file name (e.g., movie.mpg) or image sequence (e.g., im%02d.jpg, to read frames "
80  "im00.jpg, im01.jpg, etc).",
81  JEVOIS_CAMERA_DEFAULT, ParamCateg);
82 
83  //! Parameter \relates jevois::Engine
84  JEVOIS_DECLARE_PARAMETER(cameranbuf, unsigned int, "Number of video input (camera) buffers, or 0 for automatic.",
85  0, ParamCateg);
86 
87  //! Parameter \relates jevois::Engine
88  JEVOIS_DECLARE_PARAMETER(gadgetdev, std::string, "Gadget device name. This is used on platform hardware only. "
89  "On host hardware, a display window will be used unless gadgetdev is None (useful "
90  "for benchmarking) or is a file stem for a movie file that does not start with /dev/ "
91  "(and which should contain a printf-style directive for a single int argument, "
92  "the movie number).",
93  JEVOIS_GADGET_DEFAULT, ParamCateg);
94 
95  //! Parameter \relates jevois::Engine
96  JEVOIS_DECLARE_PARAMETER(gadgetnbuf, unsigned int, "Number of video output (USB video) buffers, or 0 for auto",
97  0, ParamCateg);
98 
99  //! Parameter \relates jevois::Engine
100  JEVOIS_DECLARE_PARAMETER(videomapping, int, "Index of Video Mapping to use, or -1 to use the default mapping",
101  -1, ParamCateg);
102 
103  //! Parameter \relates jevois::Engine
104  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(serialdev, std::string, "Hardware (4-pin connector) serial device name, "
105  "or 'stdio' to use the console, or empty for no hardware serial port",
106  JEVOIS_SERIAL_DEFAULT, ParamCateg);
107 
108  //! Parameter \relates jevois::Engine
109  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(usbserialdev, std::string, "Over-the-USB serial device name, or empty",
110  JEVOIS_USBSERIAL_DEFAULT, ParamCateg);
111 
112  //! Parameter \relates jevois::Engine
113  JEVOIS_DECLARE_PARAMETER(camreg, bool, "Enable raw access to camera registers through setcamreg and getcamreg",
114  false, ParamCateg);
115 
116  //! Parameter \relates jevois::Engine
117  JEVOIS_DECLARE_PARAMETER(camturbo, bool, "Enable camera turbo mode by relaxing the need for DMA-coherent video "
118  "buffer memory. This can accelerate severalfolds access to the captured image data, but "
119  "it may also yield stripe artifacts with some modules, such as PassThrough. The stripes "
120  "are pieces of incorrect data in the cache. You should experiment with each particular "
121  "module. Turbo mode is not recommended for any production-grade application.",
122  false, ParamCateg);
123 
124  //! Enum for Parameter \relates jevois::Engine
125  JEVOIS_DEFINE_ENUM_CLASS(SerPort, (None) (All) (Hard) (USB) );
126 
127  //! Parameter \relates jevois::Engine
128  JEVOIS_DECLARE_PARAMETER(serlog, SerPort, "Show log and debug messages on selected serial port(s)",
129  SerPort::None, SerPort_Values, ParamCateg);
130 
131  //! Parameter \relates jevois::Engine
132  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(videoerrors, bool, "Show any machine vision module errors (exceptions) "
133  "in the video stream. Only takes effect if streaming video to USB.",
134  true, ParamCateg);
135 
136  //! Parameter \relates jevois::Engine
137  JEVOIS_DECLARE_PARAMETER(serout, SerPort, "Send module serial messages to selected serial port(s)",
138  SerPort::None, SerPort_Values, ParamCateg);
139 
140  //! Enum for Parameter \relates jevois::Engine
141  JEVOIS_DEFINE_ENUM_CLASS(CPUmode, (PowerSave) (Conservative) (OnDemand) (Interactive) (Performance) );
142 
143  //! Parameter \relates jevois::Engine
144  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumode, CPUmode, "CPU frequency modulation mode",
145  CPUmode::Performance, CPUmode_Values, ParamCateg);
146 
147  // keep this in sync with sunxi-cpufreq.c
148  //! Parameter \relates jevois::Engine
149  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumax, unsigned int, "CPU maximum frequency in MHz",
150  1344, { 120, 240, 312, 408, 480, 504, 600, 648, 720, 816, 912, 1008,
151  1044, 1056, 1080, 1104, 1116, 1152, 1200, 1224, 1248, 1296, 1344 },
152  ParamCateg);
153 
154  //! Parameter \relates jevois::Engine
155  JEVOIS_DECLARE_PARAMETER(multicam, bool, "Allow up to 3 JeVois cameras on one USB bus. Enabling this "
156  "reduces the amount of USB bandwidth used by each JeVois camera, from 3kb "
157  "per USB isochronous microframe to 1kb. All 3 JeVois cameras must have this "
158  "option enabled, and the JeVois linux kernel module should also have "
159  "been loaded with multicam on.",
160  false, ParamCateg);
161 
162  //! Parameter \relates jevois::Engine
163  JEVOIS_DECLARE_PARAMETER(quietcmd, bool, "When true, do not issue a message 'OK' after every correct command "
164  "received at the command-line interface. Recommended for advanced users only.",
165  false, ParamCateg);
166 
167  //! Parameter \relates jevois::Engine
168  JEVOIS_DECLARE_PARAMETER(python, bool, "When true, enable support for modules written in Python. Otherwise, "
169  "attempting to load a python module will throw an exception. Disabling python saves "
170  "a lot of memory and may be useful when using C++ modules that run large deep neural "
171  "networks.",
172  true, ParamCateg);
173  }
174 
175  //! JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB
176  /*! The Engine is orchestrating the execution of vision processing software. It is a Manager, i.e., it is the root of
177  a hierarchy of Component objects and it handles access to their Parameter settings and their construction, init(),
178  unInit(), and destruction. The component hierarchy consists of Engine at the root, then one Module which is
179  selected by the user at runtime, e.g., by selecting a given video format on video camera software running on a
180  host computer connected to the JeVois hardware. The Module may then contain an arbitrarily complex hierarchy of
181  Component objects with Parameter settings in them. Module derives from Component and thus may also have its own
182  Parameter settings.
183 
184  Engine contains the following basic elements:
185 
186  - A VideoInput, instantiated as either a Camera for live video streaming or a MovieInput for processing of
187  pre-recorded video files or sequences of images (useful during algorithm development, to test and optimize on
188  reproducible inputs);
189 
190  - A VideoOutput, instantiated either as a USB Gadget driver when running on the JeVois hardware platform, or as a
191  VideoDisplay when running on a computer that has a graphics display, or as a MovieOutput to save output video
192  frames to disk, or as a VideoOutputNone if desired for benchmarking of vision algorithms while discounting any
193  work related to transmitting output frames.
194 
195  - A DynamicLoader which handles loading the chosen vision processing Module at runtime depending on user
196  selections;
197 
198  - Any number of UserInterface objects, instantiated as either a hardware Serial port (for the 4-pin JST 1.0mm
199  connector on the platform hardware), a serial-over-USB Serial port (visible on the host computer to which the
200  JeVois hardware is connected by USB), or an StdioInterface (used to accept commands and print results directly
201  in the terminal where the JeVois Engine was started, particularly useful when running on a generic computer as
202  opposed to the platform hardware). When running on platform hardware, usually two UserInterface objects are
203  created (one hardware Serial, one serial-over-USB Serial), while, when running on a generic computer, usually
204  only one UserInterface is created (of type StdioInterface to accept commands directly in the terminal in which
205  the jevois-daemon was started);
206 
207  - The list of VideoMapping definitions imported from your videomappings.cfg file. These definitions specify which
208  video output modes are available over USB and their corresponding Camera settings and which Module to use, as
209  well as which modes are available that do not have any sreaming video output over USB (e.g., when connecting the
210  hardware platform to an Arduino only).
211 
212  The main loop of Engine runs until the user decides to quit, and basically goes through the following steps:
213 
214  - Create an InputFrame object which is an exception-safe wrapper around the next available Camera frame. The frame
215  may not have been captured yet. The InputFrame can be understood as a mechanism to gain access to that frame in
216  the future, when it has become available (i.e., has been captured by the camera). This is very similar to the
217  std::future framework of C++11.
218 
219  - When the current VideoMapping specifies that we will be streaming video frames out over USB, also create an
220  OutputFrame object which is an exception-safe wrapper around the next available Gadget frame. This is also just
221  a mechanism for gaining access to the next blank video buffer that is available from the USB driver and that we
222  should fill with interesting pixel data before sending it over USB to a host computer.
223 
224  - Call the currently-loaded Module's process() function, either as process(InputFrame, OutputFrame) when the
225  current VideoMapping specifies that some video output is to be sent over USB, or as process(InputFrame) when the
226  current VideoMapping specifies no video output. Any exception thrown by the Module's process() function will be
227  caught, reported, and ignored. The process() function would typically request the next available camera image
228  through the InputFrame wrapper (this request may block until the frame has been captured by the camera sensor
229  hardware), process that image, request the next available output image through the OutputFrame wrapper (when
230  VideoMapping specifies that there is USB video output), and paint some results into that output image, which
231  will then be sent to the host coputer over USB, for display by some webcam program or for further processing by
232  some custom vision software running on that computer. In addition, the currently loaded Module may issue
233  messages over the UserInterface ports (e.g., indicating the location at which an object was found, to let an
234  Arduino know about it).
235 
236  - Read any new commands issued by users over the UserInterface ports and execute the appropriate commands.
237 
238  - Handle user requests to change VideoMapping, when they select a different video mode in their webcam software
239  running on the host computer connected to the JeVois hardware. Such requests may trigger unloading of the
240  current Module and loading a new one, and changing camera pixel format, image size, etc. These changes are
241  guaranteed to occur when the Module's process() function is not running, i.e., Module programmers do not have to
242  worry about possible changes in image dimensions or pixel formats during execution of their process() function.
243 
244  - Pass any user requests received over USB or UserInterface to adjust camera parameters to the actual Camera
245  hardware driver (e.g., when users change contrast in their webcam program, that request is sent to the Engine
246  over USB, and the Engine then forwards it to the Camera hardware driver).
247 
248  \ingroup core */
249  class Engine : public Manager,
250  public Parameter<engine::cameradev, engine::cameranbuf, engine::gadgetdev, engine::gadgetnbuf,
251  engine::videomapping, engine::serialdev, engine::usbserialdev, engine::camreg,
252  engine::camturbo, engine::serlog, engine::videoerrors, engine::serout,
253  engine::cpumode, engine::cpumax, engine::multicam, engine::quietcmd,
254  engine::python>
255  {
256  public:
257  //! Constructor
258  Engine(std::string const & instance);
259 
260  //! Constructor with command-line parsing
261  Engine(int argc, char const* argv[], std::string const & instance);
262 
263  //! Destructor
264  ~Engine();
265 
266  //! Find the VideoMapping that has the given output specs, or throw if not found
267  VideoMapping const & findVideoMapping(unsigned int oformat, unsigned int owidth, unsigned int oheight,
268  float oframespersec) const;
269 
270  //! Get the current video mapping
271  /*! Note that the current mapping may not have an entry in our list of mappings obtained from videomappings.cfg,
272  if the current one was set on the fly by the setmapping2 CLI command. */
273  VideoMapping const & getCurrentVideoMapping() const;
274 
275  //! Return the number of video mappings
276  size_t numVideoMappings() const;
277 
278  //! Allow access to our video mappings which are parsed from file at construction
279  VideoMapping const & getVideoMapping(size_t idx) const;
280 
281  //! Get the video mapping index for a given UVC iformat, iframe and interval
282  size_t getVideoMappingIdx(unsigned int iformat, unsigned int iframe, unsigned int interval) const;
283 
284  //! Allow access to the default video mapping
285  VideoMapping const & getDefaultVideoMapping() const;
286 
287  //! Allow access to the default video mapping index
288  size_t getDefaultVideoMappingIdx() const;
289 
290  //! Callback for when the user selects a new output video format
291  /*! Here, we stop streaming, nuke any current processing module, set the camera format, set the gadget output
292  format, load the new processing module, and start streaming again. The given VideoMapping will typically be
293  obtained using findVideoMapping() from output specs received over the USB link. */
294  void setFormat(size_t idx);
295 
296  //! Start streaming on video from camera, processing, and USB
297  void streamOn();
298 
299  //! Stop streaming on video from camera, processing, and USB
300  void streamOff();
301 
302  //! Main loop: grab, process, send over USB. Should be called by main application thread
303  void mainLoop();
304 
305  //! Send a string to all serial ports
306  /*! \note When islog is true, this is assumes to be a log message, and it will be sent to the port(s) specified by
307  parameter serlog. Otherwise, the message will be sent to the ports specified by parameter serout. */
308  void sendSerial(std::string const & str, bool islog = false);
309 
310  protected:
311  //! Run a script from file
312  /*! The filename should be absolute. The file should have any of the commands supported by Engine, one per
313  line. Filename should be relative to the current module's path. */
314  void runScriptFromFile(std::string const & filename, std::shared_ptr<UserInterface> ser,
315  bool throw_no_file);
316 
317  //! Parameter callback
318  void onParamChange(engine::cameradev const & param, std::string const & newval);
319 
320  //! Parameter callback
321  void onParamChange(engine::gadgetdev const & param, std::string const & newval);
322 
323  //! Parameter callback
324  void onParamChange(engine::serialdev const & param, std::string const & newval);
325 
326  //! Parameter callback
327  void onParamChange(engine::usbserialdev const & param, std::string const & newval);
328 
329  //! Parameter callback
330  void onParamChange(engine::cpumode const & param, engine::CPUmode const & newval);
331 
332  //! Parameter callback
333  void onParamChange(engine::cpumax const & param, unsigned int const & newval);
334 
335  //! Parameter callback
336  void onParamChange(engine::videoerrors const & param, bool const & newval);
337 
338  size_t itsDefaultMappingIdx; //!< Index of default mapping
339  std::vector<VideoMapping> const itsMappings; //!< All our mappings from videomappings.cfg
340  VideoMapping itsCurrentMapping; //!< Current video mapping, may not match any in itsMappings if setmapping2 used
341 
342  std::shared_ptr<VideoInput> itsCamera; //!< Our camera
343  std::shared_ptr<VideoOutput> itsGadget; //!< Our gadget
344 
345  std::unique_ptr<DynamicLoader> itsLoader; //!< Our module loader
346  std::shared_ptr<Module> itsModule; //!< Our current module
347 
348  std::atomic<bool> itsRunning; //!< True when we are running
349  std::atomic<bool> itsStreaming; //!< True when we are streaming video
350  std::atomic<bool> itsStopMainLoop; //!< Flag used to stop the main loop
351 
352  mutable std::timed_mutex itsMtx; //!< Mutex to protect our internals
353 
354  void preInit() override; //!< Override of Manager::preInit()
355  void postInit() override; //!< Override of Manager::postInit()
356 
357  //! Parse a user command received over serial port
358  /*! Throw upon receiving an incorrect command (eg, bad parameter value), return true if success, return false if
359  command was not recognized and should be tried by Module. */
360  bool parseCommand(std::string const & str, std::shared_ptr<UserInterface> s);
361 
362  private:
363  std::list<std::shared_ptr<UserInterface> > itsSerials;
364 
365  void setFormatInternal(size_t idx); // itsMtx should be locked by caller
366  void setFormatInternal(jevois::VideoMapping const & m); // itsMtx should be locked by caller
367 
368  // Return help string for a camera control or throw
369  std::string camCtrlHelp(struct v4l2_queryctrl & qc, std::set<int> & doneids);
370 
371  // Get short name from V4L2 ID, long name is a backup in case we don't find the control in our list
372  std::string camctrlname(int id, char const * longname) const;
373 
374  // Get V4L2 ID from short name
375  int camctrlid(std::string const & shortname);
376 
377  bool itsTurbo;
378  bool itsManualStreamon; // allow manual streamon when outputing video to None or file
379  std::atomic<bool> itsVideoErrors; // fast cached value for engine::videoerrors
380  jevois::RawImage itsVideoErrorImage;
381  std::string itsModuleConstructionError; // Non-empty error message if module constructor threw
382 
383 #ifdef JEVOIS_PLATFORM
384  // Things related to mass storage gadget to export our /jevois partition as a virtual USB flash drive:
385  void checkMassStorage(); // thread to check mass storage gadget status
386  std::future<void> itsCheckMassStorageFut;
387  std::atomic<bool> itsCheckingMassStorage;
388  std::atomic<bool> itsMassStorageMode;
389  void startMassStorageMode();
390  void stopMassStorageMode();
391  void reboot();
392 #endif
393  };
394 
395 } // namespace jevois
396 
size_t itsDefaultMappingIdx
Index of default mapping.
Definition: Engine.H:338
std::shared_ptr< VideoInput > itsCamera
Our camera.
Definition: Engine.H:342
Generic variadic template class template definition for Component Parameters.
#define JEVOIS_GADGET_DEFAULT
On generic computer hardware, device for the USB gadget driver should always be empty.
Definition: Engine.H:55
std::unique_ptr< DynamicLoader > itsLoader
Our module loader.
Definition: Engine.H:345
std::atomic< bool > itsStreaming
True when we are streaming video.
Definition: Engine.H:349
Manager of a hierarchy of Component objects.
Definition: Manager.H:73
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
std::shared_ptr< VideoOutput > itsGadget
Our gadget.
Definition: Engine.H:343
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
VideoMapping itsCurrentMapping
Current video mapping, may not match any in itsMappings if setmapping2 used.
Definition: Engine.H:340
Simple struct to hold video mapping definitions for the processing Engine.
Definition: VideoMapping.H:41
std::atomic< bool > itsStopMainLoop
Flag used to stop the main loop.
Definition: Engine.H:350
#define JEVOIS_SERIAL_DEFAULT
On generic computer hardware, device for serial port should always be stdio to use an StdioInterface...
Definition: Engine.H:58
std::vector< VideoMapping > const itsMappings
All our mappings from videomappings.cfg.
Definition: Engine.H:339
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:249
std::timed_mutex itsMtx
Mutex to protect our internals.
Definition: Engine.H:352
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Parameter callback.
#define JEVOIS_CAMERA_DEFAULT
On generic computer hardware, device for the camera sensor.
Definition: Engine.H:52
std::atomic< bool > itsRunning
True when we are running.
Definition: Engine.H:348
#define JEVOIS_USBSERIAL_DEFAULT
On generic computer hardware, device for the serial-over-USB port should always be empty...
Definition: Engine.H:61
std::shared_ptr< Module > itsModule
Our current module.
Definition: Engine.H:346