JeVois  1.5
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
Engine.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
22 #include <jevois/Types/Enum.H>
23 #include <jevois/Image/RawImage.H>
24 
25 #include <memory>
26 #include <mutex>
27 #include <vector>
28 #include <list>
29 #include <atomic>
30 
31 #ifdef JEVOIS_PLATFORM
32 // On the platform (JeVois hardware), we use a gadget driver by default to send output frames over USB, one hardware
33 // serial driver, and one serial-over-USB driver:
34 
35 //! On platform hardware, device for the camera sensor
36 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
37 
38 //! On platform hardware, device for the USB gadget driver (which sends video frames over USB to a host computer)
39 #define JEVOIS_GADGET_DEFAULT "/dev/video1"
40 
41 //! On platform hardware, device for the 4-pin hardware serial port
42 #define JEVOIS_SERIAL_DEFAULT "/dev/ttyS0"
43 
44 //! On platform hardware, device for serial-over-USB port
45 #define JEVOIS_USBSERIAL_DEFAULT "/dev/ttyGS0"
46 
47 #else
48 // On the host, we have no gadget (which will trigger displaying output frames to a window) and we use the terminal in
49 // which jevois-daemon was started for serial commands:
50 
51 //! On generic computer hardware, device for the camera sensor
52 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
53 
54 //! On generic computer hardware, device for the USB gadget driver should always be empty
55 #define JEVOIS_GADGET_DEFAULT ""
56 
57 //! On generic computer hardware, device for serial port should always be stdio to use an StdioInterface
58 #define JEVOIS_SERIAL_DEFAULT "stdio"
59 
60 //! On generic computer hardware, device for the serial-over-USB port should always be empty
61 #define JEVOIS_USBSERIAL_DEFAULT ""
62 
63 #endif
64 
65 namespace jevois
66 {
67  class VideoInput;
68  class VideoOutput;
69  class Module;
70  class DynamicLoader;
71  class UserInterface;
72 
73  namespace engine
74  {
75  static ParameterCategory const ParamCateg("Engine Options");
76 
77  //! Parameter \relates jevois::Engine
78  JEVOIS_DECLARE_PARAMETER(cameradev, std::string, "Camera device name (if starting with /dev/v...), or movie "
79  "file name (e.g., movie.mpg) or image sequence (e.g., im%02d.jpg, to read frames "
80  "im00.jpg, im01.jpg, etc).",
81  JEVOIS_CAMERA_DEFAULT, ParamCateg);
82 
83  //! Parameter \relates jevois::Engine
84  JEVOIS_DECLARE_PARAMETER(cameranbuf, unsigned int, "Number of video input (camera) buffers, or 0 for automatic.",
85  0, ParamCateg);
86 
87  //! Parameter \relates jevois::Engine
88  JEVOIS_DECLARE_PARAMETER(gadgetdev, std::string, "Gadget device name. This is used on platform hardware only. "
89  "On host hardware, a display window will be used unless gadgetdev is None (useful "
90  "for benchmarking) or is a file stem for a movie file that does not start with /dev/ "
91  "(and which should contain a printf-style directive for a single int argument, "
92  "the movie number).",
93  JEVOIS_GADGET_DEFAULT, ParamCateg);
94 
95  //! Parameter \relates jevois::Engine
96  JEVOIS_DECLARE_PARAMETER(gadgetnbuf, unsigned int, "Number of video output (USB video) buffers, or 0 for auto",
97  0, ParamCateg);
98 
99  //! Parameter \relates jevois::Engine
100  JEVOIS_DECLARE_PARAMETER(videomapping, int, "Index of Video Mapping to use, or -1 to use the default mapping",
101  -1, ParamCateg);
102 
103  //! Parameter \relates jevois::Engine
104  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(serialdev, std::string, "Hardware (4-pin connector) serial device name, "
105  "or 'stdio' to use the console, or empty for no hardware serial port",
106  JEVOIS_SERIAL_DEFAULT, ParamCateg);
107 
108  //! Parameter \relates jevois::Engine
109  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(usbserialdev, std::string, "Over-the-USB serial device name, or empty",
110  JEVOIS_USBSERIAL_DEFAULT, ParamCateg);
111 
112  //! Parameter \relates jevois::Engine
113  JEVOIS_DECLARE_PARAMETER(camreg, bool, "Enable raw access to camera registers through setcamreg and getcamreg",
114  false, ParamCateg);
115 
116  //! Parameter \relates jevois::Engine
117  JEVOIS_DECLARE_PARAMETER(camturbo, bool, "Enable camera turbo mode by relaxing the need for DMA-coherent video "
118  "buffer memory. This can accelerate severalfolds access to the captured image data, but "
119  "it may also yield stripe artifacts with some modules, such as PassThrough. The stripes "
120  "are pieces of incorrect data in the cache. You should experiment with each particular "
121  "module. Turbo mode is not recommended for any production-grade application.",
122  false, ParamCateg);
123 
124  //! Enum for Parameter \relates jevois::Engine
125  JEVOIS_DEFINE_ENUM_CLASS(SerPort, (None) (All) (Hard) (USB) );
126 
127  //! Parameter \relates jevois::Engine
128  JEVOIS_DECLARE_PARAMETER(serlog, SerPort, "Show log and debug messages on selected serial port(s)",
129  SerPort::None, SerPort_Values, ParamCateg);
130 
131  //! Parameter \relates jevois::Engine
132  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(videoerrors, bool, "Show any machine vision module errors (exceptions) "
133  "in the video stream. Only takes effect if streaming video to USB.",
134  true, ParamCateg);
135 
136  //! Parameter \relates jevois::Engine
137  JEVOIS_DECLARE_PARAMETER(serout, SerPort, "Send module serial messages to selected serial port(s)",
138  SerPort::None, SerPort_Values, ParamCateg);
139 
140  //! Enum for Parameter \relates jevois::Engine
141  JEVOIS_DEFINE_ENUM_CLASS(CPUmode, (PowerSave) (Conservative) (OnDemand) (Interactive) (Performance) );
142 
143  //! Parameter \relates jevois::Engine
144  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumode, CPUmode, "CPU frequency modulation mode",
145  CPUmode::Performance, CPUmode_Values, ParamCateg);
146 
147  // keep this in sync with sunxi-cpufreq.c
148  //! Parameter \relates jevois::Engine
149  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumax, unsigned int, "CPU maximum frequency in MHz",
150  1344, { 120, 240, 312, 408, 480, 504, 600, 648, 720, 816, 912, 1008,
151  1044, 1056, 1080, 1104, 1116, 1152, 1200, 1224, 1248, 1296, 1344 },
152  ParamCateg);
153  }
154 
155  //! JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB
156  /*! The Engine is orchestrating the execution of vision processing software. It is a Manager, i.e., it is the root of
157  a hierarchy of Component objects and it handles access to their Parameter settings and their construction, init(),
158  unInit(), and destruction. The component hierarchy consists of Engine at the root, then one Module which is
159  selected by the user at runtime, e.g., by selecting a given video format on video camera software running on a
160  host computer connected to the JeVois hardware. The Module may then contain an arbitrarily complex hierarchy of
161  Component objects with Parameter settings in them. Module derives from Component and thus may also have its own
162  Parameter settings.
163 
164  Engine contains the following basic elements:
165 
166  - A VideoInput, instantiated as either a Camera for live video streaming or a MovieInput for processing of
167  pre-recorded video files or sequences of images (useful during algorithm development, to test and optimize on
168  reproducible inputs);
169 
170  - A VideoOutput, instantiated either as a USB Gadget driver when running on the JeVois hardware platform, or as a
171  VideoDisplay when running on a computer that has a graphics display, or as a MovieOutput to save output video
172  frames to disk, or as a VideoOutputNone if desired for benchmarking of vision algorithms while discounting any
173  work related to transmitting output frames.
174 
175  - A DynamicLoader which handles loading the chosen vision processing Module at runtime depending on user
176  selections;
177 
178  - Any number of UserInterface objects, instantiated as either a hardware Serial port (for the 4-pin JST 1.0mm
179  connector on the platform hardware), a serial-over-USB Serial port (visible on the host computer to which the
180  JeVois hardware is connected by USB), or an StdioInterface (used to accept commands and print results directly
181  in the terminal where the JeVois Engine was started, particularly useful when running on a generic computer as
182  opposed to the platform hardware). When running on platform hardware, usually two UserInterface objects are
183  created (one hardware Serial, one serial-over-USB Serial), while, when running on a generic computer, usually
184  only one UserInterface is created (of type StdioInterface to accept commands directly in the terminal in which
185  the jevois-daemon was started);
186 
187  - The list of VideoMapping definitions imported from your videomappings.cfg file. These definitions specify which
188  video output modes are available over USB and their corresponding Camera settings and which Module to use, as
189  well as which modes are available that do not have any sreaming video output over USB (e.g., when connecting the
190  hardware platform to an Arduino only).
191 
192  The main loop of Engine runs until the user decides to quit, and basically goes through the following steps:
193 
194  - Create an InputFrame object which is an exception-safe wrapper around the next available Camera frame. The frame
195  may not have been captured yet. The InputFrame can be understood as a mechanism to gain access to that frame in
196  the future, when it has become available (i.e., has been captured by the camera). This is very similar to the
197  std::future framework of C++11.
198 
199  - When the current VideoMapping specifies that we will be streaming video frames out over USB, also create an
200  OutputFrame object which is an exception-safe wrapper around the next available Gadget frame. This is also just
201  a mechanism for gaining access to the next blank video buffer that is available from the USB driver and that we
202  should fill with interesting pixel data before sending it over USB to a host computer.
203 
204  - Call the currently-loaded Module's process() function, either as process(InputFrame, OutputFrame) when the
205  current VideoMapping specifies that some video output is to be sent over USB, or as process(InputFrame) when the
206  current VideoMapping specifies no video output. Any exception thrown by the Module's process() function will be
207  caught, reported, and ignored. The process() function would typically request the next available camera image
208  through the InputFrame wrapper (this request may block until the frame has been captured by the camera sensor
209  hardware), process that image, request the next available output image through the OutputFrame wrapper (when
210  VideoMapping specifies that there is USB video output), and paint some results into that output image, which
211  will then be sent to the host coputer over USB, for display by some webcam program or for further processing by
212  some custom vision software running on that computer. In addition, the currently loaded Module may issue
213  messages over the UserInterface ports (e.g., indicating the location at which an object was found, to let an
214  Arduino know about it).
215 
216  - Read any new commands issued by users over the UserInterface ports and execute the appropriate commands.
217 
218  - Handle user requests to change VideoMapping, when they select a different video mode in their webcam software
219  running on the host computer connected to the JeVois hardware. Such requests may trigger unloading of the
220  current Module and loading a new one, and changing camera pixel format, image size, etc. These changes are
221  guaranteed to occur when the Module's process() function is not running, i.e., Module programmers do not have to
222  worry about possible changes in image dimensions or pixel formats during execution of their process() function.
223 
224  - Pass any user requests received over USB or UserInterface to adjust camera parameters to the actual Camera
225  hardware driver (e.g., when users change contrast in their webcam program, that request is sent to the Engine
226  over USB, and the Engine then forwards it to the Camera hardware driver).
227 
228  \ingroup core */
229  class Engine : public Manager,
230  public Parameter<engine::cameradev, engine::cameranbuf, engine::gadgetdev, engine::gadgetnbuf,
231  engine::videomapping, engine::serialdev, engine::usbserialdev, engine::camreg,
232  engine::camturbo, engine::serlog, engine::videoerrors, engine::serout,
233  engine::cpumode, engine::cpumax>
234  {
235  public:
236  //! Constructor
237  Engine(std::string const & instance);
238 
239  //! Constructor with command-line parsing
240  Engine(int argc, char const* argv[], std::string const & instance);
241 
242  //! Destructor
243  ~Engine();
244 
245  //! Find the VideoMapping that has the given output specs, or throw if not found
246  VideoMapping const & findVideoMapping(unsigned int oformat, unsigned int owidth, unsigned int oheight,
247  float oframespersec) const;
248 
249  //! Get the current video mapping
250  /*! Note that the current mapping may not have an entry in our list of mappings obtained from videomappings.cfg,
251  if the current one was set on the fly by the setmapping2 CLI command. */
252  VideoMapping const & getCurrentVideoMapping() const;
253 
254  //! Return the number of video mappings
255  size_t numVideoMappings() const;
256 
257  //! Allow access to our video mappings which are parsed from file at construction
258  VideoMapping const & getVideoMapping(size_t idx) const;
259 
260  //! Get the video mapping index for a given UVC iformat, iframe and interval
261  size_t getVideoMappingIdx(unsigned int iformat, unsigned int iframe, unsigned int interval) const;
262 
263  //! Allow access to the default video mapping
264  VideoMapping const & getDefaultVideoMapping() const;
265 
266  //! Allow access to the default video mapping index
267  size_t getDefaultVideoMappingIdx() const;
268 
269  //! Callback for when the user selects a new output video format
270  /*! Here, we stop streaming, nuke any current processing module, set the camera format, set the gadget output
271  format, load the new processing module, and start streaming again. The given VideoMapping will typically be
272  obtained using findVideoMapping() from output specs received over the USB link. */
273  void setFormat(size_t idx);
274 
275  //! Start streaming on video from camera, processing, and USB
276  void streamOn();
277 
278  //! Stop streaming on video from camera, processing, and USB
279  void streamOff();
280 
281  //! Main loop: grab, process, send over USB. Should be called by main application thread
282  void mainLoop();
283 
284  //! Send a string to all serial ports
285  /*! \note When islog is true, this is assumes to be a log message, and it will be sent to the port(s) specified by
286  parameter serlog. Otherwise, the message will be sent to the ports specified by parameter serout. */
287  void sendSerial(std::string const & str, bool islog = false);
288 
289  protected:
290  //! Run a script from file
291  /*! The filename should be absolute. The file should have any of the commands supported by Engine, one per
292  line. Filename should be relative to the current module's path. */
293  void runScriptFromFile(std::string const & filename, std::shared_ptr<UserInterface> ser,
294  bool throw_no_file);
295 
296  //! Parameter callback
297  void onParamChange(engine::cameradev const & param, std::string const & newval);
298 
299  //! Parameter callback
300  void onParamChange(engine::gadgetdev const & param, std::string const & newval);
301 
302  //! Parameter callback
303  void onParamChange(engine::serialdev const & param, std::string const & newval);
304 
305  //! Parameter callback
306  void onParamChange(engine::usbserialdev const & param, std::string const & newval);
307 
308  //! Parameter callback
309  void onParamChange(engine::cpumode const & param, engine::CPUmode const & newval);
310 
311  //! Parameter callback
312  void onParamChange(engine::cpumax const & param, unsigned int const & newval);
313 
314  //! Parameter callback
315  void onParamChange(engine::videoerrors const & param, bool const & newval);
316 
317  size_t itsDefaultMappingIdx; //!< Index of default mapping
318  std::vector<VideoMapping> const itsMappings; //!< All our mappings from videomappings.cfg
319  VideoMapping itsCurrentMapping; //!< Current video mapping, may not match any in itsMappings if setmapping2 used
320 
321  std::shared_ptr<VideoInput> itsCamera; //!< Our camera
322  std::shared_ptr<VideoOutput> itsGadget; //!< Our gadget
323 
324  std::unique_ptr<DynamicLoader> itsLoader; //!< Our module loader
325  std::shared_ptr<Module> itsModule; //!< Our current module
326 
327  std::atomic<bool> itsRunning; //!< True when we are running
328  std::atomic<bool> itsStreaming; //!< True when we are streaming video
329  std::atomic<bool> itsStopMainLoop; //!< Flag used to stop the main loop
330 
331  mutable std::timed_mutex itsMtx; //!< Mutex to protect our internals
332 
333  void preInit() override; //!< Override of Manager::preInit()
334  void postInit() override; //!< Override of Manager::postInit()
335 
336  //! Parse a user command received over serial port
337  /*! Throw upon receiving an incorrect command (eg, bad parameter value), return true if success, return false if
338  command was not recognized and should be tried by Module. */
339  bool parseCommand(std::string const & str, std::shared_ptr<UserInterface> s);
340 
341  private:
342  std::list<std::shared_ptr<UserInterface> > itsSerials;
343 
344  void setFormatInternal(size_t idx); // itsMtx should be locked by caller
345  void setFormatInternal(jevois::VideoMapping const & m); // itsMtx should be locked by caller
346 
347  // Return help string for a camera control or throw
348  std::string camCtrlHelp(struct v4l2_queryctrl & qc, std::set<int> & doneids);
349 
350  // Get short name from V4L2 ID, long name is a backup in case we don't find the control in our list
351  std::string camctrlname(int id, char const * longname) const;
352 
353  // Get V4L2 ID from short name
354  int camctrlid(std::string const & shortname);
355 
356  bool itsTurbo;
357  bool itsManualStreamon; // allow manual streamon when outputing video to None or file
358  std::atomic<bool> itsVideoErrors; // fast cached value for engine::videoerrors
359  jevois::RawImage itsVideoErrorImage;
360  std::string itsModuleConstructionError; // Non-empty error message if module constructor threw
361 
362 #ifdef JEVOIS_PLATFORM
363  // Things related to mass storage gadget to export our /jevois partition as a virtual USB flash drive:
364  void checkMassStorage(); // thread to check mass storage gadget status
365  std::future<void> itsCheckMassStorageFut;
366  std::atomic<bool> itsCheckingMassStorage;
367  std::atomic<bool> itsMassStorageMode;
368  void startMassStorageMode();
369  void stopMassStorageMode();
370  void reboot();
371 #endif
372  };
373 
374 } // namespace jevois
375 
size_t itsDefaultMappingIdx
Index of default mapping.
Definition: Engine.H:317
std::shared_ptr< VideoInput > itsCamera
Our camera.
Definition: Engine.H:321
Generic variadic template class template definition for Component Parameters.
#define JEVOIS_GADGET_DEFAULT
On generic computer hardware, device for the USB gadget driver should always be empty.
Definition: Engine.H:55
std::unique_ptr< DynamicLoader > itsLoader
Our module loader.
Definition: Engine.H:324
std::atomic< bool > itsStreaming
True when we are streaming video.
Definition: Engine.H:328
Manager of a hierarchy of Component objects.
Definition: Manager.H:73
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool, "Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
JEVOIS_DECLARE_PARAMETER(thresh1, double, "First threshold for hysteresis", 50.0, ParamCateg)
std::shared_ptr< VideoOutput > itsGadget
Our gadget.
Definition: Engine.H:322
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
VideoMapping itsCurrentMapping
Current video mapping, may not match any in itsMappings if setmapping2 used.
Definition: Engine.H:319
Simple struct to hold video mapping definitions for the processing Engine.
Definition: VideoMapping.H:41
std::atomic< bool > itsStopMainLoop
Flag used to stop the main loop.
Definition: Engine.H:329
#define JEVOIS_SERIAL_DEFAULT
On generic computer hardware, device for serial port should always be stdio to use an StdioInterface...
Definition: Engine.H:58
std::vector< VideoMapping > const itsMappings
All our mappings from videomappings.cfg.
Definition: Engine.H:318
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:229
std::timed_mutex itsMtx
Mutex to protect our internals.
Definition: Engine.H:331
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Parameter callback.
#define JEVOIS_CAMERA_DEFAULT
On generic computer hardware, device for the camera sensor.
Definition: Engine.H:52
std::atomic< bool > itsRunning
True when we are running.
Definition: Engine.H:327
#define JEVOIS_USBSERIAL_DEFAULT
On generic computer hardware, device for the serial-over-USB port should always be empty...
Definition: Engine.H:61
std::shared_ptr< Module > itsModule
Our current module.
Definition: Engine.H:325