JeVois  1.0
JeVois Smart Embedded Machine Vision Toolkit
Engine.H
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #pragma once
19 
22 #include <jevois/Types/Enum.H>
23 
24 #include <memory>
25 #include <mutex>
26 #include <vector>
27 #include <list>
28 #include <atomic>
29 
30 #ifdef JEVOIS_PLATFORM
31 // On the platform (JeVois hardware), we use a gadget driver by default to send output frames over USB, one hardware
32 // serial driver, and one serial-over-USB driver:
33 
34 //! On platform hardware, device for the camera sensor
35 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
36 
37 //! On platform hardware, device for the USB gadget driver (which sends video frames over USB to a host computer)
38 #define JEVOIS_GADGET_DEFAULT "/dev/video1"
39 
40 //! On platform hardware, device for the 4-pin hardware serial port
41 #define JEVOIS_SERIAL_DEFAULT "/dev/ttyS0"
42 
43 //! On platform hardware, device for serial-over-USB port
44 #define JEVOIS_USBSERIAL_DEFAULT "/dev/ttyGS0"
45 
46 #else
47 // On the host, we have no gadget (which will trigger displaying output frames to a window) and we use the terminal in
48 // which jevois-daemon was started for serial commands:
49 
50 //! On generic computer hardware, device for the camera sensor
51 #define JEVOIS_CAMERA_DEFAULT "/dev/video0"
52 
53 //! On generic computer hardware, device for the USB gadget driver should always be empty
54 #define JEVOIS_GADGET_DEFAULT ""
55 
56 //! On generic computer hardware, device for serial port should always be stdio to use an StdioInterface
57 #define JEVOIS_SERIAL_DEFAULT "stdio"
58 
59 //! On generic computer hardware, device for the serial-over-USB port should always be empty
60 #define JEVOIS_USBSERIAL_DEFAULT ""
61 
62 #endif
63 
64 namespace jevois
65 {
66  class VideoInput;
67  class VideoOutput;
68  class Module;
69  class DynamicLoader;
70  class UserInterface;
71 
72  namespace engine
73  {
74  static ParameterCategory const ParamCateg("Engine Options");
75 
76  //! Parameter \relates jevois::Engine
77  JEVOIS_DECLARE_PARAMETER(cameradev, std::string, "Camera device name (if starting with /dev/v...), or movie "
78  "file name (e.g., movie.mpg) or image sequence (e.g., im%02d.jpg, to read frames "
79  "im00.jpg, im01.jpg, etc).",
80  JEVOIS_CAMERA_DEFAULT, ParamCateg);
81 
82  //! Parameter \relates jevois::Engine
83  JEVOIS_DECLARE_PARAMETER(cameranbuf, unsigned int, "Number of video input (camera) buffers, or 0 for automatic.",
84  0, ParamCateg);
85 
86  //! Parameter \relates jevois::Engine
87  JEVOIS_DECLARE_PARAMETER(gadgetdev, std::string, "Gadget device name. This is used on platform hardware only. "
88  "On host hardware, a display window will be used unless gadgetdev is None (useful "
89  "for benchmarking) or is a file stem for a movie file that does not start with /dev/ "
90  "(and which should contain a printf-style directive for a single int argument, "
91  "the movie number).",
92  JEVOIS_GADGET_DEFAULT, ParamCateg);
93 
94  //! Parameter \relates jevois::Engine
95  JEVOIS_DECLARE_PARAMETER(gadgetnbuf, unsigned int, "Number of video output (USB video) buffers, or 0 for auto",
96  0, ParamCateg);
97 
98  //! Parameter \relates jevois::Engine
99  JEVOIS_DECLARE_PARAMETER(videomapping, int, "Index of Video Mapping to use, or -1 to use the default mapping",
100  -1, ParamCateg);
101 
102  //! Parameter \relates jevois::Engine
103  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(serialdev, std::string, "Hardware (4-pin connector) serial device name, "
104  "or 'stdio' to use the console, or empty for no hardware serial port",
105  JEVOIS_SERIAL_DEFAULT, ParamCateg);
106 
107  //! Parameter \relates jevois::Engine
108  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(usbserialdev, std::string, "Over-the-USB serial device name, or empty",
109  JEVOIS_USBSERIAL_DEFAULT, ParamCateg);
110 
111  //! Parameter \relates jevois::Engine
112  JEVOIS_DECLARE_PARAMETER(camreg, bool, "Enable raw access to camera registers through setcamreg and getcamreg",
113  false, ParamCateg);
114 
115  //! Parameter \relates jevois::Engine
116  JEVOIS_DECLARE_PARAMETER(camturbo, bool, "Enable camera turbo mode by relaxing the need for DMA-coherent video "
117  "buffer memory. This can accelerate severalfolds access to the captured image data, but "
118  "it may also yield stripe artifacts with some modules, such as PassThrough. The stripes "
119  "are pieces of incorrect data in the cache. You should experiment with each particular "
120  "module. Turbo mode is not recommended for any production-grade application.",
121  false, ParamCateg);
122 
123  //! Enum for Parameter \relates jevois::Engine
124  JEVOIS_DEFINE_ENUM_CLASS(SerPort, (None) (All) (Hard) (USB) );
125 
126  //! Parameter \relates jevois::Engine
127  JEVOIS_DECLARE_PARAMETER(serlog, SerPort, "Show log and debug messages on selected serial port(s)",
128  SerPort::None, SerPort_Values, ParamCateg);
129 
130  //! Parameter \relates jevois::Engine
131  JEVOIS_DECLARE_PARAMETER(serout, SerPort, "Send module serial messages to selected serial port(s)",
132  SerPort::None, SerPort_Values, ParamCateg);
133 
134  //! Enum for Parameter \relates jevois::Engine
135  JEVOIS_DEFINE_ENUM_CLASS(CPUmode, (PowerSave) (Conservative) (OnDemand) (Interactive) (Performance) )
136 
137  //! Parameter \relates jevois::Engine
138  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumode, CPUmode, "CPU frequency modulation mode",
139  CPUmode::Performance, CPUmode_Values, ParamCateg);
140 
141  // keep this in sync with sunxi-cpufreq.c
142  //! Parameter \relates jevois::Engine
143  JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(cpumax, unsigned int, "CPU maximum frequency in MHz",
144  1344, { 120, 240, 312, 408, 480, 504, 600, 648, 720, 816, 912, 1008,
145  1044, 1056, 1080, 1104, 1116, 1152, 1200, 1224, 1248, 1296, 1344 },
146  ParamCateg)
147  }
148 
149  //! JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB
150  /*! The Engine is orchestrating the execution of vision processing software. It is a Manager, i.e., it is the root of
151  a hierarchy of Component objects and it handles access to their Parameter settings and their construction, init(),
152  unInit(), and destruction. The component hierarchy consists of Engine at the root, then one Module which is
153  selected by the user at runtime, e.g., by selecting a given video format on video camera software running on a
154  host computer connected to the JeVois hardware. The Module may then contain an arbitrarily complex hierarchy of
155  Component objects with Parameter settings in them. Module derives from Component and thus may also have its own
156  Parameter settings.
157 
158  Engine contains the following basic elements:
159 
160  - A VideoInput, instantiated as either a Camera for live video streaming or a MovieInput for processing of
161  pre-recorded video files or sequences of images (useful during algorithm development, to test and optimize on
162  reproducible inputs);
163 
164  - A VideoOutput, instantiated either as a USB Gadget driver when running on the JeVois hardware platform, or as a
165  VideoDisplay when running on a computer that has a graphics display, or as a MovieOutput to save output video
166  frames to disk, or as a VideoOutputNone if desired for benchmarking of vision algorithms while discounting any
167  work related to transmitting output frames.
168 
169  - A DynamicLoader which handles loading the chosen vision processing Module at runtime depending on user
170  selections;
171 
172  - Any number of UserInterface objects, instantiated as either a hardware Serial port (for the 4-pin JST 1.0mm
173  connector on the platform hardware), a serial-over-USB Serial port (visible on the host computer to which the
174  JeVois hardware is connected by USB), or an StdioInterface (used to accept commands and print results directly
175  in the terminal where the JeVois Engine was started, particularly useful when running on a generic computer as
176  opposed to the platform hardware). When running on platform hardware, usually two UserInterface objects are
177  created (one hardware Serial, one serial-over-USB Serial), while, when running on a generic computer, usually
178  only one UserInterface is created (of type StdioInterface to accept commands directly in the terminal in which
179  the jevois-daemon was started);
180 
181  - The list of VideoMapping definitions imported from your videomappings.cfg file. These definitions specify which
182  video output modes are available over USB and their corresponding Camera settings and which Module to use, as
183  well as which modes are available that do not have any sreaming video output over USB (e.g., when connecting the
184  hardware platform to an Arduino only).
185 
186  The main loop of Engine runs until the user decides to quit, and basically goes through the following steps:
187 
188  - Create an InputFrame object which is an exception-safe wrapper around the next available Camera frame. The frame
189  may not have been captured yet. The InputFrame can be understood as a mechanism to gain access to that frame in
190  the future, when it has become available (i.e., has been captured by the camera). This is very similar to the
191  std::future framework of C++11.
192 
193  - When the current VideoMapping specifies that we will be streaming video frames out over USB, also create an
194  OutputFrame object which is an exception-safe wrapper around the next available Gadget frame. This is also just
195  a mechanism for gaining access to the next blank video buffer that is available from the USB driver and that we
196  should fill with interesting pixel data before sending it over USB to a host computer.
197 
198  - Call the currently-loaded Module's process() function, either as process(InputFrame, OutputFrame) when the
199  current VideoMapping specifies that some video output is to be sent over USB, or as process(InputFrame) when the
200  current VideoMapping specifies no video output. Any exception thrown by the Module's process() function will be
201  caught, reported, and ignored. The process() function would typically request the next available camera image
202  through the InputFrame wrapper (this request may block until the frame has been captured by the camera sensor
203  hardware), process that image, request the next available output image through the OutputFrame wrapper (when
204  VideoMapping specifies that there is USB video output), and paint some results into that output image, which
205  will then be sent to the host coputer over USB, for display by some webcam program or for further processing by
206  some custom vision software running on that computer. In addition, the currently loaded Module may issue
207  messages over the UserInterface ports (e.g., indicating the location at which an object was found, to let an
208  Arduino know about it).
209 
210  - Read any new commands issued by users over the UserInterface ports and execute the appropriate commands.
211 
212  - Handle user requests to change VideoMapping, when they select a different video mode in their webcam software
213  running on the host computer connected to the JeVois hardware. Such requests may trigger unloading of the
214  current Module and loading a new one, and changing camera pixel format, image size, etc. These changes are
215  guaranteed to occur when the Module's process() function is not running, i.e., Module programmers do not have to
216  worry about possible changes in image dimensions or pixel formats during execution of their process() function.
217 
218  - Pass any user requests received over USB or UserInterface to adjust camera parameters to the actual Camera
219  hardware driver (e.g., when users change contrast in their webcam program, that request is sent to the Engine
220  over USB, and the Engine then forwards it to the Camera hardware driver).
221 
222  \ingroup core */
223  class Engine : public Manager,
224  public Parameter<engine::cameradev, engine::cameranbuf, engine::gadgetdev, engine::gadgetnbuf,
225  engine::videomapping, engine::serialdev, engine::usbserialdev, engine::camreg,
226  engine::camturbo, engine::serlog, engine::serout, engine::cpumode, engine::cpumax>
227  {
228  public:
229  //! Constructor
230  Engine(std::string const & instance);
231 
232  //! Constructor with command-line parsing
233  Engine(int argc, char const* argv[], std::string const & instance);
234 
235  //! Destructor
236  ~Engine();
237 
238  //! Find the VideoMapping that has the given output specs, or throw if not found
239  VideoMapping const & findVideoMapping(unsigned int oformat, unsigned int owidth, unsigned int oheight,
240  float oframespersec) const;
241 
242  //! Return the number of video mappings
243  size_t numVideoMappings() const;
244 
245  //! Allow access to our video mappings, which are parsed from file at construction
246  VideoMapping const & getVideoMapping(size_t idx) const;
247 
248  //! Get the video mapping index for a given UVC iformat, iframe and interval
249  size_t getVideoMappingIdx(unsigned int iformat, unsigned int iframe, unsigned int interval) const;
250 
251  //! Allow access to the default video mapping
252  VideoMapping const & getDefaultVideoMapping() const;
253 
254  //! Allow access to the default video mapping index
255  size_t getDefaultVideoMappingIdx() const;
256 
257  //! Callback for when the user selects a new output video format
258  /*! Here, we stop streaming, nuke any current processing module, set the camera format, set the gadget output
259  format, load the new processing module, and start streaming again. The given VideoMapping will typically be
260  obtained using findVideoMapping() from output specs received over the USB link. */
261  void setFormat(size_t idx);
262 
263  //! Start streaming on video from camera, processing, and USB
264  void streamOn();
265 
266  //! Stop streaming on video from camera, processing, and USB
267  void streamOff();
268 
269  //! Main loop: grab, process, send over USB. Should be called by main application thread
270  void mainLoop();
271 
272  //! Send a string to all serial ports
273  /*! \note When islog is true, this is assumes to be a log message, and it will be sent to the port(s) specified by
274  parameter serlog. Otherwise, the message will be sent to the ports specified by parameter serout. */
275  void sendSerial(std::string const & str, bool islog = false);
276 
277  protected:
278  //! Run a script from file
279  /*! The filename should be absolute. The file should have any of the commands supported by Engine, one per
280  line. Filename should be relative to the current module's path. */
281  void runScriptFromFile(std::string const & filename, std::shared_ptr<UserInterface> ser,
282  bool throw_no_file);
283 
284  //! Parameter callback
285  void onParamChange(engine::cameradev const & param, std::string const & newval);
286 
287  //! Parameter callback
288  void onParamChange(engine::gadgetdev const & param, std::string const & newval);
289 
290  //! Parameter callback
291  void onParamChange(engine::serialdev const & param, std::string const & newval);
292 
293  //! Parameter callback
294  void onParamChange(engine::usbserialdev const & param, std::string const & newval);
295 
296  //! Parameter callback
297  void onParamChange(engine::cpumode const & param, engine::CPUmode const & newval);
298 
299  //! Parameter callback
300  void onParamChange(engine::cpumax const & param, unsigned int const & newval);
301 
302  size_t itsDefaultMappingIdx; //!< Index of default mapping
303  std::vector<VideoMapping> const itsMappings; //!< All our mappings from videomappings.cfg
304  bool itsUSBout; //!< True when current mapping has some USB output
305 
306  std::shared_ptr<VideoInput> itsCamera; //!< Our camera
307  std::shared_ptr<VideoOutput> itsGadget; //!< Our gadget
308 
309  std::unique_ptr<DynamicLoader> itsLoader; //!< Our module loader
310  std::shared_ptr<Module> itsModule; //!< Our current module
311 
312  std::atomic<bool> itsRunning; //!< True when we are running
313  std::atomic<bool> itsStreaming; //!< True when we are streaming video
314  std::atomic<bool> itsStopMainLoop; //!< Flag used to stop the main loop
315 
316  mutable std::timed_mutex itsMtx; //!< Mutex to protect our internals
317 
318  void preInit() override; //!< Override of Manager::preInit()
319  void postInit() override; //!< Override of Manager::postInit()
320 
321  //! Parse a user command received over serial port
322  /*! Throw upon receiving an incorrect command (eg, bad parameter value), return true if success, return false if
323  command was not recognized and should be tried by Module. */
324  bool parseCommand(std::string const & str, std::shared_ptr<UserInterface> s);
325 
326  private:
327  std::list<std::shared_ptr<UserInterface> > itsSerials;
328 
329  void setFormatInternal(size_t idx); // itsMtx should be locked by caller
330 
331  // Return help string for a camera control or throw
332  std::string camCtrlHelp(struct v4l2_queryctrl & qc, std::set<int> & doneids);
333 
334  // Get short name from V4L2 ID, long name is a backup in case we don't find the control in our list
335  std::string camctrlname(int id, char const * longname) const;
336 
337  // Get V4L2 ID from short name
338  int camctrlid(std::string const & shortname);
339 
340  bool itsTurbo;
341  bool itsManualStreamon; // allow manusl streamon when outputing video to None or file
342  };
343 
344 } // namespace jevois
345 
CPU frequency modulation CPUmode_Values
Definition: Engine.H:138
size_t itsDefaultMappingIdx
Index of default mapping.
Definition: Engine.H:302
std::shared_ptr< VideoInput > itsCamera
Our camera.
Definition: Engine.H:306
Generic variadic template class template definition for Component Parameters.
JEVOIS_DECLARE_PARAMETER_WITH_CALLBACK(l2grad, bool,"Use more accurate L2 gradient norm if true, L1 if false", false, ParamCateg)
#define JEVOIS_GADGET_DEFAULT
On generic computer hardware, device for the USB gadget driver should always be empty.
Definition: Engine.H:54
std::unique_ptr< DynamicLoader > itsLoader
Our module loader.
Definition: Engine.H:309
std::atomic< bool > itsStreaming
True when we are streaming video.
Definition: Engine.H:313
Manager of a hierarchy of Component objects.
Definition: Manager.H:66
A category to which multiple ParameterDef definitions can belong.
Definition: ParameterDef.H:33
std::shared_ptr< VideoOutput > itsGadget
Our gadget.
Definition: Engine.H:307
CPU frequency modulation mode
Definition: Engine.H:138
Simple struct to hold video mapping definitions for the processing Engine.
Definition: VideoMapping.H:41
bool itsUSBout
True when current mapping has some USB output.
Definition: Engine.H:304
std::atomic< bool > itsStopMainLoop
Flag used to stop the main loop.
Definition: Engine.H:314
#define JEVOIS_SERIAL_DEFAULT
On generic computer hardware, device for serial port should always be stdio to use an StdioInterface...
Definition: Engine.H:57
std::vector< VideoMapping > const itsMappings
All our mappings from videomappings.cfg.
Definition: Engine.H:303
JeVois processing engine - gets images from camera sensor, processes them, and sends results over USB...
Definition: Engine.H:223
std::timed_mutex itsMtx
Mutex to protect our internals.
Definition: Engine.H:316
void onParamChange(manager::loglevel const &param, manager::LogLevel const &newval)
Parameter callback.
#define JEVOIS_CAMERA_DEFAULT
On generic computer hardware, device for the camera sensor.
Definition: Engine.H:51
std::atomic< bool > itsRunning
True when we are running.
Definition: Engine.H:312
#define JEVOIS_USBSERIAL_DEFAULT
On generic computer hardware, device for the serial-over-USB port should always be empty...
Definition: Engine.H:60
std::shared_ptr< Module > itsModule
Our current module.
Definition: Engine.H:310