Surprise-based recording of events. More...

Inheritance diagram for SurpriseRecorder:

[legend]

Collaboration diagram for SurpriseRecorder:

[legend]

Public Member Functions
	SurpriseRecorder (std::string const &instance)
	Constructor.

virtual	~SurpriseRecorder ()
	Virtual destructor for safe inheritance.

void	postInit () override
	Get started.

void	postUninit () override
	Get stopped.

void	process (jevois::InputFrame &&inframe) override
	Processing function, version with no video output.

Public Member Functions inherited from jevois::Module
	Module (std::string const &instance)

virtual	~Module ()

virtual void	process (InputFrame &&inframe, OutputFrame &&outframe)

virtual void	process (InputFrame &&inframe, GUIhelper &helper)

virtual void	sendSerial (std::string const &str)

virtual void	parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)

virtual void	supportedCommands (std::ostream &os)

Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)

virtual	~Component ()

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

bool	initialized () const

std::string const &	className () const

std::string const &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor, bool doit)

void	freezeAllParams (bool doit)

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

bool	initialized () const

std::string const &	className () const

std::string const &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor, bool doit)

void	freezeAllParams (bool doit)

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Protected Member Functions
void	run ()
	Video writer thread.

Protected Member Functions inherited from jevois::Component
virtual void	preInit ()

virtual void	preUninit ()

virtual void	preInit ()

virtual void	preUninit ()

Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)

void	removeParameter (ParameterBase *const param)

void	callbackInitCall ()

Protected Attributes
std::shared_ptr< Surprise >	itsSurprise

std::future< void >	itsRunFut
	Future for our run() thread.

std::deque< cv::Mat >	itsCtxBuf
	Buffer for context frames before event start.

jevois::BoundedBuffer< cv::Mat, jevois::BlockingBehavior::Block, jevois::BlockingBehavior::Block >	itsBuf
	Buffer for frames to save.

int	itsToSave
	Number of context frames after end of event that remain to be saved.

int	itsFileNum
	Video file number.

std::atomic< bool >	itsRunning
	Flag to let run thread when to quit.

std::string	itsFilename
	Current video file name.

Related Symbols
(Note that these are not member symbols.)
	JEVOIS_DECLARE_PARAMETER (filename, std::string, "Name of the video file to write. If path is not absolute, " PATHPREFIX " will be prepended to it. Name should contain a printf-like directive for " "one int argument, which will start at 0 and be incremented on each streamoff command.", "video%06d.avi", ParamCateg)
	Parameter.

	JEVOIS_DECLARE_PARAMETER (fourcc, std::string, "FourCC of the codec to use. The OpenCV VideoWriter doc is unclear " "as to which codecs are supported. Presumably, the ffmpeg library is used inside OpenCV. " "Hence any video encoder supported by ffmpeg should work. Tested codecs include: MJPG, " "MP4V, AVC1. Make sure you also pick the right filename extension (e.g., .avi for MJPG, " ".mp4 for MP4V, etc)", "MJPG", boost::regex("^\\w{4}$"), ParamCateg)
	Parameter.

	JEVOIS_DECLARE_PARAMETER (fps, double, "Video frames/sec as stored in the file and to be used both for recording and " "playback. Beware that the video writer will drop frames if you are capturing faster than " "the frame rate specified here. For example, if capturing at 120fps, be sure to set this " "parameter to 120, otherwise by default the saved video will be at 30fps even though capture " "was running at 120fps.", 15.0, ParamCateg)
	Parameter.

	JEVOIS_DECLARE_PARAMETER (thresh, double, "Surprise threshold. Lower values will record more events.", 1.0e7, ParamCateg)
	Parameter.

	JEVOIS_DECLARE_PARAMETER (ctxframes, unsigned int, "Number of context video frames recorded before and after " "each surprising event.", 150, ParamCateg)
	Parameter.

Detailed Description

Surprise-based recording of events.

This module detects surprising events in the live video feed from the camera, and records short video clips of each detected event.

Surprising is here defined according to Itti and Baldi's mathematical theory of surprise (see, e.g., http://ilab.usc.edu/surprise/) which is applied to monitoring live video streams. When a surprising event is detected, a short video clip of that event is saved to the microSD card inside JeVois, for later review.

It was created in this JeVois tutorial: http://jevois.org/tutorials/ProgrammerSurprise.html

Using this module

This module does not send any video output to USB. Rather, it just saves surprising events to microSD for later review. Hence, you may want to try the following:

mount the JeVois camera where you want it to detect surprising events
run it connected to a laptop computer, using any mode which does have some video output over USB (e.g., 640x500 YUYV). Adjust the camera orientation to best fit your needs.
edit JEVOIS:/config/initscript.cfg to contain:
```
  setmapping2 YUYV 640 480 15.0 JeVois SurpriseRecorder
  setpar thresh 1e7
  setpar channels S
  streamon
```
and see the above tutorial for more details. Next time you power JeVois, it will immediately start detecting and recording surprising events in its view.

Example

Here is one hour of video surveillance footage. It is very boring overall. Except that a few brief surprising things occur (a few seconds each). Can you find them?

Here is what the SurpriseRecorder module found (4 true events plus 2 false alarms):

With only 6 surprising events, and assuming +/- 10 seconds of context frames around each event, we have achieved a compression of the surveillance footage from 60 minutes to 2 minutes (a factor 30x).

Author: Laurent Itti

Videomapping:: NONE 0 0 0 YUYV 640 480 15.0 JeVois SurpriseRecorder

Email:: itti@usc.edu

Address:: University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA

Copyright: Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California

Main URL:: http://jevois.org

Support URL:: http://jevois.org/doc

Other URL:: http://iLab.usc.edu

License:: GPL v3

Distribution:: Unrestricted

Restrictions:: None

Definition at line 124 of file SurpriseRecorder.C.

Constructor & Destructor Documentation

◆ SurpriseRecorder()

SurpriseRecorder::SurpriseRecorder ( std::string const & instance )

inline

Constructor.

Definition at line 132 of file SurpriseRecorder.C.

References itsSurprise.

◆ ~SurpriseRecorder()

virtual SurpriseRecorder::~SurpriseRecorder ( )

inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 139 of file SurpriseRecorder.C.

Member Function Documentation

◆ postInit()

void SurpriseRecorder::postInit ( )

inlineoverridevirtual

Get started.

Reimplemented from jevois::Component.

Definition at line 145 of file SurpriseRecorder.C.

References jevois::async(), itsRunFut, itsRunning, and run().

◆ postUninit()

void SurpriseRecorder::postUninit ( )

inlineoverridevirtual

Get stopped.

Reimplemented from jevois::Component.

Definition at line 156 of file SurpriseRecorder.C.

References jevois::BoundedBuffer< typename T, BlockingBehavior WhenFull, BlockingBehavior WhenEmpty >::filled_size(), itsBuf, itsFilename, itsRunFut, itsRunning, LERROR, LINFO, jevois::BoundedBuffer< typename T, BlockingBehavior WhenFull, BlockingBehavior WhenEmpty >::push(), and jevois::warnAndIgnoreException().

◆ process()

void SurpriseRecorder::process ( jevois::InputFrame && inframe )

inlineoverridevirtual

Processing function, version with no video output.

Reimplemented from jevois::Module.

Definition at line 175 of file SurpriseRecorder.C.

References jevois::async(), jevois::Profiler::checkpoint(), jevois::rawimage::convertToCvBGR(), h, jevois::RawImage::height, itsBuf, itsCtxBuf, itsSurprise, itsToSave, jevois::BoundedBuffer< typename T, BlockingBehavior WhenFull, BlockingBehavior WhenEmpty >::push(), jevois::RawImage::require(), jevois::Profiler::start(), jevois::Profiler::stop(), and jevois::RawImage::width.

◆ run()

void SurpriseRecorder::run ( )

inlineprotected

Video writer thread.

Definition at line 256 of file SurpriseRecorder.C.

References itsBuf, itsFilename, itsFileNum, itsRunning, LERROR, LFATAL, PATHPREFIX, jevois::BoundedBuffer< typename T, BlockingBehavior WhenFull, BlockingBehavior WhenEmpty >::pop(), and jevois::Module::sendSerial().

Referenced by postInit().

Friends And Related Symbol Documentation

◆ JEVOIS_DECLARE_PARAMETER() [1/5]

JEVOIS_DECLARE_PARAMETER	(	ctxframes	,
		unsigned int	,
		"Number of context video frames recorded before and after " "each surprising event."	,
		150	,
		ParamCateg
	)

◆ JEVOIS_DECLARE_PARAMETER() [2/5]

JEVOIS_DECLARE_PARAMETER	(	filename	,
		std::string	,
		"Name of the video file to write. If path is not	absolute,
		" PATHPREFIX " will be prepended to it. Name should contain a printf-like directive for " "one int	argument,
		which will start at 0 and be incremented on each streamoff command."	,
		"video%06d.avi"	,
		ParamCateg
	)

◆ JEVOIS_DECLARE_PARAMETER() [3/5]

JEVOIS_DECLARE_PARAMETER	(	fourcc	,
		std::string	,
		"FourCC of the codec to use. The OpenCV VideoWriter doc is unclear " "as to which codecs are supported.	Presumably,
		the ffmpeg library is used inside OpenCV. " "Hence any video encoder supported by ffmpeg should work. Tested codecs include:MJPG	,
		" "	MP4V,
		AVC1. Make sure you also pick the right filename extension(e.g.,.avi for MJPG, " ".mp4 for MP4V, etc)"	,
		"MJPG"	,
		boost::regex("^\\w{4}$")	,
		ParamCateg
	)

◆ JEVOIS_DECLARE_PARAMETER() [4/5]

JEVOIS_DECLARE_PARAMETER	(	fps	,
		double	,
		"Video frames/sec as stored in the file and to be used both for recording and " "playback. Beware that the video writer will drop frames if you are capturing faster than " "the frame rate specified here. For	example,
		if capturing at	120fps,
		be sure to set this " "parameter to	120,
		otherwise by default the saved video will be at 30fps even though capture " "was running at 120fps."	,
		15.	0,
		ParamCateg
	)

◆ JEVOIS_DECLARE_PARAMETER() [5/5]

JEVOIS_DECLARE_PARAMETER	(	thresh	,
		double	,
		"Surprise threshold. Lower values will record more events."	,
		1.	0e7,
		ParamCateg
	)

Member Data Documentation

◆ itsBuf

jevois::BoundedBuffer<cv::Mat, jevois::BlockingBehavior::Block, jevois::BlockingBehavior::Block> SurpriseRecorder::itsBuf

protected

Buffer for frames to save.

Definition at line 323 of file SurpriseRecorder.C.

Referenced by postUninit(), process(), and run().

◆ itsCtxBuf

std::deque<cv::Mat> SurpriseRecorder::itsCtxBuf

protected

Buffer for context frames before event start.

Definition at line 321 of file SurpriseRecorder.C.

Referenced by process().

◆ itsFilename

std::string SurpriseRecorder::itsFilename

protected

Current video file name.

Definition at line 327 of file SurpriseRecorder.C.

Referenced by postUninit(), and run().

◆ itsFileNum

int SurpriseRecorder::itsFileNum

protected

Video file number.

Definition at line 325 of file SurpriseRecorder.C.

Referenced by run().

◆ itsRunFut

std::future<void> SurpriseRecorder::itsRunFut

protected

Future for our run() thread.

Definition at line 320 of file SurpriseRecorder.C.

Referenced by postInit(), and postUninit().

◆ itsRunning

std::atomic<bool> SurpriseRecorder::itsRunning

protected

Flag to let run thread when to quit.

Definition at line 326 of file SurpriseRecorder.C.

Referenced by postInit(), postUninit(), and run().

◆ itsSurprise

std::shared_ptr<Surprise> SurpriseRecorder::itsSurprise

protected

Definition at line 251 of file SurpriseRecorder.C.

Referenced by process(), and SurpriseRecorder().

◆ itsToSave

int SurpriseRecorder::itsToSave

protected

Number of context frames after end of event that remain to be saved.

Definition at line 324 of file SurpriseRecorder.C.

Referenced by process().

The documentation for this class was generated from the following file:

src/Modules/SurpriseRecorder/SurpriseRecorder.C

Public Member Functions

Protected Member Functions

Protected Attributes

Related Symbols

Detailed Description

Using this module

Example

Constructor & Destructor Documentation

◆ SurpriseRecorder()

◆ ~SurpriseRecorder()

Member Function Documentation

◆ postInit()

◆ postUninit()

◆ process()

◆ run()

Friends And Related Symbol Documentation

◆ JEVOIS_DECLARE_PARAMETER() [1/5]

◆ JEVOIS_DECLARE_PARAMETER() [2/5]

◆ JEVOIS_DECLARE_PARAMETER() [3/5]

◆ JEVOIS_DECLARE_PARAMETER() [4/5]

◆ JEVOIS_DECLARE_PARAMETER() [5/5]

Member Data Documentation

◆ itsBuf

◆ itsCtxBuf

◆ itsFilename

◆ itsFileNum

◆ itsRunFut

◆ itsRunning

◆ itsSurprise

◆ itsToSave