Live saliency computation and image filtering using 4-core CPU and OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU. More...

Inheritance diagram for DemoCPUGPU:

[legend]

Collaboration diagram for DemoCPUGPU:

[legend]

Public Member Functions
	DemoCPUGPU (std::string const &instance)
	Constructor. More...

virtual	~DemoCPUGPU ()
	Virtual destructor for safe inheritance. More...

void	postInit () override
	Set our GPU program after we are fully constructed and our Component path has been set. More...

virtual void	process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
	Processing function. More...

void	pasteGrayMap (jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int dy, unsigned int bitshift)
	Paste a map and add its width to the dx offset. More...

Public Member Functions inherited from jevois::StdModule
	StdModule (std::string const &instance)

virtual	~StdModule ()

void	sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")

void	sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")

void	sendSerialObjReco (std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)

void	sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)

	JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine)) JEVOIS_DECLARE_PARAMETER(serstyle

	JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)

	JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime)) JEVOIS_DECLARE_PARAMETER(serstamp

	JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both)) JEVOIS_DECLARE_PARAMETER(sermark

Public Member Functions inherited from jevois::Module
	Module (std::string const &instance)

virtual	~Module ()

virtual void	process (InputFrame &&inframe)

virtual void	process (InputFrame &&inframe, GUIhelper &helper)

virtual void	sendSerial (std::string const &str)

virtual void	parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)

virtual void	supportedCommands (std::ostream &os)

Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)

virtual	~Component ()

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

Engine *	engine () const

bool	initialized () const

const std::string &	className () const

const std::string &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor)

void	unFreezeParam (std::string const &paramdescriptor)

void	freezeAllParams ()

void	unFreezeAllParams ()

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

Engine *	engine () const

bool	initialized () const

const std::string &	className () const

const std::string &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor)

void	unFreezeParam (std::string const &paramdescriptor)

void	freezeAllParams ()

void	unFreezeAllParams ()

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Additional Inherited Members
Protected Member Functions inherited from jevois::StdModule
void	sendSerialMarkStart ()

void	sendSerialMarkStop ()

std::string	getStamp () const

Protected Member Functions inherited from jevois::Component
virtual void	preInit ()

virtual void	preUninit ()

virtual void	postUninit ()

virtual void	preInit ()

virtual void	preUninit ()

virtual void	postUninit ()

Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)

void	removeParameter (ParameterBase *const param)

void	callbackInitCall ()

Detailed Description

Live saliency computation and image filtering using 4-core CPU and OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU.

This module computes saliency and gist over our 4 CPU cores while also computing 4 different image filters over the GPU, finally combining all results into a single grayscale image:

saliency: multicore CPU-based detection of the most conspicuous (most attention-grabbing) object in the field of view.
GPU filter 1: Sobel edge detector
GPU filter 2: Median filter
GPU filter 3: Morphological erosion filter
GPU filter 4: Morphological dilation filter

For an introduction to visual saliency, see http://ilab.usc.edu/bu/

Also see DemoSaliency, JeVoisIntro, DarknetSaliency for more about saliency.

Video output

The video output is arranged vertically, with, from top to bottom:

Sobel filter results (same size as input image)
Median filter results (same size as input image)
Morphological erosion filter results (same size as input image)
Morphological dilation filter results (same size as input image)
Saliency results (those are very small): from left to right: saliency map, color map, intensity map, orientation map, flicker map, motion map. Map size is input size divided by 8 horizontally and vertically. Gist vector is appended to the right but note that at 160x120 it is truncated (see source code for details).

Serial Messages

This module can send standardized serial messages as described in Standardized serial messages formatting, where all coordinates and sizes are standardized using Helper functions to convert coordinates from camera resolution to standardized. One message is issued on every video frame at the temporally filtered attended (most salient) location:

Serial message type: 2D
id: always sm (shorthand for saliency map)
x, y: standardized 2D coordinates of temporally-filtered most salient point
w, h: always 0, 0
extra: none (empty string)

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Author: Laurent Itti

Display Name:: Demo CPU GPU

Videomapping:: GREY 160 495 60.0 YUYV 160 120 60.0 JeVois DemoCPUGPU

Email:: itti@usc.edu

Address:: University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA

Copyright: Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California

Main URL:: http://jevois.org

Support URL:: http://jevois.org/doc

Other URL:: http://iLab.usc.edu

License:: GPL v3

Distribution:: Unrestricted

Restrictions:: None

Definition at line 90 of file DemoCPUGPU.C.

Constructor & Destructor Documentation

◆ DemoCPUGPU()

DemoCPUGPU::DemoCPUGPU ( std::string const & instance )

inline

Constructor.

Definition at line 94 of file DemoCPUGPU.C.

◆ ~DemoCPUGPU()

virtual DemoCPUGPU::~DemoCPUGPU ( )

inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 106 of file DemoCPUGPU.C.

Member Function Documentation

◆ pasteGrayMap()

void DemoCPUGPU::pasteGrayMap	(	jevois::RawImage &	outimg,
		env_image const &	fmap,
		unsigned int &	dx,
		unsigned int	dy,
		unsigned int	bitshift
	)

inline

Paste a map and add its width to the dx offset.

Beware this is for a gray outimg only.

Definition at line 202 of file DemoCPUGPU.C.

References env_image::dims, env_dims::h, jevois::RawImage::height, LFATAL, env_image::pixels, jevois::RawImage::pixelsw(), env_dims::w, and jevois::RawImage::width.

◆ postInit()

void DemoCPUGPU::postInit ( )

inlineoverridevirtual

Set our GPU program after we are fully constructed and our Component path has been set.

Reimplemented from jevois::Component.

Definition at line 109 of file DemoCPUGPU.C.

◆ process()

virtual void DemoCPUGPU::process	(	jevois::InputFrame &&	inframe,
		jevois::OutputFrame &&	outframe
	)

inlineoverridevirtual

Processing function.

Reimplemented from jevois::Module.

Definition at line 117 of file DemoCPUGPU.C.

References jevois::async(), h, jevois::RawImage::height, jevois::RawImage::require(), jevois::StdModule::sendSerialStd2D(), demo::w, and jevois::RawImage::width.

The documentation for this class was generated from the following file:

src/Modules/DemoCPUGPU/DemoCPUGPU.C

Public Member Functions

Additional Inherited Members