JeVoisBase  1.22
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Loading...
Searching...
No Matches
Public Member Functions | List of all members
DemoCPUGPU Class Reference
JeVoisBase machine vision modules

Live saliency computation and image filtering using 4-core CPU and OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU. More...

Inheritance diagram for DemoCPUGPU:
Inheritance graph
[legend]
Collaboration diagram for DemoCPUGPU:
Collaboration graph
[legend]

Public Member Functions

 DemoCPUGPU (std::string const &instance)
 Constructor.
 
virtual ~DemoCPUGPU ()
 Virtual destructor for safe inheritance.
 
void postInit () override
 Set our GPU program after we are fully constructed and our Component path has been set.
 
virtual void process (jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
 Processing function.
 
void pasteGrayMap (jevois::RawImage &outimg, env_image const &fmap, unsigned int &dx, unsigned int dy, unsigned int bitshift)
 Paste a map and add its width to the dx offset.
 
- Public Member Functions inherited from jevois::StdModule
 StdModule (std::string const &instance)
 
virtual ~StdModule ()
 
void sendSerialImg1Dx (unsigned int camw, float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dx (float x, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg1Dy (unsigned int camh, float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd1Dy (float y, float size=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialImg2D (unsigned int camw, unsigned int camh, float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd2D (float x, float y, float w=0.0F, float h=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialContour2D (unsigned int camw, unsigned int camh, std::vector< cv::Point_< T > > points, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (float x, float y, float z, float w=0.0F, float h=0.0F, float d=0.0F, float q1=0.0F, float q2=0.0F, float q3=0.0f, float q4=0.0F, std::string const &id="", std::string const &extra="")
 
void sendSerialStd3D (std::vector< cv::Point3f > points, std::string const &id="", std::string const &extra="")
 
void sendSerialObjReco (std::vector< ObjReco > const &res)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, float x, float y, float w, float h, std::vector< ObjReco > const &res)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetect const &det)
 
void sendSerialObjDetImg2D (unsigned int camw, unsigned int camh, ObjDetectOBB const &det)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStyle,(Terse)(Normal)(Detail)(Fine))
 
 JEVOIS_DECLARE_PARAMETER (serstyle, SerStyle, "Style for standardized serial messages as defined in " "http://jevois.org/doc/UserSerialStyle.html", SerStyle::Terse, SerStyle_Values, ParamCateg)
 
 JEVOIS_DECLARE_PARAMETER (serprec, unsigned int, "Number of decimal points in standardized serial messages as " "defined in http://jevois.org/doc/UserSerialStyle.html", 0U, jevois::Range< unsigned int >(0U, 10U), ParamCateg)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerStamp,(None)(Frame)(Time)(FrameTime)(FrameDateTime))
 
 JEVOIS_DECLARE_PARAMETER (serstamp, SerStamp, "Prepend standardized serial messages with a frame number, " "time, frame+time, or frame+date+time. See details in " "http://jevois.org/doc/UserSerialStyle.html", SerStamp::None, SerStamp_Values, ParamCateg)
 
 JEVOIS_DEFINE_ENUM_CLASS (SerMark,(None)(Start)(Stop)(Both))
 
 JEVOIS_DECLARE_PARAMETER (sermark, SerMark, "Send serial message to mark the beginning (MARK START) of the " "processing of a video frame from the camera sensor, the end (MARK STOP), or both. " "Useful, among others, if one needs to know when no results were sent over serial " "on a given frame. Combine with parameter serstamp if you need to know the frame number.", SerMark::None, SerMark_Values, ParamCateg)
 
- Public Member Functions inherited from jevois::Module
 Module (std::string const &instance)
 
virtual ~Module ()
 
virtual void process (InputFrame &&inframe)
 
virtual void process (InputFrame &&inframe, GUIhelper &helper)
 
virtual void sendSerial (std::string const &str)
 
virtual void parseSerial (std::string const &str, std::shared_ptr< UserInterface > s)
 
virtual void supportedCommands (std::ostream &os)
 
- Public Member Functions inherited from jevois::Component
 Component (std::string const &instance)
 
virtual ~Component ()
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
T getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor, bool doit)
 
void freezeAllParams (bool doit)
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)
 
void setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)
 
void removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)
 
void setPath (std::string const &path)
 
std::filesystem::path absolutePath (std::filesystem::path const &path="")
 
std::shared_ptr< Comp > addSubComponent (std::string const &instance, Args &&...args)
 
void removeSubComponent (std::shared_ptr< Comp > &component)
 
void removeSubComponent (std::string const &instance, bool warnIfNotFound=true)
 
std::shared_ptr< Comp > getSubComponent (std::string const &instance) const
 
bool isTopLevel () const
 
bool initialized () const
 
std::string const & className () const
 
std::string const & instanceName () const
 
std::vector< std::string > setParamVal (std::string const &paramdescriptor, T const &val)
 
void setParamValUnique (std::string const &paramdescriptor, T const &val)
 
std::vector< std::pair< std::string, T > > getParamVal (std::string const &paramdescriptor) const
 
T getParamValUnique (std::string const &paramdescriptor) const
 
std::vector< std::string > setParamString (std::string const &paramdescriptor, std::string const &val)
 
void setParamStringUnique (std::string const &paramdescriptor, std::string const &val)
 
std::vector< std::pair< std::string, std::string > > getParamString (std::string const &paramdescriptor) const
 
std::string getParamStringUnique (std::string const &paramdescriptor) const
 
void freezeParam (std::string const &paramdescriptor, bool doit)
 
void freezeAllParams (bool doit)
 
std::string descriptor () const
 
void setParamsFromFile (std::string const &filename)
 
std::istream & setParamsFromStream (std::istream &is, std::string const &absfile)
 
virtual void paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")
 
void foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)
 
std::shared_ptr< DynamicParameter< T > > addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)
 
void setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)
 
void removeDynamicParameter (std::string const &name, bool throw_if_not_found=true)
 
void setPath (std::string const &path)
 
std::filesystem::path absolutePath (std::filesystem::path const &path="")
 
- Public Member Functions inherited from jevois::ParameterRegistry
virtual ~ParameterRegistry ()
 

Additional Inherited Members

- Protected Member Functions inherited from jevois::StdModule
void sendSerialMarkStart ()
 
void sendSerialMarkStop ()
 
std::string getStamp () const
 
- Protected Member Functions inherited from jevois::Component
virtual void preInit ()
 
virtual void preUninit ()
 
virtual void postUninit ()
 
virtual void preInit ()
 
virtual void preUninit ()
 
virtual void postUninit ()
 
- Protected Member Functions inherited from jevois::ParameterRegistry
void addParameter (ParameterBase *const param)
 
void removeParameter (ParameterBase *const param)
 
void callbackInitCall ()
 

Detailed Description

Live saliency computation and image filtering using 4-core CPU and OpenGL-ES 2.0 shaders on the Mali-400MP2 GPU.

This module computes saliency and gist over our 4 CPU cores while also computing 4 different image filters over the GPU, finally combining all results into a single grayscale image:

  • saliency: multicore CPU-based detection of the most conspicuous (most attention-grabbing) object in the field of view.
  • GPU filter 1: Sobel edge detector
  • GPU filter 2: Median filter
  • GPU filter 3: Morphological erosion filter
  • GPU filter 4: Morphological dilation filter

For an introduction to visual saliency, see http://ilab.usc.edu/bu/

Also see DemoSaliency, JeVoisIntro, DarknetSaliency for more about saliency.

Video output

The video output is arranged vertically, with, from top to bottom:

  • Sobel filter results (same size as input image)
  • Median filter results (same size as input image)
  • Morphological erosion filter results (same size as input image)
  • Morphological dilation filter results (same size as input image)
  • Saliency results (those are very small): from left to right: saliency map, color map, intensity map, orientation map, flicker map, motion map. Map size is input size divided by 8 horizontally and vertically. Gist vector is appended to the right but note that at 160x120 it is truncated (see source code for details).

Serial Messages

This module can send standardized serial messages as described in Standardized serial messages formatting, where all coordinates and sizes are standardized using Helper functions to convert coordinates from camera resolution to standardized. One message is issued on every video frame at the temporally filtered attended (most salient) location:

  • Serial message type: 2D
  • id: always sm (shorthand for saliency map)
  • x, y: standardized 2D coordinates of temporally-filtered most salient point
  • w, h: always 0, 0
  • extra: none (empty string)

See Standardized serial messages formatting for more on standardized serial messages, and Helper functions to convert coordinates from camera resolution to standardized for more info on standardized coordinates.

Author
Laurent Itti
Display Name:
Demo CPU GPU
Videomapping:
GREY 160 495 60.0 YUYV 160 120 60.0 JeVois DemoCPUGPU
Email:
itti@usc.edu
Address:
University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
Copyright
Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
Main URL:
http://jevois.org
Support URL:
http://jevois.org/doc
Other URL:
http://iLab.usc.edu
License:
GPL v3
Distribution:
Unrestricted
Restrictions:
None

Definition at line 90 of file DemoCPUGPU.C.

Constructor & Destructor Documentation

◆ DemoCPUGPU()

DemoCPUGPU::DemoCPUGPU ( std::string const &  instance)
inline

Constructor.

Definition at line 94 of file DemoCPUGPU.C.

◆ ~DemoCPUGPU()

virtual DemoCPUGPU::~DemoCPUGPU ( )
inlinevirtual

Virtual destructor for safe inheritance.

Definition at line 106 of file DemoCPUGPU.C.

Member Function Documentation

◆ pasteGrayMap()

void DemoCPUGPU::pasteGrayMap ( jevois::RawImage &  outimg,
env_image const &  fmap,
unsigned int &  dx,
unsigned int  dy,
unsigned int  bitshift 
)
inline

Paste a map and add its width to the dx offset.

Beware this is for a gray outimg only.

Definition at line 202 of file DemoCPUGPU.C.

References env_image::dims, env_dims::h, jevois::RawImage::height, LFATAL, env_image::pixels, jevois::RawImage::pixelsw(), env_dims::w, and jevois::RawImage::width.

Referenced by process().

◆ postInit()

void DemoCPUGPU::postInit ( )
inlineoverridevirtual

Set our GPU program after we are fully constructed and our Component path has been set.

Reimplemented from jevois::Component.

Definition at line 109 of file DemoCPUGPU.C.

◆ process()

virtual void DemoCPUGPU::process ( jevois::InputFrame &&  inframe,
jevois::OutputFrame &&  outframe 
)
inlineoverridevirtual

Processing function.

Reimplemented from jevois::Module.

Definition at line 117 of file DemoCPUGPU.C.

References jevois::async(), jevois::rawimage::convertToCvGray(), h, jevois::RawImage::height, pasteGrayMap(), jevois::RawImage::pixelsw(), jevois::RawImage::require(), jevois::StdModule::sendSerialStd2D(), jevois::rawimage::unpackCvRGBAtoGrayRawImage(), and jevois::RawImage::width.

The documentation for this class was generated from the following file: