Simple image filtering using OpenGL-ES on the GPU. More...

#include <jevoisbase/Components/FilterGPU/FilterGPU.H>

Inheritance diagram for FilterGPU:

Collaboration diagram for FilterGPU:

Public Member Functions
	FilterGPU (std::string const &instance)
	Constructor. More...

	~FilterGPU ()
	Destructor. More...

void	setProgram (std::string const &vertex_shader, std::string const &frag_shader)
	Set an image processing program. More...

void	setProgramParam2f (std::string name, float val1, float val2)
	Set some program parameters, 2 float version. More...

void	setProgramParam1f (std::string name, float val)
	Set some program parameters, 1 float version. More...

void	setProgramParam2i (std::string name, int val1, int val2)
	Set some program parameters, 2 int version. More...

void	setProgramParam1i (std::string name, int val)
	Set some program parameters, 1 int version. More...

void	process (cv::Mat const &src, cv::Mat &dst)
	Process an image. The dst image should be allocated with correct image size and pixel type. More...

Public Member Functions inherited from jevois::Component
	Component (std::string const &instance)

virtual	~Component ()

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

Engine *	engine () const

bool	initialized () const

const std::string &	className () const

const std::string &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor)

void	unFreezeParam (std::string const &paramdescriptor)

void	freezeAllParams ()

void	unFreezeAllParams ()

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

std::shared_ptr< Comp >	addSubComponent (std::string const &instance, Args &&...args)

void	removeSubComponent (std::shared_ptr< Comp > &component)

void	removeSubComponent (std::string const &instance, bool warnIfNotFound=true)

std::shared_ptr< Comp >	getSubComponent (std::string const &instance) const

bool	isTopLevel () const

Engine *	engine () const

bool	initialized () const

const std::string &	className () const

const std::string &	instanceName () const

std::vector< std::string >	setParamVal (std::string const &paramdescriptor, T const &val)

void	setParamValUnique (std::string const &paramdescriptor, T const &val)

std::vector< std::pair< std::string, T > >	getParamVal (std::string const &paramdescriptor) const

T	getParamValUnique (std::string const &paramdescriptor) const

std::vector< std::string >	setParamString (std::string const &paramdescriptor, std::string const &val)

void	setParamStringUnique (std::string const &paramdescriptor, std::string const &val)

std::vector< std::pair< std::string, std::string > >	getParamString (std::string const &paramdescriptor) const

std::string	getParamStringUnique (std::string const &paramdescriptor) const

void	freezeParam (std::string const &paramdescriptor)

void	unFreezeParam (std::string const &paramdescriptor)

void	freezeAllParams ()

void	unFreezeAllParams ()

std::string	descriptor () const

void	setParamsFromFile (std::string const &filename)

std::istream &	setParamsFromStream (std::istream &is, std::string const &absfile)

virtual void	paramInfo (std::shared_ptr< UserInterface > s, std::map< std::string, std::string > &categs, bool skipFrozen, std::string const &cname="", std::string const &pfx="")

void	foreachParam (std::function< void(std::string const &compname, ParameterBase *p)> func, std::string const &cname="")

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ParameterCategory const &category)

std::shared_ptr< DynamicParameter< T > >	addDynamicParameter (std::string const &name, std::string const &description, T const &defaultValue, ValidValuesSpec< T > const &validValuesSpec, ParameterCategory const &category)

void	setDynamicParameterCallback (std::string const &name, std::function< void(T const &)> cb, bool callnow=true)

void	removeDynamicParameter (std::string const &name)

void	setPath (std::string const &path)

std::filesystem::path	absolutePath (std::filesystem::path const &path="")

Public Member Functions inherited from jevois::ParameterRegistry
virtual	~ParameterRegistry ()

Additional Inherited Members
Protected Member Functions inherited from jevois::Component
virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

virtual void	postUninit ()

virtual void	preInit ()

virtual void	postInit ()

virtual void	preUninit ()

virtual void	postUninit ()

Protected Member Functions inherited from jevois::ParameterRegistry
void	addParameter (ParameterBase *const param)

void	removeParameter (ParameterBase *const param)

void	callbackInitCall ()

Detailed Description

Simple image filtering using OpenGL-ES on the GPU.

This class first copies the given input image into an OpenGL texture, then applies OpenGL-ES vertex and fragment shaders to achieve some image processing, and finally gets the resulting pixels back into an image.

We support two types of input:

CV_8UC1 greyscale image. This is uploaded to GPU as a luminance texture. In the shaders, R=G=B and alpha=1.
CV-8UC4 color images. This is uploaded to GPU as an RGBA image. In the shaders, R,G,B,A are as in the input.

We support two types of output:

CV_8UC2 RGB565 images. This is useful to avoid any conversion of color results. GPU results are directly rendered into an RGB565 color buffer, which may directly be the buffer of an RGB565 output RawImage, i.e., the pixel memory was allocated and "belongs to" the USB gadget driver. Thus, the rendered pixels can be directly read back from the GPU into the RawImage and send over USB with no additional copy or conversion. See DemoGPU for an example of this.
CV-8UC4 RGBA images. This may be used either for higher-accuracy color output, but may require conversion since we do not support sending RGBA images over the USB link. Using jevois::rawimage::unpackCvRGBAtoGrayRawImage() one may unpack such a packed RGBA image into 4 single-channel planes. This then makes it possible to run one RGBA shader that produces 4 different filtering effects in each of the 4 color/alpha channels, and to then unpack the results into a 4-up greyscale RawImage buffer for sending over USB. DemoCPUGPU uses this, uploading a greyscale input image and then running 4 different filters to create the R, G, B, and A outputs, finally unpacking those into 4 greyscale images sent over USB.

Shader conventions:

input texture is in "tex"
current coordinates within texture are in "tcoord"
size of one texture pixel is set in "texelsize" (uniform 2 float)
see DemoGPU and DemoCPUGPU for example shaders

This code is inspired from the tutorial and code examples found on this page: http://robotblogging.blogspot.com/2013/10/gpu-accelerated-camera-processing-on.html

on host, mesa may be in the way of other drivers, eg nvidia. You need to get them out.

sudo find / -name libEGL*
sudo find / -name libGLESv2*
 
cd /usr/lib/x86_64-linux-gnu
ls -l | grep mesa # we will fix those with what we found above
sudo ln -fs /usr/lib/nvidia-352-updates/libEGL.so libEGL.so
sudo ln -fs /usr/lib/nvidia-352-updates/libGLESv2.so libGLESv2.so
sudo ln -fs /usr/lib/nvidia-352-updates/libGL.so libGL.so 
sudo mv mesa-egl/ld.so.conf mesa-egl/ld.so.conf.disabled
sudo ldconfig
 
# back to jevoisbase:
ldd hbuild/libjevoisbase.so |grep GL # there should not be any mesa in there

FIXME: yet, this code still does not run on the host. Maybe OpenCV which is using highgui on the host which uses Qt which uses OpenGL (not the ES version) is the issue.

Definition at line 80 of file FilterGPU.H.

Constructor & Destructor Documentation

◆ FilterGPU()

FilterGPU::FilterGPU ( std::string const & instance )

Constructor.

Definition at line 21 of file FilterGPU.C.

◆ ~FilterGPU()

FilterGPU::~FilterGPU ( )

Destructor.

Definition at line 62 of file FilterGPU.C.

Member Function Documentation

◆ process()

void FilterGPU::process	(	cv::Mat const &	src,
		cv::Mat &	dst
	)

Process an image. The dst image should be allocated with correct image size and pixel type.

Definition at line 124 of file FilterGPU.C.

References GL_CHECK, LFATAL, and LINFO.

◆ setProgram()

void FilterGPU::setProgram	(	std::string const &	vertex_shader,
		std::string const &	frag_shader
	)

Set an image processing program.

Definition at line 80 of file FilterGPU.C.

◆ setProgramParam1f()

void FilterGPU::setProgramParam1f	(	std::string	name,
		float	val
	)

Set some program parameters, 1 float version.

Definition at line 100 of file FilterGPU.C.

◆ setProgramParam1i()

void FilterGPU::setProgramParam1i	(	std::string	name,
		int	val
	)

Set some program parameters, 1 int version.

Definition at line 116 of file FilterGPU.C.

◆ setProgramParam2f()

void FilterGPU::setProgramParam2f	(	std::string	name,
		float	val1,
		float	val2
	)

Set some program parameters, 2 float version.

OpenGL is not very thread-friendly. Hence the program chosen using setProgram() will actually be instantiated inside the process() function so everything happens in one thread. Here, users can set program parameters, those are just memorized and will be set in process() as well. Each time setProgram() is called, all cached parameters are cleared and they should be set again.

Definition at line 92 of file FilterGPU.C.

◆ setProgramParam2i()

void FilterGPU::setProgramParam2i	(	std::string	name,
		int	val1,
		int	val2
	)

Set some program parameters, 2 int version.

Definition at line 108 of file FilterGPU.C.

The documentation for this class was generated from the following files:

include/jevoisbase/Components/FilterGPU/FilterGPU.H
src/Components/FilterGPU/FilterGPU.C

Public Member Functions

Additional Inherited Members