JeVoisBase  1.18
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
FilterGPU.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
20 
21 // ####################################################################################################
22 FilterGPU::FilterGPU(std::string const & instance) :
23  jevois::Component(instance), itsProgramChanged(false), itsQuadVertexBuffer(0), itsDisplay(EGL_NO_DISPLAY),
24  itsConfig(0), itsContext(0), itsSurface(0), itsFramebufferId(0), itsRenderbufferId(0),
25  itsRenderWidth(0), itsRenderHeight(0), itsRenderType(0)
26 { }
27 
28 // ####################################################################################################
29 void FilterGPU::initDisplay()
30 {
31  // Get an EGL display connection:
32  GL_CHECK(itsDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY));
33  if (itsDisplay == EGL_NO_DISPLAY) LFATAL("Could not get an OpenGL display");
34 
35  // Initialize the EGL display connection:
36  EGLint major, minor;
37  GL_CHECK_BOOL(eglInitialize(itsDisplay, &major, &minor););
38  LINFO("Initialized OpenGL-ES with EGL v" << major << '.' << minor);
39 
40  // Get an appropriate EGL configuration:
41  EGLint num_config;
42  static EGLint const cfg_attr[] =
43  { EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, EGL_NONE };
44  GL_CHECK_BOOL(eglChooseConfig(itsDisplay, cfg_attr, &itsConfig, 1, &num_config));
45  if (num_config < 1) LFATAL("Could not find a suitable OpenGL config");
46 
47  // Create a pbuffer surface:
48  GL_CHECK(itsSurface = eglCreatePbufferSurface(itsDisplay, itsConfig, NULL));
49 
50  // Bind to OpenGL-ES API:
51  GL_CHECK_BOOL(eglBindAPI(EGL_OPENGL_ES_API));
52 
53  // Create an EGL rendering context:
54  static EGLint const ctx_attr[] = { EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE };
55  GL_CHECK(itsContext = eglCreateContext(itsDisplay, itsConfig, EGL_NO_CONTEXT, ctx_attr));
56  if (itsContext == EGL_NO_CONTEXT) LFATAL("Failed to create OpenGL context");
57 
58  // Bind the context to the surface:
59  GL_CHECK(eglMakeCurrent(itsDisplay, itsSurface, itsSurface, itsContext));
60 }
61 
62 // ####################################################################################################
64 {
65  // Kill our member variables before we close down OpenGL:
66  itsProgram.reset(); itsSrcTex.reset();
67 
68  if (itsRenderbufferId) glDeleteRenderbuffers(1, &itsRenderbufferId);
69  if (itsFramebufferId) glDeleteFramebuffers(1, &itsFramebufferId);
70 
71  glDeleteBuffers(1, &itsQuadVertexBuffer);
72 
73  // Delete surface, context, etc and close down:
74  eglMakeCurrent(itsDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT);
75  if (itsSurface) eglDestroySurface(itsDisplay, itsSurface);
76  eglDestroyContext(itsDisplay, itsContext);
77  eglTerminate(itsDisplay);
78 }
79 
80 // ####################################################################################################
81 void FilterGPU::setProgram(std::string const & vertex_shader, std::string const & frag_shader)
82 {
83  // We cannot load the program here as our display may not have been initialized yet, and we may be in a different
84  // thread. Let's just remember the file names and we will do the work in process():
85  std::lock_guard<std::mutex> _(itsMutex);
86  itsVshader = absolutePath(vertex_shader);
87  itsFshader = absolutePath(frag_shader);
88  itsProgramChanged = true;
89  itsProgramParams.clear();
90 }
91 
92 // ####################################################################################################
93 void FilterGPU::setProgramParam2f(std::string name, float val1, float val2)
94 {
95  std::lock_guard<std::mutex> _(itsMutex);
96  itsProgramParams[name] = { F2, { val1, val2 } };
97  itsProgramChanged = true;
98 }
99 
100 // ####################################################################################################
101 void FilterGPU::setProgramParam1f(std::string name, float val)
102 {
103  std::lock_guard<std::mutex> _(itsMutex);
104  itsProgramParams[name] = { F1, { val, 0.0F } };
105  itsProgramChanged = true;
106 }
107 
108 // ####################################################################################################
109 void FilterGPU::setProgramParam2i(std::string name, int val1, int val2)
110 {
111  std::lock_guard<std::mutex> _(itsMutex);
112  itsProgramParams[name] = { I2, { float(val1), float(val2) } };
113  itsProgramChanged = true;
114 }
115 
116 // ####################################################################################################
117 void FilterGPU::setProgramParam1i(std::string name, int val)
118 {
119  std::lock_guard<std::mutex> _(itsMutex);
120  itsProgramParams[name] = { I2, { float(val), 0.0F } };
121  itsProgramChanged = true;
122 }
123 
124 // ####################################################################################################
125 void FilterGPU::process(cv::Mat const & src, cv::Mat & dst)
126 {
127  // We init the display here so that it is in the same thread as the subsequent processing, as OpenGL is not very
128  // thread-friendly. Yet, see here for an alternative, which is basically to create some sort of shadow context in the
129  // process() thread after having created the context in the constructor or init() thread:
130  // http://stackoverflow.com/questions/11726650/egl-can-context-be-shared-between-threads
131  if (itsDisplay == EGL_NO_DISPLAY) initDisplay();
132 
133  if (src.type() != CV_8UC1 && src.type() != CV_8UC4) LFATAL("Source pixel format must be CV_8UC1 or CV_8UC4");
134  if (dst.type() != CV_8UC2 && dst.type() != CV_8UC4) LFATAL("Dest pixel format must be CV_8UC2 or CV_8UC4");
135  GLuint const srcformat = (src.channels() == 4 ? GL_RGBA : GL_LUMINANCE); // GL_ALPHA also works
136  GLuint const dstformat = (dst.channels() == 4 ? GL_RGBA4 : GL_RGB565);
137 
138  // Allocate our textures if needed, eg, first time we are called or sizes have changed:
139  if (!itsSrcTex || itsSrcTex->Width != src.cols || itsSrcTex->Height != src.rows || itsSrcTex->Format != srcformat)
140  {
141  itsSrcTex.reset(new GPUtexture(src.cols, src.rows, srcformat, false));
142  LINFO("Input texture " << itsSrcTex->Width << 'x' << itsSrcTex->Height << ' ' <<
143  (srcformat == GL_RGBA ? "RGBA" : "LUMINANCE") << " ready.");
144  }
145 
146  // Create our framebuffer and renderbuffer if needed:
147  if (itsRenderbufferId == 0 || itsRenderWidth != dst.cols || itsRenderHeight != dst.rows ||
148  itsRenderType != dst.type())
149  {
150  if (itsRenderbufferId) glDeleteRenderbuffers(1, &itsRenderbufferId);
151  if (itsFramebufferId) glDeleteFramebuffers(1, &itsFramebufferId);
152 
153  GL_CHECK(glGenFramebuffers(1, &itsFramebufferId));
154  GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, itsFramebufferId));
155 
156  GL_CHECK(glGenRenderbuffers(1, &itsRenderbufferId));
157  GL_CHECK(glBindRenderbuffer(GL_RENDERBUFFER, itsRenderbufferId));
158  GL_CHECK(glRenderbufferStorage(GL_RENDERBUFFER, dstformat, dst.cols, dst.rows));
159  GL_CHECK(glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, itsRenderbufferId));
160 
161  if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
162  LFATAL("Framebuffer creation failed");
163 
164  itsRenderWidth = dst.cols; itsRenderHeight = dst.rows; itsRenderType = dst.type();
165  LINFO("Render buffer " << itsRenderWidth << 'x' << itsRenderHeight << ' ' <<
166  (dst.channels() == 2 ? "RGB565" : "RGBA") << " ready.");
167  }
168 
169  // Create our vertex buffer if needed:
170  if (itsQuadVertexBuffer == 0)
171  {
172  // Create an ickle vertex buffer:
173  static GLfloat const qv[] =
174  { 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f };
175 
176  GL_CHECK(glGenBuffers(1, &itsQuadVertexBuffer));
177  glBindBuffer(GL_ARRAY_BUFFER, itsQuadVertexBuffer);
178  glBufferData(GL_ARRAY_BUFFER, sizeof(qv), qv, GL_STATIC_DRAW);
179  GL_CHECK(glBindBuffer(GL_ARRAY_BUFFER, 0));
180  }
181 
182  // Set background color and clear buffers:
183  glClearColor(0.15f, 0.25f, 0.35f, 1.0f);
184  glClear(GL_COLOR_BUFFER_BIT);
185 
186  // Copy source pixel data to source texture:
187  itsSrcTex->setPixels(src.data);
188 
189  // Tell OpenGL to render into our destination framebuffer:
190  GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, itsFramebufferId));
191  GL_CHECK(glViewport(0, 0, itsRenderWidth, itsRenderHeight));
192 
193  // Load the shader program if needed:
194  if (itsProgramChanged)
195  {
196  std::lock_guard<std::mutex> _(itsMutex);
197 
198  // Nuke any old program first:
199  itsProgram.reset();
200 
201  // Create, load and compile the program:
202  itsProgram.reset(new GPUprogram(itsVshader.c_str(), itsFshader.c_str()));
203  itsProgramChanged = false;
204 
205  // Tell OpenGL to use our program:
206  glUseProgram(itsProgram->id());
207 
208  // Set all the parameters:
209  GLuint i = itsProgram->id();
210  for (auto const & p : itsProgramParams)
211  {
212  char const * n = p.first.c_str();
213  float const * v = &p.second.val[0];
214  switch (p.second.type)
215  {
216  case F2: GL_CHECK(glUniform2f(glGetUniformLocation(i, n), v[0], v[1])); break;
217  case F1: GL_CHECK(glUniform1f(glGetUniformLocation(i, n), v[0])); break;
218  case I2: GL_CHECK(glUniform2i(glGetUniformLocation(i, n), int(v[0]), int(v[1]))); break;
219  case I1: GL_CHECK(glUniform1i(glGetUniformLocation(i, n), int(v[0]))); break;
220  default: LFATAL("Unsupported GPU program parameter type " << p.second.type);
221  }
222  }
223  }
224 
225  if (!itsProgram) LFATAL("You need to set a program before processing frames");
226 
227  // Set program parameters that all programs should always use, ignore any error:
228  glUniform2f(glGetUniformLocation(itsProgram->id(), "texelsize"), 1.0f / dst.cols, 1.0f / dst.rows);
229  glUniform1i(glGetUniformLocation(itsProgram->id(), "tex"), 0);
230 
231  // Draw the texture onto the triangle strip, applying the program:
232  GL_CHECK(glBindBuffer(GL_ARRAY_BUFFER, itsQuadVertexBuffer));
233  GL_CHECK(glActiveTexture(GL_TEXTURE0));
234  GL_CHECK(glBindTexture(GL_TEXTURE_2D, itsSrcTex->Id));
235 
236  GLuint loc = glGetAttribLocation(itsProgram->id(), "vertex");
237  GL_CHECK(glVertexAttribPointer(loc, 4, GL_FLOAT, 0, 16, 0));
238  GL_CHECK(glEnableVertexAttribArray(loc));
239  GL_CHECK(glDrawArrays(GL_TRIANGLE_STRIP, 0, 4));
240 
241  // Copy the rendered pixels to destination image:
242  GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, itsFramebufferId));
243  if (dstformat == GL_RGBA4) GL_CHECK(glReadPixels(0, 0, dst.cols, dst.rows, GL_RGBA, GL_UNSIGNED_BYTE, dst.data));
244  else GL_CHECK(glReadPixels(0, 0, dst.cols, dst.rows, GL_RGB, GL_UNSIGNED_SHORT_5_6_5, dst.data));
245  GL_CHECK(glBindFramebuffer(GL_FRAMEBUFFER, 0));
246 }
247 
FilterGPU::setProgramParam2i
void setProgramParam2i(std::string name, int val1, int val2)
Set some program parameters, 2 int version.
Definition: FilterGPU.C:109
FilterGPU::setProgramParam1f
void setProgramParam1f(std::string name, float val)
Set some program parameters, 1 float version.
Definition: FilterGPU.C:101
FilterGPU::setProgram
void setProgram(std::string const &vertex_shader, std::string const &frag_shader)
Set an image processing program.
Definition: FilterGPU.C:81
GL_CHECK
#define GL_CHECK(stmt)
Simple macro to check for OpenGL errors.
Definition: OpenGL.H:49
FilterGPU.H
OpenGL.H
GL_CHECK_BOOL
#define GL_CHECK_BOOL(stmt)
Simple macro to check for OpenGL errors when a boolean result is expected.
Definition: OpenGL.H:53
jevois
FilterGPU::setProgramParam1i
void setProgramParam1i(std::string name, int val)
Set some program parameters, 1 int version.
Definition: FilterGPU.C:117
FilterGPU::FilterGPU
FilterGPU(std::string const &instance)
Constructor.
Definition: FilterGPU.C:22
FilterGPU::setProgramParam2f
void setProgramParam2f(std::string name, float val1, float val2)
Set some program parameters, 2 float version.
Definition: FilterGPU.C:93
LFATAL
#define LFATAL(msg)
FilterGPU::process
void process(cv::Mat const &src, cv::Mat &dst)
Process an image. The dst image should be allocated with correct image size and pixel type.
Definition: FilterGPU.C:125
GPUprogram
Simple class to load and compile some OpenGL-ES program.
Definition: GPUprogram.H:23
GPUtexture
Simple class to hold an OpenGL texture.
Definition: GPUtexture.H:26
demo.float
float
Definition: demo.py:39
jevois::Component::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &path="")
FilterGPU::~FilterGPU
~FilterGPU()
Destructor.
Definition: FilterGPU.C:63
name
std::string name
LINFO
#define LINFO(msg)