JeVois  1.16
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
GPUimage.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2020 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #ifdef JEVOIS_PRO
19 
20 #include <jevois/GPU/GPUimage.H>
21 
22 #include <jevois/Util/Utils.H>
23 #include <jevois/GPU/GPUtexture.H>
25 #include <jevois/GPU/GPUprogram.H>
26 #include <glm/glm.hpp>
27 #include <glm/gtc/type_ptr.hpp>
28 
29 // ##############################################################################################################
30 // Shaders used here:
31 namespace jevois
32 {
33  namespace shader
34  {
35  extern char const * vert; // Vertex shader
36  extern char const * frag_rgba; // To display an RGBA image
37  extern char const * frag_rgb; // To display an RGB image
38  extern char const * frag_grey; // To display a greyscale image
39  extern char const * frag_yuyv; // To display a YUYV image
40  extern char const * frag_oes; // To display a DMABUF camera image
41  }
42 }
43 
44 // ##############################################################################################################
46 { }
47 
48 // ##############################################################################################################
50 {
51  if (itsVertexArray) glDeleteVertexArrays(1, &itsVertexArray);
52  if (itsVertexBuffers[0]) glDeleteBuffers(2, itsVertexBuffers);
53 }
54 
55 // ##############################################################################################################
56 void jevois::GPUimage::setInternal(unsigned int width, unsigned int height, unsigned int fmt,
57  unsigned char const * data)
58 {
59 
60 
61 
62  // on platform, try EGLcreateImageKHR with EGL_NATIVE_PIXMAP_KHR and fbdev_pixmap
63  // https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_image_pixmap.txt
64  // https://github.com/peak3d/egltest/blob/master/egl_ump.cpp
65 
66 
67  if (width == 0 || height == 0) LFATAL("Cannot handle zero image width or height");
68 
69  // If image format has changed, load the appropriate program:
70  if (fmt != itsFormat)
71  {
72  char const * frag_shader;
73  switch (fmt)
74  {
75  case V4L2_PIX_FMT_YUYV: // YUYV shader gets YUYV (2 pixels) from one RGBA texture value (1 texel)
76  frag_shader = jevois::shader::frag_yuyv; itsGLtextureWidth = width / 2; itsGLtextureFmt = GL_RGBA;
77  break;
78 
79  case V4L2_PIX_FMT_RGB32: // RGBA shader is simple pass-through
80  frag_shader = jevois::shader::frag_rgba; itsGLtextureWidth = width; itsGLtextureFmt = GL_RGBA;
81  break;
82 
83  case V4L2_PIX_FMT_GREY: // GRAY shader just converts from greyscale to RGBA
84  frag_shader = jevois::shader::frag_grey; itsGLtextureWidth = width; itsGLtextureFmt = GL_LUMINANCE;
85  break;
86 
87  case V4L2_PIX_FMT_RGB24: // RGB shader gets R,G,B from 3 successive texels in a 3x wide luminance texture
88  frag_shader = jevois::shader::frag_rgb; itsGLtextureWidth = width * 3; itsGLtextureFmt = GL_LUMINANCE;
89  break;
90 
91  case V4L2_PIX_FMT_BGR24:
92  case V4L2_PIX_FMT_SRGGB8:
93  case V4L2_PIX_FMT_RGB565:
94  case V4L2_PIX_FMT_MJPEG:
95  case V4L2_PIX_FMT_UYVY:
96  case V4L2_PIX_FMT_SBGGR16:
97  case V4L2_PIX_FMT_NV12:
98  case V4L2_PIX_FMT_YUV444:
99  case 0:
100  default: LFATAL("Unsupported pixel format " << jevois::fccstr(fmt));
101  }
102 
103  // Load the appropriate program:
104  itsProgram.reset(new GPUprogram(jevois::shader::vert, frag_shader));
105 
106  // Get a handle to s_texture variable in the fragment shader, to update the texture with each new camera frame:
107  itsLocation = glGetUniformLocation(itsProgram->id(), "s_texture");
108  }
109 
110  // Generate a texture for incoming image data if needed:
111  if (width != itsTextureWidth || height != itsTextureHeight || fmt != itsFormat || !itsTexture)
112  {
113  // Now create our texture:
114  itsTexture.reset(new jevois::GPUtexture(itsGLtextureWidth, height, itsGLtextureFmt, false));
115  LDEBUG("Input texture for " << width << 'x' << height << ' ' << jevois::fccstr(fmt) << " ready.");
116  }
117 
118 #ifdef JEVOIS_PLATFORM_PRO
119  if (itsTextureDmaBuf) itsTextureDmaBuf.reset(); // invalidate any previously used dmabuf texture
120 #endif
121 
122  // Assign pixel data to our texture:
123  itsTexture->setPixels(data);
124 
125  // Remember our latest size and format:
126  itsTextureWidth = width; itsTextureHeight = height; itsFormat = fmt;
127 }
128 
129 // ##############################################################################################################
131 {
132  setInternal(img.width, img.height, img.fmt, static_cast<unsigned char const *>(img.buf->data()));
133 }
134 
135 // ##############################################################################################################
136 void jevois::GPUimage::set(cv::Mat const & img, bool rgb)
137 {
138  unsigned int fmt;
139 
140  switch(img.type())
141  {
142  case CV_8UC4: if (rgb) fmt = V4L2_PIX_FMT_RGB32; else fmt = V4L2_PIX_FMT_BGR32; break;
143  case CV_8UC3: if (rgb) fmt = V4L2_PIX_FMT_RGB24; else fmt = V4L2_PIX_FMT_BGR24; break;
144  case CV_8UC2: fmt = V4L2_PIX_FMT_YUYV; break;
145  case CV_8UC1: fmt = V4L2_PIX_FMT_GREY; break;
146  default: LFATAL("Unsupported OpenCV image format: " << img.type());
147  }
148 
149  setInternal(img.cols, img.rows, fmt, img.data);
150 }
151 
152 #ifdef JEVOIS_PLATFORM_PRO
153 // ##############################################################################################################
154 void jevois::GPUimage::set(jevois::InputFrame const & frame, EGLDisplay display)
155 {
156  jevois::RawImage const img = frame.get();
157 
158  // EGLimageKHR which we use with DMAbuf requires width to be a multiple of 32; otherwise revert to normal texture:
159  if (img.width % 32) { set(img); return; }
160 
161  // DMAbuf only supports some formats, otherwise revert to normal texture. Keep in sync with GPUtextureDmaBuf:
162  switch (img.fmt)
163  {
164  case V4L2_PIX_FMT_YUYV:
165  case V4L2_PIX_FMT_RGB32:
166  case V4L2_PIX_FMT_RGB565:
167  case V4L2_PIX_FMT_BGR24:
168  case V4L2_PIX_FMT_RGB24:
169  case V4L2_PIX_FMT_UYVY:
170  {
171  int const dmafd = frame.getDmaFd();
172  setWithDmaBuf(img, dmafd, display);
173  break;
174  }
175  default:
176  set(img);
177  }
178 }
179 
180 // ##############################################################################################################
181 void jevois::GPUimage::set2(jevois::InputFrame const & frame, EGLDisplay display)
182 {
183  jevois::RawImage const img = frame.get2();
184 
185  // EGLimageKHR which we use with DMAbuf requires width to be a multiple of 32; otherwise revert to normal texture:
186  if (img.width % 32) { set(img); return; }
187 
188  // DMAbuf only supports some formats, otherwise revert to normal texture. Keep in sync with GPUtextureDmaBuf:
189  switch (img.fmt)
190  {
191  case V4L2_PIX_FMT_YUYV:
192  case V4L2_PIX_FMT_RGB32:
193  case V4L2_PIX_FMT_RGB565:
194  case V4L2_PIX_FMT_BGR24:
195  case V4L2_PIX_FMT_RGB24:
196  case V4L2_PIX_FMT_UYVY:
197  {
198  int const dmafd = frame.getDmaFd2();
199  setWithDmaBuf(img, dmafd, display);
200  break;
201  }
202  default:
203  set(img);
204  }
205 }
206 
207 // ##############################################################################################################
208 void jevois::GPUimage::setWithDmaBuf(jevois::RawImage const & img, int dmafd, EGLDisplay display)
209 {
210  if (img.width == 0 || img.height == 0) LFATAL("Cannot handle zero image width or height");
211 
212  // Generate a dmabuf texture for incoming image data if needed:
213  if (img.width != itsTextureWidth || img.height != itsTextureHeight || img.fmt != itsFormat ||
214  itsTexture || !itsTextureDmaBuf)
215  {
216  itsTextureDmaBuf.reset(new jevois::GPUtextureDmaBuf(display, img.width, img.height, img.fmt, dmafd));
217  itsTexture.reset(); // invalidate any previously used regular texture
218  LDEBUG("Input DMABUF texture for " << img.width <<'x'<< img.height << ' ' << jevois::fccstr(img.fmt) << " ready.");
219 
220  // Remember our latest size:
221  itsTextureWidth = img.width; itsTextureHeight = img.height;
222 
223  // Load the appropriate program:
224  itsProgram.reset(new GPUprogram(jevois::shader::vert, jevois::shader::frag_oes));
225 
226  // Get a handle to s_texture variable in the fragment shader, to update the texture with each new camera frame:
227  itsLocation = glGetUniformLocation(itsProgram->id(), "s_texture");
228 
229  // Remember our latest format:
230  itsFormat = img.fmt;
231  }
232 
233  // All done. No need to assign pixel data, it will be DMA'd over.
234 }
235 
236 #else // JEVOIS_PLATFORM_PRO
237 
238 // ##############################################################################################################
239 void jevois::GPUimage::set(jevois::InputFrame const & frame, EGLDisplay JEVOIS_UNUSED_PARAM(display))
240 {
241  // DMABUF acceleration not supported:
242  jevois::RawImage const img = frame.get();
243  set(img);
244 }
245 
246 // ##############################################################################################################
247 void jevois::GPUimage::set2(jevois::InputFrame const & frame, EGLDisplay JEVOIS_UNUSED_PARAM(display))
248 {
249  // DMABUF acceleration not supported:
250  jevois::RawImage const img = frame.get2();
251  set(img);
252 }
253 
254 #endif // JEVOIS_PLATFORM_PRO
255 
256 // ##############################################################################################################
257 void jevois::GPUimage::draw(int & x, int & y, unsigned short & w, unsigned short & h, bool noalias,
258  glm::mat4 const & pvm)
259 {
260  if (itsTextureWidth == 0) throw std::runtime_error("You must call set() before draw()");
261 
262  // Enable blending, used for RGBA textures that have transparency:
263  glEnable(GL_BLEND);
264  glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
265 
266  // Get the current viewport size:
267  GLint viewport[4];
268  glGetIntegerv(GL_VIEWPORT, viewport);
269 
270  if (w == 0 || h == 0)
271  {
272  // Letterbox the image into the window to occupy as much space as possible without changing aspect ratio:
273  unsigned int winw = viewport[2], winh = viewport[3];
274  itsDrawWidth = itsTextureWidth; itsDrawHeight = itsTextureHeight;
275  jevois::applyLetterBox(itsDrawWidth, itsDrawHeight, winw, winh, noalias);
276  x = (winw - itsDrawWidth) / 2; y = (winh - itsDrawHeight) / 2; w = itsDrawWidth; h = itsDrawHeight;
277  }
278 
279  // Flip the ordinate. Our OpenGL texture rendering uses 0,0 at the bottom-left corner, with increasing y going up on
280  // the screen. But machine vision (and our users) assumes 0,0 at top left of the screen with increasing y going down:
281  int const yy = viewport[3] - y - h;
282 
283  // Allocate/update vertex arrays if needed (including on first frame):
284  if (itsDrawX != x || itsDrawY != y || itsDrawWidth != w || itsDrawHeight != h)
285  {
286  // Compute vertex coordinates: We here assume that the projection matrix is such that at the pixel perfect plane,
287  // our vertex x,y coordinates match their pixel counterparts, except that image center is at 0,0. The view matrix is
288  // responsible for translating our z from 0 here to the pixel pixel perfect plane:
289  float const tx = x - 0.5 * viewport[2];
290  float const ty = yy - 0.5 * viewport[3];
291  float const bx = tx + w;
292  float const by = ty + h;
293 
294  // Create a new vertex array. A group of vertices and indices stored in GPU memory. The frame being drawn is static
295  // for the program duration, upload all the information to the GPU at the beginning then reference the location each
296  // frame without copying the data each frame.
297  GLfloat const vertices[] = { tx, by, 0.0f, 0.0f, 0.0f,
298  tx, ty, 0.0f, 0.0f, 1.0f,
299  bx, ty, 0.0f, 1.0f, 1.0f,
300  bx, by, 0.0f, 1.0f, 0.0f };
301  static GLushort const indices[] = { 0, 1, 2, 0, 2, 3 };
302 
303  if (itsVertexArray) { glDeleteVertexArrays(1, &itsVertexArray); glDeleteBuffers(2, itsVertexBuffers); }
304  glGenVertexArrays(1, &itsVertexArray);
305 
306  // Select the vertex array that was just created and bind the new vertex buffers to the array:
307  glBindVertexArray(itsVertexArray);
308 
309  // Generate vertex buffers in GPU memory. First buffer is for vertex data, second for index data:
310  if (itsVertexBuffers[0]) glDeleteBuffers(2, itsVertexBuffers);
311  glGenBuffers(2, itsVertexBuffers);
312 
313  // Bind the first vertex buffer with the vertices to the vertex array:
314  glBindBuffer(GL_ARRAY_BUFFER, itsVertexBuffers[0]);
315  glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
316 
317  // Bind the second vertex buffer with the indices to the vertex array:
318  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, itsVertexBuffers[1]);
319  glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW);
320 
321  // Enable a_position (location 0) and a_tex_coord (location 1) in the vertex shader:
322  glEnableVertexAttribArray(0);
323  glEnableVertexAttribArray(1);
324 
325  // Copy the vertices to the GPU to be used in a_position:
326  GL_CHECK(glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(GLfloat), 0));
327 
328  // Copy the texture co-ordinates to the GPU to be used in a_tex_coord:
329  GL_CHECK(glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(GLfloat), (void const *)(3 * sizeof(GLfloat))));
330 
331  // Select the default vertex array, allowing the application's array to be unbound:
332  glBindVertexArray(0);
333 
334  // Remember our latest draw location and size:
335  itsDrawX = x; itsDrawY = y; itsDrawWidth = w; itsDrawHeight = h;
336  }
337 
338  if (!itsProgram) throw std::runtime_error("You must call set() before draw()");
339 
340  // Tell OpenGL to use our program:
341  glUseProgram(itsProgram->id());
342 
343  // Select the vertex array which includes the vertices and indices describing the window rectangle:
344  glBindVertexArray(itsVertexArray);
345 
346  // Bind our texture, regular or DMABUF:
347  GL_CHECK(glActiveTexture(GL_TEXTURE0));
348  if (itsTexture)
349  GL_CHECK(glBindTexture(GL_TEXTURE_2D, itsTexture->Id));
350 #ifdef JEVOIS_PLATFORM_PRO
351  else if (itsTextureDmaBuf)
352  GL_CHECK(glBindTexture(GL_TEXTURE_EXTERNAL_OES, itsTextureDmaBuf->Id));
353 #endif
354  else throw std::runtime_error("You must call set() before draw()");
355 
356  // Indicate that GL_TEXTURE0 is s_texture from previous lookup:
357  glUniform1i(itsLocation, 0);
358 
359  // Let the fragment shader know the true (unscaled by bpp) image width and height. Also set the PVM matrix. Ignore any
360  // errors as some shaders may not have these variables:
361  glUniform2f(glGetUniformLocation(itsProgram->id(), "tdim"), GLfloat(itsTextureWidth), GLfloat(itsTextureHeight));
362  glUniformMatrix4fv(glGetUniformLocation(itsProgram->id(), "pvm"), 1, GL_FALSE, glm::value_ptr(pvm));
363 
364  // Draw the two triangles from 6 indices to form a rectangle from the data in the vertex array.
365  // The fourth parameter, indices value here is passed as null since the values are already
366  // available in the GPU memory through the vertex array
367  // GL_TRIANGLES - draw each set of three vertices as an individual triangle.
368  glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
369 
370  // Select the default vertex array, allowing the application array to be unbound:
371  glBindVertexArray(0);
372 }
373 
374 // ##############################################################################################################
375 ImVec2 jevois::GPUimage::i2d(ImVec2 const & p)
376 {
377  if (itsDrawWidth == 0) throw std::runtime_error("Need to call set() then draw() first");
378  return ImVec2(itsDrawX + p.x * itsDrawWidth / itsTextureWidth, itsDrawY + p.y * itsDrawHeight / itsTextureHeight);
379 }
380 
381 // ##############################################################################################################
382 ImVec2 jevois::GPUimage::i2ds(ImVec2 const & p)
383 {
384  if (itsDrawWidth == 0) throw std::runtime_error("Need to call set() then draw() first");
385  return ImVec2(p.x * itsDrawWidth / itsTextureWidth, p.y * itsDrawHeight / itsTextureHeight);
386 }
387 
388 #endif // JEVOIS_PRO
jevois::GPUtexture
Simple class to hold an OpenGL texture.
Definition: GPUtexture.H:28
jevois::shader::frag_grey
const char * frag_grey
GPUprogram.H
GPUimage.H
LDEBUG
#define LDEBUG(msg)
Convenience macro for users to print out console or syslog messages, DEBUG level.
Definition: Log.H:160
GPUtextureDmaBuf.H
jevois::InputFrame::getDmaFd
int getDmaFd(bool casync=false) const
Get the DMA-BUF file descriptor of the camera frame.
Definition: InputFrame.C:86
jevois::GPUimage::~GPUimage
~GPUimage()
Destructor.
Definition: GPUimage.C:49
jevois::shader::frag_rgba
const char * frag_rgba
jevois::shader::frag_rgb
const char * frag_rgb
jevois::RawImage
A raw image as coming from a V4L2 Camera and/or being sent out to a USB Gadget.
Definition: RawImage.H:110
jevois::GPUprogram
Simple class to load and compile some OpenGL-ES program.
Definition: GPUprogram.H:25
GPUtexture.H
GL_CHECK
#define GL_CHECK(stmt)
Simple macro to check for OpenGL errors.
Definition: OpenGL.H:77
jevois::InputFrame::get
const RawImage & get(bool casync=false) const
Get the next captured camera image.
Definition: InputFrame.C:50
jevois::RawImage::width
unsigned int width
Image width in pixels.
Definition: RawImage.H:145
jevois::shader::vert
const char * vert
jevois
Definition: Concepts.dox:1
jevois::GPUimage::GPUimage
GPUimage()
Constructor.
Definition: GPUimage.C:45
jevois::GPUimage::draw
void draw(int &x, int &y, unsigned short &w, unsigned short &h, bool noalias, glm::mat4 const &pvm)
Draw to OpenGL.
Definition: GPUimage.C:257
jevois::fccstr
std::string fccstr(unsigned int fcc)
Convert a V4L2 four-cc code (V4L2_PIX_FMT_...) to a 4-char string.
Definition: Utils.C:44
jevois::GPUimage::i2ds
ImVec2 i2ds(ImVec2 const &p)
Convert a 2D size from within a rendered image to on-screen.
Definition: GPUimage.C:382
jevois::InputFrame::get2
const RawImage & get2(bool casync=false) const
Get the next captured camera image, ISP-scaled second frame.
Definition: InputFrame.C:67
jevois::applyLetterBox
void applyLetterBox(unsigned int &imw, unsigned int &imh, unsigned int const winw, unsigned int const winh, bool noalias)
Apply a letterbox resizing to fit an image into a window.
Definition: Utils.C:209
jevois::shader::frag_yuyv
const char * frag_yuyv
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
Definition: Log.H:217
jevois::InputFrame::getDmaFd2
int getDmaFd2(bool casync=false) const
Get the DMA-BUF file descriptor of the ISP-scaled second camera frame.
Definition: InputFrame.C:94
jevois::RawImage::height
unsigned int height
Image height in pixels.
Definition: RawImage.H:146
jevois::InputFrame
Exception-safe wrapper around a raw camera input frame.
Definition: InputFrame.H:50
jevois::GPUimage::setInternal
void setInternal(unsigned int width, unsigned int height, unsigned int fmt, unsigned char const *data)
Definition: GPUimage.C:56
jevois::RawImage::fmt
unsigned int fmt
Pixel format as a V4L2_PIX_FMT_XXX.
Definition: RawImage.H:147
Utils.H
jevois::GPUimage::i2d
ImVec2 i2d(ImVec2 const &p)
Convert coordinates of a point from within a rendered image to on-screen.
Definition: GPUimage.C:375
jevois::RawImage::buf
std::shared_ptr< VideoBuf > buf
The pixel data buffer.
Definition: RawImage.H:149
h
int h
Definition: GUIhelper.C:2150
jevois::GPUimage::set
void set(RawImage const &img)
Set pixel data from a vanilla RawImage, pixel data will be copied to texture.
Definition: GPUimage.C:130
jevois::shader::frag_oes
const char * frag_oes
jevois::GPUimage::set2
void set2(InputFrame const &frame, EGLDisplay display)
Set pixel data from camera input second (scaled) frame, sharing data with camera kernel driver using ...
Definition: GPUimage.C:247