JeVoisBase  1.3
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
EyeTracker.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 
20 // This code adpated from (see Contrib directory for full source):
21 
22 /*
23  *
24  * cvEyeTracker is free software; you can redistribute it and/or modify
25  * it under the terms of the GNU General Public License as published by
26  * the Free Software Foundation; either version 2 of the License, or
27  * (at your option) any later version.
28  *
29  * cvEyeTracker is distributed in the hope that it will be useful,
30  * but WITHOUT ANY WARRANTY; without even the implied warranty of
31  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32  * GNU General Public License for more details.
33  *
34  * You should have received a copy of the GNU General Public License
35  * along with cvEyeTracker; if not, write to the Free Software
36  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
37  *
38  *
39  * cvEyeTracker - Version 1.2.5
40  * Part of the openEyes ToolKit -- http://hcvl.hci.iastate.edu/openEyes
41  * Release Date:
42  * Authors : Dongheng Li <dhli@iastate.edu>
43  * Derrick Parkhurst <derrick.parkhurst@hcvl.hci.iastate.edu>
44  * Jason Babcock <babcock@nyu.edu>
45  * David Winfield <dwinfiel@iastate.edu>
46  * Copyright (c) 2004-2006
47  * All Rights Reserved.
48  *
49  */
50 
51 #include <stdio.h>
52 #include <stdlib.h>
53 #include <math.h>
54 #include <jevoisbase/Contrib/cvEyeTracker-1.2.5/remove_corneal_reflection.h>
55 #include <jevoisbase/Contrib/cvEyeTracker-1.2.5/svd.h>
56 
57 #include <opencv/cv.h>
58 
59 #define UINT8 unsigned char
60 
61 #define FIX_UINT8(x) ( (x)<0 ? 0 : ((x)>255 ? 255:(x)) )
62 
63 void Draw_Cross(IplImage *image, int centerx, int centery, int x_cross_length, int y_cross_length, double color)
64 {
65  CvPoint pt1,pt2,pt3,pt4;
66 
67  pt1.x = centerx - x_cross_length;
68  pt1.y = centery;
69  pt2.x = centerx + x_cross_length;
70  pt2.y = centery;
71 
72  pt3.x = centerx;
73  pt3.y = centery - y_cross_length;
74  pt4.x = centerx;
75  pt4.y = centery + y_cross_length;
76 
77  cvLine(image,pt1,pt2,color,1,8);
78  cvLine(image,pt3,pt4,color,1,8);
79 }
80 
81 void Normalize_Line_Histogram(IplImage *in_image)
82 {
83  unsigned char *s=(unsigned char *)in_image->imageData;
84  int x,y;
85  int linesum;
86  double factor=0;
87  int subsample=10;
88  double hwidth = (100.0f * ((double)in_image->width) / (double)subsample);
89 
90  for (y=0;y<in_image->height;y++) {
91  linesum=1;
92  for (x=0;x<in_image->width;x+=subsample) {
93  linesum+=*s;
94  s+=subsample;
95  }
96  s-=in_image->width;
97  factor=hwidth/((double)linesum);
98  for (x=0;x<in_image->width;x++) {
99  *s=(unsigned char)(((double)*s)*factor);
100  s++;
101  }
102  }
103 }
104 
105 
106 void Calculate_Avg_Intensity_Hori(IplImage* in_image, double * avg_intensity_hori)
107 {
108  UINT8 *pixel = (UINT8*)in_image->imageData;
109  int sum;
110  for (int j = 0; j < in_image->height; ++j)
111  {
112  sum = 0; for (int i = 0; i < in_image->width; ++i) sum += *pixel++;
113  avg_intensity_hori[j] = double(sum) / in_image->width;
114  }
115 }
116 
117 void Reduce_Line_Noise(IplImage* in_image, double * avg_intensity_hori, double * intensity_factor_hori)
118 {
119  const double beta = 0.2; // hysteresis factor for noise reduction
120 
121  UINT8 *pixel = (UINT8*)in_image->imageData;
122  double beta2 = 1 - beta;
123  int adjustment;
124 
125  Calculate_Avg_Intensity_Hori(in_image, avg_intensity_hori);
126 
127  for (int j = 0; j < in_image->height; ++j)
128  {
129  intensity_factor_hori[j] = avg_intensity_hori[j] * beta + intensity_factor_hori[j] * beta2;
130 
131  adjustment = (int)(intensity_factor_hori[j] - avg_intensity_hori[j]);
132 
133  for (int i = 0; i < in_image->width; ++i) { *pixel = FIX_UINT8(*pixel + adjustment); ++pixel; }
134  }
135 }
136 
137 // ##############################################################################################################
138 EyeTracker::EyeTracker(std::string const & instance) :
139  jevois::Component(instance), currWidth(0), currHeight(0), intensity_factor_hori(nullptr),
140  avg_intensity_hori(nullptr)
141 { }
142 
143 // ##############################################################################################################
145 {
146  if (intensity_factor_hori) free(intensity_factor_hori);
147  if (avg_intensity_hori) free(avg_intensity_hori);
148 }
149 
150 // ##############################################################################################################
151 void EyeTracker::process(cv::Mat & eyeimg, double pupell[5], bool debugdraw)
152 {
153  int *inliers_index;
154  CvSize ellipse_axis;
155  CvPoint gaze_point;
156  static int lost_frame_num = 0;
157 
158  // Initialize avg intensity if needed:
159  if (currWidth != eyeimg.cols || currHeight != eyeimg.rows)
160  {
161  // New image size, reset a bunch of things:
162  currWidth = eyeimg.cols; currHeight = eyeimg.rows;
163  if (intensity_factor_hori) { free(intensity_factor_hori); intensity_factor_hori = nullptr; }
164  if (avg_intensity_hori) { free(avg_intensity_hori); avg_intensity_hori = nullptr; }
165  start_point.x = currWidth / 2; start_point.y = currHeight / 2;
166  for (int k = 0; k < 5; ++k) pupil_param[k] = 0.0;
167  }
168 
169  // Make the eye image (in monochrome):
170  IplImage * eye_image = cvCreateImageHeader(cvSize(eyeimg.cols, eyeimg.rows), 8, 1);
171  eye_image->imageData = reinterpret_cast<char *>(eyeimg.data);
172 
173  // Make the threshold image (in monochrome):
174  IplImage * threshold_image = cvCloneImage(eye_image);
175 
176  if (avg_intensity_hori == nullptr)
177  {
178  avg_intensity_hori = (double *)malloc(currHeight * sizeof(double));
179  Calculate_Avg_Intensity_Hori(eye_image, avg_intensity_hori);
180  }
181 
182  if (intensity_factor_hori == nullptr)
183  {
184  intensity_factor_hori = (double *)malloc(currHeight * sizeof(double));
185  memcpy(intensity_factor_hori, avg_intensity_hori, currHeight * sizeof(double));
186  }
187 
188  cvSmooth(eye_image, eye_image, CV_GAUSSIAN, 3, 3); // JEVOIS: was 5x5
189 
190  Reduce_Line_Noise(eye_image, avg_intensity_hori, intensity_factor_hori);
191 
192  // corneal reflection
193  int crwin = eyetracker::corneal::get() & 1; // enforce odd
194  if (crwin >= eyeimg.cols) crwin = (eyeimg.cols - 1) & 1;
195  if (crwin >= eyeimg.rows) crwin = (eyeimg.rows - 1) & 1;
196 
197  int corneal_reflection_r = 0; //the radius of corneal reflection
198  CvPoint corneal_reflection = { 0, 0 }; //coordinates of corneal reflection in tracker coordinate system
199 
200  remove_corneal_reflection(eye_image, threshold_image, (int)start_point.x, (int)start_point.y,
201  crwin, (int)eye_image->height/10, corneal_reflection.x,
202  corneal_reflection.y, corneal_reflection_r);
203  /////printf("corneal reflection: (%d, %d)\n", corneal_reflection.x, corneal_reflection.y);
204 
205  //starburst pupil contour detection
206  starburst_pupil_contour_detection((UINT8*)eye_image->imageData, eye_image->width, eye_image->height,
207  eyetracker::edgethresh::get(), eyetracker::numrays::get(),
208  eyetracker::mincand::get(), start_point, edge_point);
209 
210  int inliers_num = 0;
211  CvPoint pupil = { 0, 0 }; // coordinates of pupil in tracker coordinate system
212 
213  inliers_index = pupil_fitting_inliers((UINT8*)eye_image->imageData, eye_image->width, eye_image->height,
214  inliers_num, pupil_param, edge_point);
215 
216  ellipse_axis.width = (int)pupil_param[0];
217  ellipse_axis.height = (int)pupil_param[1];
218  pupil.x = (int)pupil_param[2];
219  pupil.y = (int)pupil_param[3];
220 
221  // JEVOIS only draw line if valid
222  if (debugdraw && corneal_reflection.x > 3 && corneal_reflection.y > 3 && pupil.x > 3 && pupil.y > 3)
223  cvLine(eye_image, pupil, corneal_reflection, 100, 4, 8);
224 
225  /*** JEVOIS comment out
226  printf("ellipse a:%lf; b:%lf, cx:%lf, cy:%lf, theta:%lf; inliers_num:%d\n\n",
227  pupil_param[0], pupil_param[1], pupil_param[2], pupil_param[3], pupil_param[4], inliers_num);
228  */
229  for (int k = 0; k < 5; ++k) pupell[k] = pupil_param[k]; // send data to caller
230 
231  if (debugdraw)
232  {
233  bool is_inliers = 0;
234  for (int i = 0; i < int(edge_point.size()); i++)
235  {
236  is_inliers = 0;
237  for (int j = 0; j < inliers_num; j++) if (i == inliers_index[j]) is_inliers = 1;
238  stuDPoint const & edge = edge_point.at(i);
239  if (is_inliers) Draw_Cross(eye_image, (int)edge.x,(int)edge.y, 5, 5, 200);
240  else Draw_Cross(eye_image, (int)edge.x, (int)edge.y, 3, 3, 100);
241  }
242  }
243  free(inliers_index);
244 
245  if (ellipse_axis.width > 0 && ellipse_axis.height > 0)
246  {
247  start_point.x = pupil.x;
248  start_point.y = pupil.y;
249  //printf("start_point: %d,%d\n", start_point.x, start_point.y);
250  Draw_Cross(eye_image, pupil.x, pupil.y, 10, 10, 255);
251  cvEllipse(eye_image, pupil, ellipse_axis, -pupil_param[4]*180/M_PI, 0, 360, 255, 2);
252 
253  lost_frame_num = 0;
254  } else {
255  lost_frame_num++;
256  }
257  if (lost_frame_num > 5) {
258  start_point.x = eyeimg.cols / 2;
259  start_point.y = eyeimg.rows / 2;
260  }
261 
262  Draw_Cross(eye_image, (int)start_point.x, (int)start_point.y, 7, 7, 128);
263 
264  // Cleanup:
265  cvReleaseImageHeader(&eye_image);
266  cvReleaseImage(&threshold_image);
267 }
268 
void Calculate_Avg_Intensity_Hori(IplImage *in_image, double *avg_intensity_hori)
Definition: EyeTracker.C:106
EyeTracker(std::string const &instance)
Constructor.
Definition: EyeTracker.C:138
void Normalize_Line_Histogram(IplImage *in_image)
Definition: EyeTracker.C:81
#define FIX_UINT8(x)
Definition: EyeTracker.C:61
void process(cv::Mat &eyeimg, double pupell[5], bool debugdraw=false)
Process grayscale byte image from camera.
Definition: EyeTracker.C:151
void Reduce_Line_Noise(IplImage *in_image, double *avg_intensity_hori, double *intensity_factor_hori)
Definition: EyeTracker.C:117
void Draw_Cross(IplImage *image, int centerx, int centery, int x_cross_length, int y_cross_length, double color)
Definition: EyeTracker.C:63
#define UINT8
Definition: EyeTracker.C:59
virtual ~EyeTracker()
Destructor.
Definition: EyeTracker.C:144