JeVoisBase  1.11
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
EyeTracker.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 
20 // This code adpated from (see Contrib directory for full source):
21 
22 /*
23  *
24  * cvEyeTracker is free software; you can redistribute it and/or modify
25  * it under the terms of the GNU General Public License as published by
26  * the Free Software Foundation; either version 2 of the License, or
27  * (at your option) any later version.
28  *
29  * cvEyeTracker is distributed in the hope that it will be useful,
30  * but WITHOUT ANY WARRANTY; without even the implied warranty of
31  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32  * GNU General Public License for more details.
33  *
34  * You should have received a copy of the GNU General Public License
35  * along with cvEyeTracker; if not, write to the Free Software
36  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
37  *
38  *
39  * cvEyeTracker - Version 1.2.5
40  * Part of the openEyes ToolKit -- http://hcvl.hci.iastate.edu/openEyes
41  * Release Date:
42  * Authors : Dongheng Li <dhli@iastate.edu>
43  * Derrick Parkhurst <derrick.parkhurst@hcvl.hci.iastate.edu>
44  * Jason Babcock <babcock@nyu.edu>
45  * David Winfield <dwinfiel@iastate.edu>
46  * Copyright (c) 2004-2006
47  * All Rights Reserved.
48  *
49  */
50 
51 #include <stdio.h>
52 #include <stdlib.h>
53 #include <math.h>
54 
55 /*
56 #include <jevoisbase/Contrib/cvEyeTracker-1.2.5/remove_corneal_reflection.h>
57 #include <jevoisbase/Contrib/cvEyeTracker-1.2.5/svd.h>
58 
59 #include <opencv/cv.h>
60 
61 #define UINT8 unsigned char
62 
63 #define FIX_UINT8(x) ( (x)<0 ? 0 : ((x)>255 ? 255:(x)) )
64 
65 void Draw_Cross(IplImage *image, int centerx, int centery, int x_cross_length, int y_cross_length, double color)
66 {
67  CvPoint pt1,pt2,pt3,pt4;
68 
69  pt1.x = centerx - x_cross_length;
70  pt1.y = centery;
71  pt2.x = centerx + x_cross_length;
72  pt2.y = centery;
73 
74  pt3.x = centerx;
75  pt3.y = centery - y_cross_length;
76  pt4.x = centerx;
77  pt4.y = centery + y_cross_length;
78 
79  cvLine(image,pt1,pt2,color,1,8);
80  cvLine(image,pt3,pt4,color,1,8);
81 }
82 
83 void Normalize_Line_Histogram(IplImage *in_image)
84 {
85  unsigned char *s=(unsigned char *)in_image->imageData;
86  int x,y;
87  int linesum;
88  double factor=0;
89  int subsample=10;
90  double hwidth = (100.0f * ((double)in_image->width) / (double)subsample);
91 
92  for (y=0;y<in_image->height;y++) {
93  linesum=1;
94  for (x=0;x<in_image->width;x+=subsample) {
95  linesum+=*s;
96  s+=subsample;
97  }
98  s-=in_image->width;
99  factor=hwidth/((double)linesum);
100  for (x=0;x<in_image->width;x++) {
101  *s=(unsigned char)(((double)*s)*factor);
102  s++;
103  }
104  }
105 }
106 
107 
108 void Calculate_Avg_Intensity_Hori(IplImage* in_image, double * avg_intensity_hori)
109 {
110  UINT8 *pixel = (UINT8*)in_image->imageData;
111  int sum;
112  for (int j = 0; j < in_image->height; ++j)
113  {
114  sum = 0; for (int i = 0; i < in_image->width; ++i) sum += *pixel++;
115  avg_intensity_hori[j] = double(sum) / in_image->width;
116  }
117 }
118 
119 void Reduce_Line_Noise(IplImage* in_image, double * avg_intensity_hori, double * intensity_factor_hori)
120 {
121  const double beta = 0.2; // hysteresis factor for noise reduction
122 
123  UINT8 *pixel = (UINT8*)in_image->imageData;
124  double beta2 = 1 - beta;
125  int adjustment;
126 
127  Calculate_Avg_Intensity_Hori(in_image, avg_intensity_hori);
128 
129  for (int j = 0; j < in_image->height; ++j)
130  {
131  intensity_factor_hori[j] = avg_intensity_hori[j] * beta + intensity_factor_hori[j] * beta2;
132 
133  adjustment = (int)(intensity_factor_hori[j] - avg_intensity_hori[j]);
134 
135  for (int i = 0; i < in_image->width; ++i) { *pixel = FIX_UINT8(*pixel + adjustment); ++pixel; }
136  }
137 }
138 */
139 
140 // ##############################################################################################################
141 EyeTracker::EyeTracker(std::string const & instance) :
142  jevois::Component(instance), currWidth(0), currHeight(0), intensity_factor_hori(nullptr),
143  avg_intensity_hori(nullptr)
144 { }
145 
146 // ##############################################################################################################
148 {
149  if (intensity_factor_hori) free(intensity_factor_hori);
150  if (avg_intensity_hori) free(avg_intensity_hori);
151 }
152 
153 // ##############################################################################################################
154 void EyeTracker::process(cv::Mat & eyeimg, double pupell[5], bool debugdraw)
155 {
156  LFATAL("Sorry this module needs to be updated to work again...");
157  /*
158 
159  int *inliers_index;
160  CvSize ellipse_axis;
161  CvPoint gaze_point;
162  static int lost_frame_num = 0;
163 
164  // Initialize avg intensity if needed:
165  if (currWidth != eyeimg.cols || currHeight != eyeimg.rows)
166  {
167  // New image size, reset a bunch of things:
168  currWidth = eyeimg.cols; currHeight = eyeimg.rows;
169  if (intensity_factor_hori) { free(intensity_factor_hori); intensity_factor_hori = nullptr; }
170  if (avg_intensity_hori) { free(avg_intensity_hori); avg_intensity_hori = nullptr; }
171  start_point.x = currWidth / 2; start_point.y = currHeight / 2;
172  for (int k = 0; k < 5; ++k) pupil_param[k] = 0.0;
173  }
174 
175  // Make the eye image (in monochrome):
176  IplImage * eye_image = cvCreateImageHeader(cvSize(eyeimg.cols, eyeimg.rows), 8, 1);
177  eye_image->imageData = reinterpret_cast<char *>(eyeimg.data);
178 
179  // Make the threshold image (in monochrome):
180  IplImage * threshold_image = cvCloneImage(eye_image);
181 
182  if (avg_intensity_hori == nullptr)
183  {
184  avg_intensity_hori = (double *)malloc(currHeight * sizeof(double));
185  Calculate_Avg_Intensity_Hori(eye_image, avg_intensity_hori);
186  }
187 
188  if (intensity_factor_hori == nullptr)
189  {
190  intensity_factor_hori = (double *)malloc(currHeight * sizeof(double));
191  memcpy(intensity_factor_hori, avg_intensity_hori, currHeight * sizeof(double));
192  }
193 
194  cvSmooth(eye_image, eye_image, CV_GAUSSIAN, 3, 3); // JEVOIS: was 5x5
195 
196  Reduce_Line_Noise(eye_image, avg_intensity_hori, intensity_factor_hori);
197 
198  // corneal reflection
199  int crwin = eyetracker::corneal::get() & 1; // enforce odd
200  if (crwin >= eyeimg.cols) crwin = (eyeimg.cols - 1) & 1;
201  if (crwin >= eyeimg.rows) crwin = (eyeimg.rows - 1) & 1;
202 
203  int corneal_reflection_r = 0; //the radius of corneal reflection
204  CvPoint corneal_reflection = { 0, 0 }; //coordinates of corneal reflection in tracker coordinate system
205 
206  remove_corneal_reflection(eye_image, threshold_image, (int)start_point.x, (int)start_point.y,
207  crwin, (int)eye_image->height/10, corneal_reflection.x,
208  corneal_reflection.y, corneal_reflection_r);
209  /////printf("corneal reflection: (%d, %d)\n", corneal_reflection.x, corneal_reflection.y);
210 
211  //starburst pupil contour detection
212  starburst_pupil_contour_detection((UINT8*)eye_image->imageData, eye_image->width, eye_image->height,
213  eyetracker::edgethresh::get(), eyetracker::numrays::get(),
214  eyetracker::mincand::get(), start_point, edge_point);
215 
216  int inliers_num = 0;
217  CvPoint pupil = { 0, 0 }; // coordinates of pupil in tracker coordinate system
218 
219  inliers_index = pupil_fitting_inliers((UINT8*)eye_image->imageData, eye_image->width, eye_image->height,
220  inliers_num, pupil_param, edge_point);
221 
222  ellipse_axis.width = (int)pupil_param[0];
223  ellipse_axis.height = (int)pupil_param[1];
224  pupil.x = (int)pupil_param[2];
225  pupil.y = (int)pupil_param[3];
226 
227  // JEVOIS only draw line if valid
228  if (debugdraw && corneal_reflection.x > 3 && corneal_reflection.y > 3 && pupil.x > 3 && pupil.y > 3)
229  cvLine(eye_image, pupil, corneal_reflection, 100, 4, 8);
230 
231  //printf("ellipse a:%lf; b:%lf, cx:%lf, cy:%lf, theta:%lf; inliers_num:%d\n\n",
232  // pupil_param[0], pupil_param[1], pupil_param[2], pupil_param[3], pupil_param[4], inliers_num);
233  for (int k = 0; k < 5; ++k) pupell[k] = pupil_param[k]; // send data to caller
234 
235  if (debugdraw)
236  {
237  bool is_inliers = 0;
238  for (int i = 0; i < int(edge_point.size()); i++)
239  {
240  is_inliers = 0;
241  for (int j = 0; j < inliers_num; j++) if (i == inliers_index[j]) is_inliers = 1;
242  stuDPoint const & edge = edge_point.at(i);
243  if (is_inliers) Draw_Cross(eye_image, (int)edge.x,(int)edge.y, 5, 5, 200);
244  else Draw_Cross(eye_image, (int)edge.x, (int)edge.y, 3, 3, 100);
245  }
246  }
247  free(inliers_index);
248 
249  if (ellipse_axis.width > 0 && ellipse_axis.height > 0)
250  {
251  start_point.x = pupil.x;
252  start_point.y = pupil.y;
253  //printf("start_point: %d,%d\n", start_point.x, start_point.y);
254  Draw_Cross(eye_image, pupil.x, pupil.y, 10, 10, 255);
255  cvEllipse(eye_image, pupil, ellipse_axis, -pupil_param[4]*180/M_PI, 0, 360, 255, 2);
256 
257  lost_frame_num = 0;
258  } else {
259  lost_frame_num++;
260  }
261  if (lost_frame_num > 5) {
262  start_point.x = eyeimg.cols / 2;
263  start_point.y = eyeimg.rows / 2;
264  }
265 
266  Draw_Cross(eye_image, (int)start_point.x, (int)start_point.y, 7, 7, 128);
267 
268  // Cleanup:
269  cvReleaseImageHeader(&eye_image);
270  cvReleaseImage(&threshold_image);
271  */
272 }
273 
EyeTracker(std::string const &instance)
Constructor.
Definition: EyeTracker.C:141
void process(cv::Mat &eyeimg, double pupell[5], bool debugdraw=false)
Process grayscale byte image from camera.
Definition: EyeTracker.C:154
#define LFATAL(msg)
virtual ~EyeTracker()
Destructor.
Definition: EyeTracker.C:147