JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
Saliency.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 
26 
27 #include <jevois/Core/VideoBuf.H>
28 #include <jevois/Debug/Log.H>
29 #include <jevois/Debug/Timer.H>
32 
33 #include <cstdlib>
34 #include <future>
35 #include <functional> // for placeholders
36 
37 #define WEIGHT_SCALEBITS ((env_size_t) 8)
38 
39 // ##############################################################################################################
40 static int computeGist(const char * tagName, env_size_t clev, env_size_t slev, struct env_image* submap,
41  const struct env_image * JEVOIS_UNUSED_PARAM(center),
42  const struct env_image * JEVOIS_UNUSED_PARAM(surround), void * vdata)
43 {
44  //LINFO(tagName << ": cs=" << clev << '-' << slev << " submap " << submap->dims.w << 'x' << submap->dims.h);
45 
46  // We should be called 72 times, once for each of our 72 feature maps. Our gist contains 16 values (4x4 grid) for each
47  // of the maps, so 72*16=1152 values in total. Each single channel has 6 maps. First, parse the tagname to get the
48  // offset in our gist vector at the single channel level:
49 
50  Saliency::visitor_data * vd = reinterpret_cast<Saliency::visitor_data *>(vdata);
51 
52  env_size_t const onechan = 6 * 16;
53  env_size_t offset; unsigned int bitshift = 0;
54  switch (tagName[0])
55  {
56  case 'r':
57  if (tagName[2] == 'd') { offset = 0; bitshift = 6; } // red/green
58  else { offset = (std::atoi(tagName + 10) + 6) * onechan; bitshift = 4; } // reichardt(%d/%d) with number 1-based
59  break;
60  case 'b': offset = onechan; bitshift = 6; break; // blue/yellow
61  case 'i': offset = 2 * onechan; bitshift = 7; break; // intensity
62  case 's': offset = (std::atoi(tagName + 10) + 2) * onechan; bitshift = 0; break; // steerable(%d/%d), 1-based
63  case 'f': offset = 7 * onechan; bitshift = 5; break; // flicker
64  default: LFATAL("Unknown channel " << tagName);
65  }
66 
67  // Now we need to compute an additional offset between 0 and 5 depending on clev and slev:
68  // clev in { 2, 3, 4 }, delta = slev - clev in { 3, 4 }
69  // FIXME: use envp to get the levels, etc
70  env_size_t const delta = slev - clev;
71  offset += ((delta - vd->envp->cs_del_min) * 3 + clev - vd->envp->cs_lev_min) * 16;
72  if (offset + 16 > vd->gist_size) LFATAL("gist offset " << offset << " out of range");
73 
74  // All right, fill in the data:
75  env_grid_average(submap, vd->gist + offset, bitshift, 4, 4);
76 
77  return 0;
78 }
79 
80 // ##############################################################################################################
81 Saliency::Saliency(std::string const & instance) :
82  jevois::Component(instance), gist_size(72 * 16), itsProfiler("Saliency", 100, LOG_DEBUG), itsInputDone(true)
83 {
85 
86  env_init_integer_math(&imath, &envp);
87 
88  env_img_init_empty(&prev_input);
89  env_pyr_init(&prev_lowpass5, 0);
90  env_motion_channel_init(&motion_chan, &envp);
91 
98  gist = new unsigned char[gist_size];
99 
100  itsVisitorData.gist = gist;
101  itsVisitorData.gist_size = gist_size;
102  itsVisitorData.envp = &envp;
103 }
104 
105 // ##############################################################################################################
107 {
108  delete [] gist;
109  env_img_make_empty(&prev_input);
110  env_pyr_make_empty(&prev_lowpass5);
111  env_motion_channel_destroy(&motion_chan);
112 }
113 
114 // ##############################################################################################################
115 void Saliency::combine_output(struct env_image* chanOut, const intg32 iweight, struct env_image* result)
116 {
117  if (!env_img_initialized(chanOut)) return;
118 
119  intg32* const sptr = env_img_pixelsw(chanOut);
120  const env_size_t sz = env_img_size(chanOut);
121 
122  // Lock here so that we combine one channel at a time:
123  std::lock_guard<std::mutex> _(itsMtx);
124 
125  if (!env_img_initialized(result))
126  {
127  env_img_resize_dims(result, chanOut->dims);
128  intg32* const dptr = env_img_pixelsw(result);
129  for (env_size_t i = 0; i < sz; ++i)
130  {
131  sptr[i] = (sptr[i] >> WEIGHT_SCALEBITS) * iweight;
132  dptr[i] = sptr[i];
133  }
134  }
135  else
136  {
137  ENV_ASSERT(env_dims_equal(chanOut->dims, result->dims));
138  intg32* const dptr = env_img_pixelsw(result);
139  const env_size_t sz = env_img_size(result);
140  for (env_size_t i = 0; i < sz; ++i)
141  {
142  sptr[i] = (sptr[i] >> WEIGHT_SCALEBITS) * iweight;
143  dptr[i] += sptr[i];
144  }
145  }
146 }
147 
148 #define SALUPDATE(envval, param) \
149  prev = envp.envval; envp.envval = saliency::param::get(); if (envp.envval != prev) nuke = true;
150 
151 // ##############################################################################################################
152 void Saliency::processStart(struct env_dims const & dims, bool do_gist)
153 {
154  // Mark our input image as being processed:
155  {
156  std::unique_lock<std::mutex> ulck(itsRawImageMtx);
157  itsInputDone = false;
158  }
159 
160  bool nuke = false; size_t prev;
161 
162  // Update envp with our current parameters:
163  envp.chan_c_weight = saliency::cweight::get();
164  envp.chan_i_weight = saliency::iweight::get();
165  envp.chan_o_weight = saliency::oweight::get();
166  envp.chan_f_weight = saliency::fweight::get();
167  envp.chan_m_weight = saliency::mweight::get();
168 
169  SALUPDATE(cs_lev_min, centermin);
170  envp.cs_lev_max = envp.cs_lev_min + 2;
171 
172  SALUPDATE(cs_del_min, deltamin);
173  envp.cs_del_max = envp.cs_del_min + 1;
174 
175  SALUPDATE(output_map_level, smscale);
176 
177  envp.motion_thresh = saliency::mthresh::get();
178 
179  envp.flicker_thresh = saliency::fthresh::get();
180 
181  prev = envp.multiscale_flicker;
182  envp.multiscale_flicker = saliency::msflick::get() ? 1 : 0;
183  if (envp.multiscale_flicker != prev) nuke = true;
184 
185  env_params_validate(&envp);
186 
187  // Zero-out all our internals:
194  memset(gist, 0, gist_size);
195 
196  // Reject bad images:
197  if (dims.w < 32 || dims.h < 32) LFATAL("input dims " << dims.w << 'x' << dims.h << " too small -- REJECTED");
198  if (dims.w > 2048 || dims.h > 2048) LFATAL("input dims " << dims.w << 'x' << dims.h << " too large -- REJECTED");
199 
200  // Check whether the input size or critical params just changed, and if so invalidate our previous stored data:
201  if (env_img_initialized(&prev_input) && (prev_input.dims.w != dims.w || prev_input.dims.h != dims.h)) nuke = true;
202 
203  if (nuke)
204  {
205  env_img_make_empty(&prev_input);
206  env_pyr_make_empty(&prev_lowpass5);
207  env_motion_channel_destroy(&motion_chan);
208  env_motion_channel_init(&motion_chan, &envp);
209  }
210 
211  // Install hook for gist computation, if desired:
212  if (do_gist) { envp.user_data_preproc = &itsVisitorData; envp.submapPreProc = &computeGist; }
213  else { envp.user_data_preproc = nullptr; envp.submapPreProc = nullptr; }
214 }
215 
216 // ##############################################################################################################
218 {
219  std::unique_lock<std::mutex> ulck(itsRawImageMtx);
220  if (itsInputDone) return; // we are done already
221  itsRawImageCond.wait(ulck, [&]() { return itsInputDone; } );
222 }
223 
224 // ##############################################################################################################
225 void Saliency::process(cv::Mat const & input, bool do_gist)
226 {
227  static env_chan_status_func * statfunc = nullptr;
228  static void * statdata = nullptr;
229 
230  // We here do what env_mt_visual_cortex_inut used to do in the original envision code, but using lambdas instead of
231  // the c-based jobs:
232  struct env_dims dims = { (env_size_t)input.cols, (env_size_t)input.rows };
233  processStart(dims, do_gist);
234  struct env_rgb_pixel * inpixels = reinterpret_cast<struct env_rgb_pixel *>(input.data);
235 
236  const intg32 total_weight = env_total_weight(&envp);
237  ENV_ASSERT(total_weight > 0);
238 
239  /* We want to compute
240 
241  * weight
242  * img * ------------
243  * total_weight
244  *
245  *
246  * To do that without overflowing, we compute it as
247  *
248  *
249  * weight 256
250  * img * ------------ * ---
251  * total_weight 256
252  *
253  * img weight * 256
254  * = ( --- ) * ( ------------ )
255  * 256 total_weight
256  *
257  * where 256 is an example of (1<<WEIGHT_SCALEBITS) for
258  * WEIGHT_SCALEBITS=8.
259  */
260 
261  // We can get the color channel started right away:
262  std::future<void> colorfut;
263  if (envp.chan_c_weight > 0)
264  colorfut = std::async(std::launch::async, [&](){
265  env_chan_color("color", &envp, &imath, inpixels, dims, statfunc, statdata, &color);
266  combine_output(&color, envp.chan_c_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
267  });
268 
269  // Compute luminance image:
270  struct env_image bwimg; env_img_init(&bwimg, dims);
271  env_c_luminance_from_byte(inpixels, dims.w * dims.h, imath.nbits, env_img_pixelsw(&bwimg));
272 
273  // Notify anyone that was waiting to free the raw input that we are done with it:
274  itsInputDone = true; itsRawImageCond.notify_all();
275 
276  // Compute a luminance pyramid:
277  struct env_pyr lowpass5; env_pyr_init(&lowpass5, env_max_pyr_depth(&envp));
278  env_pyr_build_lowpass_5(&bwimg, envp.cs_lev_min, &imath, &lowpass5);
279 
280  // Now parallelize the other channels:
281  std::future<void> motfut;
282  if (envp.chan_m_weight > 0)
283  motfut = std::async(std::launch::async, [&](){
284  env_mt_motion_channel_input(&motion_chan, "motion", bwimg.dims, &lowpass5, statfunc, statdata, &motion);
285  combine_output(&motion, envp.chan_m_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
286  });
287 
288  std::future<void> orifut;
289  if (envp.chan_o_weight > 0)
290  orifut = std::async(std::launch::async, [&](){
291  env_mt_chan_orientation("orientation", &bwimg, statfunc, statdata, &ori);
292  combine_output(&ori, envp.chan_o_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
293  });
294 
295  std::future<void> flickfut;
296  if (envp.chan_f_weight > 0)
297  flickfut = std::async(std::launch::async, [&](){
298  if (envp.multiscale_flicker)
299  env_chan_msflicker("flicker", &envp, &imath, bwimg.dims, &prev_lowpass5, &lowpass5,
300  statfunc, statdata, &flicker);
301  else
302  env_chan_flicker("flicker", &envp, &imath, &prev_input, &bwimg, statfunc, statdata, &flicker);
303 
304  combine_output(&flicker, envp.chan_f_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
305 
306  if (envp.multiscale_flicker) env_pyr_copy_src_dst(&lowpass5, &prev_lowpass5);
307  else env_pyr_make_empty(&prev_lowpass5);
308  });
309 
310  // Intensity is the fastest one and we here just run it in the current thread:
311  if (envp.chan_i_weight > 0)
312  {
313  env_chan_intensity("intensity", &envp, &imath, bwimg.dims, &lowpass5, 1, statfunc, statdata, &intens);
314  combine_output(&intens, envp.chan_i_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
315  }
316 
317  // Wait for all channels to finish up:
318  if (colorfut.valid()) colorfut.get();
319  if (orifut.valid()) orifut.get();
320  if (flickfut.valid()) flickfut.get();
321  if (motfut.valid()) motfut.get();
322 
323  // Cleanup and get ready for next frame:
324  if (!envp.multiscale_flicker) env_img_swap(&prev_input, &bwimg); else env_img_make_empty(&prev_input);
325 
326  if (statfunc) (*statfunc)(statdata, "saliency", &salmap);
327 
328  // We transfer our lowpass5 to the motion channel as unshifted prev:
329  env_pyr_swap(&lowpass5, &motion_chan.unshifted_prev);
330  env_pyr_make_empty(&lowpass5);
331 
332  env_img_make_empty(&bwimg);
333  /*
334  env_visual_cortex_rescale_ranges(&salmap, &intens, &color, &ori, &flicker, &motion);
335  */
336 }
337 
338 // ##############################################################################################################
339 void Saliency::process(jevois::RawImage const & input, bool do_gist)
340 {
341  itsProfiler.start();
342 
343  static env_chan_status_func * statfunc = nullptr;
344  static void * statdata = nullptr;
345 
346  // We here do what env_mt_visual_cortex_inut used to do in the original envision code, but using lambdas instead of
347  // the c-based jobs:
348  struct env_dims dims = { input.width, input.height };
349  processStart(dims, do_gist);
350  itsProfiler.checkpoint("processStart");
351 
352  // Compute Lum, RG, BY, parallelizing over rows:
353  const intg32 lumthresh = (3*255) / 10;
354  struct env_image rgimg; env_img_init(&rgimg, dims);
355  struct env_image byimg; env_img_init(&byimg, dims);
356  struct env_image bwimg; env_img_init(&bwimg, dims);
357 
358  int const nthreads = 4;
359  int hh = dims.h / nthreads;
360  std::vector<std::future<void> > rgbyfut;
361  unsigned char const * inpix = input.pixels<unsigned char>();
362  intg32 * rgpix = env_img_pixelsw(&rgimg);
363  intg32 * bypix = env_img_pixelsw(&byimg);
364  intg32 * bwpix = env_img_pixelsw(&bwimg);
365  for (int i = 0; i < nthreads-1; ++i)
366  rgbyfut.push_back(std::async(std::launch::async, [&](int ii) {
367  int offset = dims.w * hh * ii;
368  convertYUYVtoRGBYL(dims.w, hh, inpix + offset*2, rgpix + offset, bypix + offset, bwpix + offset,
369  lumthresh, imath.nbits);
370  }, i));
371 
372  // Do the last bit in the current thread:
373  int offset = dims.w * hh * (nthreads - 1);
374  convertYUYVtoRGBYL(dims.w, dims.h - hh * (nthreads-1), inpix + offset*2, rgpix + offset, bypix + offset,
375  bwpix + offset, lumthresh, imath.nbits);
376 
377  const intg32 total_weight = env_total_weight(&envp);
378  ENV_ASSERT(total_weight > 0);
379 
380  // We can get the color channels started right away. Here we split rg and by into two threads then combine later in a
381  // manner similar to what env_chan_color_rgby() does:
382  const env_size_t firstlevel = envp.cs_lev_min;
383  const env_size_t depth = env_max_pyr_depth(&envp);
384  std::future<void> rgfut, byfut;
385  struct env_image byOut = env_img_initializer;
386 
387  // Wait for rgbylum computation to be complete:
388  for (auto & f : rgbyfut) f.get();
389  rgbyfut.clear();
390  itsProfiler.checkpoint("rgby");
391 
392  // Notify anyone that was waiting to free the raw input that we are done with it:
393  itsInputDone = true; itsRawImageCond.notify_all();
394 
395  // Launch RG and BY in threads:
396  if (envp.chan_c_weight > 0)
397  {
398  rgfut = std::async(std::launch::async, [&]() {
399  struct env_pyr rgpyr;
400  env_pyr_init(&rgpyr, depth);
401  env_pyr_build_lowpass_5(&rgimg, firstlevel, &imath, &rgpyr);
402  env_chan_intensity("red/green", &envp, &imath, rgimg.dims, &rgpyr, 0, statfunc, statdata, &color);
403  env_pyr_make_empty(&rgpyr);
404  });
405 
406  byfut = std::async(std::launch::async, [&]() {
407  struct env_pyr bypyr;
408  env_pyr_init(&bypyr, depth);
409  env_pyr_build_lowpass_5(&byimg, firstlevel, &imath, &bypyr);
410  env_chan_intensity("blue/yellow", &envp, &imath, byimg.dims, &bypyr, 0, statfunc, statdata, &byOut);
411  env_pyr_make_empty(&bypyr);
412  });
413  }
414 
415  // Compute a luminance pyramid:
416  struct env_pyr lowpass5; env_pyr_init(&lowpass5, env_max_pyr_depth(&envp));
417  env_pyr_build_lowpass_5(&bwimg, envp.cs_lev_min, &imath, &lowpass5);
418 
419  itsProfiler.checkpoint("lowpass pyr");
420 
421  // Now parallelize the other channels:
422  std::future<void> motfut;
423  if (envp.chan_m_weight > 0)
424  motfut = std::async(std::launch::async, [&]() {
425  env_mt_motion_channel_input(&motion_chan, "motion", bwimg.dims, &lowpass5, statfunc, statdata, &motion);
426  combine_output(&motion, envp.chan_m_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
427  });
428 
429  std::future<void> orifut;
430  if (envp.chan_o_weight > 0)
431  orifut = std::async(std::launch::async, [&]() {
432  env_mt_chan_orientation("orientation", &bwimg, statfunc, statdata, &ori);
433  combine_output(&ori, envp.chan_o_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
434  });
435 
436  std::future<void> flickfut;
437  if (envp.chan_f_weight > 0)
438  flickfut = std::async(std::launch::async, [&]() {
439  if (envp.multiscale_flicker)
440  env_chan_msflicker("flicker", &envp, &imath, bwimg.dims, &prev_lowpass5, &lowpass5,
441  statfunc, statdata, &flicker);
442  else
443  env_chan_flicker("flicker", &envp, &imath, &prev_input, &bwimg, statfunc, statdata, &flicker);
444 
445  combine_output(&flicker, envp.chan_f_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
446 
447  if (envp.multiscale_flicker) env_pyr_copy_src_dst(&lowpass5, &prev_lowpass5);
448  else env_pyr_make_empty(&prev_lowpass5);
449  });
450 
451  // Intensity is the fastest one and we here just run it in the current thread:
452  if (envp.chan_i_weight > 0)
453  {
454  env_chan_intensity("intensity", &envp, &imath, bwimg.dims, &lowpass5, 1, statfunc, statdata, &intens);
455  combine_output(&intens, envp.chan_i_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
456  }
457  itsProfiler.checkpoint("intens");
458 
459  // Wait for all channels to finish up:
460  if (rgfut.valid()) rgfut.get();
461  itsProfiler.checkpoint("red-green");
462 
463  if (byfut.valid())
464  {
465  byfut.get();
466 
467  // Finish up the color channel by combining rg and by:
468  const intg32 * const byptr = env_img_pixels(&byOut);
469  intg32 * const dptr = env_img_pixelsw(&color);
470  const env_size_t sz = env_img_size(&color);
471  for (env_size_t i = 0; i < sz; ++i) dptr[i] = (dptr[i] + byptr[i]) >> 1;
472 
474 
475  if (statfunc) (*statfunc)(statdata, "color", &color);
476  env_img_make_empty(&byOut);
477 
478  // Add color channel to the saliency map:
479  combine_output(&color, envp.chan_c_weight * (1<<WEIGHT_SCALEBITS) / total_weight, &salmap);
480  }
481  itsProfiler.checkpoint("blue-yellow");
482 
483  if (orifut.valid()) orifut.get();
484  itsProfiler.checkpoint("orientation");
485 
486  if (flickfut.valid()) flickfut.get();
487  itsProfiler.checkpoint("flicker");
488 
489  if (motfut.valid()) motfut.get();
490  itsProfiler.checkpoint("motion");
491 
492  // Cleanup and get ready for next frame:
493  if (!envp.multiscale_flicker) env_img_swap(&prev_input, &bwimg); else env_img_make_empty(&prev_input);
494 
495  if (statfunc) (*statfunc)(statdata, "saliency", &salmap);
496 
497  // We transfer our lowpass5 to the motion channel as unshifted prev:
498  env_pyr_swap(&lowpass5, &motion_chan.unshifted_prev);
499  env_pyr_make_empty(&lowpass5);
500 
501  env_img_make_empty(&bwimg);
502  env_img_make_empty(&rgimg);
503  env_img_make_empty(&byimg);
504  /*
505  env_visual_cortex_rescale_ranges(&salmap, &intens, &color, &ori, &flicker, &motion);
506  */
507  itsProfiler.stop();
508 }
509 
510 // ##############################################################################################################
511 void Saliency::env_mt_chan_orientation(const char* tagName, const struct env_image* img,
512  env_chan_status_func* status_func, void* status_userdata,
513  struct env_image* result)
514 {
515  env_img_make_empty(result);
516 
517  if (envp.num_orientations == 0) return;
518 
519  struct env_pyr hipass9;
520  env_pyr_init(&hipass9, env_max_pyr_depth(&envp));
521  env_pyr_build_hipass_9(img, envp.cs_lev_min, &imath, &hipass9);
522 
523  char buf[17] = {
524  's', 't', 'e', 'e', 'r', 'a', 'b', 'l', 'e', // 0--8
525  '(', '_', '_', // 9--11
526  '/', '_', '_', ')', '\0' // 12--16
527  };
528 
529  ENV_ASSERT(envp.num_orientations <= 99);
530 
531  buf[13] = '0' + (envp.num_orientations / 10);
532  buf[14] = '0' + (envp.num_orientations % 10);
533 
534  std::vector<std::future<void> > fut;
535  std::mutex mtx;
536  for (env_size_t i = 0; i < envp.num_orientations; ++i)
537  fut.push_back(std::async(std::launch::async, [&](env_size_t ii) {
538  struct env_image chanOut; env_img_init_empty(&chanOut);
539 
540  char tagname[17]; memcpy(tagname, buf, 17);
541  tagname[10] = '0' + ((ii+1) / 10);
542  tagname[11] = '0' + ((ii+1) % 10);
543 
544  // theta = (180.0 * i) / envp.num_orientations + 90.0, where ENV_TRIG_TABSIZ is equivalent to 360.0 or 2*pi
545  const env_size_t thetaidx = (ENV_TRIG_TABSIZ * ii) / (2 * envp.num_orientations) + (ENV_TRIG_TABSIZ / 4);
546  ENV_ASSERT(thetaidx < ENV_TRIG_TABSIZ);
547 
548  env_chan_steerable(tagname, &envp, &imath, img->dims, &hipass9, thetaidx,
549  status_func, status_userdata, &chanOut);
550 
551  // Access result image one thread at a time:
552  std::lock_guard<std::mutex> _(mtx);
553  if (!env_img_initialized(result))
554  {
555  env_img_resize_dims(result, chanOut.dims);
556  env_c_image_div_scalar(env_img_pixels(&chanOut), env_img_size(&chanOut), (intg32)envp.num_orientations,
557  env_img_pixelsw(result));
558  }
559  else
560  {
561  ENV_ASSERT(env_dims_equal(chanOut.dims, result->dims));
562  env_c_image_div_scalar_accum(env_img_pixels(&chanOut), env_img_size(&chanOut),
563  (intg32)envp.num_orientations, env_img_pixelsw(result));
564  }
565  env_img_make_empty(&chanOut);
566  }, i));
567 
568  // Wait for all the jobs to complete:
569  for (std::future<void> & f : fut) f.get();
570 
571  env_pyr_make_empty(&hipass9);
572 
573  if (env_img_initialized(result))
575 
576  if (status_func) (*status_func)(status_userdata, tagName, result);
577 }
578 
579 // ##############################################################################################################
580 void Saliency::env_mt_motion_channel_input(struct env_motion_channel* chan, const char* tagName,
581  const struct env_dims inputdims, struct env_pyr* unshiftedCur,
582  env_chan_status_func* status_func, void* status_userdata,
583  struct env_image* result)
584 {
585  env_img_make_empty(result);
586 
587  if (chan->num_directions != envp.num_motion_directions)
588  {
590  env_motion_channel_init(chan, &envp);
591  }
592 
593  if (chan->num_directions == 0) return;
594 
595  char buf[17] =
596  {
597  'r', 'e', 'i', 'c', 'h', 'a', 'r', 'd', 't', // 0--8
598  '(', '_', '_', // 9--11
599  '/', '_', '_', ')', '\0' // 12--16
600  };
601 
602  ENV_ASSERT(chan->num_directions <= 99);
603 
604  buf[13] = '0' + (chan->num_directions / 10);
605  buf[14] = '0' + (chan->num_directions % 10);
606 
607  // compute Reichardt motion detection into several directions
608  std::vector<std::future<void> > fut;
609  std::mutex mtx;
610  for (env_size_t dir = 0; dir < chan->num_directions; ++dir)
611  fut.push_back(std::async(std::launch::async, [&](env_size_t d) {
612  struct env_image chanOut; env_img_init_empty(&chanOut);
613 
614  char tagname[17]; memcpy(tagname, buf, 17);
615  tagname[10] = '0' + ((d+1) / 10);
616  tagname[11] = '0' + ((d+1) % 10);
617 
618  const env_size_t firstlevel = envp.cs_lev_min;
619  const env_size_t depth = env_max_pyr_depth(&envp);
620 
621  // theta = (360.0 * i) / chan->num_directions;
622  const env_size_t thetaidx = (d * ENV_TRIG_TABSIZ) / chan->num_directions;
623  ENV_ASSERT(thetaidx < ENV_TRIG_TABSIZ);
624 
625  // create an empty pyramid:
626  struct env_pyr shiftedCur; env_pyr_init(&shiftedCur, depth);
627 
628  // fill the empty pyramid with the shifted version
629  for (env_size_t i = firstlevel; i < depth; ++i)
630  {
631  env_img_resize_dims(env_pyr_imgw(&shiftedCur, i), env_pyr_img(unshiftedCur, i)->dims);
632  env_shift_image(env_pyr_img(unshiftedCur, i), imath.costab[thetaidx], -imath.sintab[thetaidx],
633  ENV_TRIG_NBITS, env_pyr_imgw(&shiftedCur, i));
634  }
635 
636  env_chan_direction(tagname, &envp, &imath, inputdims, &chan->unshifted_prev, unshiftedCur,
637  &chan->shifted_prev[d], &shiftedCur, status_func, status_userdata, &chanOut);
638 
639  env_pyr_swap(&chan->shifted_prev[d], &shiftedCur);
640  env_pyr_make_empty(&shiftedCur);
641 
642  // Access result image one thread at a time:
643  std::lock_guard<std::mutex> _(mtx);
644  if (env_img_initialized(&chanOut))
645  {
646  if (!env_img_initialized(result))
647  {
648  env_img_resize_dims(result, chanOut.dims);
649  env_c_image_div_scalar(env_img_pixels(&chanOut), env_img_size(&chanOut), (intg32)chan->num_directions,
650  env_img_pixelsw(result));
651  }
652  else
653  {
654  ENV_ASSERT(env_dims_equal(chanOut.dims, result->dims));
655  env_c_image_div_scalar_accum(env_img_pixels(&chanOut), env_img_size(&chanOut),
656  (intg32)chan->num_directions, env_img_pixelsw(result));
657  }
658  }
659  env_img_make_empty(&chanOut);
660  }, dir));
661 
662  // Wait for all the jobs to complete:
663  for (std::future<void> & f : fut) f.get();
664 
665  if (env_img_initialized(result))
667 
668  if (status_func) (*status_func)(status_userdata, tagName, result);
669 }
670 
671 // ##############################################################################################################
672 void Saliency::getSaliencyMax(int & x, int & y, intg32 & value)
673 {
674  if (env_img_initialized(&salmap) == false) LFATAL("Saliency map has not yet been computed");
675 
676  intg32 *sm = salmap.pixels; int const smw = int(salmap.dims.w), smh = int(salmap.dims.h);
677 
678  value = *sm;
679 
680  for (int j = 0; j < smh; ++j)
681  for (int i = 0; i < smw; ++i)
682  if (*sm > value) { value = *sm++; x = i; y = j; } else ++sm;
683 }
684 
685 // ##############################################################################################################
686 void Saliency::inhibitionOfReturn(int const x, int const y, float const sigma)
687 {
688  if (env_img_initialized(&salmap) == false) LFATAL("Saliency map has not yet been computed");
689 
690  intg32 *sm = salmap.pixels; int const smw = int(salmap.dims.w), smh = int(salmap.dims.h);
691  float const sigsq = sigma * sigma;
692 
693  for (int j = 0; j < smh; ++j)
694  for (int i = 0; i < smw; ++i)
695  {
696  float const distsq = (i-x)*(i-x) + (j-y)*(j-y);
697  if (distsq < sigsq)
698  *sm++ = 0; // hard kill in a disk up to sigma
699  else
700  {
701  float val = *sm;
702  val *= 1.0F - expf( -0.5F * (distsq - sigsq ) / sigsq); // smooth decay
703  *sm++ = static_cast<intg32>(val + 0.4999F);
704  }
705  }
706 }
707 
708 // ####################################################################################################
709 void drawMap(jevois::RawImage & img, env_image const * fmap, unsigned int xoff, unsigned int yoff,
710  unsigned int scale)
711 {
712  unsigned int const imgw = img.width;
713  unsigned short * d = img.pixelsw<unsigned short>() + xoff + yoff * imgw;
714  intg32 *s = fmap->pixels;
715  const env_size_t w = fmap->dims.w, h = fmap->dims.h;
716  const env_size_t ws = w * scale;
717 
718  for (env_size_t jj = 0; jj < h; ++jj)
719  {
720  unsigned short const * dd = d;
721 
722  // Copy and scale the first row one pixel at a time:
723  for (env_size_t ii = 0; ii < w; ++ii)
724  {
725  intg32 v = *s++;
726  unsigned short const val = 0x8000 | v;
727  for (env_size_t k = 0; k < scale; ++k) *d++ = val;
728  }
729  d += imgw - ws;
730 
731  // Then just use memcpy to duplicate it to achieve the scaling factor vertically:
732  for (env_size_t k = 1; k < scale; ++k) { memcpy(d, dd, ws * 2); d += imgw; }
733  }
734 
735  // Draw a rectangle to delinate the map:
736  jevois::rawimage::drawRect(img, xoff, yoff, scale * w, scale * h, 0x80a0);
737 }
738 
739 // ####################################################################################################
740 void drawMap(jevois::RawImage & img, env_image const * fmap, unsigned int xoff, unsigned int yoff,
741  unsigned int scale, unsigned int bitshift)
742 {
743  unsigned int const imgw = img.width;
744  unsigned short * d = img.pixelsw<unsigned short>() + xoff + yoff * imgw;
745  intg32 *s = fmap->pixels;
746  const env_size_t w = fmap->dims.w, h = fmap->dims.h;
747  const env_size_t ws = w * scale;
748 
749  for (env_size_t jj = 0; jj < h; ++jj)
750  {
751  unsigned short const * dd = d;
752 
753  // Copy and scale the first row one pixel at a time:
754  for (env_size_t ii = 0; ii < w; ++ii)
755  {
756  intg32 v = (*s++) >> bitshift; if (v > 255) v = 255;
757  unsigned short const val = 0x8000 | v;
758  for (env_size_t k = 0; k < scale; ++k) *d++ = val;
759  }
760  d += imgw - ws;
761 
762  // Then just use memcpy to duplicate it to achieve the scaling factor vertically:
763  for (env_size_t k = 1; k < scale; ++k) { memcpy(d, dd, ws * 2); d += imgw; }
764  }
765 
766  // Draw a rectangle to delinate the map:
767  jevois::rawimage::drawRect(img, xoff, yoff, scale * w, scale * h, 0x80a0);
768 }
769 
770 // ####################################################################################################
771 void drawGist(jevois::RawImage & img, unsigned char const * gist, size_t gistsize, unsigned int xoff,
772  unsigned int yoff, unsigned int width, unsigned int scale)
773 {
774  unsigned int const height = gistsize / width;
775  unsigned int const imgw = img.width;
776  unsigned short * d = img.pixelsw<unsigned short>() + xoff + yoff * imgw;
777  unsigned char const * const dataend = gist + gistsize;
778  unsigned int const ws = width * scale;
779 
780  for (env_size_t jj = 0; jj < height; ++jj)
781  {
782  unsigned short const * dd = d;
783 
784  // Copy and scale the first row one pixel at a time:
785  for (env_size_t ii = 0; ii < width; ++ii)
786  {
787  intg32 v = gist >= dataend ? 0 : *gist++;
788  unsigned short const val = 0x8000 | v;
789  for (env_size_t k = 0; k < scale; ++k) *d++ = val;
790  }
791  d += imgw - ws;
792 
793  // Then just use memcpy to duplicate it to achieve the scaling factor vertically:
794  for (env_size_t k = 1; k < scale; ++k) { memcpy(d, dd, ws * 2); d += imgw; }
795  }
796 
797  // Draw a rectangle to delinate the map:
798  //jevois::rawimage::drawRect(img, xoff, yoff, scale * width, scale * height, 0x80a0);
799 }
800 
void process(jevois::RawImage const &input, bool do_gist)
Process a raw YUYV image. Results are stored in the Saliency class.
Definition: Saliency.C:339
byte chan_i_weight
Definition: env_params.h:59
int(* submapPreProc)(const char *tagName, env_size_t clev, env_size_t slev, struct env_image *submap, const struct env_image *center, const struct env_image *surround, void *user_data)
Definition: env_params.h:65
#define SALUPDATE(envval, param)
Definition: Saliency.C:148
#define INTMAXNORMMIN
Definition: env_image_ops.h:47
env_size_t w
The width.
Definition: env_types.h:82
void() env_chan_status_func(void *userdata, const char *tagName, const struct env_image *img)
Definition: env_channel.h:55
env_size_t num_motion_directions
Definition: env_params.h:49
size_t const gist_size
Definition: Saliency.H:152
byte chan_m_weight
Definition: env_params.h:63
struct env_pyr * shifted_prev
void * user_data_preproc
Definition: env_params.h:80
unsigned int height
env_size_t cs_lev_min
Definition: env_params.h:54
struct env_image motion
Definition: Saliency.H:130
std::string value
void env_c_luminance_from_byte(const struct env_rgb_pixel *const src, const env_size_t sz, const env_size_t nbits, intg32 *const dst)
get the luminance with nbits of precision of the input image
void env_pyr_build_hipass_9(const struct env_image *image, env_size_t firstlevel, const struct env_math *imath, struct env_pyr *result)
struct env_image ori
Definition: Saliency.H:128
void env_init_integer_math(struct env_math *imath, const struct env_params *envp)
Definition: env_math.c:44
void env_motion_channel_destroy(struct env_motion_channel *chan)
void env_img_make_empty(struct env_image *img)
Definition: env_image.c:56
void env_img_swap(struct env_image *img1, struct env_image *img2)
Definition: env_image.c:48
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Draw a saliency map or feature map in a YUYV image.
Definition: Saliency.C:709
byte flicker_thresh
Definition: env_params.h:51
env_size_t cs_del_min
Definition: env_params.h:56
void getSaliencyMax(int &x, int &y, intg32 &value)
Get location and value of max point in the saliency map.
Definition: Saliency.C:672
Basic image class.
Definition: env_image.h:43
intg32 env_total_weight(const struct env_params *envp)
Definition: env_params.c:85
void env_motion_channel_init(struct env_motion_channel *chan, const struct env_params *envp)
A composite channel containing a set of direction channels.
byte chan_c_weight
Definition: env_params.h:60
#define env_img_initializer
Definition: env_image.h:49
void env_max_normalize_inplace(struct env_image *src, const intg32 mi, const intg32 ma, const enum env_maxnorm_type normtyp, const intg32 rangeThresh)
struct env_image flicker
Definition: Saliency.H:129
env_size_t cs_lev_max
Definition: env_params.h:55
void env_pyr_init(struct env_pyr *pyr, const env_size_t n)
Construct with a given number of empty images.
Definition: env_pyr.c:41
void checkpoint(char const *description)
void env_c_image_div_scalar_accum(const intg32 *const a, const env_size_t sz, intg32 val, intg32 *const dst)
result += a / val
void env_img_init(struct env_image *img, const struct env_dims d)
Definition: env_image.c:41
void env_pyr_copy_src_dst(const struct env_pyr *src, struct env_pyr *dst)
Definition: env_pyr.c:68
void env_chan_msflicker(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputDims, const struct env_pyr *prev_lowpass5, const struct env_pyr *cur_lowpass5, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A true multi-scale temporal flicker channel.
Definition: env_channel.c:442
Saliency(std::string const &instance)
Constructor.
Definition: Saliency.C:81
struct env_pyr unshifted_prev
void env_params_set_defaults(struct env_params *envp)
Definition: env_params.c:43
struct env_dims dims
Definition: env_image.h:45
#define WEIGHT_SCALEBITS
Definition: Saliency.C:37
void waitUntilDoneWithInput() const
Wait until process() is done using the input image.
Definition: Saliency.C:217
struct env_image color
Definition: Saliency.H:127
env_size_t h
The height.
Definition: env_types.h:83
void env_chan_color(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_rgb_pixel *const colimg, const struct env_dims dims, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A double opponent color channel that combines r/g, b/y subchannels.
Definition: env_channel.c:218
#define INTMAXNORMMAX
Definition: env_image_ops.h:48
unsigned long env_size_t
Definition: env_types.h:71
void env_params_validate(const struct env_params *envp)
Definition: env_params.c:97
void env_pyr_make_empty(struct env_pyr *dst)
Definition: env_pyr.c:51
struct env_image intens
Definition: Saliency.H:126
void env_chan_flicker(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_image *prev, const struct env_image *cur, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A temporal flicker channel.
Definition: env_channel.c:404
virtual ~Saliency()
Destructor.
Definition: Saliency.C:106
RGB pixel class.
Definition: env_types.h:74
byte chan_o_weight
Definition: env_params.h:61
void env_grid_average(const struct env_image *src, unsigned char *dest, unsigned int bitshift, env_size_t nx, env_size_t ny)
Compute average values in each tile of a grid.
intg32 * pixels
Definition: env_image.h:46
env_size_t env_max_pyr_depth(const struct env_params *envp)
Definition: env_params.c:79
env_size_t cs_del_max
Definition: env_params.h:57
void env_img_resize_dims(struct env_image *img, const struct env_dims d)
Definition: env_image.c:64
void env_chan_intensity(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputdims, const struct env_pyr *lowpass5, const int normalizeOutput, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
An intensity channel.
Definition: env_channel.c:208
#define LFATAL(msg)
ENV_INTG32_TYPE intg32
32-bit signed integer
Definition: env_types.h:52
void convertYUYVtoRGBYL(unsigned int w, unsigned int h, unsigned char const *src, int *dstrg, int *dstby, int *dstlum, int thresh, int inputbits)
struct env_image salmap
The saliency map.
Definition: Saliency.H:118
enum env_maxnorm_type maxnorm_type
Definition: env_params.h:46
unsigned char * gist
Definition: Saliency.H:155
env_size_t num_orientations
number of Gabor subchannels
Definition: env_params.h:53
byte multiscale_flicker
Definition: env_params.h:52
A simple struct to hold a pair of width/height dimensions.
Definition: env_types.h:80
void env_chan_direction(const char *tagName, const struct env_params *envp, const struct env_math *imath, const struct env_dims inputdims, const struct env_pyr *unshiftedPrev, const struct env_pyr *unshiftedCur, const struct env_pyr *shiftedPrev, const struct env_pyr *shiftedCur, env_chan_status_func *status_func, void *status_userdata, struct env_image *result)
A motion sensitive channel with direction selectivity.
Definition: env_channel.c:472
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
Draw a gist vector in a YUYV image as a rectangle of width width*scale and correct height...
Definition: Saliency.C:771
This class implements a set of images, often used as a dyadic pyramid.
Definition: env_pyr.h:45
void env_pyr_build_lowpass_5(const struct env_image *image, env_size_t firstlevel, const struct env_math *imath, struct env_pyr *result)
Wrapper for _cpu or _cuda version.
T const * pixels() const
intg32 range_thresh
Definition: env_params.h:47
unsigned char * gist
Gist vector has 1152 entries, 72 feature maps * 16 values/map.
Definition: Saliency.H:151
void env_pyr_swap(struct env_pyr *pyr1, struct env_pyr *pyr2)
Swap contents with another env_pyr.
Definition: env_pyr.c:60
byte motion_thresh
Definition: env_params.h:50
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
env_size_t nbits
Definition: env_math.h:49
void env_c_image_div_scalar(const intg32 *const a, const env_size_t sz, intg32 val, intg32 *const dst)
result = a / val
void inhibitionOfReturn(int const x, int const y, float const sigma)
Inhibit the saliency map around a point, sigma is in pixels at the sacle of the map.
Definition: Saliency.C:686
unsigned int width
#define ENV_ASSERT(expr)
Definition: env_log.h:63
byte chan_f_weight
Definition: env_params.h:62
env_params * envp
Definition: Saliency.H:155