env_channel.c

Go to the documentation of this file.

/*!@file Envision/env_channel.c Base class for channels that will use integer math */
 
// //////////////////////////////////////////////////////////////////// //
// The iLab Neuromorphic Vision C++ Toolkit - Copyright (C) 2000-2005   //
// by the University of Southern California (USC) and the iLab at USC.  //
// See http://iLab.usc.edu for information about this project.          //
// //////////////////////////////////////////////////////////////////// //
// Major portions of the iLab Neuromorphic Vision Toolkit are protected //
// under the U.S. patent ``Computation of Intrinsic Perceptual Saliency //
// in Visual Environments, and Applications'' by Christof Koch and      //
// Laurent Itti, California Institute of Technology, 2001 (patent       //
// pending; application number 09/912,225 filed July 23, 2001; see      //
// http://pair.uspto.gov/cgi-bin/final/home.pl for current status).     //
// //////////////////////////////////////////////////////////////////// //
// This file is part of the iLab Neuromorphic Vision C++ Toolkit.       //
//                                                                      //
// The iLab Neuromorphic Vision C++ Toolkit is free software; you can   //
// redistribute it and/or modify it under the terms of the GNU General  //
// Public License as published by the Free Software Foundation; either  //
// version 2 of the License, or (at your option) any later version.     //
//                                                                      //
// The iLab Neuromorphic Vision C++ Toolkit is distributed in the hope  //
// that it will be useful, but WITHOUT ANY WARRANTY; without even the   //
// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR      //
// PURPOSE.  See the GNU General Public License for more details.       //
//                                                                      //
// You should have received a copy of the GNU General Public License    //
// along with the iLab Neuromorphic Vision C++ Toolkit; if not, write   //
// to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,   //
// Boston, MA 02111-1307 USA.                                           //
// //////////////////////////////////////////////////////////////////// //
//
// Primary maintainer for this file: Rob Peters <rjpeters at usc dot edu>
// $HeadURL: svn://isvn.usc.edu/software/invt/trunk/saliency/src/Envision/env_channel.c $
// $Id: env_channel.c 9830 2008-06-18 18:50:22Z lior $
//
 
#include <jevoisbase/Components/Saliency/env_channel.h>
 
#include <jevoisbase/Components/Saliency/env_c_math_ops.h>
#include <jevoisbase/Components/Saliency/env_image_ops.h>
#include <jevoisbase/Components/Saliency/env_log.h>
#include <jevoisbase/Components/Saliency/env_params.h>
 
#ifndef ENV_NO_DEBUG
 
//! Check whether the pyramid is dyadic.
/*! A dyadic pyramid is one in which each level is one half the width and one half the height of the preceding level. */
static int is_dyadic(const struct env_pyr* pyr, const env_size_t first, const env_size_t last)
{
  if (first == last) return 0;
  
  for (env_size_t i = first + 1; i < last; ++i)
  {
    const struct env_dims prevdims = env_pyr_img(pyr, i-1)->dims;
    const struct env_dims curdims = env_pyr_img(pyr, i)->dims;
    
    // make sure we don't go below 1
    const env_size_t pw2 = ENV_MAX(prevdims.w/2,((env_size_t) 1));
    const env_size_t ph2 = ENV_MAX(prevdims.h/2,((env_size_t) 1));
    
    if (curdims.w != pw2) return 0;
    if (curdims.h != ph2) return 0;
  }
  
  return 1;
}
 
#endif
 
// ######################################################################
static void abs_diff_thresh(const struct env_image* b, const struct env_image* c,
                            const intg32 thresh, struct env_image* result)
{
  ENV_ASSERT(env_dims_equal(b->dims, c->dims));
  ENV_ASSERT(env_dims_equal(b->dims, result->dims));
  
  const intg32* const bptr = env_img_pixels(b);
  const intg32* const cptr = env_img_pixels(c);
  intg32* const dptr = env_img_pixelsw(result);
  
  const env_size_t sz = env_img_size(b);
  
  for (env_size_t i = 0; i < sz; ++i)
  {
    dptr[i] = (bptr[i] < cptr[i]) ? cptr[i] - bptr[i] : bptr[i] - cptr[i];
    if (dptr[i] < thresh) dptr[i] = 0;
  }
}
 
// ######################################################################
static void abs_diff_thresh_pyr(const struct env_pyr* b, const struct env_pyr* c,
                                const intg32 thresh, struct env_pyr* result)
{
  ENV_ASSERT(env_pyr_depth(b) == env_pyr_depth(c));
  ENV_ASSERT(env_pyr_depth(c) == env_pyr_depth(result));
  
  const env_size_t n = env_pyr_depth(b);
  
  if (env_pyr_depth(result) != n)
  {
    env_pyr_make_empty(result);
    env_pyr_init(result, n);
  }
  
  for (env_size_t i = 0; i < n; ++i)
  {
    const struct env_image* bimg = env_pyr_img(b, i);
    
    if (!env_img_initialized(bimg)) continue;
    
    // else...
    
    const struct env_image* cimg = env_pyr_img(c, i);
    struct env_image* rimg = env_pyr_imgw(result, i);
    
    env_img_resize_dims(rimg, bimg->dims);
    
    abs_diff_thresh(bimg, cimg, thresh, rimg);
  }
}
 
// ######################################################################
void env_chan_process_pyr(const char* tagName, const struct env_dims inputDims, const struct env_pyr* pyr,
                          const struct env_params* envp, const struct env_math* imath, const int takeAbs,
                          const int normalizeOutput, struct env_image* result)
{
  const struct env_dims mapDims =
    { ENV_MAX(inputDims.w / (1 << envp->output_map_level), 1),
      ENV_MAX(inputDims.h / (1 << envp->output_map_level), 1) };
  
  if (env_pyr_depth(pyr) == 0)
    // OK, our pyramid wasn't ready to give us any output yet, so just return an empty output image:
  {
    env_img_make_empty(result);
    return;
  }
 
  // We only want dyadic pyramids here:
  ENV_ASSERT(is_dyadic(pyr, envp->cs_lev_min, env_max_pyr_depth(envp)));
  
  env_img_resize_dims(result, mapDims);
  
  {
    const env_size_t mapSize = mapDims.w * mapDims.h;
    intg32* const rptr = env_img_pixelsw(result);
    for (env_size_t i = 0; i < mapSize; ++i)
      rptr[i] = 0;
  }
  
  // compute max-normalized weighted sum of center-surround at all levels:
  for (env_size_t clev = envp->cs_lev_min; clev <= envp->cs_lev_max; ++clev)
    for (env_size_t delta = envp->cs_del_min; delta <= envp->cs_del_max; ++delta)
    {
      const env_size_t slev = clev + delta;
      
      // submap is computed from a center-surround difference:
      struct env_image submap;
      env_img_init(&submap, env_pyr_img(pyr, clev)->dims);
      env_center_surround(env_pyr_img(pyr, clev), env_pyr_img(pyr, slev), takeAbs, &submap);
 
      if (envp->submapPreProc != 0)
        (*envp->submapPreProc)(tagName, clev, slev, &submap, env_pyr_img(pyr, clev), env_pyr_img(pyr, slev),
                               envp->user_data_preproc );
      
      // resize submap to fixed scale if necessary:
      if (submap.dims.w > mapDims.w || submap.dims.h > mapDims.h)
      {
        // how many levels to we need to downscale the current submap to get to the output map resolution?
        const env_size_t n = envp->output_map_level - clev;
        
        env_downsize_9_inplace(&submap, n, imath);
      }
      else if (submap.dims.w < mapDims.w || submap.dims.h < mapDims.h)
      {
        struct env_image tmp;
        env_img_init(&tmp, mapDims);
        env_rescale(&submap, &tmp);
        env_img_swap(&submap, &tmp);
      }
      
      // make sure that the resizing came out precisely:
      ENV_ASSERT(env_dims_equal(submap.dims, mapDims));
      
      // first normalize the submap to a fixed dynamic range and then apply spatial competition for salience to the
      // submap:
      env_max_normalize_inplace(&submap, INTMAXNORMMIN, INTMAXNORMMAX, envp->maxnorm_type, envp->range_thresh);
      
      if (envp->submapPostNormProc != 0)
        (*envp->submapPostNormProc)(tagName, clev, slev, &submap, env_pyr_img(pyr, clev), env_pyr_img(pyr, slev),
                                    envp->user_data_postnorm);
      
      // add submap to our sum
      env_c_image_div_scalar_accum(env_img_pixels(&submap), env_img_size(&submap), (intg32) env_max_cs_index(envp),
                                   env_img_pixelsw(result));
      
      env_img_make_empty(&submap);
    }
  
  if (envp->submapPostProc != 0) (*envp->submapPostProc)(tagName, result, envp->user_data_postproc);
  
  // apply max-normalization on the result as needed:
  if (normalizeOutput)
    env_max_normalize_inplace(result, INTMAXNORMMIN, INTMAXNORMMAX, envp->maxnorm_type, envp->range_thresh);
}
 
// ######################################################################
void env_chan_intensity(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                        const struct env_dims inputdims, const struct env_pyr* lowpass5, const int normalizeOutput,
                        env_chan_status_func* status_func, void* status_userdata, struct env_image* result)
{
  env_chan_process_pyr(tagName, inputdims, lowpass5, envp, imath, 1 /* takeAbs */, normalizeOutput, result);
 
  if (status_func) (*status_func)(status_userdata, tagName, result);
}
 
// ######################################################################
void env_chan_color(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                    const struct env_rgb_pixel* const colimg,
                    const struct env_dims dims, env_chan_status_func* status_func, void* status_userdata,
                    struct env_image* result)
{
  struct env_image rg; env_img_init(&rg, dims);
  struct env_image by; env_img_init(&by, dims);
  
  const intg32 lumthresh = (3*255) / 10;
  env_get_rgby(colimg, dims.w * dims.h, &rg, &by, lumthresh, imath->nbits);
  
  const env_size_t firstlevel = envp->cs_lev_min;
  const env_size_t depth = env_max_pyr_depth(envp);
  
  {
    struct env_pyr rgpyr;
    env_pyr_init(&rgpyr, depth);
    env_pyr_build_lowpass_5(&rg, firstlevel, imath, &rgpyr);
    
    env_chan_intensity("red/green", envp, imath, rg.dims, &rgpyr, 0, status_func, status_userdata, result);
 
    env_pyr_make_empty(&rgpyr);
  }
 
  struct env_image byOut = env_img_initializer;
  
  {
    struct env_pyr bypyr;
    env_pyr_init(&bypyr, depth);
    env_pyr_build_lowpass_5(&by, firstlevel, imath, &bypyr);
    
    env_chan_intensity("blue/yellow", envp, imath, by.dims, &bypyr, 0, status_func, status_userdata, &byOut);
    env_pyr_make_empty(&bypyr);
  }
 
  env_img_make_empty(&rg);
  env_img_make_empty(&by);
  
  const intg32* const byptr = env_img_pixels(&byOut);
  intg32* const dptr = env_img_pixelsw(result);
  const env_size_t sz = env_img_size(result);
  
  for (env_size_t i = 0; i < sz; ++i) dptr[i] = (dptr[i] / 2) + (byptr[i] / 2);
  
  env_max_normalize_inplace(result, INTMAXNORMMIN, INTMAXNORMMAX, envp->maxnorm_type, envp->range_thresh);
  if (status_func) (*status_func)(status_userdata, tagName, result);
 
  env_img_make_empty(&byOut);
}
 
// ######################################################################
void env_chan_color_rgby(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                         const struct env_image *rg, const struct env_image *by,
                         env_chan_status_func* status_func, void* status_userdata,
                         struct env_image* result)
{
  ENV_ASSERT(env_dims_equal(rg->dims, by->dims));
  
  const env_size_t firstlevel = envp->cs_lev_min;
  const env_size_t depth = env_max_pyr_depth(envp);
  
  {
    struct env_pyr rgpyr;
    env_pyr_init(&rgpyr, depth);
    env_pyr_build_lowpass_5(rg, firstlevel, imath, &rgpyr);
    
    env_chan_intensity("red/green", envp, imath, rg->dims, &rgpyr, 0, status_func, status_userdata, result);
 
    env_pyr_make_empty(&rgpyr);
  }
 
  struct env_image byOut = env_img_initializer;
  
  {
    struct env_pyr bypyr;
    env_pyr_init(&bypyr, depth);
    env_pyr_build_lowpass_5(by, firstlevel, imath, &bypyr);
    
    env_chan_intensity("blue/yellow", envp, imath, by->dims, &bypyr, 0, status_func, status_userdata, &byOut);
    env_pyr_make_empty(&bypyr);
  }
 
  const intg32* const byptr = env_img_pixels(&byOut);
  intg32* const dptr = env_img_pixelsw(result);
  const env_size_t sz = env_img_size(result);
  
  for (env_size_t i = 0; i < sz; ++i) dptr[i] = (dptr[i] + byptr[i]) >> 1;
  
  env_max_normalize_inplace(result, INTMAXNORMMIN, INTMAXNORMMAX, envp->maxnorm_type, envp->range_thresh);
  if (status_func) (*status_func)(status_userdata, tagName, result);
 
  env_img_make_empty(&byOut);
}
 
// ######################################################################
void env_chan_steerable(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                        const struct env_dims inputdims, const struct env_pyr* hipass9, const env_size_t thetaidx,
                        env_chan_status_func* status_func, void* status_userdata, struct env_image* result)
{
  const env_size_t kdenombits = ENV_TRIG_NBITS;
  
  // spatial_freq = 2.6 / (2*pi) ~= 0.41380285203892792 ~= 2069/5000
  
  const intg32 sfnumer = 2069;
  const intg32 sfdenom = 5000;
  
  const intg32 kxnumer = ((intg32) (sfnumer * imath->costab[thetaidx] * ENV_TRIG_TABSIZ)) / sfdenom;
  const intg32 kynumer = ((intg32) (sfnumer * imath->sintab[thetaidx] * ENV_TRIG_TABSIZ)) / sfdenom;
  
  // Compute our pyramid:
  struct env_pyr pyr = env_pyr_initializer;
  env_pyr_build_steerable_from_hipass_9(hipass9, kxnumer, kynumer, kdenombits, imath, &pyr);
  
  env_chan_process_pyr(tagName, inputdims, &pyr, envp, imath, 0 /* takeAbs */, 1 /* normalizeOutput */, result);
 
  if (status_func) (*status_func)(status_userdata, tagName, result);
 
  env_pyr_make_empty(&pyr);
}
 
// ######################################################################
void env_chan_orientation(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                          const struct env_image* img, env_chan_status_func* status_func,
                          void* status_userdata, struct env_image* result)
{
  env_img_make_empty(result);
  
  if (envp->num_orientations == 0) return;
  
  struct env_pyr hipass9;
  env_pyr_init(&hipass9, env_max_pyr_depth(envp));
  env_pyr_build_hipass_9(img, envp->cs_lev_min, imath, &hipass9);
  
  struct env_image chanOut = env_img_initializer;
  
  char buf[17] =
    {
      's', 't', 'e', 'e', 'r', 'a', 'b', 'l', 'e', // 0--8
      '(', '_', '_', // 9--11
      '/', '_', '_', ')', '\0' // 12--16
    };
  
  ENV_ASSERT(envp->num_orientations <= 99);
  
  buf[13] = '0' + (envp->num_orientations / 10);
  buf[14] = '0' + (envp->num_orientations % 10);
  
  for (env_size_t i = 0; i < envp->num_orientations; ++i)
  {
    // theta = (180.0 * i) / envp->num_orientations + 90.0, where ENV_TRIG_TABSIZ is equivalent to 360.0 or 2*pi
    const env_size_t thetaidx = (ENV_TRIG_TABSIZ * i) / (2 * envp->num_orientations) + (ENV_TRIG_TABSIZ / 4);
 
    ENV_ASSERT(thetaidx < ENV_TRIG_TABSIZ);
    
    buf[10] = '0' + ((i+1) / 10);
    buf[11] = '0' + ((i+1) % 10);
    
    env_chan_steerable(buf, envp, imath, img->dims, &hipass9, thetaidx, status_func, status_userdata, &chanOut);
    
    ENV_ASSERT(env_img_initialized(&chanOut));
    
    if (!env_img_initialized(result))
    {
      env_img_resize_dims(result, chanOut.dims);
      env_c_image_div_scalar(env_img_pixels(&chanOut), env_img_size(&chanOut), (intg32) envp->num_orientations,
                             env_img_pixelsw(result));
    }
    else
    {
      ENV_ASSERT(env_dims_equal(chanOut.dims, result->dims));
      env_c_image_div_scalar_accum(env_img_pixels(&chanOut), env_img_size(&chanOut),
                                   (intg32) envp->num_orientations, env_img_pixelsw(result));
    }
  }
  
  env_img_make_empty(&chanOut);
  env_pyr_make_empty(&hipass9);
  
  ENV_ASSERT(env_img_initialized(result));
  
  env_max_normalize_inplace(result, INTMAXNORMMIN, INTMAXNORMMAX, envp->maxnorm_type, envp->range_thresh);
  
  if (status_func) (*status_func)(status_userdata, tagName, result);
}
 
// ######################################################################
void env_chan_flicker(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                      const struct env_image* prev, const struct env_image* cur, env_chan_status_func* status_func,
                      void* status_userdata, struct env_image* result)
{
  // If this is the first time the flicker channel has seen input, then prev will be uninitialized; obviously we can't
  // compute any flicker with only one frame, so we just store the current input as the next iteration's previous input
  if (!env_img_initialized(prev))
  {
    env_img_make_empty(result);
  }
  else
  {
    const intg32 lowthresh = (envp->scale_bits > 8) ? (envp->flicker_thresh << (envp->scale_bits - 8))
      : (envp->flicker_thresh >> (8 - envp->scale_bits));
    
    // take thresholded abs difference between current and previous frame:
    struct env_image fli;
    env_img_init(&fli, prev->dims);
    abs_diff_thresh(cur, prev, lowthresh, &fli);
    
    const env_size_t firstlevel = envp->cs_lev_min;
    const env_size_t depth = env_max_pyr_depth(envp);
    
    // Compute our pyramid:
    struct env_pyr pyr;
    env_pyr_init(&pyr, depth);
    env_pyr_build_lowpass_5(&fli, firstlevel, imath, &pyr);
    
    env_chan_process_pyr(tagName, fli.dims, &pyr, envp, imath, 1 /* takeAbs */, 1 /* normalizeOutput */, result);
    
    if (status_func) (*status_func)(status_userdata, tagName, result);
    
    env_img_make_empty(&fli);
    env_pyr_make_empty(&pyr);
  }
}
 
// ######################################################################
void env_chan_msflicker(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                        const struct env_dims inputDims, const struct env_pyr* prev_lowpass5,
                        const struct env_pyr* cur_lowpass5, env_chan_status_func* status_func,
                        void* status_userdata, struct env_image* result)
{
  // If this is the first time the flicker channel has seen input, then prev will be uninitialized; obviously we can't
  // compute any flicker with only one frame, so we just store the current input as the next iteration's previous input
  if (env_pyr_depth(prev_lowpass5) == 0)
  {
    env_img_make_empty(result);
  }
  else
  {
    const intg32 lowthresh = (envp->scale_bits > 8) ? (envp->flicker_thresh << (envp->scale_bits - 8))
      : (envp->flicker_thresh >> (8 - envp->scale_bits));
    
    // take thresholded abs difference between current and previous frame:
    struct env_pyr fli;
    env_pyr_init(&fli, env_pyr_depth(cur_lowpass5));
    abs_diff_thresh_pyr(cur_lowpass5, prev_lowpass5, lowthresh, &fli);
    
    env_chan_process_pyr(tagName, inputDims, &fli, envp, imath, 1 /* takeAbs */, 1 /* normalizeOutput */, result);
    
    if (status_func) (*status_func)(status_userdata, tagName, result);
    
    env_pyr_make_empty(&fli);
  }
}
 
// ######################################################################
void env_chan_direction(const char* tagName, const struct env_params* envp, const struct env_math* imath,
                        const struct env_dims inputdims, const struct env_pyr* unshiftedPrev,
                        const struct env_pyr* unshiftedCur, const struct env_pyr* shiftedPrev,
                        const struct env_pyr* shiftedCur, env_chan_status_func* status_func,
                        void* status_userdata, struct env_image* result)
{
  const env_size_t firstlevel = envp->cs_lev_min;
  const env_size_t depth = env_max_pyr_depth(envp);
  
  const env_size_t nshift = (imath->nbits+1)/2;
  
  if (env_pyr_depth(unshiftedPrev) == 0)
  {
    // it's our first time, so just return an empty image:
    env_img_make_empty(result);
  }
  else
  {
    struct env_pyr pyr;
    env_pyr_init(&pyr, depth);
    
    const intg32 lowthresh = (envp->scale_bits > 8) ? (envp->motion_thresh << (envp->scale_bits - 8))
      : (envp->motion_thresh >> (8 - envp->scale_bits));
    
    // compute the Reichardt maps
    for (env_size_t i = firstlevel; i < depth; i++)
    {
      env_img_resize_dims(env_pyr_imgw(&pyr, i), env_pyr_img(unshiftedCur, i)->dims);
      
      const intg32* const ucurr = env_img_pixels(env_pyr_img(unshiftedCur, i));
      const intg32* const uprev = env_img_pixels(env_pyr_img(unshiftedPrev, i));
      const intg32* const scurr = env_img_pixels(env_pyr_img(shiftedCur, i));
      const intg32* const sprev = env_img_pixels(env_pyr_img(shiftedPrev, i));
      intg32* const dptr = env_img_pixelsw(env_pyr_imgw(&pyr, i));
      
      const env_size_t sz = env_img_size(env_pyr_img(&pyr, i));
      
      for (env_size_t c = 0; c < sz; ++c)
      {
        dptr[c] = ((ucurr[c] >> nshift) * (sprev[c] >> nshift)) - ((uprev[c] >> nshift) * (scurr[c] >> nshift));
        
        if (dptr[c] < lowthresh) dptr[c] = 0;
      }
    }
    
    env_chan_process_pyr(tagName, inputdims, &pyr, envp, imath, 1 /* takeAbs */, 1 /* normalizeOutput */, result);
    
    if (status_func) (*status_func)(status_userdata, tagName, result);
    
    env_pyr_make_empty(&pyr);
  }
}