JeVoisIntro.C

Go to the documentation of this file.

// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//
// JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
// California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
//
// This file is part of the JeVois Smart Embedded Machine Vision Toolkit.  This program is free software; you can
// redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
// Foundation, version 2.  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
// License for more details.  You should have received a copy of the GNU General Public License along with this program;
// if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
//
// Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
// Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*! \file */

#include <jevois/Core/Module.H>

#include <jevois/Debug/Log.H>
#include <jevois/Debug/Timer.H>
#include <jevois/Image/RawImageOps.H>
#include <jevoisbase/Components/Saliency/Saliency.H>
#include <jevoisbase/Components/FaceDetection/FaceDetector.H>
#include <jevoisbase/Components/ObjectRecognition/ObjectRecognitionMNIST.H>
#include <jevoisbase/Components/Tracking/Kalman2D.H>
#include <jevoisbase/Components/Utilities/BufferedVideoReader.H>

#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <linux/videodev2.h> // for v4l2 pixel types
//#include <opencv2/highgui/highgui.hpp> // used for debugging only, see imshow below

// icon by Freepik in interface at flaticon

struct ScriptItem { char const * msg; int blinkx, blinky; };
static ScriptItem const TheScript[] = {
  { "Hello! Welcome to this simple demonstration of JeVois", 0, 0 },
  { "JeVois = camera sensor + quad-core processor + USB video output", 0, 0 },
  { "This demo is running on the small processor inside JeVois", 0, 0 },
  { "Neat, isn't it?", 0, 0 },
  { "", 0, 0 },
  { "We will help you discover what you see on this screen", 0, 0 },
  { "We will use this blinking marker to point at things:", 600, 335 },
  { "", 0, 0 },
  { "Now a brief tutorial...", 0, 0 },
  { "", 0, 0 },
  { "This demo: Attention + Gist + Faces + Objects", 0, 0 },
  { "Attention: detect things that catch the human eye", 0, 0 },
  { "Pink square in video above: most interesting (salient) location", 0, 0 },
  { "Green circle in video above: smoothed attention trajectory", 0, 0 },
  { "Try it: wave at JeVois, show it some objects, move it around", 0, 0 },
  { "", 0, 0 },
  { "Did you catch the attention of JeVois?", 0, 0 },
  { "", 0, 0 },
  { "Attention is guided by color contrast, ...", 40, 270 },
  { "by luminance (intensity) contrast, ...", 120, 270 },
  { "by oriented edges, ...", 200, 270 },
  { "by flickering or blinking lights, ...", 280, 270 },
  { "and by moving objects.", 360, 270 },
  { "All these visual cues combine into a measure of saliency", 480, 120 },
  { "or visual interest for every location in view.", 480, 120 },
  { "", 0, 0 },
  { "", 0, 0 },
  { "Gist: statistical summary of a scene, also based on ...", 440, 270 },
  { "color, intensity, orientation, flicker and motion features.", 0, 0 },
  { "Gist can be used to recognize places, such as a kitchen or ...", 0, 0 },
  { "a bathroom, or a road turning left versus turning right.", 0, 0 },
  { "Try it: point JeVois to different things and see gist change", 440, 270 },
  { "", 0, 0 },
  { "", 0, 0 },
  { "Face detection finds human faces in the camera's view", 612, 316 },
  { "Try it: point JeVois towards a face. Adjust distance until ...", 612, 316 },
  { "the face fits inside the attention pink square. When a face ...", 0, 0 },
  { "is detected, it will appear in the bottom-right corner.", 612, 316 },
  { "", 0, 0 },
  { "", 0, 0 },
  { "Objects: Here we recognize handwritten digits using ...", 525, 316 },
  { "deep neural networks. Try it! Draw a number on paper ...", 0, 0 },
  { "and point JeVois towards it. Adjust distance until the", 0, 0 },
  { "number fits in the attention pink square.", 0, 0 },
  { "Recognized digits are shown near the detected faces.", 525, 316 },
  { "", 0, 0 },
  { "Recognition scores for digits 0 to 9 are shown above", 464, 310 },
  { "Sometimes the neural network makes mistakes and thinks it ...", 0, 0 },
  { "found a digit when actually it is looking at something else.", 0, 0 },
  { "This is still a research issue", 0, 0 },
  { "but machine vision is improving fast, so stay tuned!", 0, 0 },
  { "", 0, 0 },
  { "With JeVois the future of machine vision is in your hands.", 0, 0 },
  { "", 0, 0 },
  { "", 0, 0 },
  { "", 0, 0 },
  { "This tutorial is now complete. It will restart.", 0, 0 },
  { "", 0, 0 },
  { nullptr, 0, 0 }
};

//! Simple introduction to JeVois and demo that combines saliency, gist, face detection, and object recognition
/*! This module plays an introduction movie, and then launches the equivalent of DemoSalGistFaceObj, but with some added
    text messages that explain what is going on on the screen.

    @author Laurent Itti

    @displayname JeVois Intro
    @videomapping YUYV 640 360 50.0 YUYV 320 240 50.0 JeVois JeVoisIntro
    @videomapping YUYV 640 480 50.0 YUYV 320 240 50.0 JeVois JeVoisIntro
    @email itti\@usc.edu
    @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
    @copyright Copyright (C) 2016 by Laurent Itti, iLab and the University of Southern California
    @mainurl http://jevois.org
    @supporturl http://jevois.org/doc
    @otherurl http://iLab.usc.edu
    @license GPL v3
    @distribution Unrestricted
    @restrictions None
    \ingroup modules */
class JeVoisIntro : public jevois::Module
{
  public:
    //! Constructor
    JeVoisIntro(std::string const & instance) : jevois::Module(instance), itsScoresStr(" ")
    {
      itsSaliency = addSubComponent<Saliency>("saliency");
      itsFaceDetector = addSubComponent<FaceDetector>("facedetect");
      itsObjectRecognition = addSubComponent<ObjectRecognitionMNIST>("MNIST");
      itsKF = addSubComponent<Kalman2D>("kalman");
      itsVideo = addSubComponent<BufferedVideoReader>("intromovie");
    }

    //! Virtual destructor for safe inheritance
    virtual ~JeVoisIntro() { }

    //! Initialization once parameters are set:
    virtual void postInit() override
    {
      // Read the banner image and convert to YUYV RawImage:
      cv::Mat banner_bgr = cv::imread(absolutePath("jevois-banner-notext.png"));
      itsBanner.width = banner_bgr.cols;
      itsBanner.height = banner_bgr.rows;
      itsBanner.fmt = V4L2_PIX_FMT_YUYV;
      itsBanner.bufindex = 0;
      itsBanner.buf.reset(new jevois::VideoBuf(-1, itsBanner.bytesize(), 0));
      jevois::rawimage::convertCvBGRtoRawImage(banner_bgr, itsBanner, 100);

      // Allow our movie to load a bit:
      std::this_thread::sleep_for(std::chrono::milliseconds(750));
    }
    
    //! Processing function
    virtual void process(jevois::InputFrame && inframe, jevois::OutputFrame && outframe) override
    {
      static jevois::Timer itsProcessingTimer("Processing");
      static cv::Mat itsLastFace(60, 60, CV_8UC2, 0x80aa) ; // Note that this one will contain raw YUV pixels
      static cv::Mat itsLastObject(60, 60, CV_8UC2, 0x80aa) ; // Note that this one will contain raw YUV pixels
      static std::string itsLastObjectCateg;
      static bool doobject = false; // alternate between object and face recognition
      static bool intromode = false; // intro mode plays a video at the beginning, then shows some info messages
      static bool intromoviedone = false; // turns true when intro movie complete
      static ScriptItem const * scriptitem = &TheScript[0];
      static int scriptframe = 0;
      
      // Wait for next available camera image:
      jevois::RawImage inimg = inframe.get();

      // We only handle one specific input format in this demo:
      inimg.require("input", 320, 240, V4L2_PIX_FMT_YUYV);
      
      itsProcessingTimer.start();
      int const roihw = 32; // face & object roi half width and height
      
      // Compute saliency, in a thread:
      auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg, true); });
      
      // While computing, wait for an image from our gadget driver into which we will put our results:
      jevois::RawImage outimg = outframe.get();
      outimg.require("output", 640, outimg.height, V4L2_PIX_FMT_YUYV);
      switch (outimg.height)
      {
      case 312: break; // normal mode
      case 360:
      case 480: intromode = true; break; // intro mode
      default: LFATAL("Incorrect output height: should be 312, 360 or 480");
      }

      // Play the intro movie first if requested:
      if (intromode && intromoviedone == false)
      {
        cv::Mat m = itsVideo->get();

        if (m.empty()) intromoviedone = true;
        else
        {
          jevois::rawimage::convertCvBGRtoRawImage(m, outimg, 75);

          // Handle bottom of th eframe: blank or banner
          if (outimg.height == 480)
            jevois::rawimage::paste(itsBanner, outimg, 0, 360);
          else if (outimg.height > 360)
            jevois::rawimage::drawFilledRect(outimg, 0, 360, outimg.width, outimg.height - 360, 0x8000);

          // If on a mac with height = 480, need to flip horizontally for photobooth to work (it will flip again):
          if (outimg.height == 480) jevois::rawimage::hFlipYUYV(outimg);

          sal_fut.get(); // yes, we are wasting CPU here, just to keep code more readable with the intro stuff added
          inframe.done();
          outframe.send();
          return;
        }
      }
      
      // Paste the original image to the top-left corner of the display:
      unsigned short const txtcol = jevois::yuyv::White;
      jevois::rawimage::paste(inimg, outimg, 0, 0);
      jevois::rawimage::writeText(outimg, "JeVois Saliency + Gist + Faces + Objects", 3, 3, txtcol);
      
      // Wait until saliency computation is complete:
      sal_fut.get();
      
      // find most salient point:
      int mx, my; intg32 msal;
      itsSaliency->getSaliencyMax(mx, my, msal);

      // Scale back to original image coordinates:
      int const smlev = itsSaliency->smscale::get();
      int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0; // half a saliency map pixel adjustment
      int const dmx = (mx << smlev) + smadj;
      int const dmy = (my << smlev) + smadj;

      // Compute instantaneous attended ROI (note: coords must be even to avoid flipping U/V when we later paste):
      int const rx = std::min(int(inimg.width) - roihw, std::max(roihw, dmx));
      int const ry = std::min(int(inimg.height) - roihw, std::max(roihw, dmy));
      
      // Asynchronously launch a bunch of saliency drawings and filter the attended locations
      auto draw_fut =
        std::async(std::launch::async, [&]() {
            // Paste the various saliency results:
            drawMap(outimg, &itsSaliency->salmap, 320, 0, 16, 20);
            jevois::rawimage::writeText(outimg, "Saliency Map", 640 - 12*6-4, 3, txtcol);
            
            drawMap(outimg, &itsSaliency->color, 0, 240, 4, 18);
            jevois::rawimage::writeText(outimg, "Color", 3, 243, txtcol);
            
            drawMap(outimg, &itsSaliency->intens, 80, 240, 4, 18);
            jevois::rawimage::writeText(outimg, "Intensity", 83, 243, txtcol);
            
            drawMap(outimg, &itsSaliency->ori, 160, 240, 4, 18);
            jevois::rawimage::writeText(outimg, "Orientation", 163, 243, txtcol);
            
            drawMap(outimg, &itsSaliency->flicker, 240, 240, 4, 18);
            jevois::rawimage::writeText(outimg, "Flicker", 243, 243, txtcol);
            
            drawMap(outimg, &itsSaliency->motion, 320, 240, 4, 18);
            jevois::rawimage::writeText(outimg, "Motion", 323, 243, txtcol);
            
            // Draw the gist vector:
            drawGist(outimg, itsSaliency->gist, itsSaliency->gist_size, 400, 242, 40, 2);
            
            // Draw a small square at most salient location in image and in saliency map:
            jevois::rawimage::drawFilledRect(outimg, mx * 16 + 5, my * 16 + 5, 8, 8, 0xffff);
            jevois::rawimage::drawFilledRect(outimg, 320 + mx * 16 + 5, my * 16 + 5, 8, 8, 0xffff);
            jevois::rawimage::drawRect(outimg, rx - roihw, ry - roihw, roihw*2, roihw*2, 0xf0f0);
            jevois::rawimage::drawRect(outimg, rx - roihw+1, ry - roihw+1, roihw*2-2, roihw*2-2, 0xf0f0);

            // Blank out free space from 480 to 519 at the bottom, and small space above and below gist vector:
            jevois::rawimage::drawFilledRect(outimg, 480, 240, 40, 60, 0x8000);
            jevois::rawimage::drawRect(outimg, 400, 240, 80, 2, 0x80a0);
            jevois::rawimage::drawRect(outimg, 400, 298, 80, 2, 0x80a0);
            jevois::rawimage::drawFilledRect(outimg, 0, 300, 640, 12, jevois::yuyv::Black);

            // If intro mode, blank out rows 312 to bottom:
            if (outimg.height == 480)
            {
              jevois::rawimage::drawFilledRect(outimg, 0, 312, outimg.width, 48, 0x8000);
              jevois::rawimage::paste(itsBanner, outimg, 0, 360);
            }
            else if (outimg.height > 312)
              jevois::rawimage::drawFilledRect(outimg, 0, 312, outimg.width, outimg.height - 312, 0x8000);
            
            // Filter the attended locations:
            itsKF->set(dmx, dmy, inimg.width, inimg.height);
            float kfxraw, kfyraw, kfximg, kfyimg;
            itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, inimg.width, inimg.height, 1.0F, 1.0F);
      
            // Draw a circle around the kalman-filtered attended location:
            jevois::rawimage::drawCircle(outimg, int(kfximg), int(kfyimg), 20, 1, jevois::yuyv::LightGreen);
            
            // Send saliency info to serial port (for arduino, etc):
            sendSerial(jevois::sformat("T2D %d %d", int(kfxraw), int(kfyraw)));

            // If intro mode, draw some text messages according to our script:
            if (intromode && intromoviedone)
            {
              // Compute fade: we do 1s fade in, 2s full luminance, 1s fade out:
              int lum = 255;
              if (scriptframe < 32) lum = scriptframe * 8;
              else if (scriptframe > 4*30 - 32) lum = std::max(0, (4*30 - scriptframe) * 8);

              // Display the text with the proper fade:
              int x = (640 - 10 * strlen(scriptitem->msg)) / 2;
              jevois::rawimage::writeText(outimg, scriptitem->msg, x, 325, 0x7700 | lum, jevois::rawimage::Font10x20);

              // Add a blinking marker if specified in the script:
              if (scriptitem->blinkx)
              {
                int phase = scriptframe / 10;
                if ((phase % 2) == 0) jevois::rawimage::drawDisk(outimg, scriptitem->blinkx, scriptitem->blinky,
                                                                 10, jevois::yuyv::LightTeal);
              }

              // Move to next video frame and possibly next script item or loop the script:
              if (++scriptframe >= 140)
              {
                scriptframe = 0; ++scriptitem;
                if (scriptitem->msg == nullptr) scriptitem = &TheScript[0];
              }
            }
          });

      // Extract a raw YUYV ROI around attended point:
      cv::Mat rawimgcv = jevois::rawimage::cvImage(inimg);
      cv::Mat rawroi = rawimgcv(cv::Rect(rx - roihw, ry - roihw, roihw * 2, roihw * 2));

      if (doobject)
      {
        // #################### Object recognition:
        
        // Prepare a color or grayscale ROI for the object recognition module:
        auto objsz = itsObjectRecognition->insize();
        cv::Mat objroi;
        switch (objsz.depth_)
        {
        case 1: // grayscale input
        {
          // mnist is white letters on black background, so invert the image before we send it for recognition, as we
          // assume here black letters on white backgrounds. We also need to provide a clean crop around the digit for
          // the deep network to work well:
          cv::cvtColor(rawroi, objroi, CV_YUV2GRAY_YUYV);

          // Find the 10th percentile gray value:
          size_t const elem = (objroi.cols * objroi.rows * 10) / 100;
          std::vector<unsigned char> v; v.assign(objroi.datastart, objroi.dataend);
          std::nth_element(v.begin(), v.begin() + elem, v.end());
          unsigned char const thresh = std::min((unsigned char)(100), std::max((unsigned char)(30), v[elem]));

          // Threshold and invert the image:
          cv::threshold(objroi, objroi, thresh, 255, cv::THRESH_BINARY_INV);

          // Find the digit and center and crop it:
          cv::Mat pts; cv::findNonZero(objroi, pts);
          cv::Rect r = cv::boundingRect(pts);
          int const cx = r.x + r.width / 2;
          int const cy = r.y + r.height / 2;
          int const siz = std::min(roihw * 2, std::max(16, 8 + std::max(r.width, r.height))); // margin of 4 pix
          int const tlx = std::max(0, std::min(roihw*2 - siz, cx - siz/2));
          int const tly = std::max(0, std::min(roihw*2 - siz, cy - siz/2));
          cv::Rect ar(tlx, tly, siz, siz);
          cv::resize(objroi(ar), objroi, cv::Size(objsz.width_, objsz.height_), 0, 0, cv::INTER_AREA);
          //cv::imshow("cropped roi", objroi);cv::waitKey(1);
        }
        break;
          
        case 3: // color input
          cv::cvtColor(rawroi, objroi, CV_YUV2RGB_YUYV);
          cv::resize(objroi, objroi, cv::Size(objsz.width_, objsz.height_), 0, 0, cv::INTER_AREA);
          break;
          
        default:
          LFATAL("Unsupported object detection input depth " << objsz.depth_);
        }
        
        // Launch object recognition on the ROI and get the recognition scores:
        auto scores = itsObjectRecognition->process(objroi);

        // Create a string to show all scores:
        std::ostringstream oss;
        for (size_t i = 0; i < scores.size(); ++i)
          oss << itsObjectRecognition->category(i) << ':' << std::fixed << std::setprecision(2) << scores[i] << ' ';
        itsScoresStr = oss.str();
        
        // Check whether the highest score is very high and significantly higher than the second best:
        float best1 = scores[0], best2 = scores[0]; size_t idx1 = 0, idx2 = 0;
        for (size_t i = 1; i < scores.size(); ++i)
        {
          if (scores[i] > best1) { best2 = best1; idx2 = idx1; best1 = scores[i]; idx1 = i; }
          else if (scores[i] > best2) { best2 = scores[i]; idx2 = i; }
        }
        
        // Update our display upon each "clean" recognition:
        if (best1 > 90.0F && best2 < 20.0F)
        {
          // Remember this recognized object for future displays:
          itsLastObjectCateg = itsObjectRecognition->category(idx1);
          itsLastObject = rawimgcv(cv::Rect(rx - 30, ry - 30, 60, 60)).clone(); // make a deep copy
          
          LINFO("Object recognition: best: " << itsLastObjectCateg <<" (" << best1 <<
                "), second best: " << itsObjectRecognition->category(idx2) << " (" << best2 << ')');
        }
      }
      else
      {
        // #################### Face detection:
        
        // Prepare a grey ROI from our raw YUYV roi:
        cv::Mat grayroi; cv::cvtColor(rawroi, grayroi, CV_YUV2GRAY_YUYV);
        cv::equalizeHist(grayroi, grayroi);
        
        // Launch the face detector:
        std::vector<cv::Rect> faces; std::vector<std::vector<cv::Rect> > eyes;
        itsFaceDetector->process(grayroi, faces, eyes, false);
        
        // Draw the faces and eyes, if any:
        if (faces.size())
        {
          LINFO("detected " << faces.size() << " faces");
          // Store the attended ROI into our last ROI, fixed size 60x60 for our display:
          itsLastFace = rawimgcv(cv::Rect(rx - 30, ry - 30, 60, 60)).clone(); // make a deep copy
        }
        
        for (size_t i = 0; i < faces.size(); ++i)
        {
          // Draw one face:
          cv::Rect const & f = faces[i];
          jevois::rawimage::drawRect(outimg, f.x + rx - roihw, f.y + ry - roihw, f.width, f.height, 0xc0ff);
          
          // Draw the corresponding eyes:
          for (auto const & e : eyes[i])
            jevois::rawimage::drawRect(outimg, e.x + rx - roihw, e.y + ry - roihw, e.width, e.height, 0x40ff);
        }
      }
      
      // Let camera know we are done processing the raw YUV input image. NOTE: rawroi is now invalid:
      inframe.done();
      
      // Paste our last attended and recognized face and object (or empty pics):
      cv::Mat outimgcv(outimg.height, outimg.width, CV_8UC2, outimg.buf->data());
      itsLastObject.copyTo(outimgcv(cv::Rect(520, 240, 60, 60)));
      itsLastFace.copyTo(outimgcv(cv::Rect(580, 240, 60, 60)));
      
      // Wait until all saliency drawings are complete (since they blank out our object label area):
      draw_fut.get();
  
      // Print all object scores:
      jevois::rawimage::writeText(outimg, itsScoresStr, 2, 301, txtcol);

      // Write any positively recognized object category:
      jevois::rawimage::writeText(outimg, itsLastObjectCateg.c_str(), 517-6*itsLastObjectCateg.length(), 263, txtcol);
      
      // FIXME do svm on gist and write resuts here
      
      // Show processing fps:
      std::string const & fpscpu = itsProcessingTimer.stop();
      jevois::rawimage::writeText(outimg, fpscpu, 3, 240 - 13, jevois::yuyv::White);

      // If on a mac with height = 480, need to flip horizontally for photobooth to work (it will flip again):
      if (outimg.height == 480) jevois::rawimage::hFlipYUYV(outimg);
      
      // Send the output image with our processing results to the host over USB:
      outframe.send();
      
      // Alternate between face and object recognition:
      doobject = ! doobject;
    }

  protected:
    std::shared_ptr<Saliency> itsSaliency;
    std::shared_ptr<FaceDetector> itsFaceDetector;
    std::shared_ptr<ObjectRecognitionBase> itsObjectRecognition;
    std::shared_ptr<Kalman2D> itsKF;
    std::shared_ptr<BufferedVideoReader> itsVideo;
    jevois::RawImage itsBanner;
    std::string itsScoresStr;
};

// Allow the module to be loaded as a shared object (.so) file:
JEVOIS_REGISTER_MODULE(JeVoisIntro);