JeVoisBase  1.17
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
PyEmotion.py
Go to the documentation of this file.
1 import pyjevois
2 if pyjevois.pro: import libjevoispro as jevois
3 else: import libjevois as jevois
4 import cv2 as cv
5 import numpy as np
6 
7 ## Human facial emotion recognition using OpenCV Deep Neural Networks (DNN)
8 #
9 # This module runs an emotion classification deep neural network using the OpenCV DNN library. The network is from the
10 # FER+ emotion recognition project, "Training Deep Networks for Facial Expression Recognition with Crowd-Sourced Label
11 # Distribution" arXiv:1608.01041
12 #
13 # The module outputs a score from -1000 to 1000 for each of: neutral, happiness, surprise, sadness, anger, disgust,
14 # fear, contempt.
15 #
16 # Note that this module does not include any face detection. Hence it always assumes that there is a face well centered
17 # in the image. You should enhance this module with first applying a face detector (see, e.g.,
18 # \jvmod{PyDetectionDNN}) and to only run the emotion recognition network on the detected faces.
19 #
20 # @author Laurent Itti
21 #
22 # @videomapping YUYV 320 336 15.0 YUYV 320 336 15.0 JeVois PyEmotion
23 # @email itti@usc.edu
24 # @address 880 W 1st St Suite 807, Los Angeles CA 90012, USA
25 # @copyright Copyright (C) 2019 by Laurent Itti
26 # @mainurl http://jevois.org
27 # @supporturl http://jevois.org
28 # @otherurl http://jevois.org
29 # @license GPL v3
30 # @distribution Unrestricted
31 # @restrictions None
32 # @ingroup modules
33 class PyEmotion:
34  # ####################################################################################################
35  ## Constructor
36  def __init__(self):
37  self.inpWidth = 64 # Resized image width passed to network
38  self.inpHeight = 64 # Resized image height passed to network
39  self.scale = 1.0 # Value scaling factor applied to input pixels
40  self.mean = [127,127,127] # Mean BGR value subtracted from input image
41  self.rgb = False # True if model expects RGB inputs, otherwise it expects BGR
42 
43  # This network takes a while to load from microSD. To avoid timouts at construction,
44  # we will load it in process() instead.
45 
46  self.timer = jevois.Timer('Neural emotion', 10, jevois.LOG_DEBUG)
47 
48  # ####################################################################################################
49  ## JeVois main processing function
50  def process(self, inframe, outframe):
51  font = cv.FONT_HERSHEY_PLAIN
52  siz = 0.8
53  white = (255, 255, 255)
54 
55  # Load the network if needed:
56  if not hasattr(self, 'net'):
57  self.classes = [ "neutral", "happiness", "surprise", "sadness", "anger", "disgust",
58  "fear", "contempt" ]
59  self.model = 'FER+ ONNX'
60  self.net = cv.dnn.readNet(pyjevois.share + "/opencv-dnn/classification/emotion_ferplus.onnx", '')
61  self.net.setPreferableBackend(cv.dnn.DNN_BACKEND_OPENCV)
62  self.net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)
63 
64  # Get the next frame from the camera sensor:
65  frame = inframe.getCvBGR()
66  self.timer.start()
67 
68  frameHeight = frame.shape[0]
69  frameWidth = frame.shape[1]
70  mid = int((frameWidth - 110) / 2) + 110 # x coord of midpoint of our bars
71  leng = frameWidth - mid - 6 # max length of our bars
72  maxconf = 999
73 
74  # Create a 4D blob from a frame.
75  gframe = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
76  blob = cv.dnn.blobFromImage(gframe, self.scale, (self.inpWidth, self.inpHeight), self.mean, self.rgb, crop=True)
77 
78  # Run the model
79  self.net.setInput(blob)
80  out = self.net.forward()
81 
82  # Create dark-gray (value 80) image for the bottom panel, 96 pixels tall and show top-1 class:
83  msgbox = np.zeros((96, frame.shape[1], 3), dtype = np.uint8) + 80
84 
85  # Show the scores for each class:
86  out = out.flatten()
87  for i in range(8):
88  conf = out[i] * 100
89  if conf > maxconf: conf = maxconf
90  if conf < -maxconf: conf = -maxconf
91  cv.putText(msgbox, self.classes[i] + ':', (3, 11*(i+1)), font, siz, white, 1, cv.LINE_AA)
92  rlabel = '%+6.1f' % conf
93  cv.putText(msgbox, rlabel, (76, 11*(i+1)), font, siz, white, 1, cv.LINE_AA)
94  cv.line(msgbox, (mid, 11*i+6), (mid + int(conf*leng/maxconf), 11*i+6), white, 4)
95 
96  # Put efficiency information.
97  cv.putText(frame, 'JeVois Emotion DNN - ' + self.model, (3, 15),
98  cv.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv.LINE_AA)
99  t, _ = self.net.getPerfProfile()
100  fps = self.timer.stop()
101  label = fps + ', %dms' % (t * 1000.0 / cv.getTickFrequency())
102  cv.putText(frame, label, (3, frameHeight-5), cv.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv.LINE_AA)
103 
104  # Stack bottom panel below main image:
105  frame = np.vstack((frame, msgbox))
106 
107  # Send output frame to host:
108  outframe.sendCv(frame)
PyEmotion.PyEmotion.__init__
def __init__(self)
Constructor.
Definition: PyEmotion.py:36
PyEmotion.PyEmotion.process
def process(self, inframe, outframe)
JeVois main processing function.
Definition: PyEmotion.py:50
PyEmotion.PyEmotion.net
net
Definition: PyEmotion.py:60
PyEmotion.PyEmotion.mean
mean
Definition: PyEmotion.py:40
PyEmotion.PyEmotion.rgb
rgb
Definition: PyEmotion.py:41
PyEmotion.PyEmotion.scale
scale
Definition: PyEmotion.py:39
PyEmotion.PyEmotion.classes
classes
Definition: PyEmotion.py:57
PyEmotion.PyEmotion.inpWidth
inpWidth
Definition: PyEmotion.py:37
PyEmotion.PyEmotion
Human facial emotion recognition using OpenCV Deep Neural Networks (DNN)
Definition: PyEmotion.py:33
PyEmotion.PyEmotion.model
model
Definition: PyEmotion.py:59
PyEmotion.PyEmotion.timer
timer
Definition: PyEmotion.py:46
PyEmotion.PyEmotion.inpHeight
inpHeight
Definition: PyEmotion.py:38
jevois::Timer