JeVoisBase  1.16
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
PyHandDetector.py
Go to the documentation of this file.
1 import pyjevois
2 if pyjevois.pro: import libjevoispro as jevois
3 else: import libjevois as jevois
4 import cv2
5 import numpy as np
6 import mediapipe as mp
7 
8 ## Hand detection using MediaPipe
9 #
10 # Detect hands using MediaPipe in Python
11 #
12 # This code is derived from sample_hand.py at https://github.com/Kazuhito00/mediapipe-python-sample
13 #
14 # @author Laurent Itti
15 #
16 # @videomapping JVUI 0 0 30.0 CropScale=RGB24@512x288:YUYV 1920 1080 30.0 JeVois PyHandDetector
17 # @email itti\@usc.edu
18 # @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
19 # @copyright Copyright (C) 2021 by Laurent Itti, iLab and the University of Southern California
20 # @mainurl http://jevois.org
21 # @supporturl http://jevois.org/doc
22 # @otherurl http://iLab.usc.edu
23 # @license GPL v3
24 # @distribution Unrestricted
25 # @restrictions None
26 # @ingroup modules
28  # ###################################################################################################
29  ## Constructor
30  def __init__(self):
31  # Instantiate a JeVois Timer to measure our processing framerate:
32  self.timer = jevois.Timer("hand", 100, jevois.LOG_INFO)
33 
34  # Instantiate mediapipe hand detector:
35  self.mp_hands = mp.solutions.hands
36  self.hands = self.mp_hands.Hands(max_num_hands = 2, min_detection_confidence = 0.7,
37  min_tracking_confidence = 0.5)
38  self.use_brect = True; # true to show a bounding rectangle around each hand
39 
40  # ###################################################################################################
41  ## Process function with GUI output
42  def processGUI(self, inframe, helper):
43  # Start a new display frame, gets its size and also whether mouse/keyboard are idle:
44  idle, winw, winh = helper.startFrame()
45 
46  # Draw full-resolution input frame from camera:
47  x, y, w, h = helper.drawInputFrame("c", inframe, False, False)
48 
49  # Get the next camera image at processing resolution (may block until it is captured):
50  image = inframe.getCvRGBp()
51  iw, ih = image.shape[1], image.shape[0]
52 
53  # Start measuring image processing time:
54  self.timer.start()
55 
56  # Detect hands:
57  results = self.hands.process(image)
58 
59  # Draw results:
60  if results.multi_hand_landmarks is not None:
61  for hand_landmarks, handedness in zip(results.multi_hand_landmarks, results.multi_handedness):
62  cx, cy = calc_palm_moment(iw, ih, hand_landmarks)
63  draw_landmarks(helper, iw, ih, cx, cy, hand_landmarks, handedness)
64  if self.use_brect:
65  brect = calc_bounding_rect(iw, ih, hand_landmarks)
66  helper.drawRect(brect[0], brect[1], brect[2], brect[3], 0x6040ffff, True)
67 
68  # Write frames/s info from our timer:
69  fps = self.timer.stop()
70  helper.iinfo(inframe, fps, winw, winh);
71 
72  # End of frame:
73  helper.endFrame()
74 
75 
76 
77 
78 # ###################################################################################################
79 def calc_palm_moment(iw, ih, landmarks):
80  palm_array = np.empty((0, 2), int)
81 
82  for index, landmark in enumerate(landmarks.landmark):
83  landmark_x = min(int(landmark.x * iw), iw - 1)
84  landmark_y = min(int(landmark.y * ih), ih - 1)
85 
86  landmark_point = [np.array((landmark_x, landmark_y))]
87 
88  if index == 0:
89  palm_array = np.append(palm_array, landmark_point, axis=0)
90  if index == 1:
91  palm_array = np.append(palm_array, landmark_point, axis=0)
92  if index == 5:
93  palm_array = np.append(palm_array, landmark_point, axis=0)
94  if index == 9:
95  palm_array = np.append(palm_array, landmark_point, axis=0)
96  if index == 13:
97  palm_array = np.append(palm_array, landmark_point, axis=0)
98  if index == 17:
99  palm_array = np.append(palm_array, landmark_point, axis=0)
100 
101  M = cv2.moments(palm_array)
102  cx, cy = 0, 0
103  if M['m00'] != 0:
104  cx = int(M['m10'] / M['m00'])
105  cy = int(M['m01'] / M['m00'])
106  return cx, cy
107 
108 # ###################################################################################################
109 def calc_bounding_rect(iw, ih, landmarks):
110  landmark_array = np.empty((0, 2), int)
111 
112  for _, landmark in enumerate(landmarks.landmark):
113  landmark_x = min(int(landmark.x * iw), iw - 1)
114  landmark_y = min(int(landmark.y * ih), ih - 1)
115  landmark_point = [np.array((landmark_x, landmark_y))]
116  landmark_array = np.append(landmark_array, landmark_point, axis=0)
117  x, y, ww, hh = cv2.boundingRect(landmark_array)
118  return [x, y, x + ww, y + hh]
119 
120 # ###################################################################################################
121 def draw_landmarks(helper, iw, ih, cx, cy, landmarks, handedness):
122  lpx = []
123  lpy = []
124  col = 0xff00ff00
125 
126  for index, landmark in enumerate(landmarks.landmark):
127  if landmark.visibility < 0 or landmark.presence < 0:
128  continue
129 
130  landmark_x = min(int(landmark.x * iw), iw - 1)
131  landmark_y = min(int(landmark.y * ih), ih - 1)
132  # landmark_z = landmark.z
133 
134  lpx.append(landmark_x)
135  lpy.append(landmark_y)
136 
137  if index == 0:
138  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
139  if index == 1:
140  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
141  if index == 2:
142  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
143  if index == 3:
144  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
145  if index == 4:
146  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
147  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
148  if index == 5:
149  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
150  if index == 6:
151  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
152  if index == 7:
153  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
154  if index == 8:
155  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
156  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
157  if index == 9:
158  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
159  if index == 10:
160  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
161  if index == 11:
162  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
163  if index == 12:
164  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
165  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
166  if index == 13:
167  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
168  if index == 14:
169  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
170  if index == 15:
171  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
172  if index == 16:
173  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
174  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
175  if index == 17:
176  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
177  if index == 18:
178  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
179  if index == 19:
180  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
181  if index == 20:
182  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
183  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
184 
185  if len(lpx) > 0:
186  helper.drawLine(lpx[2], lpy[2], lpx[3], lpy[3], col)
187  helper.drawLine(lpx[3], lpy[3], lpx[4], lpy[4], col)
188 
189  helper.drawLine(lpx[5], lpy[5], lpx[6], lpy[6], col)
190  helper.drawLine(lpx[6], lpy[6], lpx[7], lpy[7], col)
191  helper.drawLine(lpx[7], lpy[7], lpx[8], lpy[8], col)
192 
193  helper.drawLine(lpx[9], lpy[9], lpx[10], lpy[10], col)
194  helper.drawLine(lpx[10], lpy[10], lpx[11], lpy[11], col)
195  helper.drawLine(lpx[11], lpy[11], lpx[12], lpy[12], col)
196 
197  helper.drawLine(lpx[13], lpy[13], lpx[14], lpy[14], col)
198  helper.drawLine(lpx[14], lpy[14], lpx[15], lpy[15], col)
199  helper.drawLine(lpx[15], lpy[15], lpx[16], lpy[16], col)
200 
201  helper.drawLine(lpx[17], lpy[17], lpx[18], lpy[18], col)
202  helper.drawLine(lpx[18], lpy[18], lpx[19], lpy[19], col)
203  helper.drawLine(lpx[19], lpy[19], lpx[20], lpy[20], col)
204 
205  helper.drawLine(lpx[0], lpy[0], lpx[1], lpy[1], col)
206  helper.drawLine(lpx[1], lpy[1], lpx[2], lpy[2], col)
207  helper.drawLine(lpx[2], lpy[2], lpx[5], lpy[5], col)
208  helper.drawLine(lpx[5], lpy[5], lpx[9], lpy[9], col)
209  helper.drawLine(lpx[9], lpy[9], lpx[13], lpy[13], col)
210  helper.drawLine(lpx[13], lpy[13], lpx[17], lpy[17], col)
211  helper.drawLine(lpx[17], lpy[17], lpx[0], lpy[0], col)
212 
213  if len(lpx) > 0:
214  helper.drawCircle(cx, cy, 5, 0xffff8080, True)
215  # If camera view was not flipped horizontally, swap left/right hands:
216  if handedness.classification[0].label[0] == 'L': hnd = 'R'
217  else: hnd = 'L'
218  helper.drawText(cx - 2, cy - 3, hnd, 0xffff0000)
PyHandDetector.PyHandDetector.processGUI
def processGUI(self, inframe, helper)
Process function with GUI output.
Definition: PyHandDetector.py:42
PyHandDetector.PyHandDetector.timer
timer
Definition: PyHandDetector.py:32
PyHandDetector.calc_bounding_rect
def calc_bounding_rect(iw, ih, landmarks)
Definition: PyHandDetector.py:109
PyHandDetector.PyHandDetector
Hand detection using MediaPipe.
Definition: PyHandDetector.py:27
PyHandDetector.PyHandDetector.hands
hands
Definition: PyHandDetector.py:36
PyHandDetector.calc_palm_moment
def calc_palm_moment(iw, ih, landmarks)
Definition: PyHandDetector.py:79
PyHandDetector.PyHandDetector.__init__
def __init__(self)
Constructor.
Definition: PyHandDetector.py:30
PyHandDetector.draw_landmarks
def draw_landmarks(helper, iw, ih, cx, cy, landmarks, handedness)
Definition: PyHandDetector.py:121
PyHandDetector.PyHandDetector.mp_hands
mp_hands
Definition: PyHandDetector.py:35
PyHandDetector.PyHandDetector.use_brect
use_brect
Definition: PyHandDetector.py:38
jevois::Timer