JeVoisBase  1.20
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
PyHandDetector.py
Go to the documentation of this file.
1 import pyjevois
2 if pyjevois.pro: import libjevoispro as jevois
3 else: import libjevois as jevois
4 import cv2
5 import numpy as np
6 import mediapipe as mp
7 
8 ## Hand detection using MediaPipe
9 #
10 # Detect hands using MediaPipe in Python
11 #
12 # This code is derived from sample_hand.py at https://github.com/Kazuhito00/mediapipe-python-sample
13 #
14 # @author Laurent Itti
15 #
16 # @videomapping JVUI 0 0 30.0 CropScale=RGB24@512x288:YUYV 1920 1080 30.0 JeVois PyHandDetector
17 # @email itti\@usc.edu
18 # @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
19 # @copyright Copyright (C) 2021 by Laurent Itti, iLab and the University of Southern California
20 # @mainurl http://jevois.org
21 # @supporturl http://jevois.org/doc
22 # @otherurl http://iLab.usc.edu
23 # @license GPL v3
24 # @distribution Unrestricted
25 # @restrictions None
26 # @ingroup modules
27 class PyHandDetector:
28  # ###################################################################################################
29  ## Constructor
30  def __init__(self):
31  # Instantiate a JeVois Timer to measure our processing framerate:
32  self.timer = jevois.Timer("hand", 100, jevois.LOG_INFO)
33 
34  # Instantiate mediapipe hand detector:
35  self.mp_hands = mp.solutions.hands
36  self.hands = self.mp_hands.Hands(max_num_hands = 2, min_detection_confidence = 0.7,
37  min_tracking_confidence = 0.5)
38  self.use_brect = True; # true to show a bounding rectangle around each hand
39 
40  # ###################################################################################################
41  ## Process function with GUI output
42  def processGUI(self, inframe, helper):
43  # Start a new display frame, gets its size and also whether mouse/keyboard are idle:
44  idle, winw, winh = helper.startFrame()
45 
46  # Draw full-resolution input frame from camera:
47  x, y, w, h = helper.drawInputFrame("c", inframe, False, False)
48  helper.itext('JeVois-Pro Hand Landmarks Detection', 0, -1)
49 
50  # Get the next camera image at processing resolution (may block until it is captured):
51  image = inframe.getCvRGBp()
52  iw, ih = image.shape[1], image.shape[0]
53 
54  # Start measuring image processing time:
55  self.timer.start()
56 
57  # Detect hands:
58  results = self.hands.process(image)
59 
60  # Draw results:
61  if results.multi_hand_landmarks is not None:
62  for hand_landmarks, handedness in zip(results.multi_hand_landmarks, results.multi_handedness):
63  cx, cy = self.calc_palm_moment(iw, ih, hand_landmarks)
64  self.draw_landmarks(helper, iw, ih, cx, cy, hand_landmarks, handedness)
65  if self.use_brect:
66  brect = self.calc_bounding_rect(iw, ih, hand_landmarks)
67  helper.drawRect(brect[0], brect[1], brect[2], brect[3], 0x6040ffff, True)
68 
69  # Write frames/s info from our timer:
70  fps = self.timer.stop()
71  helper.iinfo(inframe, fps, winw, winh);
72 
73  # End of frame:
74  helper.endFrame()
75 
76  # ###################################################################################################
77  def calc_palm_moment(self, iw, ih, landmarks):
78  palm_array = np.empty((0, 2), int)
79 
80  for index, landmark in enumerate(landmarks.landmark):
81  landmark_x = min(int(landmark.x * iw), iw - 1)
82  landmark_y = min(int(landmark.y * ih), ih - 1)
83 
84  landmark_point = [np.array((landmark_x, landmark_y))]
85 
86  if index == 0:
87  palm_array = np.append(palm_array, landmark_point, axis=0)
88  if index == 1:
89  palm_array = np.append(palm_array, landmark_point, axis=0)
90  if index == 5:
91  palm_array = np.append(palm_array, landmark_point, axis=0)
92  if index == 9:
93  palm_array = np.append(palm_array, landmark_point, axis=0)
94  if index == 13:
95  palm_array = np.append(palm_array, landmark_point, axis=0)
96  if index == 17:
97  palm_array = np.append(palm_array, landmark_point, axis=0)
98 
99  M = cv2.moments(palm_array)
100  cx, cy = 0, 0
101  if M['m00'] != 0:
102  cx = int(M['m10'] / M['m00'])
103  cy = int(M['m01'] / M['m00'])
104  return cx, cy
105 
106  # ###################################################################################################
107  def calc_bounding_rect(self, iw, ih, landmarks):
108  landmark_array = np.empty((0, 2), int)
109 
110  for _, landmark in enumerate(landmarks.landmark):
111  landmark_x = min(int(landmark.x * iw), iw - 1)
112  landmark_y = min(int(landmark.y * ih), ih - 1)
113  landmark_point = [np.array((landmark_x, landmark_y))]
114  landmark_array = np.append(landmark_array, landmark_point, axis=0)
115  x, y, ww, hh = cv2.boundingRect(landmark_array)
116  return [x, y, x + ww, y + hh]
117 
118  # ###################################################################################################
119  def draw_landmarks(self, helper, iw, ih, cx, cy, landmarks, handedness):
120  lpx = []
121  lpy = []
122  col = 0xff00ff00
123 
124  for index, landmark in enumerate(landmarks.landmark):
125  if landmark.visibility < 0 or landmark.presence < 0: continue
126 
127  landmark_x = min(int(landmark.x * iw), iw - 1)
128  landmark_y = min(int(landmark.y * ih), ih - 1)
129  # landmark_z = landmark.z
130 
131  lpx.append(landmark_x)
132  lpy.append(landmark_y)
133 
134  if index == 0:
135  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
136  if index == 1:
137  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
138  if index == 2:
139  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
140  if index == 3:
141  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
142  if index == 4:
143  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
144  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
145  if index == 5:
146  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
147  if index == 6:
148  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
149  if index == 7:
150  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
151  if index == 8:
152  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
153  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
154  if index == 9:
155  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
156  if index == 10:
157  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
158  if index == 11:
159  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
160  if index == 12:
161  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
162  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
163  if index == 13:
164  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
165  if index == 14:
166  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
167  if index == 15:
168  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
169  if index == 16:
170  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
171  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
172  if index == 17:
173  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
174  if index == 18:
175  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
176  if index == 19:
177  helper.drawCircle(landmark_x, landmark_y, 5, col, True)
178  if index == 20:
179  helper.drawCircle(landmark_x, landmark_y, 5, col, False)
180  helper.drawCircle(landmark_x, landmark_y, 12, col, True)
181 
182  if len(lpx) > 0:
183  helper.drawLine(lpx[2], lpy[2], lpx[3], lpy[3], col)
184  helper.drawLine(lpx[3], lpy[3], lpx[4], lpy[4], col)
185 
186  helper.drawLine(lpx[5], lpy[5], lpx[6], lpy[6], col)
187  helper.drawLine(lpx[6], lpy[6], lpx[7], lpy[7], col)
188  helper.drawLine(lpx[7], lpy[7], lpx[8], lpy[8], col)
189 
190  helper.drawLine(lpx[9], lpy[9], lpx[10], lpy[10], col)
191  helper.drawLine(lpx[10], lpy[10], lpx[11], lpy[11], col)
192  helper.drawLine(lpx[11], lpy[11], lpx[12], lpy[12], col)
193 
194  helper.drawLine(lpx[13], lpy[13], lpx[14], lpy[14], col)
195  helper.drawLine(lpx[14], lpy[14], lpx[15], lpy[15], col)
196  helper.drawLine(lpx[15], lpy[15], lpx[16], lpy[16], col)
197 
198  helper.drawLine(lpx[17], lpy[17], lpx[18], lpy[18], col)
199  helper.drawLine(lpx[18], lpy[18], lpx[19], lpy[19], col)
200  helper.drawLine(lpx[19], lpy[19], lpx[20], lpy[20], col)
201 
202  helper.drawLine(lpx[0], lpy[0], lpx[1], lpy[1], col)
203  helper.drawLine(lpx[1], lpy[1], lpx[2], lpy[2], col)
204  helper.drawLine(lpx[2], lpy[2], lpx[5], lpy[5], col)
205  helper.drawLine(lpx[5], lpy[5], lpx[9], lpy[9], col)
206  helper.drawLine(lpx[9], lpy[9], lpx[13], lpy[13], col)
207  helper.drawLine(lpx[13], lpy[13], lpx[17], lpy[17], col)
208  helper.drawLine(lpx[17], lpy[17], lpx[0], lpy[0], col)
209 
210  helper.drawCircle(cx, cy, 5, 0xffff8080, True)
211 
212  # If camera view was not flipped horizontally, swap left/right hands:
213  if handedness.classification[0].label[0] == 'L': hnd = 'R'
214  else: hnd = 'L'
215  helper.drawText(cx - 2, cy - 3, hnd, 0xffff0000)
demo.int
int
Definition: demo.py:37
PyHandDetector.PyHandDetector.processGUI
def processGUI(self, inframe, helper)
Process function with GUI output.
Definition: PyHandDetector.py:42
PyHandDetector.PyHandDetector.timer
timer
Definition: PyHandDetector.py:32
PyHandDetector.PyHandDetector
Hand detection using MediaPipe.
Definition: PyHandDetector.py:27
PyHandDetector.PyHandDetector.hands
hands
Definition: PyHandDetector.py:36
PyHandDetector.PyHandDetector.calc_bounding_rect
def calc_bounding_rect(self, iw, ih, landmarks)
Definition: PyHandDetector.py:107
PyHandDetector.PyHandDetector.__init__
def __init__(self)
Constructor.
Definition: PyHandDetector.py:30
PyHandDetector.PyHandDetector.calc_palm_moment
def calc_palm_moment(self, iw, ih, landmarks)
Definition: PyHandDetector.py:77
PyHandDetector.PyHandDetector.draw_landmarks
def draw_landmarks(self, helper, iw, ih, cx, cy, landmarks, handedness)
Definition: PyHandDetector.py:119
PyHandDetector.PyHandDetector.mp_hands
mp_hands
Definition: PyHandDetector.py:35
PyHandDetector.PyHandDetector.use_brect
use_brect
Definition: PyHandDetector.py:38
jevois::Timer