JeVoisBase  1.18
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
PyObjectron.py
Go to the documentation of this file.
1 import pyjevois
2 if pyjevois.pro: import libjevoispro as jevois
3 else: import libjevois as jevois
4 import cv2
5 import numpy as np
6 import mediapipe as mp
7 
8 ## 3D object detection using MediaPipe
9 #
10 # Detect objects and draw estimated 3D bounding boxes, using MediaPipe in Python
11 #
12 # Only works for a few pre-trained objects: 'Shoe', 'Chair', 'Cup', 'Camera', with Shoe selected by default. So
13 # point the camera to your shoes and see what happens...
14 #
15 # This code is derived from sample_objectron.py at https://github.com/Kazuhito00/mediapipe-python-sample
16 #
17 # @author Laurent Itti
18 #
19 # @videomapping JVUI 0 0 30.0 CropScale=RGB24@512x288:YUYV 1920 1080 30.0 JeVois PyObjectron
20 # @email itti\@usc.edu
21 # @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
22 # @copyright Copyright (C) 2022 by Laurent Itti, iLab and the University of Southern California
23 # @mainurl http://jevois.org
24 # @supporturl http://jevois.org/doc
25 # @otherurl http://iLab.usc.edu
26 # @license GPL v3
27 # @distribution Unrestricted
28 # @restrictions None
29 # @ingroup modules
31  # ###################################################################################################
32  ## Constructor
33  def __init__(self):
34  # Parameters:
35  max_num_objects = 5
36  min_detection_confidence = 0.5
37  min_tracking_confidence = 0.99
38  model_name = 'Shoe' # 'Shoe', 'Chair', 'Cup', 'Camera'
39 
40  # Instantiate a JeVois Timer to measure our processing framerate:
41  self.timer = jevois.Timer("objectron", 30, jevois.LOG_DEBUG)
42 
43  # Instantiate mediapipe model:
44  self.mp_objectron = mp.solutions.objectron
45  self.objectron = self.mp_objectron.Objectron(
46  static_image_mode = False,
47  max_num_objects = max_num_objects,
48  min_detection_confidence = min_detection_confidence,
49  min_tracking_confidence = min_tracking_confidence,
50  model_name = model_name)
51 
52  # ###################################################################################################
53  ## Process function with GUI output
54  def processGUI(self, inframe, helper):
55  # Start a new display frame, gets its size and also whether mouse/keyboard are idle:
56  idle, winw, winh = helper.startFrame()
57 
58  # Draw full-resolution input frame from camera:
59  x, y, w, h = helper.drawInputFrame("c", inframe, False, False)
60 
61  # Get the next camera image at processing resolution (may block until it is captured):
62  image = inframe.getCvRGBp()
63  iw, ih = image.shape[1], image.shape[0]
64 
65  # Start measuring image processing time:
66  self.timer.start()
67 
68  # Run graph:
69  results = self.objectron.process(image)
70 
71  # Draw results:
72  if results.detected_objects is not None:
73  for detected_object in results.detected_objects:
74  self.draw_landmarks(helper, iw, ih, detected_object.landmarks_2d)
75  self.draw_axis(helper, iw, ih, detected_object.rotation, detected_object.translation, 0.1)
76 
77  # Write frames/s info from our timer:
78  fps = self.timer.stop()
79  helper.iinfo(inframe, fps, winw, winh);
80 
81  # End of frame:
82  helper.endFrame()
83 
84 
85  # ###################################################################################################
86  def draw_landmarks(self, helper, iw, ih, landmarks):
87  col = 0xff00ff00
88  idx_to_coordinates = {}
89 
90  for index, landmark in enumerate(landmarks.landmark):
91  lm = helper.i2d(landmark.x * iw, landmark.y * ih, "c")
92  helper.drawCircle(lm.x, lm.y, 5, col, True)
93  idx_to_coordinates[index] = lm
94 
95  # See https://github.com/google/mediapipe/blob/master/mediapipe/modules/objectron/calculators/box.h
96  # for the 8 vertex locations. Code here derived from mediapipe/python/solutions/drawing_utils.py
97  connections = self.mp_objectron.BOX_CONNECTIONS
98  if connections:
99  num_landmarks = len(landmarks.landmark)
100  for connection in connections:
101  start_idx = connection[0]
102  end_idx = connection[1]
103  if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks):
104  raise ValueError(f'Landmark index is out of range. Invalid connection '
105  f'from landmark #{start_idx} to landmark #{end_idx}.')
106  if start_idx in idx_to_coordinates and end_idx in idx_to_coordinates:
107  helper.drawLine(idx_to_coordinates[start_idx].x, idx_to_coordinates[start_idx].y,
108  idx_to_coordinates[end_idx].x, idx_to_coordinates[end_idx].y, col)
109 
110  # ###################################################################################################
111  def draw_axis(self, helper, iw, ih, rotation, translation, axis_length):
112  focal_length = (1.0, 1.0)
113  principal_point = (0.0, 0.0)
114  axis_world = np.float32([[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]])
115  axis_cam = np.matmul(rotation, axis_length * axis_world.T).T + translation
116  x = axis_cam[..., 0]
117  y = axis_cam[..., 1]
118  z = axis_cam[..., 2]
119 
120  # Project 3D points to NDC space.
121  fx, fy = focal_length
122  px, py = principal_point
123  x_ndc = np.clip(-fx * x / (z + 1e-5) + px, -1., 1.)
124  y_ndc = np.clip(-fy * y / (z + 1e-5) + py, -1., 1.)
125 
126  # Convert from NDC space to image space.
127  x_im = (1 + x_ndc) * 0.5 * iw
128  y_im = (1 - y_ndc) * 0.5 * ih
129 
130  # Draw, converting coords from low-res processing frame to full-res display frame:
131  orig = helper.i2d(x_im[0], y_im[0], "c")
132  xa = helper.i2d(x_im[1], y_im[1], "c")
133  ya = helper.i2d(x_im[2], y_im[2], "c")
134  za = helper.i2d(x_im[3], y_im[3], "c")
135  helper.drawLine(orig.x, orig.y, xa.x, xa.y, 0xff0000ff)
136  helper.drawLine(orig.x, orig.y, ya.x, ya.y, 0xff00ff00)
137  helper.drawLine(orig.x, orig.y, za.x, za.y, 0xffff0000)
138 
PyObjectron.PyObjectron.__init__
def __init__(self)
Constructor.
Definition: PyObjectron.py:33
PyObjectron.PyObjectron.draw_axis
def draw_axis(self, helper, iw, ih, rotation, translation, axis_length)
Definition: PyObjectron.py:111
PyObjectron.PyObjectron.objectron
objectron
Definition: PyObjectron.py:45
PyObjectron.PyObjectron
3D object detection using MediaPipe
Definition: PyObjectron.py:30
PyObjectron.PyObjectron.draw_landmarks
def draw_landmarks(self, helper, iw, ih, landmarks)
Definition: PyObjectron.py:86
PyObjectron.PyObjectron.mp_objectron
mp_objectron
Definition: PyObjectron.py:44
PyObjectron.PyObjectron.processGUI
def processGUI(self, inframe, helper)
Process function with GUI output.
Definition: PyObjectron.py:54
PyObjectron.PyObjectron.timer
timer
Definition: PyObjectron.py:41
jevois::Timer