JeVoisBase  1.20
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
PyObjectron.py
Go to the documentation of this file.
1 import pyjevois
2 if pyjevois.pro: import libjevoispro as jevois
3 else: import libjevois as jevois
4 import cv2
5 import numpy as np
6 import mediapipe as mp
7 
8 ## 3D object detection using MediaPipe
9 #
10 # Detect objects and draw estimated 3D bounding boxes, using MediaPipe in Python
11 #
12 # Only works for a few pre-trained objects: 'Shoe', 'Chair', 'Cup', 'Camera', with Shoe selected by default. So
13 # point the camera to your shoes and see what happens...
14 #
15 # This code is derived from sample_objectron.py at https://github.com/Kazuhito00/mediapipe-python-sample
16 #
17 # @author Laurent Itti
18 #
19 # @videomapping JVUI 0 0 30.0 CropScale=RGB24@512x288:YUYV 1920 1080 30.0 JeVois PyObjectron
20 # @email itti\@usc.edu
21 # @address University of Southern California, HNB-07A, 3641 Watt Way, Los Angeles, CA 90089-2520, USA
22 # @copyright Copyright (C) 2022 by Laurent Itti, iLab and the University of Southern California
23 # @mainurl http://jevois.org
24 # @supporturl http://jevois.org/doc
25 # @otherurl http://iLab.usc.edu
26 # @license GPL v3
27 # @distribution Unrestricted
28 # @restrictions None
29 # @ingroup modules
31  # ###################################################################################################
32  ## Constructor
33  def __init__(self):
34  # Parameters:
35  max_num_objects = 5
36  min_detection_confidence = 0.5
37  min_tracking_confidence = 0.99
38  model_name = 'Shoe' # 'Shoe', 'Chair', 'Cup', 'Camera'
39 
40  # Instantiate a JeVois Timer to measure our processing framerate:
41  self.timer = jevois.Timer("objectron", 30, jevois.LOG_DEBUG)
42 
43  # Instantiate mediapipe model:
44  self.mp_objectron = mp.solutions.objectron
45  self.objectron = self.mp_objectron.Objectron(
46  static_image_mode = False,
47  max_num_objects = max_num_objects,
48  min_detection_confidence = min_detection_confidence,
49  min_tracking_confidence = min_tracking_confidence,
50  model_name = model_name)
51 
52  # ###################################################################################################
53  ## Process function with GUI output
54  def processGUI(self, inframe, helper):
55  # Start a new display frame, gets its size and also whether mouse/keyboard are idle:
56  idle, winw, winh = helper.startFrame()
57 
58  # Draw full-resolution input frame from camera:
59  x, y, w, h = helper.drawInputFrame("c", inframe, False, False)
60  helper.itext('JeVois-Pro 3D Object Detection', 0, -1)
61 
62  # Get the next camera image at processing resolution (may block until it is captured):
63  image = inframe.getCvRGBp()
64  iw, ih = image.shape[1], image.shape[0]
65 
66  # Start measuring image processing time:
67  self.timer.start()
68 
69  # Run graph:
70  results = self.objectron.process(image)
71 
72  # Draw results:
73  if results.detected_objects is not None:
74  for detected_object in results.detected_objects:
75  self.draw_landmarks(helper, iw, ih, detected_object.landmarks_2d)
76  self.draw_axis(helper, iw, ih, detected_object.rotation, detected_object.translation, 0.1)
77 
78  # Write frames/s info from our timer:
79  fps = self.timer.stop()
80  helper.iinfo(inframe, fps, winw, winh);
81 
82  # End of frame:
83  helper.endFrame()
84 
85 
86  # ###################################################################################################
87  def draw_landmarks(self, helper, iw, ih, landmarks):
88  col = 0xff00ff00
89  idx_to_coordinates = {}
90 
91  for index, landmark in enumerate(landmarks.landmark):
92  lm = helper.i2d(landmark.x * iw, landmark.y * ih, "c")
93  helper.drawCircle(lm.x, lm.y, 5, col, True)
94  idx_to_coordinates[index] = lm
95 
96  # See https://github.com/google/mediapipe/blob/master/mediapipe/modules/objectron/calculators/box.h
97  # for the 8 vertex locations. Code here derived from mediapipe/python/solutions/drawing_utils.py
98  connections = self.mp_objectron.BOX_CONNECTIONS
99  if connections:
100  num_landmarks = len(landmarks.landmark)
101  for connection in connections:
102  start_idx = connection[0]
103  end_idx = connection[1]
104  if not (0 <= start_idx < num_landmarks and 0 <= end_idx < num_landmarks):
105  raise ValueError(f'Landmark index is out of range. Invalid connection '
106  f'from landmark #{start_idx} to landmark #{end_idx}.')
107  if start_idx in idx_to_coordinates and end_idx in idx_to_coordinates:
108  helper.drawLine(idx_to_coordinates[start_idx].x, idx_to_coordinates[start_idx].y,
109  idx_to_coordinates[end_idx].x, idx_to_coordinates[end_idx].y, col)
110 
111  # ###################################################################################################
112  def draw_axis(self, helper, iw, ih, rotation, translation, axis_length):
113  focal_length = (1.0, 1.0)
114  principal_point = (0.0, 0.0)
115  axis_world = np.float32([[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]])
116  axis_cam = np.matmul(rotation, axis_length * axis_world.T).T + translation
117  x = axis_cam[..., 0]
118  y = axis_cam[..., 1]
119  z = axis_cam[..., 2]
120 
121  # Project 3D points to NDC space.
122  fx, fy = focal_length
123  px, py = principal_point
124  x_ndc = np.clip(-fx * x / (z + 1e-5) + px, -1., 1.)
125  y_ndc = np.clip(-fy * y / (z + 1e-5) + py, -1., 1.)
126 
127  # Convert from NDC space to image space.
128  x_im = (1 + x_ndc) * 0.5 * iw
129  y_im = (1 - y_ndc) * 0.5 * ih
130 
131  # Draw, converting coords from low-res processing frame to full-res display frame:
132  orig = helper.i2d(x_im[0], y_im[0], "c")
133  xa = helper.i2d(x_im[1], y_im[1], "c")
134  ya = helper.i2d(x_im[2], y_im[2], "c")
135  za = helper.i2d(x_im[3], y_im[3], "c")
136  helper.drawLine(orig.x, orig.y, xa.x, xa.y, 0xff0000ff)
137  helper.drawLine(orig.x, orig.y, ya.x, ya.y, 0xff00ff00)
138  helper.drawLine(orig.x, orig.y, za.x, za.y, 0xffff0000)
139 
PyObjectron.PyObjectron.__init__
def __init__(self)
Constructor.
Definition: PyObjectron.py:33
PyObjectron.PyObjectron.draw_axis
def draw_axis(self, helper, iw, ih, rotation, translation, axis_length)
Definition: PyObjectron.py:112
PyObjectron.PyObjectron.objectron
objectron
Definition: PyObjectron.py:45
PyObjectron.PyObjectron
3D object detection using MediaPipe
Definition: PyObjectron.py:30
PyObjectron.PyObjectron.draw_landmarks
def draw_landmarks(self, helper, iw, ih, landmarks)
Definition: PyObjectron.py:87
PyObjectron.PyObjectron.mp_objectron
mp_objectron
Definition: PyObjectron.py:44
PyObjectron.PyObjectron.processGUI
def processGUI(self, inframe, helper)
Process function with GUI output.
Definition: PyObjectron.py:54
PyObjectron.PyObjectron.timer
timer
Definition: PyObjectron.py:41
jevois::Timer