JeVoisBase  1.16
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
PyDetectionDNN.py
Go to the documentation of this file.
1 import pyjevois
2 if pyjevois.pro: import libjevoispro as jevois
3 else: import libjevois as jevois
4 import cv2 as cv
5 import numpy as np
6 import sys
7 
8 ## Object detection and recognition using OpenCV Deep Neural Networks (DNN)
9 #
10 # This module runs an object detection deep neural network using the OpenCV DNN
11 # library. Detection networks analyze a whole scene and produce a number of
12 # bounding boxes around detected objects, together with identity labels
13 # and confidence scores for each detected box.
14 #
15 # This module supports detection networks implemented in TensorFlow, Caffe,
16 # Darknet, Torch, etc as supported by the OpenCV DNN module.
17 #
18 # Included with the standard JeVois distribution are:
19 #
20 # - OpenCV Face Detector, Caffe model
21 # - MobileNet + SSD trained on Pascal VOC (20 object classes), Caffe model
22 # - MobileNet + SSD trained on Coco (80 object classes), TensorFlow model
23 # - MobileNet v2 + SSD trained on Coco (80 object classes), TensorFlow model
24 # - Darknet Tiny YOLO v3 trained on Coco (80 object classes), Darknet model
25 # - Darknet Tiny YOLO v2 trained on Pascal VOC (20 object classes), Darknet model
26 #
27 # See the module's constructor (__init__) code and select a value for \b model to switch network. Object categories are
28 # as follows:
29 # - The 80 COCO object categories are: person, bicycle, car, motorbike, aeroplane, bus, train, truck, boat, traffic,
30 # fire, stop, parking, bench, bird, cat, dog, horse, sheep, cow, elephant, bear, zebra, giraffe, backpack, umbrella,
31 # handbag, tie, suitcase, frisbee, skis, snowboard, sports, kite, baseball, baseball, skateboard, surfboard, tennis,
32 # bottle, wine, cup, fork, knife, spoon, bowl, banana, apple, sandwich, orange, broccoli, carrot, hot, pizza, donut,
33 # cake, chair, sofa, pottedplant, bed, diningtable, toilet, tvmonitor, laptop, mouse, remote, keyboard, cell,
34 # microwave, oven, toaster, sink, refrigerator, book, clock, vase, scissors, teddy, hair, toothbrush.
35 #
36 # - The 20 Pascal-VOC object categories are: aeroplane, bicycle, bird, boat, bottle, bus, car, cat, chair, cow,
37 # diningtable, dog, horse, motorbike, person, pottedplant, sheep, sofa, train, tvmonitor.
38 #
39 # Sometimes it will make mistakes! The performance of yolov3-tiny is about 33.1% correct (mean average precision) on
40 # the COCO test set. The OpenCV Face Detector is quite fast and robust!
41 #
42 # This module is adapted from the sample OpenCV code:
43 # https://github.com/opencv/opencv/blob/master/samples/dnn/object_detection.py
44 #
45 # More pre-trained models are available on github in opencv_extra
46 #
47 #
48 # @author Laurent Itti
49 #
50 # @videomapping YUYV 640 502 20.0 YUYV 640 480 20.0 JeVois PyDetectionDNN
51 # @email itti@usc.edu
52 # @address 880 W 1st St Suite 807, Los Angeles CA 90012, USA
53 # @copyright Copyright (C) 2018 by Laurent Itti
54 # @mainurl http://jevois.org
55 # @supporturl http://jevois.org
56 # @otherurl http://jevois.org
57 # @license GPL v3
58 # @distribution Unrestricted
59 # @restrictions None
60 # @ingroup modules
62  # ####################################################################################################
63  ## Constructor
64  def __init__(self):
65  self.confThreshold = 0.5 # Confidence threshold (0..1), higher for stricter detection confidence.
66  self.nmsThreshold = 0.4 # Non-maximum suppression threshold (0..1), higher to remove more duplicate boxes.
67  self.inpWidth = 160 # Resized image width passed to network
68  self.inpHeight = 120 # Resized image height passed to network
69  self.scale = 2/255 # Value scaling factor applied to input pixels
70  self.mean = [127.5, 127.5, 127.5] # Mean BGR value subtracted from input image
71  self.rgb = True # True if model expects RGB inputs, otherwise it expects BGR
72 
73  # Select one of the models:
74  model = 'Face' # OpenCV Face Detector, Caffe model
75  #model = 'MobileNetV2SSD' # MobileNet v2 + SSD trained on Coco (80 object classes), TensorFlow model
76  #model = 'MobileNetSSD' # MobileNet + SSD trained on Pascal VOC (20 object classes), Caffe model
77  #model = 'MobileNetSSDcoco' # MobileNet + SSD trained on Coco (80 object classes), TensorFlow model
78  #model = 'YOLOv3' # Darknet Tiny YOLO v3 trained on Coco (80 object classes), Darknet model
79  #model = 'YOLOv2' # Darknet Tiny YOLO v2 trained on Pascal VOC (20 object classes), Darknet model
80 
81  # You should not have to edit anything beyond this point.
82  backend = cv.dnn.DNN_BACKEND_OPENCV
83  target = cv.dnn.DNN_TARGET_CPU
84  self.classes = None
85  classnames = None
86  if (model == 'MobileNetSSD'):
87  classnames = pyjevois.share + '/darknet/yolo/data/voc.names'
88  modelname = pyjevois.share + '/opencv-dnn/detection/MobileNetSSD_deploy.caffemodel'
89  configname = pyjevois.share + '/opencv-dnn/detection/MobileNetSSD_deploy.prototxt'
90  self.rgb = False
91  elif (model == 'MobileNetV2SSD'):
92  classnames = pyjevois.share + '/darknet/yolo/data/coco.names'
93  modelname = pyjevois.share + '/opencv-dnn/detection/ssd_mobilenet_v2_coco_2018_03_29.pb'
94  configname = pyjevois.share + '/opencv-dnn/detection/ssd_mobilenet_v2_coco_2018_03_29.pbtxt'
95  elif (model == 'MobileNetSSDcoco'):
96  classnames = pyjevois.share + '/darknet/yolo/data/coco.names'
97  modelname = pyjevois.share + '/opencv-dnn/detection/ssd_mobilenet_v1_coco_2017_11_17.pb'
98  configname = pyjevois.share + '/opencv-dnn/detection/ssd_mobilenet_v1_coco_2017_11_17.pbtxt'
99  self.rgb = False
100  self.nmsThreshold = 0.1
101  elif (model == 'YOLOv3'):
102  classnames = pyjevois.share + '/darknet/yolo/data/coco.names'
103  modelname = pyjevois.share + '/darknet/yolo/weights/yolov3-tiny.weights'
104  configname = pyjevois.share + '/darknet/yolo/cfg/yolov3-tiny.cfg'
105  elif (model == 'YOLOv2'):
106  classnames = pyjevois.share + '/darknet/yolo/data/voc.names'
107  modelname = pyjevois.share + '/darknet/yolo/weights/yolov2-tiny-voc.weights'
108  configname = pyjevois.share + '/darknet/yolo/cfg/yolov2-tiny-voc.cfg'
109  self.inpWidth = 320
110  self.inpHeight = 240
111  else:
112  classnames = pyjevois.share + '/opencv-dnn/detection/opencv_face_detector.classes'
113  modelname = pyjevois.share + '/opencv-dnn/detection/opencv_face_detector.caffemodel'
114  configname = pyjevois.share + '/opencv-dnn/detection/opencv_face_detector.prototxt'
115  self.scale = 1.0
116  self.mean = [104.0, 177.0, 123.0]
117  self.rgb = False
118 
119  # Load names of classes
120  if classnames:
121  with open(classnames, 'rt') as f:
122  self.classes = f.read().rstrip('\n').split('\n')
123 
124  # Load a network
125  self.net = cv.dnn.readNet(modelname, configname)
126  self.net.setPreferableBackend(backend)
127  self.net.setPreferableTarget(target)
128  self.timer = jevois.Timer('Neural detection', 10, jevois.LOG_DEBUG)
129  self.model = model
130 
131  # ####################################################################################################
132  ## Get names of the network's output layers
133  def getOutputsNames(self, net):
134  layersNames = self.net.getLayerNames()
135  return [layersNames[i[0] - 1] for i in net.getUnconnectedOutLayers()]
136 
137  # ####################################################################################################
138  ## Analyze and draw boxes, object names, and confidence scores
139  def postprocess(self, frame, outs):
140  frameHeight = frame.shape[0]
141  frameWidth = frame.shape[1]
142 
143  def drawPred(classId, conf, left, top, right, bottom):
144  # Draw a bounding box.
145  cv.rectangle(frame, (left, top), (right, bottom), (0, 255, 0), 2)
146 
147  label = '%.2f' % (conf * 100)
148 
149  # Print a label of class.
150  if self.classes:
151  if (classId >= len(self.classes)):
152  label = 'Oooops id=%d: %s' % (classId, label)
153  else:
154  label = '%s: %s' % (self.classes[classId], label)
155 
156  labelSize, baseLine = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.4, 1)
157  top = max(top, labelSize[1])
158  cv.rectangle(frame, (left, top - labelSize[1]-2), (left + labelSize[0], top + baseLine),
159  (255, 255, 255), cv.FILLED)
160  cv.putText(frame, label, (left, top), cv.FONT_HERSHEY_SIMPLEX, 0.4, (0, 0, 0))
161 
162  layerNames = self.net.getLayerNames()
163  lastLayerId = self.net.getLayerId(layerNames[-1])
164  lastLayer = self.net.getLayer(lastLayerId)
165 
166  classIds = []
167  confidences = []
168  boxes = []
169  if self.net.getLayer(0).outputNameToIndex('im_info') != -1: # Faster-RCNN or R-FCN
170  # Network produces output blob with a shape 1x1xNx7 where N is a number of
171  # detections and an every detection is a vector of values
172  # [batchId, classId, confidence, left, top, right, bottom]
173  for out in outs:
174  for detection in out[0, 0]:
175  confidence = detection[2]
176  if confidence > self.confThreshold:
177  left = int(detection[3])
178  top = int(detection[4])
179  right = int(detection[5])
180  bottom = int(detection[6])
181  width = right - left + 1
182  height = bottom - top + 1
183  classIds.append(int(detection[1]) - 1) # Skip background label
184  confidences.append(float(confidence))
185  boxes.append([left, top, width, height])
186  elif lastLayer.type == 'DetectionOutput':
187  # Network produces output blob with a shape 1x1xNx7 where N is a number of
188  # detections and an every detection is a vector of values
189  # [batchId, classId, confidence, left, top, right, bottom]
190  for out in outs:
191  for detection in out[0, 0]:
192  confidence = detection[2]
193  if confidence > self.confThreshold:
194  left = int(detection[3] * frameWidth)
195  top = int(detection[4] * frameHeight)
196  right = int(detection[5] * frameWidth)
197  bottom = int(detection[6] * frameHeight)
198  width = right - left + 1
199  height = bottom - top + 1
200  classIds.append(int(detection[1]) - 1) # Skip background label
201  confidences.append(float(confidence))
202  boxes.append([left, top, width, height])
203  elif lastLayer.type == 'Region':
204  # Network produces output blob with a shape NxC where N is a number of
205  # detected objects and C is a number of classes + 4 where the first 4
206  # numbers are [center_x, center_y, width, height]
207  classIds = []
208  confidences = []
209  boxes = []
210  for out in outs:
211  for detection in out:
212  scores = detection[5:]
213  classId = np.argmax(scores)
214  confidence = scores[classId]
215  if confidence > self.confThreshold:
216  center_x = int(detection[0] * frameWidth)
217  center_y = int(detection[1] * frameHeight)
218  width = int(detection[2] * frameWidth)
219  height = int(detection[3] * frameHeight)
220  left = int(center_x - width / 2)
221  top = int(center_y - height / 2)
222  classIds.append(classId)
223  confidences.append(float(confidence))
224  boxes.append([left, top, width, height])
225  else:
226  jevois.LERROR('Unknown output layer type: ' + lastLayer.type)
227  return
228 
229  indices = cv.dnn.NMSBoxes(boxes, confidences, self.confThreshold, self.nmsThreshold)
230  for i in indices:
231  i = i[0]
232  box = boxes[i]
233  left = box[0]
234  top = box[1]
235  width = box[2]
236  height = box[3]
237  drawPred(classIds[i], confidences[i], left, top, left + width, top + height)
238 
239  # ####################################################################################################
240  ## JeVois main processing function
241  def process(self, inframe, outframe):
242  frame = inframe.getCvBGR()
243  self.timer.start()
244 
245  frameHeight = frame.shape[0]
246  frameWidth = frame.shape[1]
247 
248  # Create a 4D blob from a frame.
249  blob = cv.dnn.blobFromImage(frame, self.scale, (self.inpWidth, self.inpHeight), self.mean, self.rgb, crop=False)
250 
251  # Run a model
252  self.net.setInput(blob)
253  if self.net.getLayer(0).outputNameToIndex('im_info') != -1: # Faster-RCNN or R-FCN
254  frame = cv.resize(frame, (self.inpWidth, self.inpHeight))
255  self.net.setInput(np.array([self.inpHeight, self.inpWidth, 1.6], dtype=np.float32), 'im_info')
256  outs = self.net.forward(self.getOutputsNames(self.net))
257 
258  self.postprocess(frame, outs)
259 
260  # Create dark-gray (value 80) image for the bottom panel, 22 pixels tall:
261  msgbox = np.zeros((22, frame.shape[1], 3), dtype = np.uint8) + 80
262 
263  # Put efficiency information.
264  cv.putText(frame, 'JeVois Python Object Detection DNN - ' + self.model, (3, 15),
265  cv.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv.LINE_AA)
266  t, _ = self.net.getPerfProfile()
267  fps = self.timer.stop()
268  label = fps + ' - Inference time: %.2fms' % (t * 1000.0 / cv.getTickFrequency())
269  cv.putText(msgbox, label, (3, 15), cv.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv.LINE_AA)
270 
271  # Stack bottom panel below main image:
272  frame = np.vstack((frame, msgbox))
273 
274  # Send output frame to host:
275  outframe.sendCv(frame)
PyDetectionDNN.PyDetectionDNN.rgb
rgb
Definition: PyDetectionDNN.py:71
PyDetectionDNN.PyDetectionDNN.timer
timer
Definition: PyDetectionDNN.py:128
PyDetectionDNN.PyDetectionDNN
Object detection and recognition using OpenCV Deep Neural Networks (DNN)
Definition: PyDetectionDNN.py:61
split
std::vector< std::string > split(std::string const &input, std::string const &regex="\\s+")
PyDetectionDNN.PyDetectionDNN.confThreshold
confThreshold
Definition: PyDetectionDNN.py:65
PyDetectionDNN.PyDetectionDNN.__init__
def __init__(self)
Constructor.
Definition: PyDetectionDNN.py:64
PyDetectionDNN.PyDetectionDNN.getOutputsNames
def getOutputsNames(self, net)
Get names of the network's output layers.
Definition: PyDetectionDNN.py:133
PyDetectionDNN.PyDetectionDNN.classes
classes
Definition: PyDetectionDNN.py:84
PyDetectionDNN.PyDetectionDNN.process
def process(self, inframe, outframe)
JeVois main processing function.
Definition: PyDetectionDNN.py:241
PyDetectionDNN.PyDetectionDNN.mean
mean
Definition: PyDetectionDNN.py:70
PyDetectionDNN.PyDetectionDNN.inpWidth
inpWidth
Definition: PyDetectionDNN.py:67
PyDetectionDNN.PyDetectionDNN.model
model
Definition: PyDetectionDNN.py:129
PyDetectionDNN.PyDetectionDNN.net
net
Definition: PyDetectionDNN.py:125
PyDetectionDNN.PyDetectionDNN.scale
scale
Definition: PyDetectionDNN.py:69
PyDetectionDNN.PyDetectionDNN.nmsThreshold
nmsThreshold
Definition: PyDetectionDNN.py:66
PyDetectionDNN.PyDetectionDNN.inpHeight
inpHeight
Definition: PyDetectionDNN.py:68
PyDetectionDNN.PyDetectionDNN.postprocess
def postprocess(self, frame, outs)
Analyze and draw boxes, object names, and confidence scores.
Definition: PyDetectionDNN.py:139
jevois::Timer