JeVoisBase  1.18
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
benchmark.py
Go to the documentation of this file.
1 import os
2 import argparse
3 
4 import yaml
5 import numpy as np
6 import cv2 as cv
7 
8 from models import MODELS
9 from utils import METRICS, DATALOADERS
10 
11 parser = argparse.ArgumentParser("Benchmarks for OpenCV Zoo.")
12 parser.add_argument('--cfg', '-c', type=str,
13  help='Benchmarking on the given config.')
14 args = parser.parse_args()
15 
16 def build_from_cfg(cfg, registery, key=None, name=None):
17  if key is not None:
18  obj_name = cfg.pop(key)
19  obj = registery.get(obj_name)
20  return obj(**cfg)
21  elif name is not None:
22  obj = registery.get(name)
23  return obj(**cfg)
24  else:
25  raise NotImplementedError()
26 
28  for k, v in cfg.items():
29  if isinstance(v, dict):
31  else:
32  if 'path' in k.lower():
33  cfg[k] = os.path.join(os.environ['PYTHONPATH'], v)
34 
35 class Benchmark:
36  def __init__(self, **kwargs):
37  self._type = kwargs.pop('type', None)
38  if self._type is None:
39  self._type = 'Base'
40  print('Benchmark[\'type\'] is omitted, set to \'Base\' by default.')
41 
42  self._data_dict = kwargs.pop('data', None)
43  assert self._data_dict, 'Benchmark[\'data\'] cannot be empty and must have path and files.'
44  if 'type' in self._data_dict:
45  self._dataloader = build_from_cfg(self._data_dict, registery=DATALOADERS, key='type')
46  else:
47  self._dataloader = build_from_cfg(self._data_dict, registery=DATALOADERS, name=self._type)
48 
49  self._metric_dict = kwargs.pop('metric', None)
50  assert self._metric_dict, 'Benchmark[\'metric\'] cannot be empty.'
51  if 'type' in self._metric_dict:
52  self._metric = build_from_cfg(self._metric_dict, registery=METRICS, key='type')
53  else:
54  self._metric = build_from_cfg(self._metric_dict, registery=METRICS, name=self._type)
55 
56  backend_id = kwargs.pop('backend', 'default')
57  available_backends = dict(
58  default=cv.dnn.DNN_BACKEND_DEFAULT,
59  # halide=cv.dnn.DNN_BACKEND_HALIDE,
60  # inference_engine=cv.dnn.DNN_BACKEND_INFERENCE_ENGINE,
61  opencv=cv.dnn.DNN_BACKEND_OPENCV,
62  # vkcom=cv.dnn.DNN_BACKEND_VKCOM,
63  cuda=cv.dnn.DNN_BACKEND_CUDA,
64  )
65 
66  target_id = kwargs.pop('target', 'cpu')
67  available_targets = dict(
68  cpu=cv.dnn.DNN_TARGET_CPU,
69  # opencl=cv.dnn.DNN_TARGET_OPENCL,
70  # opencl_fp16=cv.dnn.DNN_TARGET_OPENCL_FP16,
71  # myriad=cv.dnn.DNN_TARGET_MYRIAD,
72  # vulkan=cv.dnn.DNN_TARGET_VULKAN,
73  # fpga=cv.dnn.DNN_TARGET_FPGA,
74  cuda=cv.dnn.DNN_TARGET_CUDA,
75  cuda_fp16=cv.dnn.DNN_TARGET_CUDA_FP16,
76  # hddl=cv.dnn.DNN_TARGET_HDDL,
77  )
78 
79  # add extra backends & targets
80  try:
81  available_backends['timvx'] = cv.dnn.DNN_BACKEND_TIMVX
82  available_targets['npu'] = cv.dnn.DNN_TARGET_NPU
83  except:
84  print('OpenCV is not compiled with TIM-VX backend enbaled. See https://github.com/opencv/opencv/wiki/TIM-VX-Backend-For-Running-OpenCV-On-NPU for more details on how to enable TIM-VX backend.')
85 
86  self._backend = available_backends[backend_id]
87  self._target = available_targets[target_id]
88 
89  self._benchmark_results = dict()
90 
91  def run(self, model):
92  model.setBackend(self._backend)
93  model.setTarget(self._target)
94 
95  for idx, data in enumerate(self._dataloader):
96  filename, input_data = data[:2]
97  if filename not in self._benchmark_results:
98  self._benchmark_results[filename] = dict()
99  if isinstance(input_data, np.ndarray):
100  size = [input_data.shape[1], input_data.shape[0]]
101  else:
102  size = input_data.getFrameSize()
103  self._benchmark_results[filename][str(size)] = self._metric.forward(model, *data[1:])
104 
105  def printResults(self):
106  for imgName, results in self._benchmark_results.items():
107  print(' image: {}'.format(imgName))
108  total_latency = 0
109  for key, latency in results.items():
110  total_latency += latency
111  print(' {}, latency ({}): {:.4f} ms'.format(key, self._metric.getReduction(), latency))
112 
113 if __name__ == '__main__':
114  assert args.cfg.endswith('yaml'), 'Currently support configs of yaml format only.'
115  with open(args.cfg, 'r') as f:
116  cfg = yaml.safe_load(f)
117 
118  # prepend PYTHONPATH to each path
119  prepend_pythonpath(cfg)
120 
121  # Instantiate benchmarking
122  benchmark = Benchmark(**cfg['Benchmark'])
123 
124  # Instantiate model
125  model = build_from_cfg(cfg=cfg['Model'], registery=MODELS, key='name')
126 
127  # Run benchmarking
128  print('Benchmarking {}:'.format(model.name))
129  benchmark.run(model)
130  benchmark.printResults()
benchmark.Benchmark
Definition: benchmark.py:35
demo.str
str
Definition: demo.py:35
benchmark.Benchmark._dataloader
_dataloader
Definition: benchmark.py:45
benchmark.prepend_pythonpath
def prepend_pythonpath(cfg)
Definition: benchmark.py:27
benchmark.Benchmark.run
def run(self, model)
Definition: benchmark.py:91
benchmark.Benchmark.printResults
def printResults(self)
Definition: benchmark.py:105
benchmark.Benchmark._metric_dict
_metric_dict
Definition: benchmark.py:49
benchmark.build_from_cfg
def build_from_cfg(cfg, registery, key=None, name=None)
Definition: benchmark.py:16
benchmark.Benchmark._benchmark_results
_benchmark_results
Definition: benchmark.py:89
benchmark.Benchmark._target
_target
Definition: benchmark.py:87
benchmark.Benchmark._data_dict
_data_dict
Definition: benchmark.py:42
benchmark.Benchmark._type
_type
Definition: benchmark.py:37
benchmark.Benchmark._backend
_backend
Definition: benchmark.py:86
benchmark.Benchmark.__init__
def __init__(self, **kwargs)
Definition: benchmark.py:36
benchmark.Benchmark._metric
_metric
Definition: benchmark.py:52