JeVois  1.18
JeVois Smart Embedded Machine Vision Toolkit
Share this page:
NetworkHailo.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2022 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
18 #ifdef JEVOIS_PRO
19 
21 #include <jevois/Util/Utils.H>
22 #include <jevois/DNN/Utils.H>
23 
24 #include <hailo/hailort.h>
25 #include <hailo/hailort.hpp>
26 
27 // This code modified from tapas/apps/native/detection/detection_app.cpp
28 // Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved.
29 // Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt)
30 
31 #define HAILO_CHECK(obj, msg) do { if (!obj) LFATAL(msg << ": " << obj.status() << " [" << \
32  hailo_get_status_message(obj.status()) << ']'); } while (0)
33 
34 // ####################################################################################################
35 jevois::dnn::NetworkHailo::NetworkHailo(std::string const & instance) :
36  jevois::dnn::Network(instance)
37 {
38 
39  /* Skip this test for now... The overlay need to be tuned up to actually bind to the hailo chip.
40 
41 #ifdef JEVOIS_PLATFORM
42  try { (void)jevois::getFileString("/proc/device-tree/reserved-memory/linux,hailo_cma/name"); }
43  catch (...)
44  {
45  LFATAL("Using the Hailo8 accelerator requires some extra Linux kernel configuration.\n\n"
46  "Please edit '/boot/env.txt' (you can use the editor from the Config tab of the JeVois-Pro GUI), "
47  "and towards the end, add 'hailo' to the space-separated list of overlays.\n\n"
48  "Then reboot and it should be ready to go.");
49  }
50 #endif
51 
52  */
53 }
54 
55 // ####################################################################################################
56 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::NetworkHailo::inputShapes()
57 { return itsInAttrs; }
58 
59 // ####################################################################################################
60 std::vector<vsi_nn_tensor_attr_t> jevois::dnn::NetworkHailo::outputShapes()
61 { return itsOutAttrs; }
62 
63 // ####################################################################################################
65 {
66  // If we are loading the network via load() running in a thread, wait until that is done before we destroy
67  // (base class jevois::dnn::Network handles this):
68  waitBeforeDestroy();
69 }
70 
71 // ####################################################################################################
73 {
74  dataroot::freeze(doit);
75  model::freeze(doit);
76 }
77 
78 // ####################################################################################################
80 {
81  // We can only load once...
82  if (itsDevice) LFATAL("Network already loaded... restart the module to load a new one.");
83 
84  // Create a device and load the HEF network file:
85  auto dev = hailort::Device::create_pcie();
86  if (!dev) LFATAL("Failed to create PCIe device:" << dev.status());
87  itsDevice = dev.release();
88 
89  /*
90  // FIXME: looks like there may be a memory leak in the Hailo PCIe driver. It may not always deallocate all DMA
91  // coherent memory. Try a reset each time we load a new network:
92  itsDevice->reset(HAILO_RESET_DEVICE_MODE_FORCED_SOFT);
93  itsDevice.reset();
94  std::this_thread::sleep_for(std::chrono::milliseconds(20));
95  (void)jevois::system("/usr/sbin/rmmod hailo_pci");
96  std::this_thread::sleep_for(std::chrono::milliseconds(20));
97  (void)jevois::system("/usr/sbin/modprobe hailo_pci");
98  std::this_thread::sleep_for(std::chrono::milliseconds(20));
99 
100  // After reset we need a new Device object:
101  auto dev2 = hailort::Device::create_pcie();
102  if (!dev2) LFATAL("Failed to create PCIe device:" << dev2.status());
103  itsDevice = dev2.release();
104  */
105 
106  // Load the network from HEF:
107  std::string const m = jevois::absolutePath(dataroot::get(), model::get());
108  LINFO("Loading HEF file " << m << " ...");
109  auto hef = hailort::Hef::create(m);
110  HAILO_CHECK(hef, "Failed to load HEF file " << m);
111 
112  std::vector<std::string> ngn = hef->get_network_groups_names();
113  for (std::string const & n : ngn) LINFO("Network Group: " << n);
114  if (ngn.size() != 1) LERROR("More than one network groups in HEF -- USING FIRST ONE");
115 
116  auto configure_params = hef->create_configure_params(HAILO_STREAM_INTERFACE_PCIE);
117  HAILO_CHECK(configure_params, "Could not configure params from HEF file " << m);
118 
119  auto network_groups = itsDevice->configure(hef.value(), configure_params.value());
120  HAILO_CHECK(network_groups, "Could not configure device");
121  if (network_groups->empty()) LFATAL("HEF file " << m << " does not contain any network groups");
122  itsNetGroup = std::move(network_groups->at(0)); // use the first network group
123 
124  // Configure the input and output virtual streams:
125  constexpr bool QUANTIZED = true;
126  constexpr hailo_format_type_t FORMAT_TYPE = HAILO_FORMAT_TYPE_AUTO;
127 
128  auto vstreams = hailort::VStreamsBuilder::create_vstreams(*itsNetGroup, QUANTIZED, FORMAT_TYPE);
129  HAILO_CHECK(vstreams, "Failed to create vstreams");
130 
131  itsInStreams = std::move(vstreams->first);
132  itsOutStreams = std::move(vstreams->second);
133 
134  // Activate the network group:
135  auto activated_network_group = itsNetGroup->activate();
136  HAILO_CHECK(activated_network_group, "Failed activating network group");
137  itsActiveNetGroup = activated_network_group.release();
138 
139  // Get the input and output attributes, allocate some cv::Mat for the outputs:
140  for (auto const & vs : itsInStreams)
141  {
142  itsInAttrs.emplace_back(jevois::dnn::tensorattr(vs.get_info()));
143  auto const & attr = itsInAttrs.back();
144  LINFO("Input " << vs.name() << ": " << jevois::dnn::attrstr(attr));
145  }
146 
147  for (auto const & vs : itsOutStreams)
148  {
149  itsOutAttrs.emplace_back(jevois::dnn::tensorattr(vs.get_info()));
150  auto const & attr = itsOutAttrs.back();
151  LINFO("Output " << vs.name() << ": " << jevois::dnn::attrstr(attr));
152  itsRawOutMats.emplace_back(jevois::dnn::attrmat(attr));
153  itsOutMats.emplace_back(cv::Mat(jevois::dnn::attrdims(attr), CV_32F));
154  }
155 
156  // Turbo parameter may have been changed before or while we loaded, so set it here:
157  itsDevice->set_throttling_state(! turbo::get());
158 }
159 
160 // ####################################################################################################
161 void jevois::dnn::NetworkHailo::onParamChange(network::turbo const & JEVOIS_UNUSED_PARAM(par), bool const & newval)
162 {
163  if (itsDevice) itsDevice->set_throttling_state(! newval);
164 }
165 
166 // ####################################################################################################
167 std::vector<cv::Mat> jevois::dnn::NetworkHailo::doprocess(std::vector<cv::Mat> const & blobs,
168  std::vector<std::string> & info)
169 {
170  if (blobs.size() != itsInStreams.size())
171  LFATAL("Received " << blobs.size() << " blobs, but network has " << itsInStreams.size() << " inputs");
172 
173  std::string err;
174  for (size_t i = 0; i < blobs.size(); ++i)
175  if (jevois::dnn::attrmatch(itsInAttrs[i], blobs[i]) == false)
176  err += "Input " + std::to_string(i) + ": received " + jevois::dnn::shapestr(blobs[i]) +
177  " but want: " + jevois::dnn::shapestr(itsInAttrs[i]) + "\n";
178  if (err.empty() == false) LFATAL(err);
179 
180  // Launch the output reader threads (device->host) first:
181  std::vector<std::future<std::string>> fvec(itsInStreams.size() + itsOutStreams.size());
182  bool const dq = dequant::get();
183 
184  for (uint32_t i = 0; i < itsOutStreams.size(); ++i)
185  fvec[i + itsInStreams.size()] = jevois::async([this](uint32_t i, bool dq) -> std::string
186  {
187  auto const & attr = itsOutAttrs[i];
188  uint8_t * tensor_data = (uint8_t *)itsRawOutMats[i].data;
189  size_t const sz = itsRawOutMats[i].total();
190 
191  auto status = itsOutStreams[i].read(hailort::MemoryView(tensor_data, sz));
192  if (status != HAILO_SUCCESS) LFATAL("Failed collecting output " << i << " from device: " << status);
193 
194  if (dq)
195  {
196  itsOutMats[i] = jevois::dnn::dequantize(itsRawOutMats[i], attr);
197  return "- Out " + std::to_string(i) + ": " + jevois::dnn::attrstr(attr) + " -> 32F";
198  }
199  else
200  {
201  itsOutMats[i] = itsRawOutMats[i];
202  return "- Out " + std::to_string(i) + ": " + jevois::dnn::attrstr(attr);
203  }
204 
205  }, i, dq);
206 
207  // Launch the input writing (host->device) threads:
208  for (size_t b = 0; b < blobs.size(); ++b)
209  fvec[b] = jevois::async([this, &blobs](size_t b) -> std::string
210  {
211  cv::Mat const & blob = blobs[b];
212  size_t const sz = blob.total() * blob.elemSize();
213 
214  // Copy blob data to device:
215  auto status = itsInStreams[b].write(hailort::MemoryView(blob.data, sz));
216  if (status != HAILO_SUCCESS) LFATAL("Failed writing input " << b << " data to device: " << status);
217 
218  return "- In " + std::to_string(b) + ": " + jevois::dnn::attrstr(itsInAttrs[b]);
219  }, b);
220 
221  // Add some info, once in a while:
222  static std::string devstr = "- Hailo8: ---W, ---C";
223  if ((jevois::frameNum() % 30) == 0)
224  {
225  bool throttle = itsDevice->get_throttling_state().value();
226  hailo_chip_temperature_info_t const temp = itsDevice->get_chip_temperature().value();
227  float pwr = itsDevice->power_measurement(HAILO_DVM_OPTIONS_AUTO, HAILO_POWER_MEASUREMENT_TYPES__POWER).value();
228 
229  devstr = jevois::sformat("- Hailo8: %.1fW, %.0fC%s", pwr, temp.ts0_temperature, throttle ? "" : " (turbo)");
230  }
231 
232  // Join all threads (may throw a single combined exception):
233  std::vector<std::string> retvec = jevois::joinall(fvec);
234  info.insert(info.end(), std::make_move_iterator(retvec.begin()), std::make_move_iterator(retvec.end()));
235  info.emplace_back(devstr);
236 
237  return itsOutMats;
238 }
239 
240 #endif // JEVOIS_PRO
jevois::imu::get
Data collection mode RAW means that the latest available raw data is returned each time get() is called
NetworkHailo.H
jevois::async
std::future< std::invoke_result_t< std::decay_t< Function >, std::decay_t< Args >... > > async(Function &&f, Args &&... args)
Async execution using a thread pool.
jevois::sformat
std::string sformat(char const *fmt,...) __attribute__((format(__printf__
Create a string using printf style arguments.
Definition: Utils.C:419
jevois::dnn::NetworkHailo::outputShapes
virtual std::vector< vsi_nn_tensor_attr_t > outputShapes() override
Get shapes of all output tensors.
Definition: NetworkHailo.C:60
jevois::dnn::NetworkHailo::doprocess
std::vector< cv::Mat > doprocess(std::vector< cv::Mat > const &blobs, std::vector< std::string > &info) override
Process input blobs and obtain output blobs.
Definition: NetworkHailo.C:167
Utils.H
jevois::dnn::dequantize
cv::Mat dequantize(cv::Mat const &m, vsi_nn_tensor_attr_t const &attr)
Dequantize an output to float32 according to the quantization spec in attr.
Definition: Utils.C:714
jevois::dnn::tensorattr
vsi_nn_tensor_attr_t tensorattr(TfLiteTensor const *t)
Get tensor shape and type attributes for a TensorFlow Lite tensor.
Definition: Utils.C:484
jevois::dnn::NetworkHailo::freeze
void freeze(bool doit) override
Freeze/unfreeze parameters that users should not change while running.
Definition: NetworkHailo.C:72
jevois::dnn::NetworkHailo::inputShapes
virtual std::vector< vsi_nn_tensor_attr_t > inputShapes() override
Get shapes of all input tensors.
Definition: NetworkHailo.C:56
LERROR
#define LERROR(msg)
Convenience macro for users to print out console or syslog messages, ERROR level.
Definition: Log.H:211
jevois::dnn::attrmatch
bool attrmatch(vsi_nn_tensor_attr_t const &attr, cv::Mat const &blob)
Check that a cv::Mat blob matches exactly the spec of an attr.
Definition: Utils.C:625
jevois::absolutePath
std::filesystem::path absolutePath(std::filesystem::path const &root, std::filesystem::path const &path)
Compute an absolute path from two paths.
Definition: Utils.C:365
jevois::dnn::shapestr
std::string shapestr(cv::Mat const &m)
Get a string of the form: "nD AxBxC... TYPE" from an n-dimensional cv::Mat with data type TYPE.
Definition: Utils.C:104
jevois
Definition: Concepts.dox:1
jevois::joinall
std::vector< T > joinall(std::vector< std::future< T >> &fvec, bool multiline=true)
Collect results from several async threads that are all returning a T result.
LFATAL
#define LFATAL(msg)
Convenience macro for users to print out console or syslog messages, FATAL level.
jevois::dnn::attrdims
std::vector< int > attrdims(vsi_nn_tensor_attr_t const &attr)
Get a tensor dims as a vector of int, useful to construct a matching cv::Mat.
Definition: Utils.C:399
HAILO_CHECK
#define HAILO_CHECK(obj, msg)
Definition: NetworkHailo.C:31
jevois::to_string
std::string to_string(T const &val)
Convert from type to string.
jevois::dnn::attrstr
std::string attrstr(vsi_nn_tensor_attr_t const &attr)
Get a string describing the specs of a tensor, including quantification specs (not provided by shapes...
Definition: Utils.C:433
jevois::dnn::attrmat
cv::Mat attrmat(vsi_nn_tensor_attr_t const &attr, void *dataptr=nullptr)
Construct a cv::Mat from attr and possibly data pointer.
Definition: Utils.C:392
Utils.H
jevois::dnn::NetworkHailo::NetworkHailo
NetworkHailo(std::string const &instance)
Constructor.
Definition: NetworkHailo.C:35
LINFO
#define LINFO(msg)
Convenience macro for users to print out console or syslog messages, INFO level.
Definition: Log.H:194
jevois::dnn::NetworkHailo::onParamChange
void onParamChange(network::turbo const &par, bool const &newval) override
Definition: NetworkHailo.C:161
jevois::dnn::NetworkHailo::~NetworkHailo
virtual ~NetworkHailo()
Destructor.
Definition: NetworkHailo.C:64
jevois::dnn::Network
Abstract class to represent a neural network.
Definition: Network.H:153
jevois::dnn::NetworkHailo::load
void load() override
Load from disk.
Definition: NetworkHailo.C:79