JeVoisBase  1.5
JeVois Smart Embedded Machine Vision Toolkit Base Modules
Share this page:
ObjectRecognition.C
Go to the documentation of this file.
1 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
2 //
3 // JeVois Smart Embedded Machine Vision Toolkit - Copyright (C) 2016 by Laurent Itti, the University of Southern
4 // California (USC), and iLab at USC. See http://iLab.usc.edu and http://jevois.org for information about this project.
5 //
6 // This file is part of the JeVois Smart Embedded Machine Vision Toolkit. This program is free software; you can
7 // redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software
8 // Foundation, version 2. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
9 // without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
10 // License for more details. You should have received a copy of the GNU General Public License along with this program;
11 // if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
12 //
13 // Contact information: Laurent Itti - 3641 Watt Way, HNB-07A - Los Angeles, CA 90089-2520 - USA.
14 // Tel: +1 213 740 3527 - itti@pollux.usc.edu - http://iLab.usc.edu - http://jevois.org
15 // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
16 /*! \file */
17 
19 #include <jevois/Debug/Log.H>
20 #include <fstream>
21 
22 #include "tiny-dnn/tiny_dnn/tiny_dnn.h"
23 
24 // ####################################################################################################
25 ObjectRecognitionBase::ObjectRecognitionBase(std::string const & instance) :
26  jevois::Component(instance)
27 { }
28 
29 // ####################################################################################################
31 { }
32 
33 // ####################################################################################################
34 template <typename NetType>
35 ObjectRecognition<NetType>::ObjectRecognition(std::string const & instance) :
36  ObjectRecognitionBase(instance), net(new tiny_dnn::network<NetType>())
37 { }
38 
39 // ####################################################################################################
40 template <typename NetType>
42 {
43  // Load from file, if available, otherwise trigger training: NOTE: With tiny-dnn as of September 2017, saving a binary
44  // archive on amd64 host and then loading it back on 32-bit ARM platform fails. So we here save as portable json as
45  // well as binary with a suffix for host vs platform. The first time the platform tries to load the binary, it will
46  // fail, and we will load the json and then overwrite the binary. */
47 #ifdef JEVOIS_PLATFORM
48  std::string const wpath = JEVOIS_SHARE_PATH "/tiny-dnn/" + instanceName() + "/weights.tnn.platform";
49 #else
50  std::string const wpath = JEVOIS_SHARE_PATH "/tiny-dnn/" + instanceName() + "/weights.tnn.host";
51 #endif
52  std::string const jpath = JEVOIS_SHARE_PATH "/tiny-dnn/" + instanceName() + ".json";
53 
54  try
55  {
56  net->load(wpath);
57  LINFO("Loaded pre-trained weights and model from " << wpath);
58  }
59  catch (...)
60  {
61  try
62  {
63  net->load(jpath, tiny_dnn::content_type::weights_and_model, tiny_dnn::file_format::json);
64  net->save(wpath);
65  LINFO("Converted and loaded pre-trained weights and model from " << wpath);
66  }
67  catch (...)
68  {
70  LINFO("Could not load pre-trained weights and model from " << wpath << " -- start training...");
71 
72  // First define the network (implemented in derived classes):
73  this->define();
74 
75  // Then train it:
76  this->train(JEVOIS_SHARE_PATH "/tiny-dnn/" + instanceName());
77 
78  // Finally save:
79  LINFO("Saving trained weights to " << wpath);
80  try
81  {
82  net->save(wpath);
83  net->save(jpath, tiny_dnn::content_type::weights_and_model, tiny_dnn::file_format::json);
84  LINFO("Weights saved -- Network ready to work.");
85  }
86  catch (...)
87  {
88  // On host, user may have forgotten to give permissions to /jevois, or to run jevois-daemon as root for
89  // training. Try to save to /tmp instead:
91  LINFO("Saving weights to " << wpath << " failed -- trying /tmp/weights.tnn");
92  try
93  {
94  net->save("/tmp/weights.tnn");
95  net->save("/tmp/weights.tnn.json", tiny_dnn::content_type::weights_and_model,
96  tiny_dnn::file_format::json);
97  LINFO("Weights saved to /tmp/weights.tnn -- Network ready to work.");
98  }
99  catch (...) { jevois::warnAndIgnoreException(); LERROR("Could not save weights..."); }
100  }
101  }
102  }
103 }
104 
105 // ####################################################################################################
106 template <typename NetType>
108 { delete net; }
109 
110 // ####################################################################################################
111 template <typename NetType>
112 typename tiny_dnn::index3d<size_t> ObjectRecognition<NetType>::insize() const
113 { return (*net)[0]->in_shape()[0]; }
114 
115 // ####################################################################################################
116 template <typename NetType>
118 ObjectRecognition<NetType>::process(cv::Mat const & img, bool normalize)
119 {
120  auto inshape = (*net)[0]->in_shape()[0];
121 
122  if (img.cols != int(inshape.width_) ||
123  img.rows != int(inshape.height_) ||
124  img.channels() != int(inshape.depth_)) LFATAL("Incorrect input image size or format");
125 
126  // Convert input image to vec_t with values in [-1..1]:
127  size_t const sz = inshape.size();
128  tiny_dnn::vec_t data(sz);
129  unsigned char const * in = img.data; tiny_dnn::float_t * out = &data[0];
130  for (size_t i = 0; i < sz; ++i) *out++ = (*in++) * (2.0F / 255.0F) - 1.0F;
131 
132  // Recognize:
133  if (normalize)
134  {
135  // Get the raw scores:
136  auto scores = net->predict(data);
137 
138  // Normalize activation values between 0...100:
139  tiny_dnn::layer * lastlayer = (*net)[net->depth() - 1];
140  std::pair<tiny_dnn::float_t, tiny_dnn::float_t> outrange = lastlayer->out_value_range();
141  tiny_dnn::float_t const mi = outrange.first;
142  tiny_dnn::float_t const ma = outrange.second;
143 
144  for (tiny_dnn::float_t & s : scores) s = tiny_dnn::float_t(100) * (s - mi) / (ma - mi);
145 
146  return scores;
147  }
148  else
149  return net->predict(data);
150 }
151 
152 // ####################################################################################################
153 // Expplicit instantiations:
ObjectRecognitionBase(std::string const &instance)
Constructor.
std::string warnAndIgnoreException()
virtual void postInit() override
Initialize the network, required before one starts using it.
vec_t process(cv::Mat const &img, bool normalize=true) override
Process an image, results are confidence for each category.
virtual ~ObjectRecognitionBase()
Virtual destructor for safe inheritance.
virtual void define()=0
Define the network structure.
#define LERROR(msg)
std::vector< tiny_dnn::float_t, tiny_dnn::aligned_allocator< tiny_dnn::float_t, 64 > > vec_t
Type used by tiny-dnn for the results:
Network to load This meta parameter sets parameters and namefile for the chosen network
Definition: Darknet.H:39
virtual ~ObjectRecognition()
Destructor.
#define LFATAL(msg)
ObjectRecognition(std::string const &instance)
Constructor allocates the (empty) network.
Abstract base class for an object recognition component.
virtual void train(std::string const &path)=0
Train the network.
tiny_dnn::network< NetType > * net
std::string const & instanceName() const
#define LINFO(msg)
virtual tiny_dnn::index3d< size_t > insize() const override
Get the input size for the current network, useful to prepare inputs to process() ...