29 #include <opencv2/core/core.hpp>
30 #include <opencv2/imgproc/imgproc.hpp>
31 #include <linux/videodev2.h>
38 {
"Hello! Welcome to this simple demonstration of JeVois", 0, 0 },
39 {
"JeVois = camera sensor + quad-core processor + USB video output", 0, 0 },
40 {
"This demo is running on the small processor inside JeVois", 0, 0 },
41 {
"Neat, isn't it?", 0, 0 },
43 {
"We will help you discover what you see on this screen", 0, 0 },
44 {
"We will use this blinking marker to point at things:", 600, 335 },
46 {
"Now a brief tutorial...", 0, 0 },
48 {
"This demo: Attention + Gist + Faces + Objects", 0, 0 },
49 {
"Attention: detect things that catch the human eye", 0, 0 },
50 {
"Pink square in video above: most interesting (salient) location", 0, 0 },
51 {
"Green circle in video above: smoothed attention trajectory", 0, 0 },
52 {
"Try it: wave at JeVois, show it some objects, move it around", 0, 0 },
54 {
"Did you catch the attention of JeVois?", 0, 0 },
56 {
"Attention is guided by color contrast, ...", 40, 270 },
57 {
"by luminance (intensity) contrast, ...", 120, 270 },
58 {
"by oriented edges, ...", 200, 270 },
59 {
"by flickering or blinking lights, ...", 280, 270 },
60 {
"and by moving objects.", 360, 270 },
61 {
"All these visual cues combine into a measure of saliency", 480, 120 },
62 {
"or visual interest for every location in view.", 480, 120 },
65 {
"Gist: statistical summary of a scene, also based on ...", 440, 270 },
66 {
"color, intensity, orientation, flicker and motion features.", 0, 0 },
67 {
"Gist can be used to recognize places, such as a kitchen or ...", 0, 0 },
68 {
"a bathroom, or a road turning left versus turning right.", 0, 0 },
69 {
"Try it: point JeVois to different things and see gist change", 440, 270 },
72 {
"Face detection finds human faces in the camera's view", 612, 316 },
73 {
"Try it: point JeVois towards a face. Adjust distance until ...", 612, 316 },
74 {
"the face fits inside the attention pink square. When a face ...", 0, 0 },
75 {
"is detected, it will appear in the bottom-right corner.", 612, 316 },
76 {
"You may have to move a bit farther than arm's length for ...", 0, 0 },
77 {
"your face to fit inside the attention pink square.", 0, 0 },
80 {
"Objects: Here we recognize handwritten digits using ...", 525, 316 },
81 {
"deep neural networks. Try it! Draw a number on paper ...", 0, 0 },
82 {
"and point JeVois towards it. Adjust distance until the", 0, 0 },
83 {
"number fits in the attention pink square.", 0, 0 },
85 {
"Recognized digits are shown near the detected faces.", 525, 316 },
87 {
"If your number is too small, too big, or not upright ...", 0, 0 },
88 {
"keep adjusting the distance and angle of the camera.", 0, 0 },
90 {
"Recognition scores for digits 0 to 9 are shown above.", 464, 310 },
91 {
"Sometimes the neural network makes mistakes and thinks it ...", 0, 0 },
92 {
"found a digit when actually it is looking at something else.", 0, 0 },
93 {
"This is still a research issue", 0, 0 },
94 {
"but machine vision is improving fast, so stay tuned!", 0, 0 },
96 {
"With JeVois the future of machine vision is in your hands.", 0, 0 },
100 {
"This tutorial is now complete. It will restart.", 0, 0 },
132 itsSaliency = addSubComponent<Saliency>(
"saliency");
135 itsKF = addSubComponent<Kalman2D>(
"kalman");
136 itsVideo = addSubComponent<BufferedVideoReader>(
"intromovie");
146 cv::Mat banner_bgr = cv::imread(
absolutePath(
"jevois-banner-notext.png"));
155 std::this_thread::sleep_for(std::chrono::milliseconds(750));
162 static cv::Mat itsLastFace(60, 60, CV_8UC2, 0x80aa) ;
163 static cv::Mat itsLastObject(60, 60, CV_8UC2, 0x80aa) ;
164 static std::string itsLastObjectCateg;
165 static bool doobject =
false;
166 static bool intromode =
false;
167 static bool intromoviedone =
false;
168 static ScriptItem const * scriptitem = &TheScript[0];
169 static int scriptframe = 0;
170 unsigned short const txtcol = jevois::yuyv::White;
176 inimg.
require(
"input", 320, 240, V4L2_PIX_FMT_YUYV);
178 itsProcessingTimer.
start();
179 int const roihw = 32;
186 outimg = outframe.get();
187 outimg.
require(
"output", 640, outimg.
height, V4L2_PIX_FMT_YUYV);
192 case 480: intromode = true; break;
193 default: LFATAL(
"Incorrect output height: should be 312, 360 or 480");
197 if (intromode && intromoviedone ==
false)
201 if (m.empty()) intromoviedone =
true;
204 jevois::rawimage::convertCvBGRtoRawImage(m, outimg, 75);
207 if (outimg.height == 480)
208 jevois::rawimage::paste(itsBanner, outimg, 0, 360);
209 else if (outimg.height > 360)
210 jevois::rawimage::drawFilledRect(outimg, 0, 360, outimg.width, outimg.height - 360, 0x8000);
213 if (outimg.height == 480) jevois::rawimage::hFlipYUYV(outimg);
228 if (intromode && intromoviedone ==
false) { outframe.send();
return; }
236 int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0;
237 int const dmx = (mx << smlev) + smadj;
238 int const dmy = (my << smlev) + smadj;
241 int const rx = std::min(
int(inimg.
width) - roihw, std::max(roihw, dmx));
242 int const ry = std::min(
int(inimg.
height) - roihw, std::max(roihw, dmy));
287 else if (outimg.
height > 312)
292 float kfxraw, kfyraw, kfximg, kfyimg;
293 itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, inimg.
width, inimg.
height, 1.0F, 1.0F);
302 if (intromode && intromoviedone)
306 if (scriptframe < 32) lum = scriptframe * 8;
307 else if (scriptframe > 4*30 - 32) lum = std::max(0, (4*30 - scriptframe) * 8);
310 int x = (640 - 10 * strlen(scriptitem->
msg)) / 2;
316 int phase = scriptframe / 10;
318 10, jevois::yuyv::LightTeal);
322 if (++scriptframe >= 140)
324 scriptframe = 0; ++scriptitem;
325 if (scriptitem->
msg ==
nullptr) scriptitem = &TheScript[0];
332 cv::Mat rawroi = rawimgcv(cv::Rect(rx - roihw, ry - roihw, roihw * 2, roihw * 2));
341 switch (objsz.depth_)
348 cv::cvtColor(rawroi, objroi, cv::COLOR_YUV2GRAY_YUYV);
351 size_t const elem = (objroi.cols * objroi.rows * 10) / 100;
352 std::vector<unsigned char> v; v.assign(objroi.datastart, objroi.dataend);
353 std::nth_element(v.begin(), v.begin() + elem, v.end());
354 unsigned char const thresh = std::min((
unsigned char)(100), std::max((
unsigned char)(30), v[elem]));
357 cv::threshold(objroi, objroi, thresh, 255, cv::THRESH_BINARY_INV);
360 cv::Mat pts; cv::findNonZero(objroi, pts);
361 cv::Rect r = cv::boundingRect(pts);
362 int const cx = r.x + r.width / 2;
363 int const cy = r.y + r.height / 2;
364 int const siz = std::min(roihw * 2, std::max(16, 8 + std::max(r.width, r.height)));
365 int const tlx = std::max(0, std::min(roihw*2 - siz, cx - siz/2));
366 int const tly = std::max(0, std::min(roihw*2 - siz, cy - siz/2));
367 cv::Rect ar(tlx, tly, siz, siz);
368 cv::resize(objroi(ar), objroi, cv::Size(objsz.width_, objsz.height_), 0, 0, cv::INTER_AREA);
374 cv::cvtColor(rawroi, objroi, cv::COLOR_YUV2RGB_YUYV);
375 cv::resize(objroi, objroi, cv::Size(objsz.width_, objsz.height_), 0, 0, cv::INTER_AREA);
379 LFATAL(
"Unsupported object detection input depth " << objsz.depth_);
386 std::ostringstream oss;
387 for (
size_t i = 0; i < scores.size(); ++i)
388 oss <<
itsObjectRecognition->category(i) <<
':' << std::fixed << std::setprecision(2) << scores[i] <<
' ';
392 float best1 = scores[0], best2 = scores[0];
size_t idx1 = 0, idx2 = 0;
393 for (
size_t i = 1; i < scores.size(); ++i)
395 if (scores[i] > best1) { best2 = best1; idx2 = idx1; best1 = scores[i]; idx1 = i; }
396 else if (scores[i] > best2) { best2 = scores[i]; idx2 = i; }
400 if (best1 > 90.0
F && best2 < 20.0
F)
404 itsLastObject = rawimgcv(cv::Rect(rx - 30, ry - 30, 60, 60)).clone();
406 LINFO(
"Object recognition: best: " << itsLastObjectCateg <<
" (" << best1 <<
415 cv::Mat grayroi; cv::cvtColor(rawroi, grayroi, cv::COLOR_YUV2GRAY_YUYV);
416 cv::equalizeHist(grayroi, grayroi);
419 std::vector<cv::Rect> faces; std::vector<std::vector<cv::Rect> > eyes;
425 LINFO(
"detected " << faces.size() <<
" faces");
427 itsLastFace = rawimgcv(cv::Rect(rx - 30, ry - 30, 60, 60)).clone();
430 for (
size_t i = 0; i < faces.size(); ++i)
433 cv::Rect
const & f = faces[i];
437 for (
auto const & e : eyes[i])
446 cv::Mat outimgcv(outimg.
height, outimg.
width, CV_8UC2, outimg.
buf->data());
447 itsLastObject.copyTo(outimgcv(cv::Rect(520, 240, 60, 60)));
448 itsLastFace.copyTo(outimgcv(cv::Rect(580, 240, 60, 60)));
462 std::string
const & fpscpu = itsProcessingTimer.
stop() +
", v" JEVOIS_VERSION_STRING;
472 doobject = ! doobject;