29 #include <opencv2/core/core.hpp> 30 #include <opencv2/imgproc/imgproc.hpp> 31 #include <linux/videodev2.h> 38 {
"Hello! Welcome to this simple demonstration of JeVois", 0, 0 },
39 {
"JeVois = camera sensor + quad-core processor + USB video output", 0, 0 },
40 {
"This demo is running on the small processor inside JeVois", 0, 0 },
41 {
"Neat, isn't it?", 0, 0 },
43 {
"We will help you discover what you see on this screen", 0, 0 },
44 {
"We will use this blinking marker to point at things:", 600, 335 },
46 {
"Now a brief tutorial...", 0, 0 },
48 {
"This demo: Attention + Gist + Faces + Objects", 0, 0 },
49 {
"Attention: detect things that catch the human eye", 0, 0 },
50 {
"Pink square in video above: most interesting (salient) location", 0, 0 },
51 {
"Green circle in video above: smoothed attention trajectory", 0, 0 },
52 {
"Try it: wave at JeVois, show it some objects, move it around", 0, 0 },
54 {
"Did you catch the attention of JeVois?", 0, 0 },
56 {
"Attention is guided by color contrast, ...", 40, 270 },
57 {
"by luminance (intensity) contrast, ...", 120, 270 },
58 {
"by oriented edges, ...", 200, 270 },
59 {
"by flickering or blinking lights, ...", 280, 270 },
60 {
"and by moving objects.", 360, 270 },
61 {
"All these visual cues combine into a measure of saliency", 480, 120 },
62 {
"or visual interest for every location in view.", 480, 120 },
65 {
"Gist: statistical summary of a scene, also based on ...", 440, 270 },
66 {
"color, intensity, orientation, flicker and motion features.", 0, 0 },
67 {
"Gist can be used to recognize places, such as a kitchen or ...", 0, 0 },
68 {
"a bathroom, or a road turning left versus turning right.", 0, 0 },
69 {
"Try it: point JeVois to different things and see gist change", 440, 270 },
72 {
"Face detection finds human faces in the camera's view", 612, 316 },
73 {
"Try it: point JeVois towards a face. Adjust distance until ...", 612, 316 },
74 {
"the face fits inside the attention pink square. When a face ...", 0, 0 },
75 {
"is detected, it will appear in the bottom-right corner.", 612, 316 },
76 {
"You may have to move a bit farther than arm's length for ...", 0, 0 },
77 {
"your face to fit inside the attention pink square.", 0, 0 },
80 {
"Objects: Here we recognize handwritten digits using ...", 525, 316 },
81 {
"deep neural networks. Try it! Draw a number on paper ...", 0, 0 },
82 {
"and point JeVois towards it. Adjust distance until the", 0, 0 },
83 {
"number fits in the attention pink square.", 0, 0 },
85 {
"Recognized digits are shown near the detected faces.", 525, 316 },
87 {
"If your number is too small, too big, or not upright ...", 0, 0 },
88 {
"keep adjusting the distance and angle of the camera.", 0, 0 },
90 {
"Recognition scores for digits 0 to 9 are shown above.", 464, 310 },
91 {
"Sometimes the neural network makes mistakes and thinks it ...", 0, 0 },
92 {
"found a digit when actually it is looking at something else.", 0, 0 },
93 {
"This is still a research issue", 0, 0 },
94 {
"but machine vision is improving fast, so stay tuned!", 0, 0 },
96 {
"With JeVois the future of machine vision is in your hands.", 0, 0 },
100 {
"This tutorial is now complete. It will restart.", 0, 0 },
132 itsSaliency = addSubComponent<Saliency>(
"saliency");
133 itsFaceDetector = addSubComponent<FaceDetector>(
"facedetect");
134 itsObjectRecognition = addSubComponent<ObjectRecognitionMNIST>(
"MNIST");
135 itsKF = addSubComponent<Kalman2D>(
"kalman");
136 itsVideo = addSubComponent<BufferedVideoReader>(
"intromovie");
146 cv::Mat banner_bgr = cv::imread(absolutePath(
"jevois-banner-notext.png"));
147 itsBanner.width = banner_bgr.cols;
148 itsBanner.height = banner_bgr.rows;
149 itsBanner.fmt = V4L2_PIX_FMT_YUYV;
150 itsBanner.bufindex = 0;
155 std::this_thread::sleep_for(std::chrono::milliseconds(750));
162 static cv::Mat itsLastFace(60, 60, CV_8UC2, 0x80aa) ;
163 static cv::Mat itsLastObject(60, 60, CV_8UC2, 0x80aa) ;
164 static std::string itsLastObjectCateg;
165 static bool doobject =
false;
166 static bool intromode =
false;
167 static bool intromoviedone =
false;
168 static ScriptItem const * scriptitem = &TheScript[0];
169 static int scriptframe = 0;
175 inimg.
require(
"input", 320, 240, V4L2_PIX_FMT_YUYV);
177 itsProcessingTimer.
start();
178 int const roihw = 32;
181 auto sal_fut = std::async(std::launch::async, [&](){ itsSaliency->process(inimg,
true); });
185 outimg.
require(
"output", 640, outimg.height, V4L2_PIX_FMT_YUYV);
186 switch (outimg.height)
190 case 480: intromode =
true;
break;
191 default:
LFATAL(
"Incorrect output height: should be 312, 360 or 480");
195 if (intromode && intromoviedone ==
false)
197 cv::Mat m = itsVideo->get();
199 if (m.empty()) intromoviedone =
true;
205 if (outimg.height == 480)
207 else if (outimg.height > 360)
221 unsigned short const txtcol = jevois::yuyv::White;
230 itsSaliency->getSaliencyMax(mx, my, msal);
233 int const smlev = itsSaliency->smscale::get();
234 int const smadj = smlev > 0 ? (1 << (smlev-1)) : 0;
235 int const dmx = (mx << smlev) + smadj;
236 int const dmy = (my << smlev) + smadj;
239 int const rx = std::min(
int(inimg.
width) - roihw, std::max(roihw, dmx));
240 int const ry = std::min(
int(inimg.
height) - roihw, std::max(roihw, dmy));
244 std::async(std::launch::async, [&]() {
246 drawMap(outimg, &itsSaliency->salmap, 320, 0, 16, 20);
249 drawMap(outimg, &itsSaliency->color, 0, 240, 4, 18);
252 drawMap(outimg, &itsSaliency->intens, 80, 240, 4, 18);
255 drawMap(outimg, &itsSaliency->ori, 160, 240, 4, 18);
258 drawMap(outimg, &itsSaliency->flicker, 240, 240, 4, 18);
261 drawMap(outimg, &itsSaliency->motion, 320, 240, 4, 18);
265 drawGist(outimg, itsSaliency->gist, itsSaliency->gist_size, 400, 242, 40, 2);
280 if (outimg.height == 480)
285 else if (outimg.height > 312)
290 float kfxraw, kfyraw, kfximg, kfyimg;
291 itsKF->get(kfxraw, kfyraw, kfximg, kfyimg, inimg.
width, inimg.
height, 1.0F, 1.0F);
297 sendSerialImg2D(inimg.
width, inimg.
height, kfximg, kfyimg, roihw * 2, roihw * 2,
"salient");
300 if (intromode && intromoviedone)
304 if (scriptframe < 32) lum = scriptframe * 8;
305 else if (scriptframe > 4*30 - 32) lum = std::max(0, (4*30 - scriptframe) * 8);
308 int x = (640 - 10 * strlen(scriptitem->
msg)) / 2;
314 int phase = scriptframe / 10;
316 10, jevois::yuyv::LightTeal);
320 if (++scriptframe >= 140)
322 scriptframe = 0; ++scriptitem;
323 if (scriptitem->
msg ==
nullptr) scriptitem = &TheScript[0];
330 cv::Mat rawroi = rawimgcv(cv::Rect(rx - roihw, ry - roihw, roihw * 2, roihw * 2));
337 auto objsz = itsObjectRecognition->insize();
339 switch (objsz.depth_)
346 cv::cvtColor(rawroi, objroi, CV_YUV2GRAY_YUYV);
349 size_t const elem = (objroi.cols * objroi.rows * 10) / 100;
350 std::vector<unsigned char> v; v.assign(objroi.datastart, objroi.dataend);
351 std::nth_element(v.begin(), v.begin() + elem, v.end());
352 unsigned char const thresh = std::min((
unsigned char)(100), std::max((
unsigned char)(30), v[elem]));
355 cv::threshold(objroi, objroi, thresh, 255, cv::THRESH_BINARY_INV);
358 cv::Mat pts; cv::findNonZero(objroi, pts);
359 cv::Rect r = cv::boundingRect(pts);
360 int const cx = r.x + r.width / 2;
361 int const cy = r.y + r.height / 2;
362 int const siz = std::min(roihw * 2, std::max(16, 8 + std::max(r.width, r.height)));
363 int const tlx = std::max(0, std::min(roihw*2 - siz, cx - siz/2));
364 int const tly = std::max(0, std::min(roihw*2 - siz, cy - siz/2));
365 cv::Rect ar(tlx, tly, siz, siz);
366 cv::resize(objroi(ar), objroi, cv::Size(objsz.width_, objsz.height_), 0, 0, cv::INTER_AREA);
372 cv::cvtColor(rawroi, objroi, CV_YUV2RGB_YUYV);
373 cv::resize(objroi, objroi, cv::Size(objsz.width_, objsz.height_), 0, 0, cv::INTER_AREA);
377 LFATAL(
"Unsupported object detection input depth " << objsz.depth_);
381 auto scores = itsObjectRecognition->process(objroi);
384 std::ostringstream oss;
385 for (
size_t i = 0; i < scores.size(); ++i)
386 oss << itsObjectRecognition->category(i) <<
':' << std::fixed << std::setprecision(2) << scores[i] <<
' ';
387 itsScoresStr = oss.str();
390 float best1 = scores[0], best2 = scores[0];
size_t idx1 = 0, idx2 = 0;
391 for (
size_t i = 1; i < scores.size(); ++i)
393 if (scores[i] > best1) { best2 = best1; idx2 = idx1; best1 = scores[i]; idx1 = i; }
394 else if (scores[i] > best2) { best2 = scores[i]; idx2 = i; }
398 if (best1 > 90.0F && best2 < 20.0F)
401 itsLastObjectCateg = itsObjectRecognition->category(idx1);
402 itsLastObject = rawimgcv(cv::Rect(rx - 30, ry - 30, 60, 60)).clone();
404 LINFO(
"Object recognition: best: " << itsLastObjectCateg <<
" (" << best1 <<
405 "), second best: " << itsObjectRecognition->category(idx2) <<
" (" << best2 <<
')');
413 cv::Mat grayroi; cv::cvtColor(rawroi, grayroi, CV_YUV2GRAY_YUYV);
414 cv::equalizeHist(grayroi, grayroi);
417 std::vector<cv::Rect> faces; std::vector<std::vector<cv::Rect> > eyes;
418 itsFaceDetector->process(grayroi, faces, eyes,
false);
423 LINFO(
"detected " << faces.size() <<
" faces");
425 itsLastFace = rawimgcv(cv::Rect(rx - 30, ry - 30, 60, 60)).clone();
428 for (
size_t i = 0; i < faces.size(); ++i)
431 cv::Rect
const & f = faces[i];
435 for (
auto const & e : eyes[i])
444 cv::Mat outimgcv(outimg.height, outimg.width, CV_8UC2, outimg.buf->data());
445 itsLastObject.copyTo(outimgcv(cv::Rect(520, 240, 60, 60)));
446 itsLastFace.copyTo(outimgcv(cv::Rect(580, 240, 60, 60)));
460 std::string
const & fpscpu = itsProcessingTimer.
stop() +
", v" JEVOIS_VERSION_STRING;
470 doobject = ! doobject;
jevois::RawImage itsBanner
std::shared_ptr< FaceDetector > itsFaceDetector
cv::Mat cvImage(RawImage const &src)
void drawCircle(RawImage &img, int x, int y, unsigned int rad, unsigned int thick, unsigned int col)
void writeText(RawImage &img, std::string const &txt, int x, int y, unsigned int col, Font font=Font6x10)
JeVoisIntro(std::string const &instance)
Constructor.
std::shared_ptr< BufferedVideoReader > itsVideo
void drawGist(jevois::RawImage &img, unsigned char const *gist, size_t gistsize, unsigned int xoff, unsigned int yoff, unsigned int width, unsigned int scale)
std::shared_ptr< ObjectRecognitionBase > itsObjectRecognition
virtual void postInit() override
Initialization once parameters are set:
std::shared_ptr< Saliency > itsSaliency
void hFlipYUYV(RawImage &img)
void drawMap(jevois::RawImage &img, env_image const *fmap, unsigned int xoff, unsigned int yoff, unsigned int scale)
Simple introduction to JeVois and demo that combines saliency, gist, face detection, and object recognition.
void convertCvBGRtoRawImage(cv::Mat const &src, RawImage &dst, int quality)
std::string const & stop()
std::shared_ptr< Kalman2D > itsKF
ENV_INTG32_TYPE intg32
32-bit signed integer
JEVOIS_REGISTER_MODULE(JeVoisIntro)
void drawFilledRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int col)
virtual void process(jevois::InputFrame &&inframe, jevois::OutputFrame &&outframe) override
Processing function.
virtual ~JeVoisIntro()
Virtual destructor for safe inheritance.
void drawDisk(RawImage &img, int x, int y, unsigned int rad, unsigned int col)
void drawRect(RawImage &img, int x, int y, unsigned int w, unsigned int h, unsigned int thick, unsigned int col)
void paste(RawImage const &src, RawImage &dest, int dx, int dy)
void require(char const *info, unsigned int w, unsigned int h, unsigned int f) const