How can we get the category using DarknetSaliency module?

Question

Hello,

I would like to use the demo DarknetSaliency module  to get the category , the confidence, the x an y position of the found object on a arduino application. Basically I want to read the serial output of JeVois with an arduino and make something when a certain object is detected in a certain spot.

I get all that information plus frame number to which I don't know yet what use to make for it but I don't get the category. I always have 0 but when I check with video output I clearly see different object detected.

Here is the part of the program I use to change modes on Jevois with an Arduino Mega:

JeVois is attached to Serial1 port. Serial port is used to print out on Arduino monitor.

// Buffer for received serial port bytes:
#define INLEN 128
char instr[INLEN + 1];
bool change_JeVoismode=false;
int Mode=0;
// #########################
void setup()
{
 Serial1.setTimeout(50);
 Serial1.begin(115200);
 Serial1.setTimeout(50);
 Serial.begin(115200);
 change_JeVoismode=false;
 Serial1.println("setpar serlog None");
 Serial1.println("setpar serout Hard");
}

void Serial_modechange()
{
 int numero;
      numero = Serial.parseInt();
      Serial.setTimeout(4);
      Serial.println(numero);
      Serial1.println("streamoff");
      delay(100);
 
 if(numero==110)
   {
    delay(100);
    Serial1.println("setmapping 10");
    delay(100);
    Serial1.println("setpar serout Hard");   delay(100);
    Serial1.println("streamon");    delay(100);
    Serial.println("DarknetSaliency");    delay(200);
    change_JeVoismode=false;
    Mode=110;
   }     
}

void DarknetSaliency()
{
  byte len = Serial1.readBytesUntil('\n', instr, INLEN);   
  instr[len] = 0;
  //Serial.print(instr);  Serial.println("  ");  //for debug
  char *tok = strtok(instr, " \r\n");
  int state = 0; int targx = 0, targy = 0 ;
  unsigned int category = 0,  frame=0;
  float confidence =0;
  //Serial.print("  "); Serial.print(tok); Serial.print("  "); Serial.println(len);  // print raw serial for debug

 while (tok)
  {
    switch (state)
    {     
      case 0:
       if (strcmp(tok, "DKS") == 0) state = 1;
         else if (strcmp(tok, "T2") == 0) state = 2;
         else if (strcmp(tok, "DKR") == 0) state = 4;
         else state = 1000;
         break;        
      case 1: frame = atoi(tok);
        Serial.print(" frame: ");   Serial.print(frame);            
        state=0;
        break;
      case 2: targx = atoi(tok);
        Serial.print(" targx ");   Serial.print(targx);
        state = 3; break;
      case 3: targy = atoi(tok);
        Serial.print(" targy ");   Serial.print(targy);
        state = 0; break;
        case 4: category = atoi(tok);
        Serial.print(" category ");   Serial.print(category);
        state = 5; break;
        case 5: confidence = atof(tok);
        Serial.print(" confidence ");   Serial.println(confidence);
        state = 6; break;     
      default: break; // Skip any additional tokens     
    }
    tok = strtok(0, " \r\n");
  }

void loop()
{
 if (Serial.available()>0)
  {     
   char Ch = Serial.read();
   if (Ch == 'M')   
    {
     change_JeVoismode=true;
     Serial.print(Ch);
    }    
   else
    {
      change_JeVoismode=false;
    }
  }
if(change_JeVoismode==true)
 {
    Serial_modechange();
 }
if(change_JeVoismode==false)
  }  
    if(Mode==110)
    {
      DarknetSaliency();
    }
  }

The result is something like this: here below It should recognize a person.

 frame: 753 targx -150 targy 350 category 0 confidence 25.90

 frame: 754 targx -50 targy 300 category 0 confidence 53.10

 frame: 755 targx -350 targy 350 category 0 confidence 0.00

 frame: 756 targx -50 targy 300 category 0 confidence 70.20

Or here below should detect a ball pen (or sometimes a screwdriver :) )

 frame: 1161 targx 375 targy -275 category 0 confidence 42.40

 frame: 1162 targx 375 targy -275 category 0 confidence 40.80

 frame: 1163 targx -25 targy -325 category 0 confidence 37.50

Anyway the category is always 0.  I am not sure if I used the correct method to read the serial data and surely I will appreciate some help.

Thank you!

Answer 1

your code looks great!

category is sent out as a string, though. So just change your "category = atoi(tok)" by just using the tok string directly.

Comments

small addition, some category names have spaces, e.g., "bald eagle" or "sports car", so you need to keep concatenating the string tokens until you get to the last one, which is the score (as a float number).

---

Thank you for your answer
I thaught the category is a int and an index in a list with categories. However I changed the category as you sugested in my code ( category=tok) and now I have prined in my arduino monitor category = 1062 all the time . I can see clearly when I use the video output that there are diffetent object detected.
I think I miss something. What exactely should I expect to see as category  if a pen is detected or a screwdriver?
Thank you!

---

did you also change the type of your category variable? since tok is a pointer to char, you should so something like

char category[100];


then

strncpy(category, tok, 100);

otherwise, I think the 1062 you are getting is just the last 2 bytes of the address of the tok pointer. As I mentioned before, you will need to be a bit fancier to deal with categories that have several tokens like "bald eagle"; you need to concatenate those tokens back using strcat() or similar until the last one (confidence score, which is a float).

---

Thank you very much!!!  It was as simple as that!!
Below is my detection function corrected. And it works.
I remarked that the composed words category are separated by dash ex: jack-o'-lantern or go-cart so I am not sure if need to add more complexity to the code.
The detection is not that accurate in the actual form but I suppose is meant to demonstrate the principle.

void DarknetSaliency()
{
  digitalWrite(LEDPIN, LOW);
  byte len = Serial1.readBytesUntil('\n', instr, INLEN);
  instr[len] = 0;
  digitalWrite(LEDPIN, HIGH);  
  char *tok = strtok(instr, " \r\n");
  int state = 0;
  static int targx = 0, targy = 0 ;
  unsigned int frame=0;
  char category[100];
  float confidence =0;
  while (tok)
  {
    switch (state)
    {     
      case 0:
       if (strcmp(tok, "DKS") == 0) state = 1;
       else if (strcmp(tok, "T2") == 0) state = 2;
       else if (strcmp(tok, "DKR") == 0) state = 4;
       else state = 1000;
       break;        
      case 1:  frame = atoi(tok);            state = 0; break;
      case 2:  targx = atoi(tok);            state = 3; break;
      case 3:  targy = atoi(tok);            state = 0; break;
      case 4:  strncpy(category, tok, 100);  state = 5; break;
      case 5:  confidence = atof(tok);       state = 6; break;     
      case 6:  break;    
      default: break; // Skip any additional tokens     
    }
    tok = strtok(0, " \r\n");
    
   if (confidence>40)
    {
      Serial.print(" frame: ");   Serial.print(frame);
      Serial.print(" targy ");   Serial.print(targy);
      Serial.print(" targx ");   Serial.print(targx);
      Serial.print(" category ");   Serial.print(category);
      Serial.print(" confidence ");   Serial.println(confidence);
      Serial.println(" ");
    }
  }
}