kinectwuerfel/configfinder/configfinder.pde

import processing.video.*;
//on linux: sudo apt-get install libgstreamer-plugins-base0.10-dev libgstreamer0.10-dev libglib2.0-dev
//          sudo apt-get install gstreamer0.10-plugins-good

import processing.serial.*;
Serial sPort;

Capture cam;

PImage cam_reference;
PImage cam_currentbox;
PGraphics canvas;

PImage image_clear;


int currentID=0;
//int MAXID = 50; //inclusive
int MAXID = 9;
int searchstate=0; //0=get reference image and ping box, 1=wait, 2=get image to compare and turn box off, 3=wait
long starttime_wait=0;

boolean running=false;

color testcolor=color(255,255,255);
color offcolor=color(0,0,0);
color successcolor=color(0,255,0);
color failcolor=color(255,0,0);

static int boxnum=19; //how many boxes are visible?
ArrayList<Box> boxes=new ArrayList<Box>();
//result will be written to the following arrays
//int boxid[] = new int[boxnum]; //for found boxes
//PVector boxpos[] = new PVector[boxnum]; //center position of box
//PVector boxsize[] = new PVector[boxnum]; //size of box (width,height)

//Settings
//int FINDBOXES_MINIMUMBRIGHTNESS; //brightness difference required
int FINDBOXES_MAXIMUMPIXELDISTANCE; //distance to add pixel to existing box
PVector MINBOXSIZE; //minimum box size to accept box
int TIME_FIND = 100; //time in ms to find box in image
int TIME_RESULT = 100; //time in ms to show success/fail
int TIME_TURNOFF = 100; //time in ms to wait to turn off
float MINBRIGHTNESS_PERCENTILE=0.98; //percentile for minbrightness calculation (how many pixels will be considered). between 0(all) and 1(none). 0=all pixels. ->1=only bright pixels 
int MINIMUMALLOWEDBRIGHTNESS=50; //minimum brightness. will be limited to this lower bound if no box in image (percentile calculation yields low value)

void setup() {
  size(640, 480);
  frameRate(30);

  String portName = Serial.list()[0];
  sPort = new Serial(this, portName, 115200);


  canvas = createGraphics(width, height);
  
  //FINDBOXES_MINIMUMBRIGHTNESS=60; //brightness difference required
  FINDBOXES_MAXIMUMPIXELDISTANCE=width/200; //distance to add pixel to existing box
  MINBOXSIZE = new PVector(width/300,width/300); //minimum box size to accept box

  String[] cameras = Capture.list();

  if (cameras.length == 0) {
    println("There are no cameras available for capture.");
    exit();
  } else {
    println("Available cameras:");
    for (int i = 0; i < cameras.length; i++) {
      println(i+"->"+cameras[i]);
    }

    // The camera can be initialized directly using an
    // element from the array returned by list():
    //cam = new Capture(this, cameras[1]); //windows
    cam = new Capture(this, cameras[1]); //linux
    cam.start();
  }
  image_clear=cam.copy();
  cam_reference=cam.copy();
}

void draw() {

  if (cam.available() == true) {
    cam.read();
  }
  //image(cam, 0, 0);
  // The following does the same, and is faster when just drawing the image
  // without any additional resizing, transformations, or tint.
  //set(0, 0, cam);

  //Subtract images
  canvas.beginDraw();
  canvas.background(0);
  canvas.blendMode(NORMAL);
  canvas.image(cam, 0, 0);
  canvas.blendMode(SUBTRACT);
  canvas.image(cam_reference, 0, 0);
  canvas.endDraw();

  image(canvas,0,0); //show camera image

  if (running) {
    checkState();
  }

  textSize(12);
  fill(0,255,150); //textcolor
  text("id: " + currentID, 10, 20);
  text("state: " + searchstate, 10, 20*2);
  
  //draw boxes
  for (Box box : boxes) {
    PVector boxcenter = box.getBoxCenter();
    PVector boxsize = box.getBoxSize();
    
    rectMode(CENTER);
    noFill();
    fill(255,255,0,100);
    stroke(255,0,0);
    rect(boxcenter.x,boxcenter.y, boxsize.x,boxsize.y);
    textSize(12);
    fill(0,255,150); //textcolor
    text(box.getBoxID(), boxcenter.x,boxcenter.y);
  }
}

void checkState() {
  switch(searchstate) {
    case 0: //0=get reference image and ping box
      cam_reference=cam.copy();
      println();
      println("Searching BoxID "+str(currentID));

      
      colorBox(currentID,testcolor);

      starttime_wait=millis();
      searchstate++;
      break;

    case 1: //1=wait
      if (millis()>starttime_wait + TIME_FIND) { //wait some time
          searchstate++;
      }
      break;

    case 2: //2=get image to compare and turn box off
      cam_currentbox=canvas.copy();

      Box box = findSingleBox(cam_currentbox); //find single box in this image
      if (box!=null && (box.getBoxSize().x<MINBOXSIZE.x || box.getBoxSize().y<MINBOXSIZE.y)) { //box too small
        box=null;
        println("  Box too small");
      }
      if (box!=null) { //if box found
        PVector boxsize = box.getBoxSize();
        float boxarea = boxsize.x * boxsize.y;
        if (boxarea >= pow(cam_currentbox.width/40,2)) { //if box is big enough
          box.setBoxID(currentID);
          boxes.add(box);
        }
      }
      
      if (box!=null) {
        println("Found box");
        println("Pos: "+box.getBoxCenter().x+", "+box.getBoxCenter().y+" | size: "+box.getBoxSize().x+", "+box.getBoxSize().y);
        colorBox(currentID, successcolor);
      }else {
        println("No Box Found!");
        colorBox(currentID, failcolor);
      } 
      
      starttime_wait=millis();
      searchstate++;
      break;

    case 3: //3=wait, color box fail or success
      if (millis()>starttime_wait + TIME_RESULT) { //wait some time
          colorBox(currentID,offcolor); //turn box off
          starttime_wait=millis();
          searchstate++;
      }
      break;
      
    case 4: //wait for box to turn off completely
      if (millis()>starttime_wait + TIME_TURNOFF) { //wait some time
          starttime_wait=millis();
          searchstate++;
      }
      break;

    case 5: //next box
      searchstate=0;
      currentID++;
      if (currentID>MAXID) {
        running=false;
        cam_reference=image_clear; //show normal camera image
      }
      break;
  }
}

void keyPressed() {
  if (key == 'n' ){
    cam_reference=cam.copy();
    println("New Reference Set");

  } else if (key == 10){ //ENTER
    running=true;
    colorAllOff(); //turn box off
    currentID=0; //reset
    searchstate=0;
    boxes.clear(); //reset found boxes
    println("Search started");
  } else if (key == 'r' ){
    running=false;
    cam_reference=image_clear; //show normal camera image
  } else if(key == 's') {
     saveBoxConfig(); 
  }
  

}

int getPercentileBrightness(PImage img, float percentile) { //percentile between 0 and 1
  percentile=min(max(percentile,0.0),1.0); //limit to 0 to 1
  int[] brightness=new int[img.pixels.length];
  for (int i=0;i<brightness.length;i++) {
    brightness[i] = int(brightness(img.pixels[i]));
  }
  int[] sortedbrightness=sort(brightness);
  return sortedbrightness[int(percentile*sortedbrightness.length)];
}

ArrayList<Box> findBoxes(PImage img) {
  ArrayList<Box> foundboxes=new ArrayList<Box>();
  int minimumbrightness=getPercentileBrightness(img, MINBRIGHTNESS_PERCENTILE); //calculate brightness by percentile based on minimum box size
  
  minimumbrightness=max(minimumbrightness, MINIMUMALLOWEDBRIGHTNESS); //limit lower value 
  
  
  println(" minimumbrightness="+minimumbrightness);
  for (int y=0;y<img.height;y++) {
      for (int x=0;x<img.width;x++) {
        color pcolor = img.get(x,y);
        
        if (brightness(pcolor)>minimumbrightness) { //pixel has changed
            Box nearestBox=null;
            float nearestBoxDistance=MAX_FLOAT;
            for (Box cb : foundboxes) { //check all known boxes
                float cdist = cb.getDistanceTo(new PVector(x,y));
                if (cdist<=FINDBOXES_MAXIMUMPIXELDISTANCE && cdist<nearestBoxDistance) { //is near this box and closer than to another box before
                  nearestBox=cb; //remeber box
                  nearestBoxDistance=cdist; //remember distance
                }
            }
            if (nearestBox==null) { //no box near this pixel
              Box newbox=new Box(); //create new box
              newbox.addPixel(new PVector(x,y));
              foundboxes.add(newbox); //add
            } else {
              nearestBox.addPixel(new PVector(x,y)); //add this pixel to near box
            }
        }
      }
    }
    
    println(" findBoxes(): foundboxes.size="+foundboxes.size());
    //merge overlapping boxes together
    for (int i=0;i<foundboxes.size();i++) {
       Box fbox = foundboxes.get(i); 
       if (fbox.getMarked()) { continue; } //skip if this box was marked
       for (int i2=i+1;i2<foundboxes.size();i2++) {
         Box fbox2 = foundboxes.get(i2);
         if (fbox.getMarked()) { continue; } //skip if this box was marked
         if (fbox2.isInside(fbox) && fbox.isInside(fbox2)) {
           fbox.merge(fbox2); //add pixels to one box
           fbox2.setMarked(true); //mark for deletion
         }
       
       }
    }
    for (int i=0;i<foundboxes.size();i++) { //remove marked boxes
      Box fbox = foundboxes.get(i); 
      if (fbox.getMarked()) { //if marked
       foundboxes.remove(fbox); //remove merged box
       i--;
      }
      
    }
    
    println(" after merge: foundboxes.size="+foundboxes.size());
    return foundboxes;
}

Box findSingleBox(PImage img) {
  ArrayList<Box> foundboxes = findBoxes(img);
  Box mostPixelBox=null;
  for (Box box : foundboxes) {
    if (mostPixelBox == null || mostPixelBox.getPixelNumber()<box.getPixelNumber()) { //found box with more pixels
      mostPixelBox=box;
    }
  }
  return mostPixelBox;
}

public class Box {
   ArrayList<PVector> boxpixels;
   int boxid;
   boolean marked=false;
   
   public Box() {
     boxpixels = new ArrayList<PVector>();
   }
   public void setMarked(boolean pm) {
      marked=pm; 
   }
   public boolean getMarked() {
      return marked; 
   }
   
   public void addPixel(PVector newpix) {
     boxpixels.add(newpix); 
   }
   
   public ArrayList<PVector> getPixels() {
     return boxpixels;
   }
   
   public void setBoxID(int pid) {
     this.boxid=pid;
   }
   public int getBoxID() {
     return this.boxid;
   }
   
   public float getDistanceTo(PVector pix) { //get distance from pix to nearest point of this box
     float mindist=MAX_FLOAT;
     for (PVector pbox : boxpixels) {
       float currentdist = sqrt(pow(pix.x-pbox.x,2)+pow(pix.y-pbox.y,2)); //calculate distance
       mindist=min(currentdist,mindist); //save new minimum
     }
     return mindist;
   }
   
   public PVector getMeanPos(){
     PVector meanvec= new PVector(0,0);
     for (PVector pbox : boxpixels) {
      meanvec.x+=pbox.x;
      meanvec.y+=pbox.y;
     }
     meanvec.x/=boxpixels.size();
     meanvec.y/=boxpixels.size();
     return meanvec;
   }
   
   public PVector getBoxSize(){ //returns widht and height as vector
     PVector upperleft= new PVector(MAX_FLOAT,MAX_FLOAT); //min x and y
     PVector lowerright= new PVector(0,0); //max x and y
     for (PVector pbox : boxpixels) {
       upperleft.x=min(upperleft.x, pbox.x);
       upperleft.y=min(upperleft.y, pbox.y);
       
       lowerright.x=max(lowerright.x, pbox.x);
       lowerright.y=max(lowerright.y, pbox.y);
     }
     PVector boxsize = new PVector(lowerright.x-upperleft.x, lowerright.y-upperleft.y);
     return boxsize;
   }
   
   public PVector getBoxCenter(){ //returns width and height as vector
     PVector upperleft= new PVector(MAX_FLOAT,MAX_FLOAT); //min x and y
     PVector lowerright= new PVector(0,0); //max x and y
     for (PVector pbox : boxpixels) {
       upperleft.x=min(upperleft.x, pbox.x);
       upperleft.y=min(upperleft.y, pbox.y);
       
       lowerright.x=max(lowerright.x, pbox.x);
       lowerright.y=max(lowerright.y, pbox.y);
     }
     PVector boxcenter = new PVector((lowerright.x+upperleft.x)/2, (lowerright.y+upperleft.y)/2);
     return boxcenter;
   }
   
   public int getPixelNumber()  {
      return this.boxpixels.size(); 
   }
   
   public boolean isInside(Box pbox) {
     int pixelsinside=0; //for counting pixels of pbox inside this box
     PVector pboxsize=pbox.getBoxSize();
     PVector pboxcenter=pbox.getBoxCenter();
     float pboxX1 = pboxcenter.x-pboxsize.x/2;
     float pboxX2 = pboxcenter.x+pboxsize.x/2;
     float pboxY1 = pboxcenter.y-pboxsize.y/2;
     float pboxY2 = pboxcenter.y+pboxsize.y/2;
     for (PVector pix : boxpixels) {
         if (pix.x>pboxX1 && pix.x<pboxX2 && pix.y>pboxY1 && pix.y<pboxY2) { //is inside
           pixelsinside++;
         }
     }
     
     return pixelsinside>0; 
     
   }
   
   public void merge(Box mergebox) { //merge pixels from mergebox inside this box
     boxpixels.addAll(mergebox.getPixels());
   }
}

void colorBox(int id, color c) 
{
  sPort.write("B,"+str(id)+","+str(0)+","+int(red(c))+","+int(green(c))+","+int(blue(c))+"\n"); 
}
void colorAllOff() 
{
  sPort.write("A,0\n"); 
}

void saveBoxConfig(){
  String[] boxconfig=new String[boxes.size()];
  for (int i=0;i<boxes.size();i++) {
    Box box=boxes.get(i);
    PVector boxcenter=box.getBoxCenter();
    PVector boxsize=box.getBoxSize();
    boxconfig[i]=box.getBoxID()+";"+(boxcenter.x/cam.width)+";"+(boxcenter.y/cam.height)+";"+(boxsize.x/cam.width)+";"+(boxsize.y/cam.height);
  }
  saveStrings("boxconfig.txt", boxconfig);
  println("saved");
}