Displaying objects underneath silhouette depth image

RyanAchten

Hello Processing community!

Apologies in advance for the code overload, if there is a better method of providing code for the community to look at kindly let me know. I am currently working on a project where I want participants to engage with data visualisation via Processing & Microsoft Kinect (using simpleOpenNI). Being very new to the project (and Processing in general) I have encountered an issue where I want to get the data vis (in this case a static placeholder table) to display behind the silhouetted depth image feed from the Kinect. Can anyone provide me with some reasons why this may be occurring or a place where I can find the answers my self? Many thanks, Ryan

import SimpleOpenNI.*;
SimpleOpenNI  kinect;

PFont font;
PImage img; //Silhouette
PVector convertedLeftHand = new PVector();
ArrayList ballCollection; 
Table stateData;
int rowCount;
//Network Display   Soma | Light Blue | Teal | Red | Lavender
color[] SomaScheme = {
  #FDE7DC, #74C8CC, #219399, #FF5C74
};
color[] palette = SomaScheme;
int SomaColour = palette[0];
int ellipseColour = palette[1];
//Circle/Table
float age;
float degree;
float circleSize = 20;

void setup() {
  //Setup misc
  size(640, 480);
  font = loadFont("font18.vlw");
  textFont(font);

  //Kinect Setup
  kinect = new SimpleOpenNI(this); //initialize kinect variable
  kinect.enableDepth(); //asks OpenNI to initialize and start receiving depth sensor's data
  kinect.enableUser();  //asks OpenNI to initialize and start receiving User data
  kinect.setMirror(true);//enable mirroring - flips the sensor's data horizontally
  img=createImage(width, height, RGB);
  img.loadPixels();

  //Initialise ArrayList
  ballCollection = new ArrayList();

  //Provide values for x & y
  for (int row = 0; row < rowCount; row++) {
    float degree = stateData.getFloat(row, 3);
    float age = stateData.getFloat(row, 4);
    float ellipseX = map(age, 30, 41, 0, width); 
    float ellipseY = map(degree, 14, 47, 0, height);
    String stateName = stateData.getString(row, 2);
    Ball myBall = new Ball (ellipseX, ellipseY, circleSize, circleSize, stateName); 
    ballCollection.add(myBall);
  }
}

void draw() {
  background(palette[0]);

  kinect.update();

  PImage depthImage=kinect.depthImage(); //retrieves depth image
  depthImage.loadPixels();
  int[] upix=kinect.userMap(); //get user pixels - array of the same size as depthImage.pixels, that gives information about the users in the depth image:
  for (int i=0; i < upix.length; i++) {   //colorize users
    if (upix[i] > 0) { //there is a user on that position NOTE: if you need to distinguish between users, check the value of the upix[i]
      img.pixels[i]=color(0, 0, 255); //!!! Silh Colour
    } else {
      //img.pixels[i]=depthImage.pixels[i]; //depth data bg
      img.pixels[i]=color(0, 0, 0, 0); //fill background
    }
  }

  img.updatePixels();

  image(img, 0, 0); //draws the depth map data a
  //tint(255, 126); // THIS MAY NOT BE NEEDED SOON
  int[] users=kinect.getUsers(); //get array of IDs of all users present

  ellipseMode(CENTER);

  for (int i=0; i < users.length; i++) { //iterate through users
    int uid=users[i];
    PVector realCoM=new PVector(); //draw center of mass of the user
    kinect.getCoM(uid, realCoM); //get the CoM in realworld (3D) coordinates
    PVector projCoM=new PVector();
    kinect.convertRealWorldToProjective(realCoM, projCoM); //convert realworld coordinates to projective (those that we can use to draw to our canvas)
    fill(255, 0, 0);
    ellipse(projCoM.x, projCoM.y, 10, 10);

    if (kinect.isTrackingSkeleton(uid)) { //check if user has a skeleton
      //draw head
      PVector realHead=new PVector();  
      kinect.getJointPositionSkeleton(uid, SimpleOpenNI.SKEL_HEAD, realHead); //get realworld coordinates of the given joint of the user (in this case Head -> SimpleOpenNI.SKEL_HEAD)
      PVector projHead=new PVector();
      kinect.convertRealWorldToProjective(realHead, projHead);
      fill(0, 255, 0);
      ellipse(projHead.x, projHead.y, 10, 10);

      //draw left hand
      PVector realLHand=new PVector();
      kinect.getJointPositionSkeleton(uid, SimpleOpenNI.SKEL_LEFT_HAND, realLHand);
      PVector convertedLeftHand=new PVector();
      kinect.convertRealWorldToProjective(realLHand, convertedLeftHand);
      fill(255, 255, 0);
      ellipse(convertedLeftHand.x, convertedLeftHand.y, 10, 10);
    }
  }

  //Graph using ball object
  for (int i = 0; i < ballCollection.size (); i++) { 
    Ball myBall = (Ball) ballCollection.get(i); 
    myBall.over();
    myBall.display();
  }
}

//is called everytime a new user appears
void onNewUser(SimpleOpenNI curkinect, int userId)
{
  println("onNewUser - userId: " + userId);

  curkinect.startTrackingSkeleton(userId);  //asks OpenNI to start tracking a skeleton data for this user NOTE: you cannot request more than 2 skeletons at the same time due to the perfomance limitation
}                                           //so some user logic is necessary (e.g. only the closest user will have a skeleton)

void onLostUser(SimpleOpenNI curkinect, int userId) //is called everytime a user disappears
{
  println("onLostUser - userId: " + userId);
}

Ball Class:

class Ball {

  // GLOBAL VARIABLES
  float x = 0; //declares these variables
  float y = 0;
  float circleSizeX;
  float circleSizeY;
  boolean over = false;
  boolean drag = false;
  String name;


  //CONSTRUCTOR
  Ball(float tempX, float tempY, float tempcircleSizeX, float tempcircleSizeY, String n) {  
    x = tempX; //intialises these variables so that they can be accessed anywhere
    y = tempY;
    circleSizeX = tempcircleSizeX;
    circleSizeY = tempcircleSizeY;
    name = n;
  }

  //FUNCTIONS
  void run() { 
    display(); 
  }  

  void display() {   
    fill(palette[1]);
    if (over) {
      fill(palette[3]);
      //println(name);
      text(name, x, y+3);
    }
    ellipse(x, y, circleSizeX, circleSizeY);
  }

  void over() {
    if (dist (x, y, convertedLeftHand.x, convertedLeftHand.y) < circleSize/2){
    over = true;
    }
    else{
    over = false;
    }
  }
}