How do i combine interactive PCA with KinectV2 using OpenCV for Processing?

virtualarchitectures

Hi,

I'm currently trying to get Greg Borenstein's interactive PCA analysis example to work with a KinectV2 stream using Greg's newer OpenCV for Processing library (had to many issues with the old library). I'm getting output from the Kinect and processing it with OpenCV successfully but I have two issues: 1. The old opencv.image() function doesn't work with the new library so I'm trying opencv.getOutput(). Unfortunately this slows performance considerably and imagine there is a better solution. 2. I'm expecting an axis to be drawn on the OpenCV output image but this isn't occurring.

Here's my code including the commented code I'm trying to replace:

//Copyright (C) 2014  Thomas Sanchez Lengeling.
//KinectPV2, Kinect one library for processing
import KinectPV2.*;
//Import OpenCV for Processing Library by Greg Borenstein
import gab.opencv.*;
//Import Processing-PCA Library from Greg Borenstein
import pca_transform.*;
//Import JAMA Java Matrix package
import Jama.Matrix;

//Declare kinect object
KinectPV2 kinect;
//Declare opencv object
OpenCV opencv;
//Declare PCA object
PCA pca;

//Declare image variables (colour image)
PImage img;
int imgWidth = 640;
int imgHeight = 360;

//Declare variable for Threshold
int threshold = 100;

//Declare PVector variables
PVector axis1;
PVector axis2;
PVector mean;
PVector centroid;

void setup() {
  size(1280, 720);

  //Initialise the Kinect including startup methods to enable colour, depth and infrared images
  kinect = new KinectPV2(this);
  kinect.enableColorImg(true);
  kinect.enableDepthImg(true);
  kinect.enableInfraredImg(true);
  kinect.init();

  //Initialise openCV for colour image
  opencv = new OpenCV(this, 1920, 1080);

  //Initialise centroid
  centroid = new PVector();
}

//Use Java Matrix for PCA
Matrix toMatrix(PImage img) {
  ArrayList<PVector> points = new ArrayList<PVector>();
  for (int x = 0; x < img.width; x++) {
    for (int y = 0; y < img.height; y++) {
      int i = y*img.width + x;
      if (brightness(img.pixels[i]) > 0) {
        points.add(new PVector(x, y));
      }
    }
  }

  Matrix result = new Matrix(points.size(), 2);

  float centerX = 0;
  float centerY = 0;

  for (int i = 0; i < points.size(); i++) {
    result.set(i, 0, points.get(i).x);
    result.set(i, 1, points.get(i).y);

    centerX += points.get(i).x;
    centerY += points.get(i).y;
  }
  centerX /= points.size();
  centerY /= points.size();
  centroid.x = centerX;
  centroid.y = centerY;

  return result;
}

//**This code is not required**
//void imageInGrid(PImage img, String message, int row, int col) {
  //int currX = col*img.width;
  //int currY = row*img.height;
  //image(img, currX, currY);
  //fill(255, 0, 0);
  //text(message, currX + 5, currY + imgHeight - 5);
//}

void draw() {
  background(0);

  //Draw the Kinect images
  image(kinect.getColorImage(), 0, 0, imgWidth, imgHeight);
  image(kinect.getDepthImage(), 855, 0, 425, 320);
  image(kinect.getInfraredImage(), 855, 321, 425, 320);

  fill(255, 0, 0);

  //**FUNCTIONS DO NOT EXIST - This code is rewritten below**
//  opencv.read();
//  opencv.convert(GRAY);
//  imageInGrid(opencv.image(), "GRAY", 0, 0);
//
//  opencv.absDiff();
//  imageInGrid(opencv.image(), "DIFF", 0, 1);
//
//  opencv.brightness(60);
//  imageInGrid(opencv.image(), "BRIGHTNESS: 60", 0, 2);
//
//  opencv.threshold(40);
//  imageInGrid(opencv.image(), "THRESHOLD: 40", 0, 3);
//
//  opencv.contrast(120);
//  imageInGrid(opencv.image(), "CONTRAST: 120", 1, 3);

  //Load the colour image from the kinect to OpenCV 
  opencv.loadImage(kinect.getColorImage());
  //Apply grey filter
  opencv.gray();
  //Apply Threshold
  opencv.threshold(threshold);
  //Display OpenCV output
  PImage img = opencv.getOutput();
  image(img, 0, imgHeight, imgWidth, imgHeight);

    //**This code is rewritten below**
//  Matrix m = toMatrix(opencv.image());

    //Add OpenCV output to Matrix **This slows down the sketch considerably**
    Matrix m = toMatrix(opencv.getOutput());

  if (m.getRowDimension() > 0) {
    pca = new PCA(m);
    Matrix eigenVectors = pca.getEigenvectorsMatrix();

    axis1 = new PVector();
    axis2 = new PVector();
    if (eigenVectors.getColumnDimension() > 1) {

      axis1.x = (float)eigenVectors.get(0, 0);
      axis1.y = (float)eigenVectors.get(1, 0);

      axis2.x = (float)eigenVectors.get(0, 1);
      axis2.y = (float)eigenVectors.get(1, 1);  

      axis1.mult((float)pca.getEigenvalue(0));
      axis2.mult((float)pca.getEigenvalue(1));
    }

    //**This code is rewritten below** 
    //image(opencv.image(), 0, opencv.image().height, opencv.image().width*3, opencv.image().height*3);

    image(opencv.getOutput(), 0, opencv.getOutput().height, opencv.getOutput().width*3, opencv.getOutput().height*3);

    stroke(200);
    pushMatrix();
    translate(0, imgHeight);
    scale(3, 3);

    translate(centroid.x, centroid.y);

    stroke(0, 255, 0);
    line(0, 0, axis1.x, axis1.y);
    stroke(255, 0, 0);
    line(0, 0, axis2.x, axis2.y);

    popMatrix();
    fill(0, 255, 0);
    text("PCA Object Axes:\nFirst two principle components centered at blob centroid", 10, height - 20);
  }
}

//Adjust threshold variable with 'A' and 'S' keys
void keyPressed() {
  if (key == 'a') {
    threshold+=1;
  }
  if (key == 's') {
    threshold-=1;
  }
}