Matrix problems while integrating Vuforia AR SDK

emnullfuenf

I am currently integrating Qualcomm's Augmented Reality SDK Vuforia for Processing's Android mode. (I already integrated Metaio SDK but they were sold to Apple and closed down ...). It works surprisingly well. I am planning to put it in a library as soon as it's stable. We need a simple and good AR solution for processing.

My current problem is the rendering of the recognized target in Processing. Currently the target is in wrong scale an drifting.

From Vuforia i get the camera's projection matrix and the modelview matrix of the recognized target. Example:

TrackableResult trackableResult = state.getTrackableResult(0);
modelViewMatrixMat44f = Tool.convertPose2GLMatrix(trackableResult.getPose());
PMatrix3D m = new PMatrix3D();
m.set(modelViewMatrixMat44f.getData());
->
Modelview Matrix
{ 0.7313617, 0.07215277, -0.67816234, 0.0, 
-0.013186403, -0.9927057, -0.11983932, 0.0, 
-0.6818624, 0.096588396, -0.7250755, 0.0, 
-63.31185, 7.3395066, 301.61255, 1.0 };
...
Projection Matrix
{ 1.69032, 0.0, 0.0, 0.0, 
0.0, -3.0050135, 0.0, 0.0, 
0.0, -0.0027777778, 1.002002, 1.0, 
0.0, 0.0, -4.004004, 0.0 };

I get a visible but not correct result when i transpose the modelview matrix, translate to the sketch's center, invert the z translation value and apply the matrix via applyMatrix();

The transposed matrix makes more sense:

 000.7314 -000.0132 -000.6819 -063.3119
 000.0722 -000.9927  000.0966  007.3395
-000.6782 -000.1198 -000.7251  301.6125
 000.0000  000.0000  000.0000  001.0000

To be honest i am a bit confused about the current OpenGL situation in Processing 2, 3 and Android mode. Do you have a hint how to apply the projection matrix and the modelview matrix?

Here is a sketch for testing. It's also on github (with images): https://github.com/ixd-hof/vuforia_for_processing/tree/master/Matrix_Simulation

A partially working sketch for Android is here: https://github.com/ixd-hof/vuforia_for_processing/tree/master/Vuforia_Basic

Thanks in advance. I hope this will result in a good AR library.

PMatrix3D pv;
PMatrix3D [] mv;
PMatrix3D mv_;
PImage [] img;
PImage img_;

int index = 0;

void setup()
{
  size(1280, 720, P3D);

  pv = new PMatrix3D(1.69032, 0.0, 0.0, 0.0, 
    0.0, -3.0050135, 0.0, 0.0, 
    0.0, -0.0027777778, 1.002002, 1.0, 
    0.0, 0.0, -4.004004, 0.0);
  pv.transpose();

  pv.print();
  printMatrix();
  //applyMatrix(pv);
  printMatrix();

  mv = new PMatrix3D[6];

  mv[0] = new PMatrix3D(0.9995132, 0.004341676, 0.03089542, 0.0, 
    0.0059285704, -0.9986576, -0.051458698, 0.0, 
    0.030630523, 0.05161681, -0.9981972, 0.0, 
    -10.422494, 4.2534933, 236.47314, 1.0);
    mv[0].transpose();

  mv[1] = new PMatrix3D(0.9998356, 0.0043885736, 0.017598558, 0.0, 
    0.0056207767, -0.9974896, -0.070590734, 0.0, 
    0.017244587, 0.07067805, -0.9973501, 0.0, 
    -14.334564, 2.556219, 453.95117, 1.0);
    mv[1].transpose();

  mv[2] = new PMatrix3D(0.77635086, 0.058464825, -0.6275838, 0.0, 
    0.019611899, -0.9974474, -0.068659954, 0.0, 
    -0.629996, 0.0409961, -0.7755157, 0.0, 
    -13.103903, 5.684967, 298.6713, 1.0);
    mv[2].transpose();

  mv[3] = new PMatrix3D(0.99740285, -0.009295926, -0.07142218, 0.0, 
    0.03843405, -0.7699544, 0.63694036, 0.0, 
    -0.06091277, -0.6380312, -0.76759744, 0.0, 
    -8.147309, -4.703187, 277.7804, 1.0);
    mv[3].transpose();

  mv[4] = new PMatrix3D(0.99991685, -0.012891272, 3.9367346E-4, 0.0, 
    -0.00911498, -0.72794336, -0.6855767, 0.0, 
    0.009124527, 0.6855161, -0.72800034, 0.0, 
    -10.364401, -4.108573, 257.59558, 1.0);
    mv[4].transpose();

  mv[5] = new PMatrix3D(0.7313617, 0.07215277, -0.67816234, 0.0, 
    -0.013186403, -0.9927057, -0.11983932, 0.0, 
    -0.6818624, 0.096588396, -0.7250755, 0.0, 
    -63.31185, 7.3395066, 301.61255, 1.0);
    mv[5].transpose();

  mv_ = mv[0];

  img = new PImage[6];
  img[0] = loadImage("vu_10394.jpg");
  img[1] = loadImage("vu_18200.jpg");
  img[2] = loadImage("vu_25181.jpg");
  img[3] = loadImage("vu_33263.jpg");
  img[4] = loadImage("vu_40123.jpg");
  img[5] = loadImage("vu_47327.jpg");

  img_ = img[0];
}

void draw()
{
  background(img[index]);
  //image(img[index], 0, 0);

  translate(width/2, height/2, -433); // -433

  //applyMatrix(new PMatrix3D());
  //applyMatrix(pv);

  PMatrix3D mv___ = mv_.get();
  //mv___.transpose();
  float [] mv__ = new float[16];
  mv___.get(mv__);
  mv__[11] = -mv__[11];
  //mv___.set(mv__);

  applyMatrix(mv___);

  /*float [] mv__ = new float[16];
   mv_.get(mv__);
   translate(mv__[3], mv__[7], -mv__[11]);

   float c = acos(mv__[0]);
   float s = asin(mv__[2]);
   rotateY(
   */
  //applyMatrix(pv);

  //translate(0, 0, mouseY);

  //translate(0, 0, mouseY);
  fill(255, 100, 0, 100);
  stroke(255, 100);
  box(200);
}

void keyPressed()
{
  if (keyCode == RIGHT)
  {
    if (index < mv.length-1)
      index ++;
    else
      index = 0;

    mv_ = mv[index];

    println(mv[index]);
  } else if (keyCode == LEFT)
  {
    if (index > 0)
      index --;
    else
      index = mv.length-1;

    mv_ = mv[index];

    println(mv[index]);
  }

  if (key == 't')
    mv_.transpose();
  else if (key == 'i')
    mv_.invert();
  else if (key == 'p')
    saveFrame("overlay_####.jpg");

  mv_.print();
}

emnullfuenf

I tried to decompose the matrix and use the values for a camera. Like it's documented here: https://developer.vuforia.com/library/articles/Solution/Get-the-Camera-Position

PMatrix3D m = new PMatrix3D();
        m.set(modelViewMatrixMat44f.getData());
        m.rotateX(PI/2.0);
        m.invert();
        m.transpose();

        float [] mv__ = new float[16];
        m.get(mv__);

        float cam_x = mv__[12];
        float cam_y = mv__[13];
        float cam_z = mv__[14];

        float cam_right_x = mv__[0];
        float cam_right_y = mv__[1];
        float cam_right_z = mv__[2];
        float cam_up_x = mv__[4];
        float cam_up_y = -mv__[5];
        float cam_up_z = -mv__[6];
        float cam_dir_x = mv__[8];
        float cam_dir_y = mv__[9];
        float cam_dir_z = mv__[10];

        camera(cam_dir_x, cam_dir_y, cam_dir_z, cam_x, cam_y, cam_z, cam_up_x, cam_up_y, cam_up_z);

Still no luck.

emnullfuenf

I tried to tinker around until the target sits well. These values are working quite well. But z rotation is a mess.

float fovy = 0.6; //PI/3.0;
float zzz = 600;

perspective(fovy, 1280.0/720.0, 1, 1000);

translate(width/2, height/2, 0);

translate(cam_x, cam_y, -cam_z+zzz);
rotateX(-cam_dir_y);
rotateY(cam_dir_x);
// rotateZ(-cam_dir_z-PI/4.0); // seems 45 deg off and in wrong scale