I am very interested in the latest peasycam features which allow you to set to handle camera operations like pan/zoom/rotate etc through other interaction besides the mouse through handlers. Although I cannot figure out how these work. I think it should work something like this

I am trying to drag verticies in 3d space along a projected plane that is parallel to the camera using peasycam

it is essentially the same problem as this

http://processing.org/discourse/yabb2/YaBB.pl?num=1251823003

while this code works great except that when adding peasycam something goes wrong, I cant figure it out. I suspect it has something to do with the way that peasycam deals with matrix transformations.

Here is my modified version

/*
click/drag to rotate entire object
click/ drag onto vertices
*/
import peasy.*;

PGraphics3D p3d;
public static PeasyCam pcam;

PMatrix3D proj = new PMatrix3D();
PMatrix3D cam = new PMatrix3D();
PMatrix3D modvwInv = new PMatrix3D();
PMatrix3D screen2Model = new PMatrix3D();

// rotation
float rotX, rotY;

// arrays hold the vertices of 3d object
float[] vertices3D = new float[0];
int[] vertices2D = new int[0];

// index of current mouseover / clicked vertex
int vertexMouseOver = -1;
int vertexKlicked= -1;

// z value in model/world space of current vertex
float zModelMouseOver;
float zModelKlick;

void setup() {
  size(800, 600, P3D);
  p3d = (PGraphics3D)g;
  frameRate(50);
 
  pcam = new PeasyCam(this, 1000);

 
  //generate 6 vertices randomly
  for(int i=0;i<18;i++) {
      vertices3D = append(vertices3D, random(-200, 200));
    }
}

void draw() {
  background(255);
 
  // 3d object space begin
 // pushMatrix();
 
  //apply mouse rotation and translation to center of screen
  //translations();
 
  //get 3d matrices
  proj = p3d.projection.get();
  cam = p3d.camera.get();
  modvwInv = p3d.modelviewInv.get();
 
  //visualize 3d axes for orientation
  drawAxes3D();
 
  //visualize vertices
  drawVertSphere();
  drawVert();
  hitDetect();

}

void mousePressed(){
  if (mouseButton==LEFT && vertexMouseOver>-1)
  {
    vertexKlicked = vertexMouseOver;
    zModelKlick = zModelMouseOver;
    pcam.setMouseControlled(false);
    // calculate transformation matrix for projecting mouse coords
    // to the plane where the current selected vertex is
    // this doesn't work!
    screen2Model = modvwInv;
    screen2Model.apply(cam);
  }
}

void mouseReleased(){
  vertexKlicked = -1;
  pcam.setMouseControlled(true);
}

void mouseDragged() {
    if (mouseButton==LEFT && vertexKlicked>-1) {
     
      float scrn[] = {mouseX, mouseY, 0};
      float model[] = new float[3];
     
      // apply transformation matrices to mouse coords
      screen2Model.mult(scrn, model);
    
      vertices3D[vertexKlicked] = model[0 ];
      vertices3D[vertexKlicked+1] = model[1];
      vertices3D[vertexKlicked+2] = model[2];
    }

}


void drawAxes3D() {
  stroke(255,0,0);
  line(0,0,0, 100,0,0);
  stroke(0,255,0);
  line(0,0,0, 0,-100,0);
  stroke(0,0,255);
  line(0,0,0, 0,0,100);
}

void drawVertSphere() {
  //3d vertices3D as spheres
  noStroke();
  fill(100);
  sphereDetail(4);
  for(int i=0; i<vertices3D.length; i=i+3)
  {
    pushMatrix();
    translate(vertices3D[i], vertices3D[i+1], vertices3D[i+2]);
    sphere(3);
    popMatrix();
  }
}

void drawVert() {
  noFill();
  stroke(100, 100, 100);
  beginShape();
  for(int i=0; i<vertices3D.length; i=i+3)
  {
    vertex(vertices3D[i], vertices3D[i+1], vertices3D[i+2]);
  }
  endShape();
}

void hitDetect() {
  // mouse hit detection using screnX, screenY
  vertices2D = new int[0];
  vertexMouseOver = -1;
 
  for(int i=0; i<vertices3D.length; i=i+3) {
    int x = int(screenX(vertices3D[i], vertices3D[i+1], vertices3D[i+2]));
    int y = int(screenY(vertices3D[i], vertices3D[i+1], vertices3D[i+2]));
   
    vertices2D = append(vertices2D, x);
    vertices2D = append(vertices2D, y);
   
    if (x > mouseX-3 && x < mouseX+3 && y > mouseY-3 && y < mouseY+3) {
       vertexMouseOver = i;
      
        pushMatrix();
        translate(vertices3D[i], vertices3D[i+1], vertices3D[i+2]);
        pushStyle();
        fill(255,0,0);
        noStroke();
        sphere(6);
    popStyle();
        popMatrix();
    }
  }
}