you can measure the distance from your mouseposition to the 2d-screencoordiantes of your 3d-objects.
search the reference for screenX() screenY() screenZ()

Finally, I used P3D, which allows to draw in a back-buffer (OpenGL doesn't).

Therefore, posting this here may be off-board, since we're in the OpenGL topic, but here is the code anyway :

Press 'b' to see the back-buffer content.

Code:

Cube[] cubes;

PGraphics buffer;
PGraphics pickbuffer;
float a = 0;

void setup() {
  
  size(100, 100);
  frameRate(30);
  
  cubes = new Cube[2];
  cubes[0] = new Cube(0, 0, 0, 0);
  cubes[1] = new Cube(100, 0, -2, -10);
  
  buffer = createGraphics(100, 100, P3D);
  buffer.camera(2, -5, 10, 0, 0, 0, 0, 1, 0);
  buffer.hint(ENABLE_DEPTH_SORT);
  
  pickbuffer = createGraphics(100, 100, P3D);
  pickbuffer.camera(2, -5, 10, 0, 0, 0, 0, 1, 0);
  pickbuffer.hint(ENABLE_DEPTH_SORT);
}

void mouseClicked() {
  drawOnPickBuffer();
  color pick = pickbuffer.get(mouseX, mouseY);
  int id = (int)((red(pick)+green(pick)+blue(pick))/3);
  for (int i = 0; i < cubes.length; i++)
    if (cubes[i].id == id) cubes[i].changeColor();
}

void draw() {
  drawOnBuffer();
  image(buffer, 0, 0);
  if (keyPressed && key == 'b') {
    drawOnPickBuffer();
    image(pickbuffer, 0, 0);
  }
  a += PI/80; if (a > TWO_PI) a -= TWO_PI;
}

void drawOnBuffer() {
  buffer.beginDraw();
    buffer.background(255);
    for (int i = 0; i < cubes.length; i++) {
	buffer.fill(cubes[i].c);
	buffer.stroke(0);
	buffer.pushMatrix();
	  buffer.translate(cubes[i].x, cubes[i].y, cubes[i].z);
	  buffer.rotateY(a);
	  buffer.box(5);
	buffer.popMatrix();  
    }
  buffer.endDraw();
}

void drawOnPickBuffer() {
  pickbuffer.beginDraw();
    pickbuffer.background(250, 200, 50);
    for (int i = 0; i < cubes.length; i++) {
	color idcolor = color(cubes[i].id);
	pickbuffer.fill(idcolor);
	pickbuffer.stroke(idcolor);
	pickbuffer.pushMatrix();
	  pickbuffer.translate(cubes[i].x, cubes[i].y, cubes[i].z);
	  pickbuffer.rotateY(a);
	  pickbuffer.box(5);
	pickbuffer.popMatrix();  
    }
  pickbuffer.endDraw();
}

class Cube {
  
  int id;
  int x, y, z;
  color c;
  
  public Cube(int id, int x, int y, int z) {
    this.id = id; this.x = x; this.y = y; this.z = z;
    changeColor();
  }
  
  public void changeColor() {
    c = color((int)random(255), (int)random(255), (int)random(255));
  }
  
} 

Edit : actually, you can use this method with OpenGL : just use OpenGL for display and P3D for the back-buffer. :-D