Picker for processing 2.0

aufsess

in Contributed Library Questions • 1 year ago

Has anyone had success getting a color buffer picker (for picking shapes) in processing 2.0 with open gl? I'm trying to use my old scripts that employ the picker library. I get this error:

The type processing.core.PGraphics3D cannot be resolved. It is indirectly referenced from required .class file.

Or use one of the other code snippets out there. I get these errors when I use this library:

http://wiki.processing.org/index.php?title=Picking_with_a_color_buffer

2012-05-23 11:20:23.585 java[2128:1f07] invalid drawable

OpenGL error 1286 at top beginDraw(): invalid framebuffer operation

OpenGL error 1286 at bot beginDraw(): invalid framebuffer operation

OpenGL error 1286 at top endDraw(): invalid framebuffer operation

java.lang.NullPointerException

at processing.mode.java.runner.Runner.findException(Runner.java:618)

Any luck with getting this to work?

Thanks!

Alex

Replies(18)

aufsess

Re: Picker for processing 2.0

1 year ago

I've been trying to just import the library directly into processing, but I haven't figured out how to get around an error I get from "createGraphics":

"You need to use "Import Library" to add picking.Buffer to your sketch.

Here is the code:

public class Picker {
/**
* Processing applet
*/
protected PApplet parent;
/**
* Main picking buffer
* Direct access to the buffer is allowed so you can draw shapes that
* you wouldn't like to draw on the main screen (like bounding boxes).
*/
public Buffer buffer;
public Picker(PApplet parent) {
this.parent = parent;
buffer = (Buffer) parent.createGraphics(parent.width, parent.height, "picking.Buffer");
buffer.callCheckSettings();
if (parent.recorder == null) {
parent.recorder = buffer;
}
buffer.background(0);
parent.registerPre(this);
parent.registerDraw(this);
}
public void pre() {
buffer.beginDraw();
if (parent.recorder == null) {
parent.recorder = buffer;
}
}
public void draw() {
buffer.endDraw();
}
/**
* Begins recording object(s)
*
* @param i Object ID
*/
public void start(int i) {
if (i < 0 || i > 16777214) {
PApplet.println("[Picking error] start(): ID out of range");
return;
}
if (parent.recorder == null) {
parent.recorder = buffer;
}
buffer.setCurrentId(i);
}
/**
* Stops/pauses recording object(s)
*/
public void stop() {
parent.recorder = null;
}
/**
* Resumes recording object(s)
*/
public void resume() {
if (parent.recorder == null) {
parent.recorder = buffer;
}
}
/**
* Reads the ID of the object at point (x, y)
* -1 means there is no object at this point
* @param x X coordinate
* @param y Y coordinate
* @return Object ID
*/
public int get(int x, int y) {
return buffer.getId(x, y);
}
/**
* Get the buffer
* @return Buffer
*/
public PGraphics getBuffer() {
return buffer;
}
}
public class Buffer extends PGraphics3D {
protected int current_color = 0;
public Buffer() {}
public boolean displayable() { return true; }
public void callCheckSettings() { super.checkSettings(); }
public void background(int arg) { super.background(0); }
public void background(float arg) { super.background(0); }
public void background(float arg, float arg_1) { super.background(0); }
public void background(int arg, float arg_1) { super.background(0); }
public void background(float arg, float arg_1, float arg_2) { super.background(0); }
public void background(float arg, float arg_1, float arg_2, float arg_3) { super.background(0); }
public void background(PImage arg) { super.background(0); }
public void lights() {}
public void smooth() {}
public void fill(int arg) {}
public void fill(float arg) {}
public void fill(float arg, float arg_1) {}
public void fill(int arg, float arg_1) {}
public void fill(float arg, float arg_1, float arg_2) {}
public void fill(float arg, float arg_1, float arg_2, float arg_3) {}
public void stroke(int arg) {}
public void stroke(float arg) {}
public void stroke(float arg, float arg_1) {}
public void stroke(int arg, float arg_1) {}
public void stroke(float arg, float arg_1, float arg_2) {}
public void stroke(float arg, float arg_1, float arg_2, float arg_3) {}
public void textureMode(int arg) {}
public void texture(PImage arg) {}
public void vertex(float x, float y, float z, float u, float v) { super.vertex(x, y, z); }
public void image(PImage arg, float arg_1, float arg_2) {}
public void image(PImage arg, float arg_1, float arg_2, float arg_3, float arg_4) {}
public void image(PImage arg, float arg_1, float arg_2, float arg_3, float arg_4, int arg_5, int arg_6, int arg_7, int arg_8) {}
protected void imageImpl(PImage image,
float x1, float y1, float x2, float y2,
int u1, int v1, int u2, int v2) {}
public void setCurrentId(int i) {
// ID 0 to 16777214 => COLOR -16777215 to -1 (white)
// -16777216 is black
current_color = i - 16777215;
super.fill(current_color);
}
public int getId(int x, int y) {
super.loadPixels();
// COLOR -16777216 (black) to -1 => ID -1 (no object) to 16777214
int id = pixels[y*width+x] + 16777215;
return id;
}
}

aufsess

Re: Picker for processing 2.0 // GLGraphics + Toxiclibs Implementation

1 year ago

I never figured out how to implement the Picker in 2.0 (I'm staying in 1.5 for the time being), but I did figure out how to implement it with GLGraphics and use it for picking faces on a mesh. Hope this is useful to someone. Hopefully will have time to make a stand alone example soon.

// CODE needs to be spliced into something else...
import codeanticode.glgraphics.*;
import toxi.geom.*;
import toxi.geom.mesh.*;
import toxi.processing.*;
GLGraphicsOffScreen pickerB = new GLGraphicsOffScreen(this, height, width, false, 4);
PImage pickerI;
GLTexture tex;
public void initGLPicker() {
// CONVERT TO GL MODEL START
currentMesh.computeVertexNormals(); // <-- Insert your mesh here from TOXILIBS (or other GLModel Shape)
float[] verts=currentMesh.getMeshAsVertexArray();
// in the array each vertex has 4 entries (XYZ + 1 spacing)
int numV=verts.length/4;
float[] norms=currentMesh.getVertexNormalsAsArray();
bSurf = new GLModel(me, numV, TRIANGLES, GLModel.STREAM);
bSurf.beginUpdateVertices();
for (int i = 0; i < numV; i++) bSurf.updateVertex(i, verts[4 * i], verts[4 * i + 1], verts[4 * i + 2]);
bSurf.endUpdateVertices();
bSurf.initNormals();
bSurf.beginUpdateNormals();
for (int i = 0; i < numV; i++) bSurf.updateNormal(i, norms[4 * i], norms[4 * i + 1], norms[4 * i + 2]);
bSurf.endUpdateNormals();
bSurf.initColors();
bSurf.beginUpdateColors();
for (int fNum=0; fNum<currentMesh.faces.size(); fNum++) {
int currentColor = fNum - 16777215;
int vNum = fNum*3;
// println("writing face # " + fNum + " vertice # " + vNum + " " + (vNum+1) + " " + (vNum+2) + " color # = " + currentColor);
if (vNum < bSurf.getSize()) {
bSurf.updateColor(vNum, currentColor, 255);
bSurf.updateColor(vNum+1, currentColor, 255);
bSurf.updateColor(vNum+2, currentColor, 255);
}
else
println("out of bounds with color vertices!");
}
bSurf.endUpdateColors();
println("Finished loading GLMODEL BUFFER");
// CONVERT TO GL MODEL END
}
public void updateGLPickerBuffer() {
// GL START
pickerB.beginDraw();
pickerB.beginGL();
pickerB.background(0);
pickerB.model(bSurf);
pickerB.endGL();
pickerB.endDraw();
// UPDATE Picker Texture TODO: THIS IS SLOWEST PART...
tex = pickerB.getTexture();
tex.updateTexture();
//tex.save("smooth.tif");
//tex.getImage(pickerI);
}
public int getId(int x, int y) {
int arrayMax = int(width*PickerRez)*int(height*PickerRez);
int pixelNumber = min(arrayMax, int(y*PickerRez*tex.width+x*PickerRez));
// COLOR -16777216 (black) to -1 => ID -1 (no object) to 16777214
int id = tex.pixels[pixelNumber] + 16777215; // 16777215
//int id = 10;
return id;
}

andres

Re: Picker for processing 2.0

1 year ago

Hi, I just tested the object picking code from the wiki using 2.0a6, and it works. Are you using an older alpha release?

aufsess

Re: Picker for processing 2.0

1 year ago

Hi Andres,

I'm not sure exactly what release I was using, but I was having trouble with the Picker library, not the wiki. My sketch needs max performance from the picker to work, so I implemented the GL Graphics version. I don't totally grasp how I will port all of these awesome GL Graphics library things to 2.0, so I'm holding off for now.

Now if only there was a way to grab one pixel from the GPU texture without having to use "loadtexture" command. I guess that's just inherent to the process.

tex = pickerB.getTexture();
tex.updateTexture();

andres

Re: Picker for processing 2.0

1 year ago

If you are using a PGraphics object created with a P2D or P3D renderer in Processing 2.0, then you should be able to retrieve the pixels array just by calling pg.loadPixels().

aufsess

Re: Picker for processing 2.0

1 year ago

My issues is that loadPixels takes some time as does updateTexture. I'm going to see whether I can multithread some of these calls and avoid the lag in my main script. The stuff I'm doing requires me to do a loadPixels every frame to access it with the picker. While it's definitely faster to run this with a glgraphics offscreen buffer the lag comes when I offload this buffer from my GPU to run a picker function on the array of pixels. I've reduced the size of the buffer, but his creates other issues.

I'm not sure if there is a way to avoid this offloading and just access it pixel by pixel directly (to check the color) directly on the GPU? I suppose that is the same problem...

andres

Re: Picker for processing 2.0

1 year ago

If you only need to check the color value at a specific pixel location, you could do:

import java.nio.*;
void setup() {
size(100, 100, P2D);
noSmooth();
}
void draw() {
background(0);
stroke(255, 0, 0);
strokeWeight(10);
strokeCap(RECT);
point(50, 50);
PGraphicsOpenGL pg = (PGraphicsOpenGL)g;
PGL pgl = pg.beginPGL();
int[] pixel = new int[1];
IntBuffer pixelBuffer = IntBuffer.wrap(pixel);
pgl.glReadPixels(mouseX, mouseY - 1, 1, 1, PGL.GL_RGBA, PGL.GL_UNSIGNED_BYTE, pixelBuffer);
int col = PGL.nativeToJavaARGB(pixel[0]);
println(red(col) + " " + green(col) + " " + blue(col));
pg.endPGL();
}

Copying all the pixels in the screen with a separate glReadPixels() call for each is going to be much slower than using loadPixels(), but if you need to check the color of only a few pixels per frame, then this should be faster.

Massimo

Re: Picker for processing 2.0

1 year ago

Hi Andres,

I tried to create a color picking function for 2D sprites (PNGs converted into white masks and then tinted with the color ID) using 2.0a8 and strange things come up:

1) if I use P3D or P2D and call loadPixels() or get() the sketch slow down terribly

2) If I implement offscreen surface using JAVA2D I get wrong color results

My friend Erika opened 2 issues about those bugs:

Issue with tint and JAVA2D
Issue with loadPixels

In the mean while I tried your sketch in this thread but something is missing:

PGL pgl = pg.beginPGL();

This method doesn't exist anymore.

Do you think is better to wait for 2.0 or do you have other suggestions about this problem?

Thank you in advance,
Massimo

andres

Re: Picker for processing 2.0

1 year ago

Hello Massimo,

Thanks for reporting the issues, I will look into them soon.

As for the beginPGL() function, it got renamed back to beginGL() in the latest alpha releases. However, the PGL class is still undergoing changes, and it is very likely that it will become beginPGL() again (and definitely) for the beta.

Massimo

Re: Re: Picker for processing 2.0

1 year ago

Thank you andres for your constant work on Processing!

Also your suggestion seems to solve my color picking problem!

Massimo

Re: Picker for processing 2.0

1 year ago

I was wrong: your suggestion works fine if you draw on main PGraphics (g) but I tried to implement an offscreen buffer and, instead of reading the offscreen pixel, it get it from displayed pixels :-(

import java.nio.*;
PGraphics offscreen;
void setup() {
size(100, 100, P2D);
offscreen = createGraphics(100, 100, P2D);
offscreen.noSmooth();
//offscreen.hint(ENABLE_ACCURATE_2D); //is it better to include this?
}
void draw() {
offscreen.beginDraw();
offscreen.background(0);
offscreen.stroke(255, 0, 0);
offscreen.strokeWeight(10);
offscreen.strokeCap(RECT);
offscreen.point(50, 50);
offscreen.endDraw();
PGraphicsOpenGL pg = (PGraphicsOpenGL) offscreen;
PGL pgl = pg.beginGL();
int[] pixel = new int[1];
IntBuffer pixelBuffer = IntBuffer.wrap(pixel);
pgl.glReadPixels(mouseX, mouseY - 1, 1, 1, PGL.GL_RGBA, PGL.GL_UNSIGNED_BYTE, pixelBuffer);
int col = PGL.nativeToJavaARGB(pixel[0]);
println(red(col) + " " + green(col) + " " + blue(col));
pg.endGL();
}

This offscreen version of your sketch should work but it does not. Am I missing something?

I'm not so confident with GL related stuff and I hadn't the time to study your latest committed code deeply but after a small travel inside opengl PGraphics implementation I found something that confused me a bit:

if (primarySurface) {
pgl.updatePrimary();
if (pgl.primaryIsDoubleBuffered()) {
pgl.drawBuffer(PGL.BACK);
}
} else {
if (!pgl.initialized) {
initOffscreen();
} else {
boolean outdated = offscreenFramebuffer != null && offscreenFramebuffer.contextIsOutdated();
boolean outdatedMulti = offscreenFramebufferMultisample != null && offscreenFramebufferMultisample.contextIsOutdated();
if (outdated || outdatedMulti) {
pgl.initialized = false;
initOffscreen();
}
}
pushFramebuffer();
if (offscreenMultisample) {
setFramebuffer(offscreenFramebufferMultisample);
} else {
setFramebuffer(offscreenFramebuffer);
}
pgl.updateOffscreen(pgPrimary.pgl);
pgl.drawBuffer(PGL.COLOR_ATTACHMENT0);
}
// We are ready to go!

This snippet is from beginDraw method inside PGraphicsOpenGL.java... my question is should I declare somehow that the offscreen PGraphics object is the FrameBuffer?

Thank you in advance,
Massimo

andres

Re: Picker for processing 2.0

1 year ago

Hello, even if you only want to do low-level operations on the offscreen surface, you still need to enclose them between beginDraw/endDraw:

PGraphicsOpenGL pg = (PGraphicsOpenGL) offscreen;
offscreen.beginDraw();
PGL pgl = pg.beginGL();
int[] pixel = new int[1];
IntBuffer pixelBuffer = IntBuffer.wrap(pixel);
pgl.glReadPixels(mouseX, mouseY - 1, 1, 1, PGL.GL_RGBA, PGL.GL_UNSIGNED_BYTE, pixelBuffer);
int col = PGL.nativeToJavaARGB(pixel[0]);
println(red(col) + " " + green(col) + " " + blue(col));
pg.endGL();
offscreen.endDraw();

Massimo

Re: Re: Picker for processing 2.0

8 months ago

Hi Andres,

I'm trying to recreate this snippet with new Processing 2.0b7 version but with no luck.

If I use main buffer it works as color picking method but if I use it with offscreen it will return always 0

import java.nio.*;
PGraphics offscreenb;
int currentColor = 0;
void setup() {
size(640, 480, P3D);
offscreenb = createGraphics(width, height, P3D);
}
void draw() {
//draw a green box in the center
translate(width / 2, height / 2);
fill(0, 128, 0);
stroke(0, 200, 0);
box(80);
drawOffscreen();
currentColor = getPixel();
}
void mousePressed() {
println(red(currentColor) + "," + green(currentColor) + "," + blue(currentColor) + ",");
}
void drawOffscreen() {
//draw a red box in the center of offscreen
offscreenb.beginDraw();
offscreenb.translate(width / 2, height / 2);
offscreenb.fill(255, 0, 0);
offscreenb.noStroke();
offscreenb.box(80);
offscreenb.endDraw();
}
int getPixel() {
PGraphicsOpenGL pg = (PGraphicsOpenGL) offscreenb; //change this with g and it works... :-(
offscreenb.beginDraw();
PGL pgl = pg.beginPGL();
IntBuffer pixelBuffer = IntBuffer.allocate(1);
pgl.readPixels(mouseX, mouseY -1, 1, 1, PGL.RGBA, PGL.UNSIGNED_BYTE, pixelBuffer);
pg.endPGL();
offscreenb.endDraw();
return pixelBuffer.get();
}

How are you folks getting color of pixels when using offscreen buffer and opengl?

Why getColorValue() is protected? :-(

Thank you in advance,

Massimo

chicoatomo

Re: Picker for processing 2.0

8 months ago

Hello

I've modified the library picking (Nicolas Clavaud) to work with processing version 2.0.b7.

You can also define class Buffer and Picker, directly on your processing editor

Buffer Class

public class Buffer extends PGraphics3D
{
protected int current_color = 0;
public Buffer(PApplet parent)
{
setParent(parent);
setSize(parent.width, parent.height);
}
@Override
public boolean displayable() { return true; }
public void callCheckSettings() { super.checkSettings(); }
@Override
public void lights() {}
@Override
public void smooth() {}
@Override
public void fill(int arg) {}
@Override
public void fill(float arg) {}
@Override
public void fill(float arg, float arg_1) {}
@Override
public void fill(int arg, float arg_1) {}
@Override
public void fill(float arg, float arg_1, float arg_2) {}
@Override
public void fill(float arg, float arg_1, float arg_2, float arg_3) {}
@Override
public void stroke(int arg) {}
@Override
public void stroke(float arg) {}
@Override
public void stroke(float arg, float arg_1) {}
@Override
public void stroke(int arg, float arg_1) {}
@Override
public void stroke(float arg, float arg_1, float arg_2) {}
@Override
public void stroke(float arg, float arg_1, float arg_2, float arg_3) {}
@Override
public void textureMode(int arg) {}
@Override
public void texture(PImage arg) {}
@Override
public void vertex(float x, float y, float z, float u, float v) { super.vertex(x, y, z); }
@Override
public void image(PImage arg, float arg_1, float arg_2) {}
@Override
public void image(PImage arg, float arg_1, float arg_2, float arg_3, float arg_4) {}
@Override
public void image(PImage arg, float arg_1, float arg_2, float arg_3, float arg_4, int arg_5, int arg_6, int arg_7, int arg_8) {}
@Override
protected void imageImpl(PImage image, float x1, float y1, float x2, float y2, int u1, int v1, int u2, int v2) {}
public void setCurrentId(int i)
{
// ID 0 to 16777214 => COLOR -16777215 to -1 (white)
// -16777216 is black
current_color = i - 16777215;
super.fill(current_color);
}
public int getId(int x, int y)
{
super.loadPixels();
// COLOR -16777216 (black) to -1 => ID -1 (no object) to 16777214
int id = pixels[y*width+x] + 16777215;
return id;
}
}

Picker Class

public class Picker
{
protected PApplet parent;
public Buffer buffer;
public Picker(PApplet parent)
{
this.parent = parent;
buffer = new Buffer(parent);
//parent.registerMethod("pre",this);
//parent.registerMethod("draw",this);
}
public void begin()
{
if(parent.recorder == null)
{
parent.recorder = buffer;
buffer.beginDraw();
}
}
public void end()
{
buffer.endDraw();
parent.recorder = null;
}
public void start(int i)
{
if (i < 0 || i > 16777214)
{
PApplet.println("[Picking error] start(): ID out of range");
return;
}
buffer.setCurrentId(i);
}
public void stop()
{
parent.recorder = null;
}
public void resume()
{
parent.recorder = buffer;
}
public int get(int x, int y)
{
return buffer.getId(x, y);
}
public PGraphics getBuffer()
{
return buffer;
}
}

Example

Picker picker;
float a = 0.0;
void setup()
{
size(200,200, P3D);
picker = new Picker(this);
}
void draw()
{
background(0,128,0);
a += 0.01;
picker.begin();
picker.start(0);
pushMatrix();
translate(80, 75);
rotateX(a);
rotateY(a);
fill(200,40,60);
box(50);
popMatrix();
picker.start(1);
pushMatrix();
translate(140, 75);
rotateX(a);
rotateY(a);
fill(60,40,200);
box(20);
popMatrix();
picker.end();
}
void mousePressed()
{
println(picker.get(mouseX, mouseY));
}

I hope that helps somewhat

chicoatomo

Re: Picker for processing 2.0

8 months ago

Important Note:

I don't know exactly why, but for some reason background can't be correctly overloaded, so is very important to call picker.begin() after the background() instruction

chicoatomo

Re: Picker for processing 2.0

8 months ago

https://forum.processing.org/topic/using-the-picking-library-in-processing-2-0

chicoatomo

Re: Picker for processing 2.0

8 months ago

Hi all

I`ve found a important error in my solution.

the background is not managed properly in the recorder object

I'll try to find a solution

Sorry

Top Reply