change colors of points data from kinect using shaders

edited July 2016 in Kinect

Sorry, I am just starting with shaders. I have read a little but not enough to understand how to change this code.

the code below would build a 3D scene from kinect point cloud data and set all points to white. I just wanted to understand how I could set a matrix of colors dynamically for each point. (basically I just want points closer to kinect to be brighter)

I have tried to implement this, but I couldn't understand how to work with textures yet.

Thank you!

import java.nio.*;
import org.openkinect.freenect.*;
import org.openkinect.processing.*;

// Kinect Library object
Kinect kinect;

// Angle for rotation
float a = PI;

//openGL
PGL pgl;
PShader sh;

int  vertLoc;
int  vertLocId;
//int  colorLoc;

void setup() {
  // Rendering in P3D
  size(800, 600, P3D);
  kinect = new Kinect(this);
  kinect.initDepth();
  kinect.enableMirror(true); //added

  //load shaders
  sh = loadShader("frag.glsl", "vert.glsl");
  PGL pgl = beginPGL();

  IntBuffer intBuffer = IntBuffer.allocate(1);
  pgl.genBuffers(1, intBuffer);

  //memory location of the VBO
  vertLocId = intBuffer.get(0);

  endPGL();
}

void draw() {

  background(0);

 image(kinect.getDepthImage(), 0, 0, 320, 240); //why this is not mirrored ?

  pushMatrix();
  translate(width/2, height/2, 600);
  scale(150);
  rotateY(a);
  int vertData = kinect.width * kinect.height;
  FloatBuffer depthPositions =  kinect.getDephToWorldPositions();
  pgl = beginPGL();
  sh.bind();

  vertLoc  = pgl.getAttribLocation(sh.glProgram, "vertex");

  //color for ALL POINTS of the point cloud
  sh.set("fragColor", 1.0f, 1.0f, 1.0f, 1.0f); //<-----

  pgl.enableVertexAttribArray(vertLoc);

  pgl.bindBuffer(PGL.ARRAY_BUFFER, vertLocId);
  pgl.bufferData(PGL.ARRAY_BUFFER, Float.BYTES * vertData *3, depthPositions, PGL.DYNAMIC_DRAW);
  pgl.vertexAttribPointer(vertLoc, 3, PGL.FLOAT, false, Float.BYTES * 3, 0);

  pgl.bindBuffer(PGL.ARRAY_BUFFER, 0);

  //draw the XYZ depth camera points
  pgl.drawArrays(PGL.POINTS, 0, vertData);

  //clean up the vertex buffers
  pgl.disableVertexAttribArray(vertLoc);

  sh.unbind();
  endPGL();

  popMatrix();

  fill(255, 0, 0);
  text(frameRate, 50, 50);
}

frag.glsl:

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

#define PROCESSING_TEXTURE_SHADER //

varying vec4 vertColor;
//varying vec4 vertTexCoord;

//input color
uniform vec4 fragColor;
//uniform sampler2D texture; 

void main() {
  //vec4 col = texture2D(texture, vertTexCoord.st);
  //outputColor
  gl_FragColor = fragColor;
  //gl_FragColor = vec4(col.rgb, 1.0);
}

vert.glsl:

uniform mat4 transform;

attribute vec4 vertex;
attribute vec4 color;

varying vec4 vertColor;


void main() {
  gl_Position = transform * vertex;    
  vertColor = color;
}
Tagged:

Answers

  • edited July 2016

    Ok, just in case someone is also looking for an solution on these regards:

    // Based on PointCloudOGL example
    // Added color per vertex based on RGB video pixels
    // Anderson Sudario 2016/Jul/19
    /*
    
    #ifdef GL_ES
    precision mediump float;
    precision mediump int;
    #endif
    
    varying vec4 vertColor;
    
    //input color
    //uniform vec4 fragColor;
    
    void main() {
    
      //outputColor
      gl_FragColor = vertColor; //changed
    }
    */
    
    import java.nio.*;
    import com.jogamp.common.nio.Buffers;
    import org.openkinect.freenect.*;
    import org.openkinect.processing.*;
    
    // Kinect Library object
    Kinect kinect;
    
    // Angle for rotation
    float a = PI;
    
    //openGL
    PGL pgl;
    PShader sh;
    
    //vertex
    int  vertLoc;
    int  vertLocId;
    //colors
    int  colorLoc;
    int  colorLocId;
    float []colors = new float[640 * 480 * 4]; 
    
    void setup() {
      // Rendering in P3D
      size(800, 600, P3D);
      kinect = new Kinect(this);
      kinect.initDepth();
      kinect.initVideo();
      kinect.enableMirror(true); 
    
      //load shaders
      sh = loadShader("frag.glsl", "vert.glsl"); //only one change in frag.glsl
      PGL pgl = beginPGL();
    
      IntBuffer intBuffer = IntBuffer.allocate(2);
      pgl.genBuffers(2, intBuffer);
    
      //memory location of the VBO
      vertLocId = intBuffer.get(0);
      colorLocId = intBuffer.get(1);
      endPGL();
    }
    
    
    void buildColors() {
      int textureOffsetY = kinect.width* 35 ; //int(map(mouseY, 0, height, 0, 100));//kinect.width * 100;
      //println(int(map(mouseY, 0, height, 0, 100)));
      for (int i = 0; i < kinect.width*kinect.height - textureOffsetY; i+=1) { //skip if wanted
    
        color c = kinect.getVideoImage().pixels[i+textureOffsetY];
        //float[] f = RGBAToOpenGL( 255, 255, 255, 255);
        colors[i*4]     = (c >> 16 & 0xff) /100f; 
        colors[i*4 + 1] = (c >> 8 & 0xff) /100f;
        colors[i*4 + 2] = (c >> 0 & 0xff) /100f;
        colors[i*4 + 3] = 1;
      }
    }
    
    
    void draw() {
    
      background(0);
    
      //image(kinect.getDepthImage(), 0, 0, 320, 240); 
      pushMatrix();
      translate(width/2, height/2, 600);
      scale(150);
      rotateY(a);
    
    
      int vertData = kinect.width * kinect.height;
      FloatBuffer depthPositions =  kinect.getDephToWorldPositions();
    
      buildColors();
      FloatBuffer colorBuffer = Buffers.newDirectFloatBuffer(640 * 480 * 4); //allocateDirectFloatBuffer(640 * 480 * 4);
      colorBuffer.put(colors, 0, 640 * 480 * 4);
      colorBuffer.rewind();
    
      pgl = beginPGL();
      sh.bind();
    
      vertLoc  = pgl.getAttribLocation(sh.glProgram, "vertex");
      colorLoc  = pgl.getAttribLocation(sh.glProgram, "color");
    
      pgl.enableVertexAttribArray(vertLoc);
      pgl.enableVertexAttribArray(colorLoc);
    
      pgl.bindBuffer(PGL.ARRAY_BUFFER, vertLocId);
      pgl.bufferData(PGL.ARRAY_BUFFER, Float.BYTES * vertData *3, depthPositions, PGL.DYNAMIC_DRAW);
      pgl.vertexAttribPointer(vertLoc, 3, PGL.FLOAT, false, Float.BYTES * 3, 0);
    
      final int colorStride  =  4 * Float.BYTES;
      final int colorOffset  =  0 * Float.BYTES;
    
      pgl.bindBuffer(PGL.ARRAY_BUFFER, colorLocId);
      pgl.bufferData(PGL.ARRAY_BUFFER, Float.BYTES * colors.length, colorBuffer, PGL.DYNAMIC_DRAW);
      pgl.vertexAttribPointer(colorLoc, 4, PGL.FLOAT, false, colorStride, colorOffset);
    
      pgl.bindBuffer(PGL.ARRAY_BUFFER, 0);
    
      //draw the XYZ depth camera points
      pgl.drawArrays(PGL.POINTS, 0, vertData);
    
      //clean up the vertex buffers
      pgl.disableVertexAttribArray(vertLoc);
      pgl.disableVertexAttribArray(colorLoc);
    
      sh.unbind();
      endPGL();
    
      popMatrix();
    
      fill(255, 0, 0);
      text(frameRate, 50, 50);
    
      // Rotate
      a += sin(millis()/1000.0) / 500.0;
    }
    
    FloatBuffer allocateDirectFloatBuffer(int n) {
      return ByteBuffer.allocateDirect(n * Float.BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer();
    }
    
    //float[] RGBAToOpenGL(int r1, int g1, int b1, int a1) {
    //  float[] tmp = new float[4];
    //  tmp[0] = r1/255f;
    //  tmp[1] = g1/255f;
    //  tmp[2] = b1/255f;
    //  tmp[3] = a1/255f;
    //  return tmp;
    //}
    
Sign In or Register to comment.