hey guys. in this time i need some help to optimize a system to pick shaders in real time (fragment shaders, and vertex shaders). This code is a mix that i recoleted in internet with code writing by myself.

Theres a lot of problem and i need help:

the idea of the program is that user can pick the shader in real time. there are two problems, first, i dont know how uploading the vertexShader, when i save it and reload it i need to click in another frag and then vertex upload.

second, it seems very inestable.the window of shaders crash very often, specially when frame rate is lowest. I wondering how to fix that. maybe with thread(), but it dont know exactly how apply this method..

If someone have time and wants to help, im sure that this code needs to be re writing. Thanks!

ShaderWindow shaderWin;

boolean recording = false;


import netP5.*;
import oscP5.*;

OscP5 oscP5;
NetAddress myRemoteLocation;

import themidibus.*;
MidiBus bus;
MidiBus apc;
float cc [] =new float [256];


boolean record, load;
double folderTime;
File folderShader;
float desmouse = 0.5;

// BUFFERS-SHADERS

String dirFolderShaders;
String dirFolderShaders2;
String dirFolderShadersvertex;

PGraphics [] buffers = new PGraphics [4];
PGraphics buffer;

LoadShader loadshader;

LoadShader loadshader2;
LoadShader loadshaderVertex;

ArrayList<ViewShader>vs = new ArrayList<ViewShader>();
ViewShaderVertex ss;

PShader sh, buffer1;
float count;
float angle = random(0, 0.005);

int choiceSize =1;
//int ww =  1920;
//int h = 1080;

void settings() {
  if (choiceSize==0) fullScreen(P3D, 2);
  if (choiceSize==1) size(1000, 1000, P3D);
  if (choiceSize==2) size(720, 1280, P3D);
}

void setup() {

  // BUFFERS-SHADERS

  dirFolderShaders = sketchPath("frag/");  //PUT SOME FRAGMENT IN THIS FOLDER

  dirFolderShadersvertex = sketchPath("vertex/"); PUT SOME VERTEX IN THIS FOLDER

  loadshader = new LoadShader(dirFolderShaders);

  loadshaderVertex = new LoadShader(dirFolderShadersvertex);


  for (int i = 0; i < buffers.length; i++) {
    buffers[i] = createGraphics(width, height, P3D);
  }


  ss = new ViewShaderVertex(loadshaderVertex.shadersPaths[1], loadshaderVertex, buffer);
  vs.add(new ViewShader(loadshader.shadersPaths[0], loadshader, buffer, ss));

  shaderWin = new ShaderWindow();
}

void draw() {

  background(0);


  for (ViewShader s : vs) {
    s.update();
  }


  if (folderShader.listFiles().length != loadshader.shadersNames.length) {
    loadshaderVertex.loadFolderShader();
  }

  if (folderShader.listFiles().length != loadshader.shadersNames.length) {
    loadshader.loadFolderShader();
  }




  buffers[0].beginDraw();
  buffers[0].background(0);
  buffers[0].translate(width/2, height/2);


  buffers[0].sphere(100);
  buffers[0].endDraw();

  image(buffers[0],0,0);
}



// LOAD SHADER CLASS

class LoadShader {

  String dirFolderShaders;

  String[] shadersNames;
  String[] shadersPaths;

  LoadShader(String _dirFolderShaders) {
    dirFolderShaders= _dirFolderShaders;
    loadFolderShader();
  }

  void loadFolderShader() {
    load = true;
    folderShader = new File(dirFolderShaders);
    File files[] = folderShader.listFiles();
    shadersNames = new String[files.length];
    shadersPaths = new String[files.length];
    for (int i = 0; i < files.length; i++) {
      shadersNames[i] = split(files[i].getName(), ".")[0];
      shadersPaths[i] = files[i].getAbsolutePath();
      //println(shadersNames[i]);
    }
    load = false;
  }
}

class ViewShader {

  boolean reload, error, loaded;
  File fileShader;

  long timeModi;
  long timeModis;
  String src;

  PShader shader;
  PShader shader1;
  PShader shader2;
  float a;
  LoadShader loadshader;
  ViewShaderVertex ss;
  PGraphics b;

  ViewShader(String _src, LoadShader _loadshader, PGraphics _b) {
    src = _src;
    fileShader = new File(src);
    openShader(fileShader);
    //shader1 = loadShader("blur.glsl", "bloomVert.glsl");
    //shader2 = loadShader("blur_vertical.glsl", "bloomVert.glsl");
    loadshader = _loadshader;
    b = _b;
  }

  ViewShader(String _src, LoadShader _loadshader, PGraphics _b, ViewShaderVertex _s) {
    ss = _s;

    //shader = loadShader("test2.glsl", "defaultVertex2.glsl");
    //shader1 = loadShader("blur.glsl", "bloomVert.glsl");
    //shader2 = loadShader("blur_vertical.glsl", "bloomVert.glsl");

    src = _src;
    fileShader = new File(src);
    openShader(fileShader);
    loadshader = _loadshader;
    b = _b;
  }


  void update() {
    if (reload) {      
      openShader(fileShader);
      reload = false;
    }
    if (shader != null) {

      //float p = map(cc[15], 0, 1, 0, 1);
      //float amt = map(cc[16], 0, 1, 1, 50);
      //float  h = map(cc[48], 0, 1, 0, 0.1);
      // //float  h = map(cc[14], 0, 1, 0, 0.5);


      shader.set("u_time", random(10));

      //shader1.set("u_time", angle);
      //shader2.set("u_time", angle);

      //shader.set("var2", p);
      //shader.set("amt", amt);

      //shader1.set("var2", p);
      //shader2.set("var2", p);


      if (fileShader.lastModified() != timeModi) {
        openShader(fileShader);
      }
    }
  }


  void newShader(String _srcc) {
    src = _srcc;
    fileShader = new File(src);
    reload = true;
    openShader(fileShader);
  }

  void openShader(File file) {
    if (file != null) {
      fileShader = file;
      timeModi = fileShader.lastModified();  
      try {
        shader = loadShader(file.getAbsolutePath(), ss.fileShader.getAbsolutePath());

        buffers[0].shader(shader);


        println(file.getAbsolutePath());
        error = false;
        loaded = true;
      }
      catch (RuntimeException e) {
        if (error == false) {
          error = true;
          // String time = nf(str(hour()),2) + ":" + nf(str(minute()),2) + ":" + nf(str(second()),2);
          println("\n");
          // println("At", time, "loadShader() returned the following error: \n");
          println("loadShader() returned the following error: \n");
          e.printStackTrace();
        }
        loaded = false;
      }
    }
  }
}

public class ShaderWindow extends PApplet {

  ShaderWindow() {
    super();
    PApplet.runSketch(new String[] {this.getClass().getSimpleName()}, this);
  }

  boolean mover;

  void settings() {
    size(600, 800, P2D);
  }

  void setup() {
    this.frameRate(30);
  }


  void draw() {

    background(25);


    for (int i = 0; i < vs.size(); i++) {
      ViewShader sss = vs.get(i);
      selector(i*100, 16, 100, width-16, sss, sss.loadshader, this);
    }

    ViewShaderVertex s = ss;  
    selector2(400, 16, 100, width-16, s, ss.loadshader, this);

  }

  void selector(float xx, float yy, float ww, float hh, ViewShader vs, LoadShader loadshader, PApplet p) {
    p.fill(150);
    p.noStroke();
    p.rectMode(CORNER);

    int cant = loadshader.shadersNames.length;
    p.rect(xx, yy+cant*16, ww, hh-cant*16);
    for (int i = 0; i < cant; i++) {
      boolean sobre = false;
      if (p.mouseX >= xx && p.mouseX < xx+ww && p.mouseY >= yy+i*16 && p.mouseY < yy+(i+1)*16) {
        sobre = true;
      }
      if (sobre) {
        p.fill(125);
      } else {
        p.fill(100, 50);
      }
      boolean selec = vs.src.equals(loadshader.shadersPaths[i]);
      if (p.mousePressed && sobre && !selec) {
        vs.newShader(loadshader.shadersPaths[i]);
      }

      if (selec) {
        p.fill(100);
        if (vs.error) {
          p.fill(200, 10, 10);
        }
      }

      p.rect(xx, yy+i*16, ww, 16);
      p.textAlign(LEFT, TOP);
      p.fill(250);
      if (i < loadshader.shadersNames.length && !load)
        text(loadshader.shadersNames[i], xx+5, yy+i*16);
    }
  }

  void selector2(float xx, float yy, float ww, float hh, ViewShaderVertex vs, LoadShader loadshader, PApplet p) {
    p.fill(150);
    p.noStroke();
    p.rectMode(CORNER);

    int cant = loadshader.shadersNames.length;
    p.rect(xx, yy+cant*16, ww, hh-cant*16);
    for (int i = 0; i < cant; i++) {
      boolean sobre = false;
      if (p.mouseX >= xx && p.mouseX < xx+ww && p.mouseY >= yy+i*16 && p.mouseY < yy+(i+1)*16) {
        sobre = true;
      }
      if (sobre) {
        p.fill(125);
      } else {
        p.fill(100, 50);
      }
      boolean selec = ss.src.equals(loadshader.shadersPaths[i]);
      if (p.mousePressed && sobre && !selec) {
        vs.newShader(loadshader.shadersPaths[i]);
      }

      if (selec) {
        p.fill(100);
        if (vs.error) {
          p.fill(200, 10, 10);
        }
      }

      p.rect(xx, yy+i*16, ww, 16);
      p.textAlign(LEFT, TOP);
      p.fill(250);
      if (i < loadshader.shadersNames.length && !load)
        p.text(loadshader.shadersNames[i], xx+5, yy+i*16);
    }
  }
}

class ViewShaderVertex {

  boolean reload, error, loaded;
  File fileShader;
  long timeModi;
  String src;
  String src2;

  PShader shader;

  float a;
  LoadShader loadshader;
  ViewShader ss;

  ViewShaderVertex(String _src, LoadShader _loadshader, PGraphics b) {

    src = _src;
    fileShader = new File(src);
    openShader(fileShader);

    loadshader = _loadshader;
  }



  void update() {
    if (reload) {      
      openShader(fileShader);
      reload = false;
    }
    if (shader != null) {

      if (fileShader.lastModified() != timeModi) {
        openShader(fileShader);

      }
    }
  }


  void newShader(String src) {
    this.src = src;
    fileShader = new File(src);
    reload = true;
    openShader(fileShader);
  }

  void openShader(File file) {

    if (file != null) {
      fileShader = file;
      timeModi = fileShader.lastModified();

      try {
                for (ViewShader s : vs) {
          s.update();
        }
        //shader = loadShader("test.glsl", file.getAbsolutePath());
        shader.setVertexShader(file.getAbsolutePath());
        buffers[0].shader(shader);

        println(file.getAbsolutePath());
        error = false;
        loaded = true;
      }
      catch (RuntimeException e) {
        if (error == false) {
          error = true;
          // String time = nf(str(hour()),2) + ":" + nf(str(minute()),2) + ":" + nf(str(second()),2);
          println("\n");
          // println("At", time, "loadShader() returned the following error: \n");
          println("loadShader() returned the following error: \n");
          e.printStackTrace();
        }
        loaded = false;
      }
    }
  }
}

I am afraid that I do not know how to do this without an offscreen buffer.

Not sure if this is exactly what you were after but is a proof of concept I suppose.

It was a challenge tuning in the depth value.

There should be another way... :-?

Most likely a step in the matrix math that I am missing.

Fragment shader:

varying vec4 vertColor;

void main() {
    gl_FragColor = vertColor;
}

Vertex shader:

uniform mat4 projection;
uniform mat4 modelview;

attribute vec4 position;
attribute vec4 color;
attribute vec2 offset;

varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform int getZ;
uniform float maxDepth;

void main() {

  vec4 pos = modelview * position;
  vec4 clip = projection * pos;
  vec4 posit = clip + projection * vec4(offset, 0., 0.);

  float depth = -vertTexCoord.z * 8.0; // Not fond of this magic multiplier...
  vec4 deep = vec4((depth/maxDepth).xxx, 1.0);

  vertColor = getZ != 0 ? deep : color; // Choose colored dots or depth value

  gl_Position = posit;
}

I wound up using the modified blur shader from your other post. https://forum.processing.org/two/discussion/27807/how-to-combine-a-z-buffer-with-a-blur-fragment-shader

Blur shader:

#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;
uniform sampler2D depthBuffer;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;
varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>
const float pi = 3.14159265;

void main() {  
  vec4 effx = texture2D(depthBuffer, vertTexCoord.st);
  //float numBlurPixelsPerSide = float(blurSize / 2); 
  int numBlurPixelsPerSide = int((blurSize - effx.x*blurSize) / 2); // <--    Using depth info here, farther away, more blur

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;

  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));

  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }

  gl_FragColor = (avgValue / coefficientSum);
}

Sketch:

import peasy.*;
import peasy.org.apache.commons.math.*;
import peasy.org.apache.commons.math.geometry.*;

// Multipass. No, not that kind. Shaders. https://en.wikipedia.org/wiki/Deferred_shading

PGraphics zbuff, starbuff;

PeasyCam rendercam, bufcam;
PShader pointShader, blurShader;
PShape shp;
ArrayList<PVector> vectors = new ArrayList<PVector>();

void setup() {
  size(900, 900, P3D);
  frameRate(120);
  //smooth(8);

  zbuff = createGraphics(width, height, P3D);
  starbuff = createGraphics(width, height, P3D);

  // Camera settings need to match
  bufcam = new PeasyCam(this, zbuff, 500);
  bufcam.setMaximumDistance(width);
  zbuff.perspective(60 * DEG_TO_RAD, width/float(height), 2, 6000);

  rendercam = new PeasyCam(this, starbuff, 500);
  rendercam.setMaximumDistance(width);
  starbuff.perspective(60 * DEG_TO_RAD, width/float(height), 2, 6000);

  double d = bufcam.getDistance()*3;

  // Shader setup, adjust as needed
  blurShader = loadShader("blurfrag.glsl");
  blurShader.set("sigma", 1.25);
  blurShader.set("blurSize", 9);

  pointShader = loadShader("pointfrag.glsl", "pointvert.glsl");
  pointShader.set("maxDepth", (float) d);

  for (int i = 0; i < 5000; i++) {
    vectors.add(new PVector(random(width), random(width), random(width)));
  }

  // Star depth
  zbuff.shader(pointShader, POINTS);
  zbuff.strokeWeight(2);
  zbuff.stroke(255);

  // Stars
  starbuff.shader(pointShader, POINTS);
  starbuff.strokeWeight(2);
  starbuff.stroke(255);

  // We can use the same shape object
  shp = createShape();
  shp.setStroke(255);
  shp.beginShape(POINTS);
  shp.translate(-width/2, -width/2, -width/2);
  for (PVector v: vectors) {
    shp.stroke(255);
    // We can color the stars too
    // The selection of depth or color is in the shader
    //shp.stroke((int)random(255),(int)random(255),(int)random(255));
    shp.vertex(v.x, v.y, v.z);
  }
  shp.endShape();

  shader(blurShader);
}

void draw(){
  zbuff.beginDraw();
  zbuff.background(0);
  pointShader.set("getZ", 1);
  zbuff.shape(shp, 0, 0);
  zbuff.endDraw();

  starbuff.beginDraw();
  starbuff.background(0);
  pointShader.set("getZ", 0);
  starbuff.shape(shp, 0, 0);
  starbuff.endDraw();

  // pass in the z depth info
  blurShader.set("depthBuffer", zbuff);

  // and draw the star field
  blurShader.set("horizontalPass", 0);
  image(starbuff, 0, 0);
  blurShader.set("horizontalPass", 1);
  filter(blurShader);

  // Match render to zbuff positions
  rendercam.rotateY(.0001);
  rendercam.rotateX(.00005);
  bufcam.rotateY(.0001);
  bufcam.rotateX(.00005);

  //println(frameRate);
}

Seems the next thing I need to do is sign up at the new forum...

I would like to display thousands of points on a 3D canvas with a Depth of Field effect. More specifically, I would like to use a z-buffer (depth buffering) to adjust the level of blur of a point based on its distance from the camera.

So far, I could come up with the following point shader:

pointfrag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

varying vec4 vertColor;
uniform float maxDepth;

void main() {

  float depth = gl_FragCoord.z / gl_FragCoord.w;
  gl_FragColor = vec4(vec3(vertColor - depth/maxDepth), 1) ;

}

pointvert.glsl

uniform mat4 projection;
uniform mat4 modelview;

attribute vec4 position;
attribute vec4 color;
attribute vec2 offset;


varying vec4 vertColor;
varying vec4 vertTexCoord;

void main() {
  vec4 pos = modelview * position;
  vec4 clip = projection * pos;

  gl_Position = clip + projection * vec4(offset, 0, 0);

  vertColor = color;
}

I also have a blur shader (originally from the PostFX library):

blurfrag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif


#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;

varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>

const float pi = 3.14159265;

void main() {  
  float numBlurPixelsPerSide = float(blurSize / 2); 

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;
  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }

  gl_FragColor = (avgValue / coefficientSum);
}

Question:

• How can I combine the blur fragment shader with the point fragment shader ?

Ideally I'd like to have one single fragment shader that computes the level of blur based on the z-coordinate of a point. Is that even possible ?

@noahbuddy @kosowski I would be really grateful if you could help me.

An example sketch displaying points using the pointfrag.glsl and pointvert.glsl shaders above:

sketch.pde

import peasy.*;
import peasy.org.apache.commons.math.*;
import peasy.org.apache.commons.math.geometry.*;

PeasyCam cam;
PShader pointShader;
PShape shp;
ArrayList<PVector> vectors = new ArrayList<PVector>();

void setup() {
  size(900, 900, P3D);
  frameRate(1000);
  smooth(8);

  cam = new PeasyCam(this, 500);
  cam.setMaximumDistance(width);
  perspective(60 * DEG_TO_RAD, width/float(height), 2, 6000);

  double d = cam.getDistance()*3;

  pointShader = loadShader("pointfrag.glsl", "pointvert.glsl");
  pointShader.set("maxDepth", (float) d);


  for (int i = 0; i < 5000; i++) {
    vectors.add(new PVector(random(width), random(width), random(width)));
  }

  shader(pointShader, POINTS);
  strokeWeight(2);
  stroke(255);

  shp = createShape();
  shp.beginShape(POINTS);
  shp.translate(-width/2, -width/2, -width/2);  
  for (PVector v: vectors) {
    shp.vertex(v.x, v.y, v.z);
  }
  shp.endShape();

}

void draw(){
  background(0);
  shape(shp, 0, 0);

  cam.rotateY(.0001);
  cam.rotateX(.00005);

  println(frameRate);
}

https://i.imgur.com/mf5dVRn.mp4

hey everybody. this time i've a problem sending invididual mensajes to a fragment shader. I've an array of spheres, and i want that every sphere have a different size of blur, witch is a uniform of the fragment. The main code is from an example, a post-processing effects.

How can i do?

import controlP5.*;

ControlP5 cp5;
import peasy.*;

ArrayList<Sphere>s;

PGraphics canvas;

PGraphics brightPass;
PGraphics horizontalBlurPass;
PGraphics verticalBlurPass;

PShader bloomFilter;
PShader blurFilter;
PeasyCam cam;
float angle = 0;

final int surfaceWidth = 250;
final int surfaceHeight = 250;

float luminanceFilter = 0.02;
float blurSize = 100;
float sigma = 200;

void setup()
{
  cam = new PeasyCam(this, 1400);
  size(1000, 1000, P3D);

  s = new ArrayList<Sphere>();

  canvas = createGraphics(width, height, P3D);

  brightPass = createGraphics(width, height, P2D);
  brightPass.noSmooth();

  horizontalBlurPass = createGraphics(width, height, P2D);
  horizontalBlurPass.noSmooth(); 

  verticalBlurPass = createGraphics(width, height, P2D);
  verticalBlurPass.noSmooth(); 

  bloomFilter = loadShader("bloomFrag.glsl");
  blurFilter = loadShader("blurFrag.glsl");
}

void draw()
{
  background(0);
  bloomFilter.set("brightPassThreshold", luminanceFilter);
  angle += 0.05;

  for(Sphere s: s){
  blurFilter.set("blurSize", s.p);
  }

  blurFilter.set("sigma", sigma); 

  canvas.beginDraw();
  render(canvas);
  canvas.endDraw();

  // bright pass
  brightPass.beginDraw();
  brightPass.shader(bloomFilter);
  brightPass.image(canvas, 0, 0);
  brightPass.endDraw();

  // blur horizontal pass
  horizontalBlurPass.beginDraw();
  blurFilter.set("horizontalPass", 1);
  horizontalBlurPass.shader(blurFilter);
  horizontalBlurPass.image(brightPass, 0, 0);
  horizontalBlurPass.endDraw();

  // blur vertical pass
  verticalBlurPass.beginDraw();
  blurFilter.set("horizontalPass", 0);
  verticalBlurPass.shader(blurFilter);
  verticalBlurPass.image(horizontalBlurPass, 0, 0);
  verticalBlurPass.endDraw();


  cam.beginHUD();
  blendMode(BLEND);
  blendMode(SCREEN);
  image(brightPass, 0, 0);
  image(verticalBlurPass, 0, 0);

  cam.endHUD();

println(frameRate);
}

void render(PGraphics pg)
{
  cam.getState().apply(pg);

  pg.background(0, 50);

  canvas.pushMatrix();
  canvas.translate(width/2, height/2);
  for(Sphere s: s){
  s.display();

  }
  canvas.popMatrix();


}

void mousePressed() {
  s.add(new Sphere(random(-width/2, width/2), random(-height/2, height/2), random(1000)));
}


class Sphere {

  float p;
  float w;
  float h;

  Sphere(float _w, float _h, float _p) {
  p = _p;
  w = _w;
  h = _h;
  }

  void display() {
    canvas.pushMatrix();
    canvas.translate(w, h);
    noFill();
    canvas.sphere(100);

    canvas.popMatrix();
  }
}

Posting the question back here since it seems I won't get any help on the new forum (?).

As some of you may already know I'm trying to make a Depth of Field effect in Python mode. I've recently stumbled onto this thread where I could get hold of a Proscene DOF demo sketch based on the following shaders:

depth.glsl

uniform float maxDepth;

void main() {
  float depth = gl_FragCoord.z / gl_FragCoord.w;
  gl_FragColor = vec4(vec3(1.0 - depth/maxDepth), 1.0);
}

dof.glsl

uniform sampler2D texture;

varying vec4 vertexture;
varying vec4 vertTexCoord;

uniform sampler2D tDepth;

uniform float maxBlur; // max blur amount
uniform float aperture; // aperture - bigger values for shallower depth of field

uniform float focus;
uniform float aspect;

void main() {
    vec2 vUv = vertTexCoord.st;

    vec2 aspectcorrect = vec2( 1.0, aspect );

    vec4 depth1 = texture2D( tDepth, vUv );

    float factor = depth1.x - focus;

    vec2 dofblur = vec2 ( clamp( factor * aperture, -maxBlur, maxBlur ) );

    vec2 dofblur9 = dofblur * 0.9;
    vec2 dofblur7 = dofblur * 0.7;
    vec2 dofblur4 = dofblur * 0.4;

    vec4 col = vec4( 0.0 );

    col += texture2D( texture, vUv.xy );
    col += texture2D( texture, vUv.xy + ( vec2( 0.0, 0.4 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.15, 0.37 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.29, 0.29 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.37, 0.15 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.40, 0.0 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.37, -0.15 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.29, -0.29 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.15, -0.37 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.0, -0.4 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.15, 0.37 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.29, 0.29 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.37, 0.15 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.4, 0.0 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.37, -0.15 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( -0.29, -0.29 ) * aspectcorrect ) * dofblur );
    col += texture2D( texture, vUv.xy + ( vec2( 0.15, -0.37 ) * aspectcorrect ) * dofblur );

    col += texture2D( texture, vUv.xy + ( vec2( 0.15, 0.37 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.37, 0.15 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.37, -0.15 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.15, -0.37 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.15, 0.37 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.37, 0.15 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.37, -0.15 ) * aspectcorrect ) * dofblur9 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.15, -0.37 ) * aspectcorrect ) * dofblur9 );

    col += texture2D( texture, vUv.xy + ( vec2( 0.29, 0.29 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.40, 0.0 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.29, -0.29 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.0, -0.4 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.29, 0.29 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.4, 0.0 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.29, -0.29 ) * aspectcorrect ) * dofblur7 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.0, 0.4 ) * aspectcorrect ) * dofblur7 );

    col += texture2D( texture, vUv.xy + ( vec2( 0.29, 0.29 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.4, 0.0 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.29, -0.29 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.0, -0.4 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.29, 0.29 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.4, 0.0 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( -0.29, -0.29 ) * aspectcorrect ) * dofblur4 );
    col += texture2D( texture, vUv.xy + ( vec2( 0.0, 0.4 ) * aspectcorrect ) * dofblur4 );

    gl_FragColor = col / 41.0;
    gl_FragColor.a = 1.0;
}

What I'd like is to be able to use those shaders using the PeasyCam library instead of the Proscene dependency.

In the sketch below I'm testing the shaders on a point cloud. I’m drawing the points on the scene PGraphics and then passing it through 2 other PGraphics (canvas and buf). In theory it should work but I’m still stuck with the PeasyCam part at the very end of the script.

If I write:

• cam.getState().apply(canvas): I get a DoF effect BUT on a 2D canvas
• cam.getState().apply(scene) : I get a 3D canvas BUT without a DoF effect

Any help would be really appreciated.

add_library('peasycam')
liste = []

def setup():
    global depthShader, dofShader, cam, canvas, buf, scene
    size(900, 900, P3D)

    cam = PeasyCam(this, 900)
    cam.setMaximumDistance(width)

    for e in range(100):
        liste.append(PVector(random(width), random(height), random(width)))

    depthShader = loadShader("depth.glsl")
    dofShader = loadShader("dof.glsl")

    depthShader.set("maxDepth", cam.getDistance())
    dofShader.set("aspect", width / float(height))
    dofShader.set("maxBlur", 0.015)
    dofShader.set("aperture", 0.02)

    canvas = createGraphics(width, height, P3D)
    canvas.shader(depthShader)
    buf = createGraphics(width, height, P3D)
    buf.shader(dofShader)

    scene = createGraphics(width, height, P3D)

    frameRate(1000)

def draw():

    scene.beginDraw()
    scene.background(255)
    scene.stroke(0)
    scene.strokeWeight(10)
    for e in liste:
        scene.point(e.x-width/2, e.y-height/2, e.z-height/2)
    scene.endDraw()

    canvas.beginDraw()
    canvas.image(scene, 0, 0)
    canvas.endDraw()

    buf.beginDraw()
    dofShader.set("focus", map(mouseX, 0, width, -0.5, 1.5))
    dofShader.set("tDepth", canvas)
    buf.image(scene, 0, 0)
    buf.endDraw()

    cam.beginHUD()
    image(buf, 0, 0)
    cam.endHUD()

    cam.getState().apply(scene)

Here is my working conversion of the water effect. I expanded it from 255 levels of brightness to 765 by utilizing all 3 rgb channels.

waterShader.pde

PShader water;
PGraphics current, previous;

final int size = 500;
final float fsize = size;

void settings() {
  size(size, size, P3D);
}

void setup() {
  current = createGraphics(size, size, P2D);
  previous = createGraphics(size, size, P2D);
  water = loadShader("water.glsl");
  water.set("resolution", fsize, fsize);  
  water.set("dissipation", 0.990);
 // water.set("current", current);
  //  water.set("previous", previous);

  previous.beginDraw();
  previous.background(0);
  previous.endDraw();

  current.beginDraw();
  current.background(0);
  current.endDraw();
  frameRate(75);
}

boolean flip = true;
void draw() {
  water.set("time", millis()/1000.0);
  if (mousePressed) water.set("mouse", mouseX/fsize, 1-mouseY/fsize);  
  else water.set("mouse",-0.1,0.0);
  println(mouseY/fsize);

  current.beginDraw();
  current.shader(water);
  current.rect(0, 0, size, size);
  current.stroke(0);
  current.strokeWeight(10);
  current.line(0, 0, size, size);

  current.endDraw();  

  image(current, 0, 0, size, size);

  PGraphics temp = previous;
  previous = current;
  current = temp;

  water.set("current", current);
  water.set("previous", previous);
  //saveFrame("data/####.png");
}

water.glsl

#ifdef GL_ES
precision highp float;
#endif

#define PROCESSING_COLOR_SHADER

uniform float time;
uniform vec2 mouse;
uniform float dissipation;
uniform vec2 resolution;
uniform sampler2D previous;
uniform sampler2D current;

float decode(vec4 color) {
    return (color.r + color.g + color.b)/3.0;
}

vec4 encode(float r){
    int k = int(floor(r*255));
    int d = int(round(k % 3));
    float up = 1.0/255.0;

    if (d==0) return vec4(r,r,r,1.0);
    if (d==1) return vec4(r,r,r+up,1.0);
    if (d==2) return vec4(r,r+up,r+up,1.0);
}

void main( void ) {
    vec2 position = ( gl_FragCoord.xy / resolution.xy );
    vec2 pixel = 1./resolution;

    if (length(position-mouse) < 0.01) {
        gl_FragColor = vec4(1., 1., 1.0, 1.);
    }
    else {
        float sum = decode(texture2D(previous, position + pixel * vec2(-1, 0.)));
        sum += decode(texture2D(previous, position + pixel * vec2(1, 0.)));
        sum += decode(texture2D(previous, position + pixel * vec2(0., -1)));
        sum += decode(texture2D(previous, position + pixel * vec2(0., 1)));

        sum *= 0.5;
        sum -= decode(texture2D(current, position + pixel * vec2(0., 0.)));
        sum *= dissipation;

        gl_FragColor = encode(sum);
    }
}

TheCodingTrain channel did two videos recently on water and fire effects. He did them in just processing, I'm doing it in processing with shaders.

For the water, I didn't have any issues converting it (posted in comment below). For the fire, I'm having issues getting it to perform like expected. It's almost working but not quite.

Edit: I fixed it, here is the working code. The original code from the video ran at 25 fps on my machine, and this shader version does 1000fps easy.

fireShader.pde

PShader fire;
PGraphics buffer1,buffer2, cooling;
int w = 400, h = 400;
float ystart = 0.0;
int speed = 1000;

void setup() {
  size(800, 400, P2D);
  buffer1 = createGraphics(w, h, P2D);
  buffer2 = createGraphics(w, h, P2D);
  cooling = createGraphics(w, h, P2D);
  cool();
  fire = loadShader("fire.glsl");
  fire.set("resolution", w*2, h);
  fire.set("buffer1", buffer1);
  fire.set("buffer2", buffer2);
  fire.set("cooling", cooling);
  frameRate(100);
}

void cool() {
  cooling.loadPixels();
  float xoff = 31.1; 
  float increment = 0.12;
  noiseDetail(3, 1.06);
  for (int x = 0; x < w; x++) {
    xoff += increment; 
    float yoff = ystart; 
    for (int y = 0; y < h; y++) {
      yoff += increment; 
      float n = noise(xoff, yoff);     
      float bright = pow(n, 2) * 22;
      cooling.pixels[x+y*w] = color(bright);
    }
  }
  cooling.updatePixels();
  ystart += increment;
}

void startFire() {
  buffer1.beginDraw();
  buffer1.resetShader();
  buffer1.noStroke();
  buffer1.fill(255);
  buffer1.rect(mouseX,mouseY,100,5);
  buffer1.rect(0,h-4,w,4);
  buffer1.endDraw();
}

void swapBuffers(){  
  PGraphics temp = buffer1;
  buffer1 = buffer2;
  buffer2 = temp;
}

void draw() {
  fire.set("time", (frameCount % speed)/(float)speed);
  startFire();
  background(0);
  buffer2.beginDraw();
  buffer2.shader(fire);
  buffer2.rect(0, 0, w, h);
  buffer2.stroke(0);
  buffer2.endDraw();
  swapBuffers();
  image(buffer2, 0, 0);
  image(cooling, w, 0);
  text((int)frameRate,2,10);
}

fire.glsl

#ifdef GL_ES
precision highp float;
#endif

#define PROCESSING_COLOR_SHADER

uniform float time;
uniform vec2 mouse;
uniform float dissipation;
uniform vec2 resolution;
uniform sampler2D buffer1;
uniform sampler2D buffer2;
uniform sampler2D cooling;

float decode(vec4 color) {
    return (color.r + color.g + color.b)/3.0;
}

vec4 encode(float r){
    int k = int(floor(r*255));
    int d = int(round(k % 3));
    float up = 1.0/255.0;

    if (d==0) return vec4(r,r,r,1.0);
    if (d==1) return vec4(r,r,r+up,1.0);
    if (d==2) return vec4(r,r+up,r+up,1.0);
}

void main( void ) {
    vec2 position = ( gl_FragCoord.xy / resolution.xy );
    vec2 pixel = 1./resolution;
        float k = -1.;
        float sum = 0;


        sum += decode(texture2D(buffer1, position + pixel * vec2(1., k)));
        sum += decode(texture2D(buffer1, position + pixel * vec2(-1, k)));
        sum += decode(texture2D(buffer1, position + pixel * vec2(0., 1+k)));
        sum += decode(texture2D(buffer1, position + pixel * vec2(0., -1+k)));

        sum *= 0.25;
        position.y = 1-position.y;
        position.y += time;
        if (position.y > 1.0) position.y = (position.y - 1.0);

        sum -= decode(texture2D(cooling, position + pixel * vec2(0., k)))/2.0;
        gl_FragColor = encode(sum);
}

Hi @cansik, thank you for your answer.

I first though it was because of the missing pushMatrix/popMatrix as well but pushing the transformation matrix doesn't solve the problem.

When drawing on a canvas:

• I get 2 pictures at 2 different locations
• One with the bloom effect (left), the other (right) without it
• The first one doesn't rotate when moving the camera, the second do
• The 3D disappears on both pictures (flattened 3D delaunay)

sketch.pyde

add_library('peasycam')
add_library('triangulate')
add_library('PostFX')
from java.util import ArrayList

step, threshold = 5, 100
liste = ArrayList()

def setup():
    global img, triangles, fx, canvas
    size(1800, 1000, P3D)
    smooth(8)

    fx = PostFX(this)
    cam = PeasyCam(this, 1600)
    sobel = loadShader('sobelFrag.glsl')
    img = loadImage("test75.jpg")

    canvas = createGraphics(width, height, P3D)

    pg = createGraphics(img.width, img.height, P2D)
    pg.beginDraw()
    pg.image(img, 0, 0)
    pg.filter(GRAY)
    pg.filter(sobel)
    pg.loadPixels()
    for x in range(0, pg.width, step):
        for y in range(0, pg.height, step):
            i = x + y * pg.width
            col = img.pixels[i]
            b = pg.pixels[i] & 0xFF
            if b > threshold:
                liste.add(PVector(x, y , brightness(col) * .5))

    for e in range(0, pg.width, int(pg.width/20)):
        liste.add(PVector(e, 0, 0))
        liste.add(PVector(e, pg.height, 0))
    for e in range(0, pg.height, int(pg.height/20)):
        liste.add(PVector(0, e, 0))
        liste.add(PVector(pg.width, e, 0))

    pg.endDraw()
    triangles = Triangulate.triangulate(liste)


def draw():
    background(0)

    canvas.pushMatrix()
    canvas.translate(-img.width/2, -img.height/2)
    canvas.beginDraw()
    canvas.beginShape(TRIANGLES)
    canvas.noStroke()
    for t in triangles:
        x = int((t.p1.x + t.p2.x + t.p3.x) / 3)
        y = int((t.p1.y + t.p2.y + t.p3.y) / 3)
        i = x + y * img.width
        col = img.pixels[i]
        canvas.fill(col)
        canvas.vertex(t.p1.x, t.p1.y, t.p1.z)
        canvas.vertex(t.p2.x, t.p2.y, t.p2.z)
        canvas.vertex(t.p3.x, t.p3.y, t.p3.z)
    canvas.endShape() 
    canvas.endDraw()
    canvas.popMatrix()

    blendMode(BLEND)
    image(canvas,0,0)

    blendMode(SCREEN)

    fx.render(canvas).bloom(0.5, 20, 40).compose()

I have an issue involving the script above so I hope you won't mind me asking a question on this thread again.

I'm trying to add a bloom effect (from the PostFX library...again) to the sketch but for some reason I have 2 canvas being displayed on the screen:

• one at the right location (center) but without the bloom effect

• the other one, smaller, at the bottom right corner with the bloom effect

Do you guys have an idea what I'm doing wrong here ?

add_library('peasycam')
add_library('triangulate')
from java.util import ArrayList

step, threshold = 5, 100
liste = ArrayList()

def setup():
    global img, triangles, fx
    size(1800, 1000, P3D)
    smooth(8)

    cam = PeasyCam(this, 1600)
    sobel = loadShader('sobelFrag.glsl')
    img = loadImage("image_file_name.jpg")
    fx = PostFX(this)

    pg = createGraphics(img.width, img.height, P2D)
    pg.beginDraw()
    pg.image(img, 0, 0)
    pg.filter(GRAY)
    pg.filter(sobel)
    pg.loadPixels()
    for x in range(0, pg.width, step):
        for y in range(0, pg.height, step):
            i = x + y * pg.width
            col = img.pixels[i]
            b = pg.pixels[i] & 0xFF
            if b > threshold:
                liste.add(PVector(x, y , brightness(col) * .5))

    for e in range(0, pg.width, int(pg.width/20)):
        liste.add(PVector(e, 0, 0))
        liste.add(PVector(e, pg.height, 0))
    for e in range(0, pg.height, int(pg.height/20)):
        liste.add(PVector(0, e, 0))
        liste.add(PVector(pg.width, e, 0))

    pg.endDraw()
    triangles = Triangulate.triangulate(liste)

    beginShape(TRIANGLES)
    noStroke()

def draw():
    background(0)

    translate(-img.width/2, -img.height/2)
    for t in triangles:
        x = int((t.p1.x + t.p2.x + t.p3.x) / 3)
        y = int((t.p1.y + t.p2.y + t.p3.y) / 3)
        i = x + y * img.width
        col = img.pixels[i]
        fill(col)
        vertex(t.p1.x, t.p1.y, t.p1.z)
        vertex(t.p2.x, t.p2.y, t.p2.z)
        vertex(t.p3.x, t.p3.y, t.p3.z)
    endShape() 

    fx.render().bloom(0.5, 20, 40).compose()

To run the script you'll need Peasycam and Triangulate libraries (download) + the sobel fragment shader below from the PostFx library (only needed for edge detection). I first had my own sobel operator to detect the edges but found that @cansik's shader implementation was much more accurate and efficient.

Also please note that this sketch is in Python.

sobelFrag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;

varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform vec2 resolution;

void main(void) {
  float x = 1.0 / resolution.x;
  float y = 1.0 / resolution.y;
  vec4 horizEdge = vec4( 0.0 );
  horizEdge -= texture2D( texture, vec2( vertTexCoord.x - x, vertTexCoord.y - y ) ) * 1.0;
  horizEdge -= texture2D( texture, vec2( vertTexCoord.x - x, vertTexCoord.y     ) ) * 2.0;
  horizEdge -= texture2D( texture, vec2( vertTexCoord.x - x, vertTexCoord.y + y ) ) * 1.0;
  horizEdge += texture2D( texture, vec2( vertTexCoord.x + x, vertTexCoord.y - y ) ) * 1.0;
  horizEdge += texture2D( texture, vec2( vertTexCoord.x + x, vertTexCoord.y     ) ) * 2.0;
  horizEdge += texture2D( texture, vec2( vertTexCoord.x + x, vertTexCoord.y + y ) ) * 1.0;
  vec4 vertEdge = vec4( 0.0 );
  vertEdge -= texture2D( texture, vec2( vertTexCoord.x - x, vertTexCoord.y - y ) ) * 1.0;
  vertEdge -= texture2D( texture, vec2( vertTexCoord.x    , vertTexCoord.y - y ) ) * 2.0;
  vertEdge -= texture2D( texture, vec2( vertTexCoord.x + x, vertTexCoord.y - y ) ) * 1.0;
  vertEdge += texture2D( texture, vec2( vertTexCoord.x - x, vertTexCoord.y + y ) ) * 1.0;
  vertEdge += texture2D( texture, vec2( vertTexCoord.x    , vertTexCoord.y + y ) ) * 2.0;
  vertEdge += texture2D( texture, vec2( vertTexCoord.x + x, vertTexCoord.y + y ) ) * 1.0;
  vec3 edge = sqrt((horizEdge.rgb * horizEdge.rgb) + (vertEdge.rgb * vertEdge.rgb));

  gl_FragColor = vec4(edge, texture2D(texture, vertTexCoord.xy).a);
}

sketch.pyde

add_library('peasycam')
add_library('triangulate')
from java.util import ArrayList

step, threshold = 5, 100
liste = ArrayList()

def setup():
    global img, triangles
    size(1800, 1000, P3D)
    smooth(8)

    cam = PeasyCam(this, 1600)
    sobel = loadShader('sobelFrag.glsl')
    img = loadImage("image_file_name.jpg")

    pg = createGraphics(img.width, img.height, P2D)
    pg.beginDraw()
    pg.image(img, 0, 0)
    pg.filter(GRAY)
    pg.filter(sobel)
    pg.loadPixels()
    for x in range(0, pg.width, step):
        for y in range(0, pg.height, step):
            i = x + y * pg.width
            col = img.pixels[i]
            b = pg.pixels[i] & 0xFF
            if b > threshold:
                liste.add(PVector(x, y , brightness(col) * .5))

    for e in range(0, pg.width, int(pg.width/20)):
        liste.add(PVector(e, 0, 0))
        liste.add(PVector(e, pg.height, 0))
    for e in range(0, pg.height, int(pg.height/20)):
        liste.add(PVector(0, e, 0))
        liste.add(PVector(pg.width, e, 0))

    pg.endDraw()
    triangles = Triangulate.triangulate(liste)

    beginShape(TRIANGLES)
    noStroke()

def draw():
    background(0)

    translate(-img.width/2, -img.height/2)
    for t in triangles:
        x = int((t.p1.x + t.p2.x + t.p3.x) / 3)
        y = int((t.p1.y + t.p2.y + t.p3.y) / 3)
        i = x + y * img.width
        col = img.pixels[i]
        fill(col)
        vertex(t.p1.x, t.p1.y, t.p1.z)
        vertex(t.p2.x, t.p2.y, t.p2.z)
        vertex(t.p3.x, t.p3.y, t.p3.z)
    endShape() 

Cheers !

This is something I had up on this forum since last summer, but it was in a question thread so I thought I should share this here in the proper category as well.

Current version: v1.2

This LaTeX listings template will color your Processing code as you see them in the Processing editor. You can just post your code like you normally would using the listings package, and you will automatically get your code in the proper colors. Except one minor inconvenience, that I will explain here.

For a function name that is the same as an already defined keyword (Such as "boolean" and "boolean()"), you have to use [escapechar = "char"] and \color{"color"}{"text"} to color them properly. Example:

\begin{lstlisting}[escapechar = ?]
boolean;
?\color{function}{boolean}?(1);
\end{lstlisting}

Copy and paste the below template if you want to use it. Alternatively, you can copy only the necessary parts. If in that case, note that \usepackage{listings} and \usepackage{color} is a must for this to work.

Also note, I have licensed this work with CreativeCommons license CC-BY-SA, so please remember to give some credit to me ;)

If you find any typos or any other errors, please tell me and I'll try to fix them as much as possible.

Download version:
http://www.mediafire.com/file/cw861uy156xftkv/article_listing_Processing_v1.2.tex
GitHub Gist:
https://gist.github.com/ebigunso/af355220e932f72d03289c576622aa29

Full template below:

\documentclass{article}

\usepackage{graphicx}
\usepackage{url}
\usepackage{verbatim}
\usepackage{listings}
\usepackage{color}

% Processing language definition template for LaTeX listings package v1.2
%
% Credits to ebigunso for creating this LaTeX listings language definition template for Processing
% This template is licensed with CreativeCommons license CC-BY-SA 4.0
% license info:
% https://creativecommons.org/licenses/by-sa/4.0/legalcode

%Define Colors
\definecolor{black}{RGB}{0,0,0}
\definecolor{gray}{RGB}{102,102,102}        %#666666
\definecolor{function}{RGB}{0,102,153}      %#006699 lightblue
\definecolor{lightgreen}{RGB}{102,153,0}    %#669900
\definecolor{bluegreen}{RGB}{51,153,126}    %#33997e
\definecolor{magenta}{RGB}{217,74,122}  %#d94a7a
\definecolor{orange}{RGB}{226,102,26}       %#e2661a
\definecolor{purple}{RGB}{125,71,147}       %#7d4793
\definecolor{green}{RGB}{113,138,98}        %#718a62

\lstdefinelanguage{Processing}{
  %keyword1&2&6
  morekeywords = [3]{abstract, break, class, continue, default, enum, extends, false, final, finally, implements, import, instanceof, interface, native, new, null, package, private, protected, public, static, strictfp, throws, transient, true, void, volatile, length, assert, case, return, super, this, throw},
  %keyword3
  morekeywords = [4]{catch, do, for, if, else, switch, synchronized, while, try},
  %keyword4
  morekeywords = [5]{width, height, pixelHight, displayHeight, displayWidth, focused, frameCount, frameRate, key, keyCode, keyPressed, mouseButton, mousePressed, mouseX, mouseY, pixels, pixelWidth, pmouseX, pmouseY},
  %keyword5
  morekeywords = [6]{Array, ArrayList, Boolean, Byte, BufferedReader, Character, Class, Double, Float, Integer, HashMap, PrintWriter, String, StringBuffer, StringBuilder, Thread, boolean, byte, char, color, double, float, int, long, short, FloatDict, FloatList, IntDict, IntList, JSONArray, JSONObject, PFont, PGraphics, PImage, PShader, PShape, PVector, StringDict, StringList, Table, TableRow, XML},
  %literal2
  morekeywords = [7]{ADD, ALIGN_CENTER, ALIGN_LEFT, ALIGN_RIGHT, ALPHA, ALPHA_MASK, ALT, AMBIENT, ARC, ARROW, ARGB, BACKSPACE, BASELINE, BEVEL, BLEND, BLUE_MASK, BLUR, BOTTOM, BOX, BURN, CENTER, CHATTER, CHORD, CLAMP, CLICK, CLOSE, CMYK, CODED, COMPLAINT, COMPOSITE, COMPONENT, CONCAVE_POLYGON, CONTROL, CONVEX_POLYGON, CORNER, CORNERS, CROSS, CUSTOM, DARKEST, DEGREES, DEG_TO_RAD, DELETE, DIAMETER, DIFFERENCE, DIFFUSE, DILATE, DIRECTIONAL, DISABLE_ACCURATE_2D, DISABLE_DEPTH_MASK, DISABLE_DEPTH_SORT, DISABLE_DEPTH_TEST, DISABLE_NATIVE_FONTS, DISABLE_OPENGL_ERRORS, DISABLE_PURE_STROKE, DISABLE_TEXTURE_MIPMAPS, DISABLE_TRANSFORM_CACHE, DISABLE_STROKE_PERSPECTIVE, DISABLED, DODGE, DOWN, DRAG, DXF, ELLIPSE, ENABLE_ACCURATE_2D, ENABLE_DEPTH_MASK, ENABLE_DEPTH_SORT, ENABLE_DEPTH_TEST, ENABLE_NATIVE_FONTS, ENABLE_OPENGL_ERRORS, ENABLE_PURE_STROKE, ENABLE_TEXTURE_MIPMAPS, ENABLE_TRANSFORM_CACHE, ENABLE_STROKE_PERSPECTIVE, ENTER, EPSILON, ERODE, ESC, EXCLUSION, EXIT, FX2D, GIF, GRAY, GREEN_MASK, GROUP, HALF, HALF_PI, HAND, HARD_LIGHT, HINT_COUNT, HSB, IMAGE, INVERT, JAVA2D, JPEG, LEFT, LIGHTEST, LINE, LINES, LINUX, MACOSX, MAX_FLOAT, MAX_INT, MIN_FOAT, MIN_INT, MITER, MODEL, MOVE, MULTIPLY, NORMAL, NORMALIZED, NO_DEPTH_TEST, NTSC, ONE, OPAQUE, OPEN, ORTHOGRAPHIC, OVERLAY, PAL, PDF, P2D, P3D, PERSPECTIVE, PI, PIE, PIXEL_CENTER, POINT, POINTS, POSTERIZE, PRESS, PROBLEM, PROJECT, QUAD, QUAD_STRIP, QUADS, QUARTER_PI, RAD_TO_DEG, RADIUS, RADIANS, RECT, RED_MASK, RELEASE, REPEAT, REPLACE, RETURN, RGB, RIGHT, ROUND, SCREEN, SECAM, SHAPE, SHIFT, SPAN, SPECULAR, SPHERE, SOFT_LIGHT, SQUARE, SUBTRACT, SVG, SVIDEO, TAB, TARGA, TAU, TEXT, TFF, THIRD_PI, THRESHOLD, TIFF, TOP, TRIANGLE, TRIANGLE_FAN, TRIANGLES, TRIANGLE_STRIP, TUNER, TWO, TWO_PI, UP, WAIT, WHITESPACE},
  %function1
  morekeywords = [8]{start, stop, breakShape, createPath, str, loadMatrix, parseBoolean, parseByte, parseChar, parseFloat, parseInt, saveFile, savePath, sketchFile, sketchPath, abs, acos, alpha, ambient, ambientLight, append, applyMatrix, arc, arrayCopy, asin, atan, atan2, background, beginCamera, beginContour, beginRaw, beginRecord, beginShape, bezier, bezierDetail, bezierPoint, bezierTangent, bezierVertex, binary, blend, blendColor, blendMode, blue, box, brightness, camera, ceil, clear, clip, color, colorMode, concat, constrain, copy, cos, createFont, createGraphics, createImage, createInput, createOutput, createReader, createShape, createWriter, cursor, curve, curveDetail, curvePoint, curveTangent, curveTightness, curveVertex, day, degrees, delay, directionalLight, displayDensity, dist, ellipse, ellipseMode, emissive, endCamera, endContour, endRaw, endRecord, endShape, exit, exp, expand, fill, filter, floor, frustum, fullScreen, get, green, hex, hint, hour, hue, image, imageMode, join, launch, lerp, lerpColor, lightFalloff, lights, lightSpecular, line, loadBytes, loadFont, loadImage, loadJSONArray, loadJSONObject, loadPixels, loadShader, loadShape, loadStrings, loadTable, loadXML, log, loop, mag, map, match, matchAll, max, millis, min, minute, modelX, modelY, modelZ, month, nf, nfc, nfp, nfs, noClip, noCursor, noFill, noise, noiseDetail, noiseSeed, noLights, noLoop, norm, normal, noSmooth, noStroke, noTint, ortho, parseJSONArray, parseJSONObject, parseXML, perspective, list, pixelDnsity, point, pointLight, popMatrix, popStyle, pow, print, printArray, printCamera, println, printMatrix, printProjection, pushMatrix, pushStyle, quad, quadraticVertex, radians, random, randomGaussian, randomSeed, rect, rectMode, red, redraw, requestImage, resetMatrix, resetShader, reverse, rotate, rotateX, rotateY, rotateZ, round, saturation, save, saveBytes, saveFrame, saveJSONArray, saveJSONObject, saveStream, saveStrings, saveTable, saveXML, scale, screenX, screenY, screenZ, second, selectFolder, selectInput, selectOutput, set, shader, shape, shapeMode, shearX, shearY, shininess, shorten, sin, size, smooth, sort, specular, sphere, sphereDetail, splice, split, splitTokens, spotLight, sq, sqrt, stroke, strokeCap, strokeJoin, strokeWeight, subset, tan, text, textAlign, textAscent, textDescent, textFont, textLeading, textMode, textSize, texture, textureMode, textureWrap, textWidth, thread, tint, translate, triangle, trim, unbinary, unhex, updatePixels, vertex, year},
  %function2
  morekeywords = [9]{cache, readLine, close, flush, print, println, charAt, equals, indexOf, substring, toLowerCase, toUpperCase, getDouble, getLong, getColumnTitles, getColumnTypes, getColumnType, setDouble, setLong, add, clear, div, get, hasKey, keyArray, keys, mult, remove, set, size, sortKeys, sortKeysReverse, sortValues, sortValuesReverse, sub, valueArray, values, append, array, hasValue, max, min, mult, remove, reverse, shuffle, sort, sortReverse, increment, getBoolean, getFloat, getInt, getIntArray, getJSONArray, getJSONObject, getString, getStringArray, isNull, setBoolean, setFloat, setInt, setJSONArray, setJSONObject, setString, beginDraw, endDraw, blend, copy, filter, loadPixels, mask, resize, save, updatePixels, addChild, beginContour, beginShape, disableStyle, enableStyle, endContour, endShape, getChild, getChildCount, getVertex, getVertexCount, isVisible, resetMatrix, rotate, rotateX, rotateY, rotateZ, scae, setFill, setStroke, setVertex, setVisible, translate, angleBetween, cross, dist, dot, fromAngle, heading, lerp, limit, mag, magSq, normalize, randm2D, random3D, setMag, lower, upper, addColumn, addRow, clearRows, findRow, findRows, getColumnCount, getRow, getRowcount, getStringColumn, matchRow, matchRows, removeColumn, removeRow, removeTokens, rows, trim, getColumnTitle, format, getAttributeCount, getChildren, getContent, getNam, getParent, hasAttribute, hasChildren, listAttributes, listChildren, removeChild, setContent, setName, toString},
  %function4
  morekeywords = [10]{draw, keyReleased, keyTyped, mouseClicked, mouseDragged, mouseMoved, mouseReleased, mouseWheel, settings, setup},
  keywordstyle = [3]\color{bluegreen},
  keywordstyle = [4]\color{lightgreen},
  keywordstyle = [5]\color{magenta},
  keywordstyle = [6]\color{orange},
  keywordstyle = [7]\color{green},
  keywordstyle = [8]\color{function},
  keywordstyle = [9]\color{function},
  keywordstyle = [10]\color{function},
  sensitive = true,
  morecomment = [l][\color{gray}]{//},
  morecomment = [s][\color{gray}]{/*}{*/},
  morecomment = [s][\color{gray}]{/**}{*/},
  morestring = [b][\color{purple}]",
  morestring = [b][\color{purple}]'
}
\renewcommand{\ttdefault}{pcr}
\lstset{
  language={Processing},
  basicstyle={\small\ttfamily},
  identifierstyle={\small},
  commentstyle={\small\itshape},
  keywordstyle={\small},
  ndkeywordstyle={\small},
  stringstyle={\small\ttfamily},
  frame={tb},
  breaklines=true,
  columns=[l]{fullflexible},
  numbers=left,
  xrightmargin=0em,
  xleftmargin=3em,
  numberstyle={\scriptsize},
  stepnumber=1,
  numbersep=1em,
  lineskip=-0.5ex,
}

% Use escapechar and \color{<color>}{<text>} to color function names properly, that is already defined as a different color keyword.
%
% \begin{lstlisting}[escapechar = ?]
% boolean;
% ?\color{function}{boolean}?(1);
% \end{lstlisting}

\title{}
\author{}
\date{}

\begin{document}
\maketitle
\section{}

\begin{thebibliography}{9}
\end{thebibliography}
\end{document}

How can I use a samplerCube and sampler2D in a single glsl Shader ?

I get the following error when I try to use both at the same time : Cannot validate shader program:Validation Failed: Sampler error: Sampler of different types use the same texture image unit. -or- A sampler's texture unit is out of range (grater than max allowed or negative)

I dont think my unit is out of range, I think the problem is because of the same unit problem.

Exemple code :

Sketch

import queasycam.*;
import java.nio.IntBuffer;

QueasyCam cam;

PShader shader;
PShape sphere;

PImage[] skybox = new PImage[6];
PImage cubeMap;
PImage texture;

void setup() {
  fullScreen(P3D);
  noStroke();

  //Create FPS Camera
  cam = new QueasyCam(this);

  //Load Shader
  shader = loadShader("frag.glsl", "vert.glsl");

  //Load Skybox Image
  cubeMap = loadImage("skybox.jpg");
  skybox = sliceCubeMap(cubeMap);

  //Load texture
  texture = loadImage("texture.png");

  //Cube Map
  glslCubeMap(2, cubeMap);
  shader.set("cubemap", 2);

  //Diffuse Map
  shader.set("diffuseMap", texture);

  sphere = createShape(SPHERE, 150);
  sphere.setFill(color(-1, 50));
} 

void draw() {
  background(0);

  //Update Camera Position
  shader.set("camPosition", cam.position);

  //Render Shape
  shader(shader);
  shape(sphere);
}

PImage[] sliceCubeMap(PImage cubeMap) {
  PImage[] textures = new PImage[6];

  int ux = cubeMap.width/4;
  int uy = cubeMap.height/3;

  //X
  textures[0] = cubeMap.get(ux*2, uy, ux, uy);
  textures[1] = cubeMap.get(0, uy, ux, uy);

  //Y
  textures[2] = cubeMap.get(ux, 0, ux, uy);
  textures[3] = cubeMap.get(ux, uy*2, ux, uy);

  //Z
  textures[4] = cubeMap.get(ux, uy, ux, uy);
  textures[5] = cubeMap.get(ux*3, uy, ux, uy);

  return textures;
}

void glslCubeMap(int unit, PImage cubeMap) {
  glslCubeMap(unit, sliceCubeMap(cubeMap));
}

void glslCubeMap(int unit, PImage[] textures) {
  glslCubeMap(unit, textures[0], textures[1], textures[2], textures[3], textures[4], textures[5]);
}

void glslCubeMap(int unit, PImage posX, PImage negX, PImage posY, PImage negY, PImage posZ, PImage negZ) {

  PGL pgl = beginPGL();
  // create the OpenGL-based cubeMap
  IntBuffer envMapTextureID = IntBuffer.allocate(1);
  pgl.genTextures(1, envMapTextureID);
  pgl.activeTexture(PGL.TEXTURE0 + unit); // Change texture unit
  pgl.enable(PGL.TEXTURE_CUBE_MAP);  
  pgl.bindTexture(PGL.TEXTURE_CUBE_MAP, envMapTextureID.get(0));
  pgl.texParameteri(PGL.TEXTURE_CUBE_MAP, PGL.TEXTURE_WRAP_S, PGL.CLAMP_TO_EDGE);
  pgl.texParameteri(PGL.TEXTURE_CUBE_MAP, PGL.TEXTURE_WRAP_T, PGL.CLAMP_TO_EDGE);
  pgl.texParameteri(PGL.TEXTURE_CUBE_MAP, PGL.TEXTURE_WRAP_R, PGL.CLAMP_TO_EDGE);
  pgl.texParameteri(PGL.TEXTURE_CUBE_MAP, PGL.TEXTURE_MIN_FILTER, PGL.LINEAR);
  pgl.texParameteri(PGL.TEXTURE_CUBE_MAP, PGL.TEXTURE_MAG_FILTER, PGL.LINEAR);

  //Load in textures
  PImage[] textures = { posX, negX, posY, negY, posZ, negZ };
  for (int i=0; i<textures.length; i++) {
    PImage texture = textures[i];
    int w = texture.width;
    int h = texture.height;
    texture.loadPixels();
    pgl.texImage2D(PGL.TEXTURE_CUBE_MAP_POSITIVE_X + i, 0, PGL.RGBA, w, h, 0, PGL.RGBA, PGL.UNSIGNED_BYTE, IntBuffer.wrap(texture.pixels));
  }

  endPGL();
}

vert.glsl

uniform mat4 transform;
uniform mat4 modelview;
uniform mat3 normalMatrix;
uniform mat4 texMatrix;

uniform vec3 camPosition;

varying vec3 reflectVector;
varying vec3 refractVector;
varying vec4 vertTexCoord;

attribute vec4 position;
attribute vec4 vertex;
attribute vec3 normal;
attribute vec2 texCoord;

void main(void) {
    gl_Position = transform * vertex;

    vec3 unitNormal = normalize(normal);

    vec3 viewVector = normalize(vertex.xyz - camPosition);

    reflectVector = reflect(viewVector, unitNormal);
    refractVector = refract(viewVector, unitNormal, 1.0/1.33);

    //2D Texture Coordinates
    vertTexCoord = texMatrix * vec4(texCoord, 1.0, 1.0);
}

frag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

//Cube texture
uniform samplerCube cubemap;

//2D texture
uniform sampler2D diffuseMap;

varying vec3 reflectVector;
varying vec3 refractVector;

//Texture Coordinates
varying vec4 vertTexCoord;

void main(void) {
    vec4 reflectColor = texture(cubemap, -reflectVector);
    vec4 refractColor = texture(cubemap, -refractVector);

    vec4 diffuse = texture2D(diffuseMap, vertTexCoord.st);

    //Switch Red and Blue Channel (ARGB != RGBA)
    reflectColor = vec4(reflectColor.b, reflectColor.g, reflectColor.r, reflectColor.a);
    refractColor = vec4(refractColor.b, refractColor.g, refractColor.r, refractColor.a);

    vec4 materialColor = mix(reflectColor, refractColor, 1.0);

    gl_FragColor = mix(diffuse, materialColor, 0.5);
}

Here I change the unit : pgl.activeTexture(PGL.TEXTURE0 + unit); // Change texture unit

thank you in advance !

@solub

sorry, no idea how to do that : )

meanwhile i found a better depthtexture shader https://forum.processing.org/two/discussion/12775/simple-shadow-mapping

keep on exploring.

// blur by discard color value
//
// blur.glsl
/*
varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform sampler2D texture;
uniform vec2 texOffset; 
uniform int horizontalPass;
uniform float time;

void main() {  

 int blurSize = 4;
 float sigma = 2.0;

 float numBlurPixelsPerSide = float(blurSize / 2); 
 vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

 // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
 vec3 incrementalGaussian;
 incrementalGaussian.x = 1.0 / (sqrt(2.0 * 3.14159265) * sigma);
 incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
 incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

 vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
 float coefficientSum = 0.0;

 // Take the central sample first...
 avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
 coefficientSum += incrementalGaussian.x;
 incrementalGaussian.xy *= incrementalGaussian.yz;

 // Go through the remaining 8 vertical samples (4 on each side of the center)
 for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
 avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * blurMultiplyVec) * incrementalGaussian.x;         
 avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * blurMultiplyVec) * incrementalGaussian.x;         
 coefficientSum += 2.0 * incrementalGaussian.x;
 incrementalGaussian.xy *= incrementalGaussian.yz;
 }

 if ( texture2D(texture, vertTexCoord.st).rgb == vec3(step(sin(time*3.0)*0.5+0.5,0.5)) )
 discard; 
 else
 gl_FragColor.rgb = avgValue.rgb / coefficientSum;
 gl_FragColor.a = 1.;
} 
*/
PImage img;
PShader blurShader;
void setup() {
  size(640, 360, P3D);
  blurShader=loadShader("blur.glsl");
  rectMode(CENTER);
  ellipseMode(RADIUS);
  noStroke();
  noSmooth();
  img = loadImage("https://"+"processing.org/examples/moonwalk.jpg");
}
void draw() {
  //background(157, 0, 257, 255);
  background(img);

  float t = frameCount*.01;

  // white box  
  pushMatrix();
  translate(width*.15f, height*.5f-120.*cos(t), -4);
  fill(255);
  box(120, 120, 1);
  popMatrix();
  pushMatrix();
  translate(width*.15f, height*.5f-120.*cos(t), 4);
  fill(0);
  ellipse(5, 0, 30, 30);
  popMatrix();

  // black box
  pushMatrix();
  translate(width*.5f, height*.5f-120.*sin(t), -4);
  fill(0);
  box(120, 120, 1);
  popMatrix();
  pushMatrix();
  translate(width*.5f, height*.5f-120.*sin(t), 4);
  fill(255);
  ellipse(0, 0, 30, 30);
  popMatrix();

  // gray box
  pushMatrix();
  translate(width*.85f, height*.5f, 0);
  fill(255/2.);
  box(120, 120, 1);
  popMatrix();

  // blur
  blurShader.set("time", t);
  blurShader.set("horizontalPass", 1);
  filter(blurShader);
  resetShader();
  blurShader.set("horizontalPass", 0);
  filter(blurShader);

  // fps
  if (frameCount%30==0)println(frameRate);
}

Nearly there.

Then add to the shader

Sorry, I meant the blur shader.

The line beginning with "vec4 effx..." will need to be inside main() so we can read depthBuffer at each pixel. You may have to play around with how to use "effx" to get the behavior you want.

In this example, I have it changing the blurSize:

Procedure:

Draw depth information to buffer.
Pass depth info to blur shader variable.
Draw image (scene, etc.) while using blur shader.
Filter existing render with blur shader in other direction. (depth info is retained)

I am using your original frag.glsl and vert.glsl to create the depth info.

pde:

PShader depthShader, blurShader;
PGraphics pbuff, sourceImage;
float angle = 0.0;

void setup(){
  // Set screen size and renderer
  size(600, 480, P3D);
  noStroke();

  // Load shader
  depthShader = loadShader("frag.glsl", "vert.glsl");
  blurShader = loadShader("blur.glsl");

  pbuff = createGraphics(width, height, P3D);
  sourceImage = createGraphics(width, height, P3D);

  // Load an image or create the scene to blur
  sourceImage.beginDraw();
  sourceImage.background(0);
  sourceImage.fill(255);
  sourceImage.ellipse(width/2, height/2, 50, 50);
  sourceImage.rect(width*2/3, height*2/3, 100, 100);
  sourceImage.endDraw();

  blurShader.set("sigma", 3.0);
  blurShader.set("blurSize", 9);

  shader(blurShader);
}

void draw(){
  pbuff.shader(depthShader);
  pbuff.beginDraw();
    pbuff.background(#000000);
    pbuff.camera(0.0, 0.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);
    pbuff.noStroke();
    // Calculate angle
    angle += 0.01;

    // Render "sky"-cube
    pbuff.pushMatrix();
    pbuff.rotate(angle, 0.0, 1.0, 0.0);
    pbuff.box(100.0);
    pbuff.popMatrix();

    // Render cubes
    pbuff.pushMatrix();
    pbuff.translate(-30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();

    // Render spheres
    pbuff.pushMatrix();
    pbuff.translate(-30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();
  pbuff.endDraw();

  blurShader.set("depthBuffer", pbuff);

  // Set this too if you use other shaders (on main display) earlier in draw
  shader(blurShader);

  // First draw the image _and_ do first blur pass on it
  blurShader.set("horizontalPass", 0);
  image(sourceImage, 0, 0);

  // Then run the second blur pass on existing render
  blurShader.set("horizontalPass", 1);
  filter(blurShader);
}

blur.glsl:

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;
uniform sampler2D depthBuffer;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;
varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>
const float pi = 3.14159265;

void main() {  
  vec4 effx = texture2D(depthBuffer, vertTexCoord.st);
  //float numBlurPixelsPerSide = float(blurSize / 2); 
  float numBlurPixelsPerSide = float((blurSize - effx.x*blurSize) / 2); // <--    Using depth info here, farther away, more blur

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;

  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));

  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }

  gl_FragColor = (avgValue / coefficientSum);
}

@noahbuddy Thank you again for your guidance and useful tips. I must admit that despite your detailed explanations I'm still having difficulties understanding what goes into what shader and how to compute the depth. For instance when you say > Then add to the shader

What shader are you referring to ? The fragment shader ?

Here is what I came up with based on your indications and my (very) poor understanding (apologies, again):

main pde file

PShader depthShader;
float angle = 0.0;

void setup(){

    size(600, 480, P3D);
    noStroke();

    pbuff = createGraphics(width, height, P3D);
    depthShader = loadShader("frag.glsl", "vert.glsl");
    blurShader = loadShader("blurFrag.glsl");

    blurShader.set("sigma", 10.5);
    blurShader.set("blurSize", 35);

}

void draw(){

    background(#000000);
    camera(0.0, 0.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);

    pbuff.pushMatrix();
    pbuff.rotate(angle, 0.0, 1.0, 0.0);
    pbuff.box(100.0);
    pbuff.popMatrix();

    pbuff.pushMatrix();
    pbuff.translate(-30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();

    pbuff.pushMatrix();
    pbuff.translate(-30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();

    pbuff.endDraw();

    blurShader.set("depthBuffer", pbuff);
    blurShader.set("horizontalPass", 0);
    image(sourceImage, 0, 0);
    blurShader.set("horizontalPass", 1);
    filter(blurShader);

}

Inside the blur shader, use a sampler2D in addition to "texture" and use that to change the blur properties.

blurFrag.glsls (just adding uniform sampler2D depthBuffer; and vec4 effx = texture2D(depthBuffer, vertTexCoord.st); on line 17 and 18.

    #ifdef GL_ES
    precision mediump float;
    precision mediump int;
    #endif


    #define PROCESSING_TEXTURE_SHADER

    uniform sampler2D texture;

    // The inverse of the texture dimensions along X and Y
    uniform vec2 texOffset;

    varying vec4 vertColor;
    varying vec4 vertTexCoord;

    uniform sampler2D depthBuffer;
    vec4 effx = texture2D(depthBuffer, vertTexCoord.st);

    uniform int blurSize;       
    uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
    uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                                // A good value for 9x9 is around 3 to 5
                                // A good value for 7x7 is around 2.5 to 4
                                // A good value for 5x5 is around 2 to 3.5
                                // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>

    const float pi = 3.14159265;

    void main() {  
      float numBlurPixelsPerSide = float(blurSize / 2); 

      vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

      // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
      vec3 incrementalGaussian;
      incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
      incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
      incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

      vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
      float coefficientSum = 0.0;

      // Take the central sample first...
      avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
      coefficientSum += incrementalGaussian.x;
      incrementalGaussian.xy *= incrementalGaussian.yz;

      // Go through the remaining 8 vertical samples (4 on each side of the center)
      for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
        avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                              blurMultiplyVec) * incrementalGaussian.x;         
        avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                              blurMultiplyVec) * incrementalGaussian.x;         
        coefficientSum += 2.0 * incrementalGaussian.x;
        incrementalGaussian.xy *= incrementalGaussian.yz;
      }

      gl_FragColor = (avgValue / coefficientSum);
    }

But also you're suggesting to add the sames lines in the fragment shader as well right ?

Then add to the shader

frag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

uniform sampler2D depthBuffer;
vec4 effx = texture2D(depthBuffer, vertTexCoord.st);


uniform vec4 nearColor = vec4(1.0, 1.0, 1.0, 1);
uniform vec4 farColor = vec4(0.0, 0.0, 0.0, 1.0);
uniform float near = 0.0;
uniform float far = 100.0;


varying vec4 vertColor;

void main() {
    gl_FragColor = mix(nearColor, farColor, smoothstep(near, far, gl_FragCoord.z / gl_FragCoord.w));
}

Also, how can I change the main() function of this this fragment shader so as it computes a level of blur instead of a change in color ? So sorry to bother you with all there questions. I wish I had a better and faster understanding of the logic behind all this.

Hi all,

I'm trying to fake a Depth of Field effect in a 3D scene. More specifically, I would like to use a z-buffer (depth buffering) to adjust the level of blur of a an object based on its distance from the camera. While searching the forum I found the following vertex and fragment shaders provided by @Poersch (from this topic ).

vert.glsl

uniform mat4 transform;

attribute vec4 vertex;
attribute vec4 color;

varying vec4 vertColor;

void main() {
    gl_Position = transform * vertex;
    vertColor = color;
}

frag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

uniform vec4 nearColor = vec4(1.0, 1.0, 1.0, 1.0);
uniform vec4 farColor = vec4(0.0, 0.0, 0.0, 1.0);
uniform float near = 0.0;
uniform float far = 100.0;

varying vec4 vertColor;

void main() {
    gl_FragColor = mix(nearColor, farColor, smoothstep(near, far, gl_FragCoord.z / gl_FragCoord.w));
}

You can see the shaders in action with this sketch example:

sketch.pde

PShader depthShader;
float angle = 0.0;


void setup(){

    // Set screen size and renderer
    size(600, 480, P3D);
    noStroke();

    // Load shader
    depthShader = loadShader("frag.glsl", "vert.glsl");
    //depthShader.set("near", 40.0); // Standard: 0.0
    //depthShader.set("far", 60.0); // Standard: 100.0
    //depthShader.set("nearColor", 1.0, 0.0, 0.0, 1.0); // Standard: white
    //depthShader.set("farColor", 0.0, 0.0, 1.0, 1.0); // Standard: black

}


void draw(){

    // Fill background and set camera
    background(#000000);
    camera(0.0, 0.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);

    // Bind shader
    shader(depthShader);

    // Calculate angle
    angle += 0.01;

    // Render "sky"-cube
    pushMatrix();
    rotate(angle, 0.0, 1.0, 0.0);
    box(100.0);
    popMatrix();

    // Render cubes
    pushMatrix();
    translate(-30.0, 20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    box(25.0);
    popMatrix();
    pushMatrix();
    translate(30.0, -20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    box(25.0);
    popMatrix();

    // Render spheres
    pushMatrix();
    translate(-30.0, -20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    sphere(20.0);
    popMatrix();
    pushMatrix();
    translate(30.0, 20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    sphere(20.0);
    popMatrix();

}

Here, the z-buffer is used to change the color of an object: the closer the lighter, the farther, the darker.

QUESTION

• How can I use the same z-buffer to change the blur level of an object ?

Ideally I would like to use the Gaussian blur shader from the PostFX library:

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif


#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;

varying vec4 vertColor;
varying vec4 vertTexCoord;


uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>

const float pi = 3.14159265;

void main() {  
  float numBlurPixelsPerSide = float(blurSize / 2); 

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;
  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }

  gl_FragColor = (avgValue / coefficientSum);
}

Unfortunately I can't figure out how to make the two (z-buffer and blur fragment shader) work together. Any hint would be greatly appreciated !

To preserve transparency, with or without shaders, use an offscreen buffer (PGraphics). For example, saving a PNG image with transparent background.

I removed the contrast matrix from @cansik 's blur shader and instead put it into a separate filter.

In the blur shader, change the assignment to:

gl_FragColor = (avgValue / coefficientSum );

I think this is close...

pde:

PShader contrast, blurry;
PGraphics buf;

void setup() {
  size(200, 200, P2D);
  buf = createGraphics(width, height, P2D);

  contrast = loadShader("colfrag.glsl");
  blurry = loadShader("blurFrag.glsl");

  // Don't forget to set these
  blurry.set("sigma", 4.5);
  blurry.set("blurSize", 9);
}

void draw() {
  background(100);

  buf.beginDraw();
    // Reset transparency
    // Note, the color used here will affect your edges
    // even with zero for alpha
    buf.background(100, 0); // set to match main background

    buf.noStroke();
    buf.fill(255, 30, 30);
    buf.ellipse(width/2, height/2, 40, 40);
    buf.ellipse(mouseX, mouseY, 40, 40);

    blurry.set("horizontalPass", 1);
    buf.filter(blurry);
    blurry.set("horizontalPass", 0);
    buf.filter(blurry);
  buf.endDraw();

  shader(contrast);
  image(buf, 0,0, width,height);
}

colfrag:

#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;
varying vec4 vertTexCoord;

uniform vec4 o = vec4(0, 0, 0, -7.0); 
uniform lowp mat4 colorMatrix = mat4(1.0, 0.0, 0.0, 0.0, 
                                     0.0, 1.0, 0.0, 0.0, 
                                     0.0, 0.0, 1.0, 0.0, 
                                     0.0, 0.0, 0.0, 18.0);

void main() {
  vec4 pix = texture2D(texture, vertTexCoord.st);

  vec4 color = (pix * colorMatrix) + o;
  gl_FragColor = color;
}

I'm trying to replicate a web design trick known as "gooey effect" (see it live here). It's a technique applying SVG filters on moving ellipses in order to get a blob-like motion. The process is rather simple:

• apply a gaussian blur
• increase the contrast of the alpha channel only

The combination of the two creates a blob effect

The last step (increasing the alpha channel contrast) is usually done through a "color matrix filter".

A color matrix is composed of 5 columns (RGBA + offset) and 4 rows.

The values in the first four columns are multiplied with the source red, green, blue, and alpha values respectively. The fifth column value is added (offset).

In CSS, increasing the alpha channel contrast is as simple as calling a SVG filter and specifying the contrast value (here 18):

<feColorMatrix in="blur" mode="matrix" values="1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 18 -7" result="goo" />

In Processing though, it seems to be a bit more complicated. I believe (I may be wrong) the only way to apply a color matrix filter is to create one in a shader. After a few tries I came up with these (very basic) vertex and fragment shaders for color rendering:

colorvert.glsl

uniform mat4 transform;
attribute vec4 position;
attribute vec4 color;
varying vec4 vertColor;

uniform vec4 o=vec4(0, 0, 0, -9); 
uniform lowp mat4 colorMatrix = mat4(1.0, 0.0, 0.0, 0.0, 
                                     0.0, 1.0, 0.0, 0.0, 
                                     0.0, 0.0, 1.0, 0.0, 
                                     0.0, 0.0, 0.0, 60.0);


void main() {
  gl_Position = transform * position; 
  vertColor = (color * colorMatrix) + o  ;
}

colorfrag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

varying vec4 vertColor;

void main() {
  gl_FragColor = vertColor;
}

PROBLEM:

The color matrix is partially working: changing the RGB values do affect the colors but changing the alpha values don't !

When trying to combine the shader with a Gaussian filter, the drawn ellipse stays blurry even after I set the alpha channel contrast to 60 (like in the codepen example):

PShader colmat;

void setup() {
  size(200, 200, P2D);
  colmat = loadShader("colorfrag.glsl", "colorvert.glsl");
}

void draw() {
  background(100);
  shader(colmat);

  noStroke();
  fill(255, 30, 30);
  ellipse(width/2, height/2, 40, 40);
  filter(BLUR,6);
}

The same thing happens when I implement the color matrix within @cansik 's Gaussian blur shader (from the PostFX library). I can see the colors changing but not the alpha contrast:

blurFrag.glsl

/ Adapted from:
// <a href="http://callumhay.blogspot.com/2010/09/gaussian-blur-shader-glsl.html" target="_blank" rel="nofollow">http://callumhay.blogspot.com/2010/09/gaussian-blur-shader-glsl.html</a>;

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

#define PROCESSING_TEXTURE_SHADER


uniform sampler2D texture;

uniform vec4 o=vec4(0, 0, 0, 0); 
uniform lowp mat4 colorMatrix = mat4(1, 0.0, 0.0, 0.0, 
                                     0.0, 1, 0.0, 0.0, 
                                     0.0, 0.0, 1, 0.0, 
                                     0, 0.0, 0.0, 60.0); //Alpha contrast set to 60


varying vec2 center;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;

varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>

const float pi = 3.14159265;

void main() {  
  float numBlurPixelsPerSide = float(blurSize / 2); 

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;
  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }
  gl_FragColor = (avgValue / coefficientSum )  * colorMatrix;
}

Questions:

• Why does the alpha channel contrast not work ? How can I "enable" it ?
• Is it possible to enable it in the blurFrag shader posted above ?
• Do you know a better way to implement that gooey effect ?

Any help would be much appreciated !

Thank you

import com.jogamp.opengl.*;
import  com.jogamp.opengl.GL4;
import com.jogamp.opengl.util.GLBuffers;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;

IntBuffer Tex;
FloatBuffer mBuf;

PJOGL pgl;
GL4 gl4 ;

final int twidth = 1;

PShader shdr;

void settings() {
  size(640, 360, P3D );
  PJOGL.profile = 4;
}

void setup() {

  /* //fragment.glsl
  #version 430
  layout(binding = 0) uniform sampler1D tex;
  layout(location = 0) uniform vec3 iResolution;
  layout(location = 0) out vec4 fragColor;
  void main() {
  vec2 uvv = (gl_FragCoord.xy*2.-iResolution.xy)/iResolution.z;
  float arr = texelFetch( tex ,0,0) .r;
  float cir=arr/length(uvv);
  fragColor = vec4(vec3(cir),1.);
  }
  */

  /* //vertex.glsl
  #version 430
  in vec2 position;
  void main(){
  gl_Position = vec4(position.xy,0.,1.);
  }
  */

  shdr = loadShader("fragment.glsl", "vertex.glsl");



  shader(shdr);
  shdr.set("iResolution", new PVector(width, height, Math.min(width, height)));

  Tex = GLBuffers.newDirectIntBuffer(1);

  pgl = (PJOGL) beginPGL();  
  gl4 = pgl.gl.getGL4();

  gl4.glGenTextures(1, Tex);

  gl4.glBindTexture(GL4.GL_TEXTURE_1D, Tex.get(0));

  mBuf = GLBuffers.newDirectFloatBuffer(twidth);

  shdr.bind();
  gl4.glTexImage1D( GL4.GL_TEXTURE_1D, 0, GL4.GL_R32F, twidth, 0, GL4.GL_RED, GL4.GL_FLOAT, mBuf );
  shdr.unbind();
  endPGL();
}

void draw() {
  pgl = (PJOGL) beginPGL(); 

  float arr = 1./random(2, 24);

  gl4 = pgl.gl.getGL4();
  shdr.bind();
  if (frameCount%12==0) mBuf.put( arr ).rewind();
  println(arr);

  gl4.glTexSubImage1D( GL4.GL_TEXTURE_1D, 0, 0, twidth, GL4.GL_RED, GL4.GL_FLOAT, mBuf );
  shdr.unbind();
  endPGL();
  rect(0, 0, width, height);
}

I got it working. I'm using a magic number, I have no idea where it comes from and it's probably because I'm doing something wrong/improperly. Any ideas where this 1.731875 number in my vert shader comes from? It's needed to match the scale. Also none of my .set("var",var) type stuff is working with my shader, any idea what could cause that?

// pointvert.glsl
 #define PROCESSING_POINT_SHADER
uniform vec2 wh;
uniform float fix;
uniform mat4 projection;
uniform mat4 modelview;
uniform sampler2D tex_position;

attribute vec4 position;
attribute vec4 color;
attribute vec2 offset;

varying vec4 vertColor;

vec2 posDecode(vec4 c) {
    return vec2(c.r + c.g/255.0, c.b + c.a/255.0);
}
void main() {

 vec4 colord = texelFetch(tex_position, ivec2(position.x, 0),0);
 vertColor = colord;
 vec4 loc = vec4(posDecode(colord),0,0); // decoded location from texture

  vec4 pos = modelview * vec4(0,0,position.zw); 
  vec4 clip = projection * pos;

  vec4 locAdjusted =  clip + projection * vec4(offset, 0, 0);

  vec2 screenWH = vec2(1000,1000); // setting this with wh from processing isn't working?
  screenWH *= 1.731875; // What is this magic numbers? Doesn't seem to relate to the screen size in a clear ratio.
  loc.x *= screenWH.x;  
  loc.y *= screenWH.y;
  locAdjusted.x += loc.x; 
  locAdjusted.y -= loc.y;

  gl_Position = locAdjusted;
}

.....

//pointfrag.glsl
#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

varying vec4 vertColor;

void main() {  
  gl_FragColor = vertColor;
}

....

sketch

// PRESS ANY KEY TO TOGGLE SHADER/CPU RENDERING
// CLICK FOR FRESH DATA

PShape psPoints; // vertex list pointing to every pixel/encoded location in pgLoc
PShader pointShader;
PImage pgLocs;
int totalPoints = 5000;

void setup() {
  size(1000, 1000, P3D);
  pgLocs = createImage(totalPoints,1, ARGB); 

  pointShader = loadShader("pointfrag.glsl", "pointvert.glsl");
  float[] f= new float[2];
  f[0] = width; 
  f[1] = height;
  pointShader.set("wh", width,height);  // this isn't working, no idea why
  randomFill();
  initShape();
}

void randomFill() {
  pgLocs.loadPixels();
  for (int i = 0; i < totalPoints; i++) {  
    PVector loc = new PVector(random(width)/width, random(height)/height);
    pgLocs.pixels[i] = xyToRGBA(loc);
  }
  pgLocs.pixels[0] = xyToRGBA(new PVector(((float)mouseX)/width, ((float)mouseY)/height));
  pgLocs.updatePixels();
  pointShader.set("tex_position", pgLocs);
}

void initShape() {
  // creates a shape with verticies with the pixel locations for the encoded locations in pgLocs;
  // this triggers the vert shader for each data point without needing gl_VertexID
  psPoints = createShape();
  psPoints.beginShape(POINTS);
  psPoints.stroke(255);
  psPoints.strokeWeight(10);
  for (int i = 0; i < totalPoints; i++) {
    psPoints.vertex(i*1.00+0.00, 0.0);
  }
  psPoints.endShape();
}

boolean useShader = true;
void keyPressed() {
  useShader = !useShader;
}

void draw() {    
  pointShader.set("fix", 800);
  initShape();
  background(20);
  if (mousePressed) {   
    randomFill();
    initShape();
  }  
  if (!useShader) { // CPU implementation
    resetShader(POINTS);
    strokeWeight(10);
    pgLocs.loadPixels();
    for (int i = 0; i < totalPoints; i++) { // What I'd like to do in a shader instead
      color c = pgLocs.pixels[i];// //get pixel color
      PVector loc = RGBAtoXY(c); // decode location
      stroke(c);                 // set color just for fun
      point(loc.x*width, loc.y*height); // show location was stored in the texture properly
    }
    text("CPU render", 10, 10);
  } 
  else {  // Shader implementation
    shader(pointShader, POINTS);
    stroke(255);
    shape(psPoints); // sends a list of point verticies, pointing to every pixel in pgLoc
    text("Shader render", 10, 30);
  }
}
color xyToRGBA(PVector loc) { // pack x into r g, y into b a
  PVector l = loc.copy().mult(255);
  int xi = floor(l.x);
  int yi = floor(l.y);
  int xr = floor((l.x-xi)*255);
  int yr = floor((l.y-yi)*255);
  return (yr << 24) | (xi << 16) | (xr << 8) | yi;
}

PVector RGBAtoXY(color c) { 
  int a = ((c >> 24) & 0xff) ;
  int r = ((c >> 16) & 0xff) ;
  int g = ((c >> 8)  & 0xff) ;
  int b =  (c        & 0xff) ;
  return new PVector((r+g/255.0)/255.0, (b+a/255.0)/255.0);
}