How can i do?

import controlP5.*;

ControlP5 cp5;
import peasy.*;

ArrayList<Sphere>s;

PGraphics canvas;

PGraphics brightPass;
PGraphics horizontalBlurPass;
PGraphics verticalBlurPass;

PShader bloomFilter;
PShader blurFilter;
PeasyCam cam;
float angle = 0;

final int surfaceWidth = 250;
final int surfaceHeight = 250;

float luminanceFilter = 0.02;
float blurSize = 100;
float sigma = 200;

void setup()
{
  cam = new PeasyCam(this, 1400);
  size(1000, 1000, P3D);

  s = new ArrayList<Sphere>();

  canvas = createGraphics(width, height, P3D);

  brightPass = createGraphics(width, height, P2D);
  brightPass.noSmooth();

  horizontalBlurPass = createGraphics(width, height, P2D);
  horizontalBlurPass.noSmooth(); 

  verticalBlurPass = createGraphics(width, height, P2D);
  verticalBlurPass.noSmooth(); 

  bloomFilter = loadShader("bloomFrag.glsl");
  blurFilter = loadShader("blurFrag.glsl");
}

void draw()
{
  background(0);
  bloomFilter.set("brightPassThreshold", luminanceFilter);
  angle += 0.05;

  for(Sphere s: s){
  blurFilter.set("blurSize", s.p);
  }

  blurFilter.set("sigma", sigma); 

  canvas.beginDraw();
  render(canvas);
  canvas.endDraw();

  // bright pass
  brightPass.beginDraw();
  brightPass.shader(bloomFilter);
  brightPass.image(canvas, 0, 0);
  brightPass.endDraw();

  // blur horizontal pass
  horizontalBlurPass.beginDraw();
  blurFilter.set("horizontalPass", 1);
  horizontalBlurPass.shader(blurFilter);
  horizontalBlurPass.image(brightPass, 0, 0);
  horizontalBlurPass.endDraw();

  // blur vertical pass
  verticalBlurPass.beginDraw();
  blurFilter.set("horizontalPass", 0);
  verticalBlurPass.shader(blurFilter);
  verticalBlurPass.image(horizontalBlurPass, 0, 0);
  verticalBlurPass.endDraw();


  cam.beginHUD();
  blendMode(BLEND);
  blendMode(SCREEN);
  image(brightPass, 0, 0);
  image(verticalBlurPass, 0, 0);

  cam.endHUD();

println(frameRate);
}

void render(PGraphics pg)
{
  cam.getState().apply(pg);

  pg.background(0, 50);

  canvas.pushMatrix();
  canvas.translate(width/2, height/2);
  for(Sphere s: s){
  s.display();

  }
  canvas.popMatrix();


}

void mousePressed() {
  s.add(new Sphere(random(-width/2, width/2), random(-height/2, height/2), random(1000)));
}


class Sphere {

  float p;
  float w;
  float h;

  Sphere(float _w, float _h, float _p) {
  p = _p;
  w = _w;
  h = _h;
  }

  void display() {
    canvas.pushMatrix();
    canvas.translate(w, h);
    noFill();
    canvas.sphere(100);

    canvas.popMatrix();
  }
}

In the example sketch, you can enable the depth pass shader for the 3D scene (left picture) with any key. So the depth buffer is correctly drawn onto the screen.

Now the problem that I have is, that I am not able to shade / filter the 3d context with the depth buffer shader, and draw it onto the 2d canvas. Without doing that, I am not able to use it inside the next texture shader I would like to apply to the 2d graphics.

I hope it is not too confusing. What I want is to have the depth information in a 2d texture shader.

Attached is the example sketch, which currently is working on the right side (no depth information is shaded onto the 2d context).

void createDepthImage(PGraphics3D graphics)
{
  if (!graphics.is3D())
    return;

  // add shader to graphics
  graphics.shader(shader);

  depthImage.beginDraw();
  depthImage.image(graphics, 0, 0);
  depthImage.endDraw();

  // reset shader after operation
  if (!shaderApplied)
    graphics.resetShader();
}

Complete sketch: gist.github.com/d6a61bcfe1ffc4f36eb1592d7143ab8f

Press 1, 2 or 3 to switch between the different demo "landscapes", s for spotlight and d for directional light.

import peasy.*;

PVector lightDir = new PVector();
PShader defaultShader;
PGraphics shadowMap;
int landscape = 1;

void setup() {
    size(800, 800, P3D);
    new PeasyCam(this, 300).rotateX(4.0);
    initShadowPass();
    initDefaultPass();
}

void draw() {

    // Calculate the light direction (actually scaled by negative distance)
    float lightAngle = frameCount * 0.002;
    lightDir.set(sin(lightAngle) * 160, 160, cos(lightAngle) * 160);

    // Render shadow pass
    shadowMap.beginDraw();
    shadowMap.camera(lightDir.x, lightDir.y, lightDir.z, 0, 0, 0, 0, 1, 0);
    shadowMap.background(0xffffffff); // Will set the depth to 1.0 (maximum depth)
    renderLandscape(shadowMap);
    shadowMap.endDraw();
    shadowMap.updatePixels();

    // Update the shadow transformation matrix and send it, the light
    // direction normal and the shadow map to the default shader.
    updateDefaultShader();

    // Render default pass
    background(0xff222222);
    renderLandscape(g);

    // Render light source
    pushMatrix();
    fill(0xffffffff);
    translate(lightDir.x, lightDir.y, lightDir.z);
    box(5);
    popMatrix();

}

public void initShadowPass() {
    shadowMap = createGraphics(2048, 2048, P3D);
    String[] vertSource = {
        "uniform mat4 transform;",

        "attribute vec4 vertex;",

        "void main() {",
            "gl_Position = transform * vertex;",
        "}"
    };
    String[] fragSource = {

        // In the default shader we won't be able to access the shadowMap's depth anymore,
        // just the color, so this function will pack the 16bit depth float into the first
        // two 8bit channels of the rgba vector.
        "vec4 packDepth(float depth) {",
            "float depthFrac = fract(depth * 255.0);",
            "return vec4(depth - depthFrac / 255.0, depthFrac, 1.0, 1.0);",
        "}",

        "void main(void) {",
            "gl_FragColor = packDepth(gl_FragCoord.z);",
        "}"
    };
    shadowMap.noSmooth(); // Antialiasing on the shadowMap leads to weird artifacts
    //shadowMap.loadPixels(); // Will interfere with noSmooth() (probably a bug in Processing)
    shadowMap.beginDraw();
    shadowMap.noStroke();
    shadowMap.shader(new PShader(this, vertSource, fragSource));
    shadowMap.ortho(-200, 200, -200, 200, 10, 400); // Setup orthogonal view matrix for the directional light
    shadowMap.endDraw();
}

public void initDefaultPass() {
    String[] vertSource = {
        "uniform mat4 transform;",
        "uniform mat4 modelview;",
        "uniform mat3 normalMatrix;",
        "uniform mat4 shadowTransform;",
        "uniform vec3 lightDirection;",

        "attribute vec4 vertex;",
        "attribute vec4 color;",
        "attribute vec3 normal;",

        "varying vec4 vertColor;",
        "varying vec4 shadowCoord;",
        "varying float lightIntensity;",

        "void main() {",
            "vertColor = color;",
            "vec4 vertPosition = modelview * vertex;", // Get vertex position in model view space
            "vec3 vertNormal = normalize(normalMatrix * normal);", // Get normal direction in model view space
            "shadowCoord = shadowTransform * (vertPosition + vec4(vertNormal, 0.0));", // Normal bias removes the shadow acne
            "lightIntensity = 0.5 + dot(-lightDirection, vertNormal) * 0.5;",
            "gl_Position = transform * vertex;",
        "}"
    };
    String[] fragSource = {
        "#version 120",

        // Used a bigger poisson disk kernel than in the tutorial to get smoother results
        "const vec2 poissonDisk[9] = vec2[] (",
            "vec2(0.95581, -0.18159), vec2(0.50147, -0.35807), vec2(0.69607, 0.35559),",
            "vec2(-0.0036825, -0.59150), vec2(0.15930, 0.089750), vec2(-0.65031, 0.058189),",
            "vec2(0.11915, 0.78449), vec2(-0.34296, 0.51575), vec2(-0.60380, -0.41527)",
        ");",

        // Unpack the 16bit depth float from the first two 8bit channels of the rgba vector
        "float unpackDepth(vec4 color) {",
            "return color.r + color.g / 255.0;",
        "}",

        "uniform sampler2D shadowMap;",

        "varying vec4 vertColor;",
        "varying vec4 shadowCoord;",
        "varying float lightIntensity;",

        "void main(void) {",

            // Project shadow coords, needed for a perspective light matrix (spotlight)
            "vec3 shadowCoordProj = shadowCoord.xyz / shadowCoord.w;",

            // Only render shadow if fragment is facing the light
            "if(lightIntensity > 0.5) {",
                "float visibility = 9.0;",

                // I used step() instead of branching, should be much faster this way
                "for(int n = 0; n < 9; ++n)",
                    "visibility += step(shadowCoordProj.z, unpackDepth(texture2D(shadowMap, shadowCoordProj.xy + poissonDisk[n] / 512.0)));",

                "gl_FragColor = vec4(vertColor.rgb * min(visibility * 0.05556, lightIntensity), vertColor.a);",
            "} else",
                "gl_FragColor = vec4(vertColor.rgb * lightIntensity, vertColor.a);",

        "}"
    };
    shader(defaultShader = new PShader(this, vertSource, fragSource));
    noStroke();
    perspective(60 * DEG_TO_RAD, (float)width / height, 10, 1000);
}

void updateDefaultShader() {

    // Bias matrix to move homogeneous shadowCoords into the UV texture space
    PMatrix3D shadowTransform = new PMatrix3D(
        0.5, 0.0, 0.0, 0.5, 
        0.0, 0.5, 0.0, 0.5, 
        0.0, 0.0, 0.5, 0.5, 
        0.0, 0.0, 0.0, 1.0
    );

    // Apply project modelview matrix from the shadow pass (light direction)
    shadowTransform.apply(((PGraphicsOpenGL)shadowMap).projmodelview);

    // Apply the inverted modelview matrix from the default pass to get the original vertex
    // positions inside the shader. This is needed because Processing is pre-multiplying
    // the vertices by the modelview matrix (for better performance).
    PMatrix3D modelviewInv = ((PGraphicsOpenGL)g).modelviewInv;
    shadowTransform.apply(modelviewInv);

    // Convert column-minor PMatrix to column-major GLMatrix and send it to the shader.
    // PShader.set(String, PMatrix3D) doesn't convert the matrix for some reason.
    defaultShader.set("shadowTransform", new PMatrix3D(
        shadowTransform.m00, shadowTransform.m10, shadowTransform.m20, shadowTransform.m30, 
        shadowTransform.m01, shadowTransform.m11, shadowTransform.m21, shadowTransform.m31, 
        shadowTransform.m02, shadowTransform.m12, shadowTransform.m22, shadowTransform.m32, 
        shadowTransform.m03, shadowTransform.m13, shadowTransform.m23, shadowTransform.m33
    ));

    // Calculate light direction normal, which is the transpose of the inverse of the
    // modelview matrix and send it to the default shader.
    float lightNormalX = lightDir.x * modelviewInv.m00 + lightDir.y * modelviewInv.m10 + lightDir.z * modelviewInv.m20;
    float lightNormalY = lightDir.x * modelviewInv.m01 + lightDir.y * modelviewInv.m11 + lightDir.z * modelviewInv.m21;
    float lightNormalZ = lightDir.x * modelviewInv.m02 + lightDir.y * modelviewInv.m12 + lightDir.z * modelviewInv.m22;
    float normalLength = sqrt(lightNormalX * lightNormalX + lightNormalY * lightNormalY + lightNormalZ * lightNormalZ);
    defaultShader.set("lightDirection", lightNormalX / -normalLength, lightNormalY / -normalLength, lightNormalZ / -normalLength);

    // Send the shadowmap to the default shader
    defaultShader.set("shadowMap", shadowMap);

}

public void keyPressed() {
    if(key != CODED) {
        if(key >= '1' && key <= '3')
            landscape = key - '0';
        else if(key == 'd') {
            shadowMap.beginDraw(); shadowMap.ortho(-200, 200, -200, 200, 10, 400); shadowMap.endDraw();
        } else if(key == 's') {
            shadowMap.beginDraw(); shadowMap.perspective(60 * DEG_TO_RAD, 1, 10, 1000); shadowMap.endDraw();
        }
    }
}

public void renderLandscape(PGraphics canvas) {
    switch(landscape) {
        case 1: {
            float offset = -frameCount * 0.01;
            canvas.fill(0xffff5500);
            for(int z = -5; z < 6; ++z)
                for(int x = -5; x < 6; ++x) {
                    canvas.pushMatrix();
                    canvas.translate(x * 12, sin(offset + x) * 20 + cos(offset + z) * 20, z * 12);
                    canvas.box(10, 100, 10);
                    canvas.popMatrix();
                }
        } break;
        case 2: {
            float angle = -frameCount * 0.0015, rotation = TWO_PI / 20;
            canvas.fill(0xffff5500);
            for(int n = 0; n < 20; ++n, angle += rotation) {
                canvas.pushMatrix();
                canvas.translate(sin(angle) * 70, cos(angle * 4) * 10, cos(angle) * 70);
                canvas.box(10, 100, 10);
                canvas.popMatrix();
            }
            canvas.fill(0xff0055ff);
            canvas.sphere(50);
        } break;
        case 3: {
            float angle = -frameCount * 0.0015, rotation = TWO_PI / 20;
            canvas.fill(0xffff5500);
            for(int n = 0; n < 20; ++n, angle += rotation) {
                canvas.pushMatrix();
                canvas.translate(sin(angle) * 70, cos(angle) * 70, 0);
                canvas.box(10, 10, 100);
                canvas.popMatrix();
            }
            canvas.fill(0xff00ff55);
            canvas.sphere(50);
        }
    }
    canvas.fill(0xff222222);
    canvas.box(360, 5, 360);
}

I mostly commented the Processing part of the sketch, the GLSL stuff is explained in the linked tutorial. Anyways, I hope you have fun with it. :)

here is the code:

  import processing.video.*;
import peasy.*;
import peasy.org.apache.commons.math.*;
import peasy.org.apache.commons.math.*;

PeasyCam cam;
PShader sh;
float count;
Movie mov;
PGraphics p;

void setup (){

size(1440, 900, P3D );
mov = new Movie(this, "osc_noc.mov");
mov.play();
p = createGraphics(width,height);
cam = new PeasyCam(this, 500);
sh = loadShader("basicShaderFrag.glsl", "basicShader.glsl");
}

void movieEvent(Movie m) {
  m.read();
}

void draw(){
background(0);
shader(sh);
count +=0.09;
sh.set("u_time", count);

sphere(100);
p.beginDraw();
p.background(0);
p.image(mov, 0, 0, 200, 200);
p.endDraw();
sh.set("tex",p);
// image(p, 5, 260 ,200, 200);
}

#version 150

uniform mat4 transform;
uniform sampler2D tex;

in vec4 position;
in vec2 texCoord;
in vec3 normal;

out vec2 TexCoord;

void main(){

  TexCoord = texCoord;
  gl_Position = transform * position;
}


#ifdef GL_ES
precision mediump float;
#endif

#define PI 3.14

in vec2 TexCoord;
uniform float u_time;
uniform sampler2D tex;

void main(){
  vec2 uv = TexCoord;

  gl_FragColor = vec4(texture(tex, TexCoord));
}

a white screen appears, and next it close. the console just say: "Finished". it may be a bug? i could pass a PImage as a texture. but when i link the fragment and shader program into the sketch folder then crash. ..

code is simple I update blendMode(ADD ) in setup and draw sphere() I am going to use shader later but with out shader there is this problem.

void setup() 
{
//....... other setup setup stuff
      blendMode(ADD);
}
int y=0;
void draw()
{
//  hint(ENABLE_DEPTH_TEST);
  noStroke();
  background(0);

  sh.set("mainPower",2.7);
  sh.set("eyePos",0.0, 0.0, 100.0);
  //shader(sh);

  translate(width/2,height/2+y,0);
  fill(250,100,0);
  sphere(90);
  //box(30);
  if(y>height/2)
  y=-height/2;
  else 
  y++;
}

tileShader.set("time", millis() / 1000.0);

If I try to modify this so that the tile will move at a speed according to my mouse,

float t =millis() / 1000.0; 
float pctX = map (mouseX, 0, width, 0, 1);
tileShader.set("time", t*pctX);

the shader will do this 'scrubbing' effect where the entire image moves very quickly, but then once you stop moving the mouse, it will go at the speed you want. Is there any way to avoid this scrubbing effect? What I'm trying to accomplish are smooth increases in speed of the tile movement.

Back in Processing 2 I was able to do this with a pair of fragment shaders without trouble:

PShader shader1;
PShader shader2;

void setup() {
  size(256, 256, P2D);
  fill(128);
  noStroke();
  shader1 = loadShader("shader1.glsl");
  shader2 = loadShader("shader2.glsl");
}

void draw() {
  shader(shader1);
  shader(shader2);
  rect(0, 0, width, height);
}

Where shader1 is as follows:

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

uniform sampler2D textureSampler;

varying vec4 vertColor;
varying vec4 vertTexCoord;

void main()
{
    vec3 pos = vertTexCoord.xyz;

    gl_FragColor = vec4(pos.x, pos.y, pos.z, 1.0);
}

And shader2 is as follows:

#ifdef GL_ES
precision mediump float;
#endif

uniform sampler2D texture;

varying vec4 vertColor;
varying vec4 vertTexCoord;

void main () {
    gl_FragColor = texture2D(texture, vertTexCoord.xy);
}

No matter what I do in Processing 3, this always results in a black image when shader2 is applied, but works fine with just shader1. I have tried all sorts of things, including different OpenGL version directives, passing the resolution to shader2 and dividing the frag coord by it before sampling, and re-arranging the shader calls. I have tried both P2D and P3D mode.

In fact, many of the examples I have tried from the documentation fail to work at all in Processing 3.3.

Any idea what's going on? If it matters, I'm on a GTX 980 Ti with the latest drivers.

When using filter(), everything works fine. I just apply it after drawing everything else to the PGraphics.
When trying to use shader(), I call pg.shader(myShader) before drawing any objects to the PGraphics (because that's how shaders work, right?), and in between pg.beginDraw() and pg.endDraw(), but without success. I've tried other combinations, but they don't work either.

What am I missing?

My sketch is trying to emulate an image "shatter" - a bunch of different PShapes fly around, all with static UV coordinates referencing the same image. Imagine a stained glass window smashing - little bits of image everywhere.

I have things working, albeit slowly, and have been told I can improve my performance with shaders.

I've tried to create a simple working version before implementing into the full sketch. Just trying to draw one PShape with a texture.

Sketch

PShader myShader;
PShape myShape;
PImage myImage;

void setup(){
    size(200, 200, P3D);

    myImage = loadImage("Grug.jpg");
    myShader = loadShader("frag.glsl", "vert.glsl");
    myShape = createShape();

    myShape.beginShape();
    myShape.noFill();
    myShape.noStroke();
    myShape.vertex(20, 20, 20, 20);
    myShape.vertex(80, 20, 80, 20);
    myShape.vertex(80, 80, 80, 80);
    myShape.vertex(20, 80, 20, 80);
    myShape.endShape();
}

void draw(){
    background(150);
    myShader.set("myImage", myImage);
    shader(myShader);
    shape(myShape);
}

Vert shader

#define PROCESSING_TEXTURE_SHADER

uniform mat4 transform;
uniform mat4 texMatrix;

attribute vec4 vertex;
attribute vec4 color;
attribute vec2 texCoord;

varying vec4 vertColor;
varying vec4 vertTexCoord;

void main() {
  gl_Position = transform * vertex;

  vertColor = color;
  vertTexCoord = texMatrix * vec4(texCoord, 1.0, 1.0);
}

Frag shader

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

uniform sampler2D texture;
uniform sampler2D myImage;

varying vec4 vertColor;
varying vec4 vertTexCoord;

void main() {
    gl_FragColor = texture2D(myImage, vertTexCoord.st);
}

However, nothing displays. Note that I want to avoid calling myShape.texture(myImage) when defining the shape. This call does make things work, but on a large scale the framerate dives.

I'm pretty sure it's possible to have a few thousand verts flying about referencing the same texture - please correct me if I'm mistaken. What am I missing? Or is my approach a bit wonky?! Shaders have been a bit to wrap my head around.

Thanks in advance! (Mac OSX 10.11.5, Processing 3.1.1.)

sketch code:

PShader myShader;
PImage  myImage;

void setup() {
  size(640, 360, P2D);
  noStroke();
  myImage = loadImage("noise512.png");
  myShader = loadShader("noise_anim.glsl");
  myShader.set("resolution", float(width), float(height));

}

void draw() {
  background(0);
  myShader.set("time", millis() / 1000.0);

  shader(myShader);
  image( myImage, 0, 0, width, height ); // when using image or text maps

}

noise_anim.glsl code:

#ifdef GL_ES
precision highp float;
#endif

#define PROCESSING_TEXTURE_SHADER

uniform float time;
uniform vec2 resolution;
uniform vec2 mouse;
uniform sampler2D texture;


// Layer between Processing and Shadertoy uniforms
vec3 iResolution = vec3(resolution,0.0);
float iGlobalTime = time;
vec4 iMouse = vec4(mouse,0.0,0.0); // zw would normally be the click status

//Noise animation - Electric
//by nimitz (stormoid.com) (twitter: @stormoid)

//The domain is displaced by two fbm calls one for each axis.
//Turbulent fbm (aka ridged) is used for better effect.

#define time iGlobalTime*0.15
#define tau 6.2831853

mat2 makem2(in float theta){
    float c = cos(theta);
    float s = sin(theta);
    return mat2(c,-s,s,c);
    }
float noise( in vec2 x ){
    return texture2D(texture, x*.01).x;
    }

float fbm(in vec2 p)
{   
    float z=2.;
    float rz = 0.;
    vec2 bp = p;
    for (float i= 1.;i < 6.;i++)
    {
        rz+= abs((noise(p)-0.5)*2.)/z;
        z = z*2.;
        p = p*2.;
    }
    return rz;
}

float dualfbm(in vec2 p)
{
    //get two rotated fbm calls and displace the domain
    vec2 p2 = p*.7;
    vec2 basis = vec2(fbm(p2-time*1.6),fbm(p2+time*1.7));
    basis = (basis-.5)*.2;
    p += basis;

    //coloring
    return fbm(p*makem2(time*0.2));
}

float circ(vec2 p) 
{
    float r = length(p);
    //r = log(sqrt(r));
    r = 0.5*log(r);
    return abs(mod(r*4.,tau)-3.14)*3.+.2;

}

void main()
{
    //setup system
    vec2 p = gl_FragCoord.xy / iResolution.xy-0.5;
    p.x *= iResolution.x/iResolution.y;
    p*=4.;

    float rz = dualfbm(p);

    //rings
    p /= exp(mod(time*10.,3.14159));
    rz *= pow(abs((0.1-circ(p))),.9);

    //final color
    vec3 col = vec3(.2,0.1,0.4)/rz;
    col=pow(abs(col),vec3(.99));
    gl_FragColor = vec4(col,1.);
}

#version 410

uniform mat4 transform;

in vec4 vertex;
in vec2 texCoord;

out vec2 vertTexCoord;

void main() {

    vertTexCoord = vec2(texCoord.x, 1.0 - texCoord.y);
    gl_Position = transform * vertex;
}

// ----

#version 410


uniform sampler2D texture;

in vec2 vertTexCoord;

out vec4 fragColor;

void main() {
    fragColor = texture(texture, vertTexCoord);
}