› › ›

How to combine a z-buffer with a blur fragment shader

solubsolub
edited April 2018 in GLSL / Shaders

Hi all,

I'm trying to fake a Depth of Field effect in a 3D scene. More specifically, I would like to use a z-buffer (depth buffering) to adjust the level of blur of a an object based on its distance from the camera. While searching the forum I found the following vertex and fragment shaders provided by @Poersch (from this topic ).

vert.glsl

uniform mat4 transform;

attribute vec4 vertex;
attribute vec4 color;

varying vec4 vertColor;

void main() {
    gl_Position = transform * vertex;
    vertColor = color;
}

frag.glsl

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

uniform vec4 nearColor = vec4(1.0, 1.0, 1.0, 1.0);
uniform vec4 farColor = vec4(0.0, 0.0, 0.0, 1.0);
uniform float near = 0.0;
uniform float far = 100.0;

varying vec4 vertColor;

void main() {
    gl_FragColor = mix(nearColor, farColor, smoothstep(near, far, gl_FragCoord.z / gl_FragCoord.w));
}

You can see the shaders in action with this sketch example:

sketch.pde

PShader depthShader;
float angle = 0.0;


void setup(){

    // Set screen size and renderer
    size(600, 480, P3D);
    noStroke();

    // Load shader
    depthShader = loadShader("frag.glsl", "vert.glsl");
    //depthShader.set("near", 40.0); // Standard: 0.0
    //depthShader.set("far", 60.0); // Standard: 100.0
    //depthShader.set("nearColor", 1.0, 0.0, 0.0, 1.0); // Standard: white
    //depthShader.set("farColor", 0.0, 0.0, 1.0, 1.0); // Standard: black

}


void draw(){

    // Fill background and set camera
    background(#000000);
    camera(0.0, 0.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);

    // Bind shader
    shader(depthShader);

    // Calculate angle
    angle += 0.01;

    // Render "sky"-cube
    pushMatrix();
    rotate(angle, 0.0, 1.0, 0.0);
    box(100.0);
    popMatrix();

    // Render cubes
    pushMatrix();
    translate(-30.0, 20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    box(25.0);
    popMatrix();
    pushMatrix();
    translate(30.0, -20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    box(25.0);
    popMatrix();

    // Render spheres
    pushMatrix();
    translate(-30.0, -20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    sphere(20.0);
    popMatrix();
    pushMatrix();
    translate(30.0, 20.0, -50.0);
    rotate(angle, 1.0, 1.0, 1.0);
    sphere(20.0);
    popMatrix();

}

Here, the z-buffer is used to change the color of an object: the closer the lighter, the farther, the darker.

QUESTION

  • How can I use the same z-buffer to change the blur level of an object ?

Ideally I would like to use the Gaussian blur shader from the PostFX library:

#ifdef GL_ES
precision mediump float;
precision mediump int;
#endif


#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;

varying vec4 vertColor;
varying vec4 vertTexCoord;


uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>

const float pi = 3.14159265;

void main() {  
  float numBlurPixelsPerSide = float(blurSize / 2); 

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;
  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }

  gl_FragColor = (avgValue / coefficientSum);
}

Unfortunately I can't figure out how to make the two (z-buffer and blur fragment shader) work together. Any hint would be greatly appreciated !

Tagged:

Answers

  • noahbuddynoahbuddy
    edited April 2018

    Short answer:

    Use the results of the depth shader as sampler2D input for a separate blur shader.

    More detail:

    You could certainly implement everything in one pass but I like to follow the deferred shading style.

    If for no other reason, I find it easier to debug. :)

    Inside the blur shader, use a sampler2D in addition to "texture" and use that to change the blur properties. (sigma or blurSize for example)

    To draw an image and use the depth to modify it, draw the depth info to a PGraphics object.

    In draw, call:

    pbuff.shader(depthShader);
pbuff.beginDraw();
    ...
pbuff.endDraw();

blurShader.set("depthBuffer", pbuff);

blurShader.set("horizontalPass", 0);
image(sourceImage, 0, 0);
blurShader.set("horizontalPass", 1);
filter(blurShader);

    Then add to the shader:

    uniform sampler2D depthBuffer;
...
vec4 effx = texture2D(depthBuffer, vertTexCoord.st);
...
// do something with effx

    GPUs are best used for doing the same operation on lots of data.

    Keep in mind that changing blurSize and therefore the time it takes to process each pixel can affect the performance.

    Newer hardware/GL may handle it better.

  • solubsolub
    edited April 2018

    @noahbuddy Thank you again for your guidance and useful tips. I must admit that despite your detailed explanations I'm still having difficulties understanding what goes into what shader and how to compute the depth. For instance when you say > Then add to the shader

    What shader are you referring to ? The fragment shader ?

    Here is what I came up with based on your indications and my (very) poor understanding (apologies, again):

    main pde file

    PShader depthShader;
float angle = 0.0;

void setup(){

    size(600, 480, P3D);
    noStroke();

    pbuff = createGraphics(width, height, P3D);
    depthShader = loadShader("frag.glsl", "vert.glsl");
    blurShader = loadShader("blurFrag.glsl");

    blurShader.set("sigma", 10.5);
    blurShader.set("blurSize", 35);

}

void draw(){

    background(#000000);
    camera(0.0, 0.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);

    pbuff.pushMatrix();
    pbuff.rotate(angle, 0.0, 1.0, 0.0);
    pbuff.box(100.0);
    pbuff.popMatrix();

    pbuff.pushMatrix();
    pbuff.translate(-30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();

    pbuff.pushMatrix();
    pbuff.translate(-30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();

    pbuff.endDraw();

    blurShader.set("depthBuffer", pbuff);
    blurShader.set("horizontalPass", 0);
    image(sourceImage, 0, 0);
    blurShader.set("horizontalPass", 1);
    filter(blurShader);

}

    Inside the blur shader, use a sampler2D in addition to "texture" and use that to change the blur properties.

    blurFrag.glsls (just adding uniform sampler2D depthBuffer; and vec4 effx = texture2D(depthBuffer, vertTexCoord.st); on line 17 and 18.

        #ifdef GL_ES
    precision mediump float;
    precision mediump int;
    #endif


    #define PROCESSING_TEXTURE_SHADER

    uniform sampler2D texture;

    // The inverse of the texture dimensions along X and Y
    uniform vec2 texOffset;

    varying vec4 vertColor;
    varying vec4 vertTexCoord;

    uniform sampler2D depthBuffer;
    vec4 effx = texture2D(depthBuffer, vertTexCoord.st);

    uniform int blurSize;       
    uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
    uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                                // A good value for 9x9 is around 3 to 5
                                // A good value for 7x7 is around 2.5 to 4
                                // A good value for 5x5 is around 2 to 3.5
                                // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>

    const float pi = 3.14159265;

    void main() {  
      float numBlurPixelsPerSide = float(blurSize / 2); 

      vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

      // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
      vec3 incrementalGaussian;
      incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
      incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
      incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

      vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
      float coefficientSum = 0.0;

      // Take the central sample first...
      avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
      coefficientSum += incrementalGaussian.x;
      incrementalGaussian.xy *= incrementalGaussian.yz;

      // Go through the remaining 8 vertical samples (4 on each side of the center)
      for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
        avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                              blurMultiplyVec) * incrementalGaussian.x;         
        avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                              blurMultiplyVec) * incrementalGaussian.x;         
        coefficientSum += 2.0 * incrementalGaussian.x;
        incrementalGaussian.xy *= incrementalGaussian.yz;
      }

      gl_FragColor = (avgValue / coefficientSum);
    }

    But also you're suggesting to add the sames lines in the fragment shader as well right ?

    Then add to the shader

    frag.glsl

    #ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

uniform sampler2D depthBuffer;
vec4 effx = texture2D(depthBuffer, vertTexCoord.st);


uniform vec4 nearColor = vec4(1.0, 1.0, 1.0, 1);
uniform vec4 farColor = vec4(0.0, 0.0, 0.0, 1.0);
uniform float near = 0.0;
uniform float far = 100.0;


varying vec4 vertColor;

void main() {
    gl_FragColor = mix(nearColor, farColor, smoothstep(near, far, gl_FragCoord.z / gl_FragCoord.w));
}

    Also, how can I change the main() function of this this fragment shader so as it computes a level of blur instead of a change in color ? So sorry to bother you with all there questions. I wish I had a better and faster understanding of the logic behind all this.

  • noahbuddynoahbuddy

    Nearly there.

    Then add to the shader

    Sorry, I meant the blur shader.

    The line beginning with "vec4 effx..." will need to be inside main() so we can read depthBuffer at each pixel. You may have to play around with how to use "effx" to get the behavior you want.

    In this example, I have it changing the blurSize:

    Procedure:

    Draw depth information to buffer.
Pass depth info to blur shader variable.
Draw image (scene, etc.) while using blur shader.
Filter existing render with blur shader in other direction. (depth info is retained)

    I am using your original frag.glsl and vert.glsl to create the depth info.

    pde:

    PShader depthShader, blurShader;
PGraphics pbuff, sourceImage;
float angle = 0.0;

void setup(){
  // Set screen size and renderer
  size(600, 480, P3D);
  noStroke();

  // Load shader
  depthShader = loadShader("frag.glsl", "vert.glsl");
  blurShader = loadShader("blur.glsl");

  pbuff = createGraphics(width, height, P3D);
  sourceImage = createGraphics(width, height, P3D);

  // Load an image or create the scene to blur
  sourceImage.beginDraw();
  sourceImage.background(0);
  sourceImage.fill(255);
  sourceImage.ellipse(width/2, height/2, 50, 50);
  sourceImage.rect(width*2/3, height*2/3, 100, 100);
  sourceImage.endDraw();

  blurShader.set("sigma", 3.0);
  blurShader.set("blurSize", 9);

  shader(blurShader);
}

void draw(){
  pbuff.shader(depthShader);
  pbuff.beginDraw();
    pbuff.background(#000000);
    pbuff.camera(0.0, 0.0, 50.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);
    pbuff.noStroke();
    // Calculate angle
    angle += 0.01;

    // Render "sky"-cube
    pbuff.pushMatrix();
    pbuff.rotate(angle, 0.0, 1.0, 0.0);
    pbuff.box(100.0);
    pbuff.popMatrix();

    // Render cubes
    pbuff.pushMatrix();
    pbuff.translate(-30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.box(25.0);
    pbuff.popMatrix();

    // Render spheres
    pbuff.pushMatrix();
    pbuff.translate(-30.0, -20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();
    pbuff.pushMatrix();
    pbuff.translate(30.0, 20.0, -50.0);
    pbuff.rotate(angle, 1.0, 1.0, 1.0);
    pbuff.sphere(20.0);
    pbuff.popMatrix();
  pbuff.endDraw();

  blurShader.set("depthBuffer", pbuff);

  // Set this too if you use other shaders (on main display) earlier in draw
  shader(blurShader);

  // First draw the image _and_ do first blur pass on it
  blurShader.set("horizontalPass", 0);
  image(sourceImage, 0, 0);

  // Then run the second blur pass on existing render
  blurShader.set("horizontalPass", 1);
  filter(blurShader);
}

    blur.glsl:

    #ifdef GL_ES
precision mediump float;
precision mediump int;
#endif

#define PROCESSING_TEXTURE_SHADER

uniform sampler2D texture;
uniform sampler2D depthBuffer;

// The inverse of the texture dimensions along X and Y
uniform vec2 texOffset;
varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform int blurSize;       
uniform int horizontalPass; // 0 or 1 to indicate vertical or horizontal pass
uniform float sigma;        // The sigma value for the gaussian function: higher value means more blur
                            // A good value for 9x9 is around 3 to 5
                            // A good value for 7x7 is around 2.5 to 4
                            // A good value for 5x5 is around 2 to 3.5
                            // ... play around with this based on what you need <span class="Emoticon Emoticon1"><span>:)</span></span>
const float pi = 3.14159265;

void main() {  
  vec4 effx = texture2D(depthBuffer, vertTexCoord.st);
  //float numBlurPixelsPerSide = float(blurSize / 2); 
  float numBlurPixelsPerSide = float((blurSize - effx.x*blurSize) / 2); // <--    Using depth info here, farther away, more blur

  vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

  // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
  vec3 incrementalGaussian;

  incrementalGaussian.x = 1.0 / (sqrt(2.0 * pi) * sigma);
  incrementalGaussian.y = exp(-0.5 / (sigma * sigma));

  incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

  vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
  float coefficientSum = 0.0;

  // Take the central sample first...
  avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
  coefficientSum += incrementalGaussian.x;
  incrementalGaussian.xy *= incrementalGaussian.yz;

  // Go through the remaining 8 vertical samples (4 on each side of the center)
  for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
    avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * 
                          blurMultiplyVec) * incrementalGaussian.x;         
    coefficientSum += 2.0 * incrementalGaussian.x;
    incrementalGaussian.xy *= incrementalGaussian.yz;
  }

  gl_FragColor = (avgValue / coefficientSum);
}
  • nabrnabr
    edited April 2018

    yeah, nice!

    i dont think Poersch was right with that solution, he was right in an artistic way. Unfortunately you can find some of these examples around the web. Acutally the shader shows you a gray value going in the OpenGLs negative z direction, could be a startig point, as your shader suggest.

    So if you archive some nice results, we fine.

    otherwise, i collected few links here: https://forum.processing.org/two/discussion/comment/122724/#Comment_122724

    I would recomend to to the math for depth from the camera to object distance on CPU first, you know, println(some nummers), and get familiar with the Processings operations, like how setup and simpel camera and also how OpenGL do that fancy 3D stuff - and then convert it to a GPU version. Could be an interessting task!

    Best

  • solubsolub
    edited April 2018

    @noahbuddy sorry for the late reply, I had some work to do this weekend + I just spend most of the day trying to make that DoF effect work... without success. For some reason the script gives me some strange results: 2D plane instead of 3D, not fitting the canvas size, only half of the primitives are drawn...etc. There's probably something I'm doing wrong that I have yet to figure out. Anyway I would like to thank you again for your tips, it really helped me learn more about how buffers and shaders work.

    @nabr Thank you for the link, I tried to implement your sketch example in my script but there's still the depth from the camera to object distance that I don't know how to compute. Do you have any link or documentation that could help me figure this out ? I would like to draw thousands of points in a 3d scene so I guess it would be best to compute the level of blur for every points on the GPU, right ?

  • nabrnabr
    edited April 2018

    @solub

    sorry, no idea how to do that : )

    meanwhile i found a better depthtexture shader https://forum.processing.org/two/discussion/12775/simple-shadow-mapping

    keep on exploring.

    // blur by discard color value
//
// blur.glsl
/*
varying vec4 vertColor;
varying vec4 vertTexCoord;

uniform sampler2D texture;
uniform vec2 texOffset; 
uniform int horizontalPass;
uniform float time;

void main() {  

 int blurSize = 4;
 float sigma = 2.0;

 float numBlurPixelsPerSide = float(blurSize / 2); 
 vec2 blurMultiplyVec = 0 < horizontalPass ? vec2(1.0, 0.0) : vec2(0.0, 1.0);

 // Incremental Gaussian Coefficent Calculation (See GPU Gems 3 pp. 877 - 889)
 vec3 incrementalGaussian;
 incrementalGaussian.x = 1.0 / (sqrt(2.0 * 3.14159265) * sigma);
 incrementalGaussian.y = exp(-0.5 / (sigma * sigma));
 incrementalGaussian.z = incrementalGaussian.y * incrementalGaussian.y;

 vec4 avgValue = vec4(0.0, 0.0, 0.0, 0.0);
 float coefficientSum = 0.0;

 // Take the central sample first...
 avgValue += texture2D(texture, vertTexCoord.st) * incrementalGaussian.x;
 coefficientSum += incrementalGaussian.x;
 incrementalGaussian.xy *= incrementalGaussian.yz;

 // Go through the remaining 8 vertical samples (4 on each side of the center)
 for (float i = 1.0; i <= numBlurPixelsPerSide; i++) { 
 avgValue += texture2D(texture, vertTexCoord.st - i * texOffset * blurMultiplyVec) * incrementalGaussian.x;         
 avgValue += texture2D(texture, vertTexCoord.st + i * texOffset * blurMultiplyVec) * incrementalGaussian.x;         
 coefficientSum += 2.0 * incrementalGaussian.x;
 incrementalGaussian.xy *= incrementalGaussian.yz;
 }

 if ( texture2D(texture, vertTexCoord.st).rgb == vec3(step(sin(time*3.0)*0.5+0.5,0.5)) )
 discard; 
 else
 gl_FragColor.rgb = avgValue.rgb / coefficientSum;
 gl_FragColor.a = 1.;
} 
*/
PImage img;
PShader blurShader;
void setup() {
  size(640, 360, P3D);
  blurShader=loadShader("blur.glsl");
  rectMode(CENTER);
  ellipseMode(RADIUS);
  noStroke();
  noSmooth();
  img = loadImage("https://"+"processing.org/examples/moonwalk.jpg");
}
void draw() {
  //background(157, 0, 257, 255);
  background(img);

  float t = frameCount*.01;

  // white box  
  pushMatrix();
  translate(width*.15f, height*.5f-120.*cos(t), -4);
  fill(255);
  box(120, 120, 1);
  popMatrix();
  pushMatrix();
  translate(width*.15f, height*.5f-120.*cos(t), 4);
  fill(0);
  ellipse(5, 0, 30, 30);
  popMatrix();

  // black box
  pushMatrix();
  translate(width*.5f, height*.5f-120.*sin(t), -4);
  fill(0);
  box(120, 120, 1);
  popMatrix();
  pushMatrix();
  translate(width*.5f, height*.5f-120.*sin(t), 4);
  fill(255);
  ellipse(0, 0, 30, 30);
  popMatrix();

  // gray box
  pushMatrix();
  translate(width*.85f, height*.5f, 0);
  fill(255/2.);
  box(120, 120, 1);
  popMatrix();

  // blur
  blurShader.set("time", t);
  blurShader.set("horizontalPass", 1);
  filter(blurShader);
  resetShader();
  blurShader.set("horizontalPass", 0);
  filter(blurShader);

  // fps
  if (frameCount%30==0)println(frameRate);
}
  • solubsolub

    Thank you, I will !

Sign In or Register to comment.