Continuing my adventures through the land of masking in OpenGL:

i have done this in the past. don't have access to the code from here though, will look it up tonight.



this is green background with a red cube drawn on it and then using the same red cube as a mask for the blue cube. the note along with the picture is kinda cryptic but says

"the final thing that needed doing was to use:
  gl.glClearStencil(0x0);
at the top of the code."

if that helps. i also seem to remember having trouble with Windows whereas it was fine in linux. something to do with how it was reporting stencil bits.

this would be processing 1.5 btw

found some code, not the above code, but it does use the stencil buffer

1. // acd 20091116
2. // Stencil Buffer
3. 
   
4. import javax.media.opengl.*;
5. import javax.media.opengl.glu.*;
6. import processing.opengl.*;
7. import java.nio.IntBuffer;
8. 
   
9. public static final int W = 512;
10. public static final int H = 512;
11. float angle, angle2;
12. 
    
13. void setup() {
14.   size(512, 512, OPENGL);
15.  
16.   fill(0);
17.   ellipseMode(CENTER);
18.   GL gl = ((PGraphicsOpenGL)g).gl;
19.   GLCapabilities capabilities = new GLCapabilities();
20.   int bits = capabilities.getStencilBits();
21.   println("StencilBits: " + bits);
22.   capabilities.setStencilBits(8);
23.   bits = capabilities.getStencilBits();
24. //  println("StencilBits: " + bits);
25.   frameRate(25);
26.  
27.   angle = random(TWO_PI);
28.   angle2 = random(TWO_PI);
29. }
30. 
    
31. void draw() { 
32.   background(255);
33.   noStroke();
    
34.   translate(width / 2, height / 2);
35. 
    
36.   GL gl = ((PGraphicsOpenGL)g).gl;
37.   GLU glu = ((PGraphicsOpenGL)g).glu;
38.   glu.gluLookAt(0, 0, 100, 0, 0, 0, 0, 1, 0);
39.   gl.glClearStencil(0x0);  /* IMPORTANT */
40.   gl.glClearColor(0.0, 1.0, 0.0, 0.0);
41.   gl.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_STENCIL_BUFFER_BIT);
42.   gl.glDisable(GL.GL_DEPTH_TEST);
43.   GLCapabilities capabilities = new GLCapabilities();
44.   int bits = capabilities.getStencilBits();
45.   capabilities.setStencilBits(8);
46.   bits = capabilities.getStencilBits();
47.   IntBuffer i = IntBuffer.allocate(100);
48.   gl.glGetIntegerv(GL.GL_STENCIL_BITS, i);
49.   // there are 8 stencil planes (in linux. windows says 0)
50.  
51.   // that said, i think this is using the standard colour buffer as the stencil anyway
52.   // draw red cube to stencil buffer
53.   gl.glEnable(GL.GL_STENCIL_TEST);
54.   gl.glStencilFunc(GL.GL_ALWAYS, 1, 0xffffffff);  /* IMPORTANT */
55.   gl.glStencilOp(GL.GL_KEEP, GL.GL_KEEP, GL.GL_REPLACE);
56.   fill(255, 0, 0);
57.   pushMatrix();
58.   rotateX(angle);
59.   rotateY(angle);
60.   box(300);
61.   angle += .05;
62.   popMatrix();
63.  
64.   // now draw blue cube to screen proper
65.   gl.glColorMask(true, true, true, true);      // re-enable colours
66.   gl.glStencilFunc(GL.GL_NOTEQUAL, 0, 0xFFFFFFFF);     // mask  /* IMPORTANT */
67.   gl.glStencilOp(GL.GL_KEEP, GL.GL_KEEP, GL.GL_KEEP);  // don't change buffer
68.   fill(0, 0, 255);
69.   pushMatrix();
70.   rotateX(angle2);
71.   rotateY(angle2);
72.   rotateZ(angle2);
73.   angle2 += .05;
74.   box(275);
75.   popMatrix();
76. }
77. 
    
78. void keyPressed() {
79.   saveFrame("Stencil_####.png");
80.   delay(1000);
81. }

Hey Kooogy,