I am developing a software were PImages have to be displayed depending on MIDI messages. At runtime, when entering into draw(), some pictures are not displayed and some pictures are displayed after an operation on their hue: depending of the MIDI messages the hue changes. All displayed pictures are blended together in ADD mode (I want black to become transparent) and displayed in a "canvas", a PGrahics object which is sent to a Syphon server.

I first tried to apply the hue change using the loadPixels() and pixels[] of the PImage but it was to much work for the CPU. I also tried to use LUTs as suggested in this post but it wasn't enough (the CPU exploses and each time I have pictures to display the framerate slow down around 20 fps). So I decided that the most powerful way to do the work was to go into glsl shaders... It is the first time I use it so I still miss a lot of experience !

After playing around with examples, tutorials and forum topics, I finally got a shader which does the hue operation very well (and it is very fast! I keep a good fps and it demands only 35% of CPU VS 70% before). The problem is that my blend mode is not working... the behaviour of my canvas is weird, with the same MIDI messages sometimes the pictures are blending, sometimes not, some pictures are not displayed...

I think that the blend shaders proposed here are not what I need as they need two pictures (textures) as parameters but what I want is to blend the canvas into itself every time I draw a picture in it between beginDraw() and endDraw(). I read on the internet about openGL and the concept of buffering for blend modes but I am still confused with all of this. I also think that asking Processing to do the blend mode as there are operations on the PImages in the shader is not a good way to go (I tried this and it doesn't work), but I don't really know about blending in openGL, still learning!

Processing 2.0b8

Here is the Processing code:

2. import java.awt.event.ActionEvent;
3. import java.util.Observable;
4. import java.util.Observer;
5. import controlP5.*;
6. import java.io.*;
7. import java.util.*;
8. import themidibus.*;
9. import codeanticode.syphon.*;
10. 
    
11. Model model;
12. ControlP5 cp5;
13. MidiBus bus;
14. SyphonServer server;
15. PGraphics canvas;
16. Group[] controlChannel;
17. 
    
18. XML config;
19. String busMIDI;
20. File[][] mediaFiles;
21. int lengthLibrary = 0;
22. PShader hueShader;
23. 
    
24. public static final int nbreChannels = 16;
25. 
    
26. void setup() {
27.   
28.   // SETUP FRAME AND SYPHON
29.   size(1280, 720, P2D);
30.   canvas = createGraphics(1280, 720, P2D);
31.   server = new SyphonServer(this, "Piano Syphon");
32.   
33.   hueShader = loadShader("hueChange2.glsl");
34.   hueShader.set("resolution", float(width), float(height));   
35.   hueShader.set("hue", 0.0);
36.   
37.   loadConfig();
38.   
39.   // SETUP THE MIDI BUS
40.   MidiBus.list();
41.   bus = new MidiBus(this, 0, 0, "bus");
42.   
43.   // SETUP MVC
44.   createGUI();
45.   
46.   noLoop();
47. }
48. 
    
49. void draw(){
50.   if (frameCount==1){
51.     try{
52.       mediaFiles = createMediasFiles();
53.     }
54.     catch(Exception e){
55.     }
56.     for (int i=0;i<mediaFiles.length;i++){
57.       for (int j=0;j<12;j++){
58.         println(mediaFiles[i][j].getAbsolutePath());
59.       }
60.     }
61.     model = new Model(this, mediaFiles);
62.     loop();
63.   }
64.   
65.   if (frame!=null){
66.     frame.setTitle(int(frameRate) + " fps");
67.   }
68.   
69.   int startTime = millis();
70.   background(0);
71.   // here I tried to use canvas.blendMode(ADD); and/or blendMode(ADD); , none of them work
72.   canvas.beginDraw();
73.   canvas.background(0);
74.   colorMode(HSB,1,1,1);
75.   
76.   /*
77. ColoredObject updates their colors depending of the MIDI messages, then the hue operation is applied depending of these colors
78. */
79.   for (ColoredObject co: model.observers){
80.     //this if() sentence is for GUI, checking if a MIDI channel is active or not
81.     if (co.isSwitchedON()&&(co.getMedia()!=-1)){
82.       
83.       // get a color to apply
84.       color[] mco = co.getMixedColors();
85.       for (int i=0;i<mco.length;i++){
86.         //checks if the color is black or not, if black don't display the picture
87.         if (saturation(mco[i]) !=0){
88.           PImage img = model.images[co.getMedia()-1][i];
89.           float h = hue(mco[i]);
90.           hueShader.set("iChannel0", img);
91.           hueShader.set("hue", h);
92.           canvas.shader(hueShader);
93.           canvas.rect(0, 0, 1280, 720);
94.           /*
95.             here I tried blend(canvas, 0, 0, 1280, 720, 0, 0, 1280, 720, ADD); but does not work properly and get               the CPU crazy again so the shader is useless
96.          */
97.         }
98.       }
99.     }
100.   }
101.   canvas.endDraw();
102.   image(canvas,0,0);
103.   server.sendImage(canvas);
104.   colorMode(RGB,255);
105.   println("draw: " + (millis() - startTime) + " milliseconds");
106. }
107. 
     
108. void loadConfig(){
109. //load some specific constants
110. }
111. 
     
112. File[][] createMediasFiles(){
113. //create an array of Files, which will be used to load the pictures when the sketch begins (just after setup() )
114. }
115. 
     
116. void createGUI(){
117.   cp5 = new ControlP5(this);
118.   cp5.setColorActive(color(255,107,107));
119.   cp5.setColorBackground(color(21,24,51,100));
120.   cp5.setColorForeground(color(111,160,206));
121.   initDashboard();
122.   initControlChannel();
123.   cp5.hide();
124. }
125. 
     
126. void initDashboard(){
127. // create a GUI element
128. }
129. 
     
130. void initControlChannel(){
131. // create a GUI element
132. }
133. 
     
134. void initLibrary(){
135. // create a GUI element
136.   cp5.show();
137. }
138. 
     
139. void noteOn(int channel, int pitch, int velocity) {
140.   try{
141.     // update ColorObject's colors
142.     model.observers[channel].update(pitch, velocity);
143.   }
144.     
145.   catch(Exception e){
146.     println("erreur noteOn");
147.   }
148.   println("noteOn:" + pitch +"vélocité note" + "," + "channel" + (channel+1) + ":" + velocity);
149.   
150. }
151. 
     
152. void noteOff(int channel, int pitch, int velocity) {
153.   try{
154.       // update ColorObject's colors
155.       model.observers[channel].update(pitch, 0);
156.   }
157.   catch(Exception e){
158.     println("erreur noteOn");
159.   }
160.   println("noteOff");
161. }
162. 
     
163. 
     
164. // ACTIONS : VUE > MODELE
165. 
     
166. void controlEvent(ControlEvent theEvent) {
167. //actions on GUI
168. }
169. 
     
170. void keyPressed() {
171.   if (key == ESC) {
172.     key = 0;
173.   }
174. }
175. 
     
176. // IMAGE LOADING LISTENER
177. 
     
178. class Model implements DataLoadCallback {
179. //basically the Model loads and store the PImages to be manipulated and displayed
180. }

and the shader code :

2. // HUE CHANGE
3. 
   
4. #ifdef GL_ES
5. precision highp float;
6. #endif
7. 
   
8. #define PROCESSING_COLOR_SHADER
9. 
   
10. uniform float time;
11. uniform vec2 resolution;
12. uniform vec2 mouse;
13. 
    
14. vec3 iResolution = vec3(resolution,0.0);
15. float iGlobalTime = time;
16. vec4 iMouse = vec4(mouse,0.0,0.0); // zw would normally be the click status
17. 
    
18. uniform sampler2D iChannel0;
19. uniform float hue;
20. 
    
21. // works well
22. vec3 HUEtoRGB(in float H){
23.   float R = abs(H * 6.0 - 3.0) - 1.0;
24.     float G = 2.0 - abs(H * 6.0 - 2.0);
25.     float B = 2.0 - abs(H * 6.0 - 4.0);
26.     return clamp(vec3(R,G,B),0.0,1.0);
27. }
28.   
29. //works well
30. vec3 HSVtoRGB(in vec3 HSV){
31.   vec3 RGB = HUEtoRGB(HSV.x);
32.     return ((RGB - 1.0) * HSV.y + 1.0) * HSV.z;
33. }
34. 
    
35. //works well
36. vec3 RGBtoHSV(vec3 rgb) {
37.   vec3 hsv;
38.   float rgb_min, rgb_max;
39.   float r = rgb.x;
40.   float g = rgb.y;
41.   float b = rgb.z;
42.   rgb_min = min(r, min(g,b));
43.   rgb_max = max(r, max(g, b));
44. 
    
45.   hsv.z = rgb_max;
46.   if (hsv.z == 0.0){
47.         hsv.x = hsv.y = 0.0;
48.         return hsv;
49.   }
50. 
    
51.   r /= hsv.z;
52.   g/= hsv.z;
53.   b /= hsv.z;
54.   rgb_min = min(r, min(g,b));
55.   rgb_max = max(r, max(g, b));
56. 
    
57.   hsv.y = rgb_max - rgb_min;
58.   if (hsv.y == 0.0){
59.     hsv.x = 0.0;
60.     return hsv;
61.   }
62. 
    
63.   r = (r - rgb_min) / (rgb_max - rgb_min);
64.   g = (g- rgb_min) / (rgb_max - rgb_min);
65.   b = (b- rgb_min) / (rgb_max - rgb_min);
66.   rgb_min = min(r, min(g,b));
67.   rgb_max = max(r, max(g, b));
68. 
    
69.   if (rgb_max == r){
70.     hsv.x = 0.0 + 60.0*(g-b);
71.     if (hsv.x < 0.0){
72.       hsv.x += 1.0;
73.     }
74.   }
75.   else if (rgb_max == g){
76.     hsv.x = 0.5 + 0.16*(b - r);
77.   }
78.   else {
79.     hsv.x = 0.6 + 0.16*(r- g);
80.   }
81.   
82.   return hsv;
83. }
84. 
    
85. void main(void){
86.       vec2 position = gl_FragCoord.xy / iResolution.xy;
87.       position.y = 1.0-position.y;
88.       vec4 color = texture2D(iChannel0, position);
89. 
    
90.       /*
91.       get the RGB info, then turn the RGB image info into a HSV image info then go back to RGB but impose       the hue. This part works very well !
92.       */
93.       vec3 rgbImage = vec3(color.r, color.g, color.b);
94.       vec3 hsvImage = RGBtoHSV(rgbImage);
95.       vec3 rgbApplied = HSVtoRGB(vec3(hue, hsvImage.y, hsvImage.z));
96.   
97.       /*
98.       the alpha is set to the V value of HSV, just a try with the blend() and blendMode() functions of Processing       to get the dark/black parts transparents
99.       */
100.       gl_FragColor = vec4(rgbApplied.x, rgbApplied.y, rgbApplied.z, hsvImage.z);
101. }

So basically I am seeking a way to do my ADD blend mode as fast as possible (as after creating the image of the canvas I want to send it through Syphon + my system has to be fast and reactive to MIDI messages at runtime)

Does anyone have some clues / suggestions / tracks for me ? It would be great.

Thanks all