I still learning the in's and outs of processing and at the same time i'm creating a tester project using brightness tracking and ESS sound. I have worked out the brightness tracking with this code:

import processing.video.*;
import krister.Ess.*;
import java.io.File;
import java.io.IOException;

Capture myCapture;


import krister.Ess.*;

AudioChannel myChannel, myChannel2;
PinkNoise myNoise;


public void stop() {
 Ess.stop();
 super.stop();
}


// Processing code for tracking the brightest pixel in a live video signal

int video_width = 320;
int video_height = 240;


//-------------------------------------------
void setup(){
size(320, 240);
 Ess.start(this);

// This line prints out all available cameras so you'll be able to
// spot if you's is there or not
println(Capture.list());
String s = "";
myCapture = new Capture(this, s, width, height, 30);
}
void captureEvent(Capture myCapture) {
myCapture.read();
 // create a new AudioChannel
 myChannel=new AudioChannel();
 myChannel2 = new AudioChannel();

 // set the channel size to 5 seconds
 myChannel.initChannel(myChannel.frames(5000));

 // generate 3 seconds of soft pink noise
 myNoise=new PinkNoise(.1);
 myNoise.generate(myChannel,0,myChannel.frames(9000));

 // play
 myChannel.play();
 myChannel.fadeTo(.5,500);
 
}

//-------------------------------------------
void draw(){
image(myCapture, 0, 0); // Draw the webcam video onto the screen.


// Declare some numbers to be computed later:
int brightest_x = 0; // the x-coordinate of the brightest video pixel
int brightest_y = 0; // the y-coordinate of the brightest video pixel
float brightest_val = 0; // the brightness of the brightest video pixel


// Now search for the brightest pixel:
for (int y=0; y<video_height; y++){ // For each row of pixels in the video image,
for (int x=0; x<video_width; x++){ // and for each pixel in the y'th row,
int index = y*video_width + x; // compute each pixel's index in the video,
int pix_val = myCapture.pixels[index];// fetch the color stored in that pixel,
float pix_bri = brightness(pix_val);


// and determine the brightness of that pixel.
if (pix_bri > brightest_val){ // If that value is brighter than any previous,
brightest_val = pix_bri; // then store the brightness of that pixel,
brightest_y = y; // as well as its (x,y) location.
brightest_x = x;
}else if (brightest_val < 200){
brightest_y = -10;
brightest_x = -10;
}


}
}




fill(255,0,0); // Set the fill color to red, and then
ellipse( brightest_x, brightest_y, 10,10); // draw a circle at the brightest pixel.
}


I would like to combine it with this code which is currently on a mouseclick event:


import krister.Ess .*;

AudioChannel myChannel1,myChannel2;
SineWave myWave;
PinkNoise myNoise;

void setup() {
 size(256,200);
 
 // start up Ess
 
 Ess.start(this);

 myChannel1=new AudioChannel();
 myChannel2=new AudioChannel();

 myChannel1.initChannel(myChannel1.frames(2000));
 myChannel2.initChannel(myChannel2.frames(2000));

 myWave=new SineWave(480,.5);
 myNoise=new PinkNoise(.5);

 myWave.generate(myChannel1);
 myNoise.generate(myChannel2);

 myChannel1.volume(0);
 myChannel2.volume(0);

 // pan only at zero crossings

 myChannel1.smoothPan=true;

 myChannel1.play(Ess.FOREVER);
 myChannel2.play(Ess.FOREVER);

 // small fade in

 myChannel1.fadeTo(.5,500);
 myChannel2.fadeTo(.5,500);

 framerate(30);

 noFill();
 smooth();
 ellipseMode(CENTER);
}

void draw() {
 background(0,0,255);

 strokeWeight(1);
 stroke(255);

 int tx=mouseX;
 int ty=mouseY;

 float newPan=1-(tx/float(width))*2;
 float newVolume=1-ty/float(height);

 if (!myChannel1.panning) myChannel1.panTo(newPan,500);
 if (!myChannel2.panning) myChannel2.panTo(-newPan,500);
 
 line(tx,0,tx,height);
 line(width-tx,0,width-tx,height);

 if (!myChannel1.fading) myChannel1.fadeTo(newVolume,500);
 if (!myChannel2.fading) myChannel2.fadeTo(1-newVolume,500);

 line(0,ty,width,ty);
 line(0,height-ty,width,height-ty);

 int d=((millis()/75) % 20)+4;

 strokeWeight(3);
 stroke(255,255-d*(255/20));

 ellipse(tx,ty,d,d);
 ellipse(width-tx,height-ty,d,d);
}

// clean up Ess before exiting

public void stop() {
 Ess.stop();
 super.stop();
}



Would anyone have any ideas on how to combine the two peices of code so that when a person moves their hand around the screen it changes the pitch and volume of the sounds. Any help would be really appeciated.

hello,
i am really into this program you made, but i was wondering if you explain something for me. how did you make the brightness tracking replace mouseinput?

thanks for the help,
gil