Problem triggering audio files with kinect!

doctorvor..

Problem triggering audio files with kinect!

in Integration and Hardware • 2 years ago

Hello!
Okay... having major struggles with this

I am triggering sounds with motion on kinect. The first problem was it using AudioSample which triggered the sound over and over again and didnt stop.

I changed to AudioSnippet which triggered the sound just once, but then would not trigger again.

When using the rewind, play functions does not help (repeats over and over again)...

I then slowed the frameRate down to 0.5 with the rewind and play functions, which gives me the result I want, but the animation slows right down and couldnt find a way to seperate frame rates for sounds and animation

I am new to Processing so im in struggle street, any help will be greatly appreciated..

My Code:

import org.openkinect.*;

import org.openkinect.processing.*;

import hypermedia.video.*;

// code ref: http://code.compartmental.net/tools/minim/quickstart/

import ddf.minim.*;

import ddf.minim.signals.*;

import javax.sound.sampled.*;

Kinect kinect;

OpenCV opencv;

// Showing how we can farm all the kinect stuff out to a separate class

KinectTracker tracker;

// code ref: http://www.openprocessing.org/visuals/?visualID=15179

// code ref: http://processing.org/discourse/yabb2/YaBB.pl?num=1191974198

int numFrames = 7; // Number of frames for full animation

int frame = 0;

PImage[] images = new PImage[numFrames];

int numFramesThree = 103; // Number of frames for full animation

int frameThree = 0;

PImage[] imagesThree = new PImage[numFramesThree];

int numFramesHat = 11; // Number of frames for full animation

int frameHat = 0;

PImage[] imagesHat = new PImage[numFramesHat];

int numFramesBhat = 11; // Number of frames for full animation

int frameBhat = 0;

PImage[] imagesBhat = new PImage[numFramesHat];

int numFramesTap = 41; // Number of frames for full animation

int frameTap = 0;

PImage[] imagesTap = new PImage[numFramesTap];

int numFramesKick = 23; // Number of frames for full animation

int frameKick = 0;

PImage[] imagesKick = new PImage[numFramesKick];

// code ref: Minim example - trigger

Minim minim; // Load in sounds/music

AudioSample wave;

AudioSnippet woo;

AudioSnippet scratch;

AudioSnippet mjsong;

AudioSnippet tah;

AudioSnippet eheh;

AudioSnippet yow;

SineWave sine;

PImage thirteen;

void setup() {

size(640, 480, P2D);

frameRate(10);

kinect = new Kinect(this);

kinect.start();

kinect.enableDepth(true);

tracker = new KinectTracker();

opencv = new OpenCV(this);

opencv.allocate(width, height);

minim = new Minim(this);

sine = new SineWave (220,

0.5,

44100);

float[] samples = new float[1024*8];

woo = minim.loadSnippet("woo01.mp3"); // Load in sounds/music

scratch = minim.loadSnippet("scratch.mp3");

mjsong = minim.loadSnippet("billyb.mp3");

tah = minim.loadSnippet("tah.mp3");

eheh = minim.loadSnippet("eheh.mp3");

yow = minim.loadSnippet("yow.mp3");

sine.generate(samples);

AudioFormat format = new AudioFormat (44100,

16,

true,

true);

wave = minim.createSample (samples,

format,

1024);

thirteen = loadImage("13.jpg");

for(int i=0; i<numFrames; i++) { // Code for animation

String imageName = "da_" + nf(i, 2) + ".jpg";

images[i] = loadImage(imageName);

}

for(int i=0; i<numFramesThree; i++) { // Code for animation

String imageName = "mw_" + nf(i, 3) + ".jpg";

imagesThree[i] = loadImage(imageName);

}

for(int i=0; i<numFramesHat; i++) { // Code for animation

String imageName = "hat_" + nf(i, 2) + ".jpg";

imagesHat[i] = loadImage(imageName);

}

for(int i=0; i<numFramesBhat; i++) { // Code for animation

String imageName = "bhat_" + nf(i, 2) + ".jpg";

imagesBhat[i] = loadImage(imageName);

}

for(int i=0; i<numFramesTap; i++) { // Code for animation

String imageName = "tap_" + nf(i, 2) + ".jpg";

imagesTap[i] = loadImage(imageName);

}

for(int i=0; i<numFramesKick; i++) { // Code for animation

String imageName = "kick_" + nf(i, 2) + ".jpg";

imagesKick[i] = loadImage(imageName);

}

void draw() {

// code ref: http://www.openprocessing.org/visuals/?visualID=15179

// code ref: http://processing.org/discourse/yabb2/YaBB.pl?num=1191974198

{frame = (frame+1) % numFrames;}

{frameThree = (frameThree+1) % numFramesThree;}

{frameHat = (frameHat+1) % numFramesHat;}

{frameBhat = (frameBhat+1) % numFramesBhat;}

{frameTap = (frameTap+1) % numFramesTap;}

{frameKick = (frameKick+1) % numFramesKick;}

opencv.copy(kinect.getDepthImage());

// image(opencv.image(), 0, 0);

// code ref: Kinect example - Blob detection

// opencv.threshold(180);

// Run the tracking analysis

tracker.track();

// Show the image

// tracker.display();

// code ref: Kinect example - Blob detection

Blob[] blobs = opencv.blobs(10, width*height/2, 100, true);

// fill(255, 200, 0);

// Let's draw the raw location

PVector v1 = tracker.getPos();

// Let's draw the "lerped" location

PVector v2 = tracker.getLerpedPos();

for( int i = 0; i < blobs.length; i++) {

// beginShape();

{image(imagesThree[frameThree], 0, 0); }

// code ref: Lynneal Santos

//left

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{image(imagesKick[frameKick], 0, 0);}

// left - trigger 'woo'

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{ mjsong.play();

}

//right

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{image(images[frame], 0, 0);}

//right - trigger 'yow

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{

yow.play();

yow.rewind();

}

// top

if ((v1.x > width/3) && (v1.x < (2*width)/3) &&

(v1.y > 0) && (v1.y < height/3))

{image(imagesHat[frameHat], 0, 0);}

// topleft

if ((mouseX > 0) && (mouseX < width/3) &&

(mouseY > 0) && (mouseY < height/3))

{image(imagesBhat[frameBhat], 0, 0);}

// bottom

if ((v1.x > width/3) && (v1.x < (2*width)/3) &&

(v1.y > (2*height)/3) && (v1.y < height))

{image(imagesTap[frameTap], 0, 0);}

for(int j = 0; j < blobs[i].points.length; j++) {

vertex(blobs[i].points[j].x, blobs[i].points[j].y);

}

void stop()

{

// code ref: Minim example - trigger

wave.close();

mjsong.close();

yow.close();

minim.stop();

super.stop();

}

Replies(5)

dackdel

Re: Problem triggering audio files with kinect!

1 year ago

did it work?

doctorvor..

Re: Problem triggering audio files with kinect!

1 year ago

yeah I eventually figured it out, cant remember what I did, was a while ago now...

here is the latest version of the code (there is also a second tab of code, will post it under this lot under dotted line)

import org.openkinect.*;

import org.openkinect.processing.*;

import hypermedia.video.*;

// code ref: http://code.compartmental.net/tools/minim/quickstart/

import ddf.minim.*;

import ddf.minim.signals.*;

import javax.sound.sampled.*;

Kinect kinect;

OpenCV opencv;

// Showing how we can farm all the kinect stuff out to a separate class

KinectTracker tracker;

// code ref: http://www.openprocessing.org/visuals/?visualID=15179

// code ref: http://processing.org/discourse/yabb2/YaBB.pl?num=1191974198

int numFrames = 7; // Number of frames for full animation

int frame = 0;

PImage[] images = new PImage[numFrames];

int numFramesThree = 103; // Number of frames for full animation

int frameThree = 0;

PImage[] imagesThree = new PImage[numFramesThree];

int numFramesHat = 11; // Number of frames for full animation

int frameHat = 0;

PImage[] imagesHat = new PImage[numFramesHat];

int numFramesBhat = 11; // Number of frames for full animation

int frameBhat = 0;

PImage[] imagesBhat = new PImage[numFramesHat];

int numFramesTap = 41; // Number of frames for full animation

int frameTap = 0;

PImage[] imagesTap = new PImage[numFramesTap];

int numFramesKick = 23; // Number of frames for full animation

int frameKick = 0;

PImage[] imagesKick = new PImage[numFramesKick];

int numFramesBackspin = 21; // Number of frames for full animation

int frameBackspin = 0;

PImage[] imagesBackspin = new PImage[numFramesBackspin];

int numFramesRobot = 41; // Number of frames for full animation

int frameRobot = 0;

PImage[] imagesRobot = new PImage[numFramesRobot];

int numFramesHump = 21; // Number of frames for full animation

int frameHump = 0;

PImage[] imagesHump = new PImage[numFramesHump];

// code ref: Minim example - trigger

Minim minim; // Load in sounds/music

AudioSample wave;

AudioSnippet woo;

AudioSnippet scratch;

AudioSnippet mjsong;

AudioSnippet tah;

AudioSnippet eheh;

AudioSnippet yow;

AudioSnippet cheer;

SineWave sine;

PImage thirteen;

void setup() {

size(640, 480, P2D);

frameRate(10);

kinect = new Kinect(this);

kinect.start();

kinect.enableDepth(true);

tracker = new KinectTracker();

opencv = new OpenCV(this);

opencv.allocate(width, height);

minim = new Minim(this);

sine = new SineWave (220,

0.5,

44100);

float[] samples = new float[1024*8];

woo = minim.loadSnippet("woo01.mp3"); // Load in sounds/music

scratch = minim.loadSnippet("scratch.mp3");

mjsong = minim.loadSnippet("billyb.mp3");

tah = minim.loadSnippet("tah.mp3");

eheh = minim.loadSnippet("eheh.mp3");

yow = minim.loadSnippet("yow.mp3");

cheer = minim.loadSnippet("cheer.mp3");

sine.generate(samples);

AudioFormat format = new AudioFormat (44100,

16,

true,

true);

wave = minim.createSample (samples,

format,

1024);

thirteen = loadImage("13.jpg");

for(int i=0; i<numFrames; i++) { // Code for animation

String imageName = "da_" + nf(i, 2) + ".jpg";

images[i] = loadImage(imageName);

}

for(int i=0; i<numFramesThree; i++) { // Code for animation

String imageName = "mw_" + nf(i, 3) + ".jpg";

imagesThree[i] = loadImage(imageName);

}

for(int i=0; i<numFramesHat; i++) { // Code for animation

String imageName = "hat_" + nf(i, 2) + ".jpg";

imagesHat[i] = loadImage(imageName);

}

for(int i=0; i<numFramesBhat; i++) { // Code for animation

String imageName = "bhat_" + nf(i, 2) + ".jpg";

imagesBhat[i] = loadImage(imageName);

}

for(int i=0; i<numFramesTap; i++) { // Code for animation

String imageName = "tap_" + nf(i, 2) + ".jpg";

imagesTap[i] = loadImage(imageName);

}

for(int i=0; i<numFramesKick; i++) { // Code for animation

String imageName = "kick_" + nf(i, 2) + ".jpg";

imagesKick[i] = loadImage(imageName);

}

for(int i=0; i<numFramesBackspin; i++) { // Code for animation

String imageName = "spi_" + nf(i, 2) + ".jpg";

imagesBackspin[i] = loadImage(imageName);

}

for(int i=0; i<numFramesRobot; i++) { // Code for animation

String imageName = "rob_" + nf(i, 2) + ".jpg";

imagesRobot[i] = loadImage(imageName);

}

for(int i=0; i<numFramesHump; i++) { // Code for animation

String imageName = "hump_" + nf(i, 2) + ".jpg";

imagesHump[i] = loadImage(imageName);

}

void draw() {

// code ref: http://www.openprocessing.org/visuals/?visualID=15179

// code ref: http://processing.org/discourse/yabb2/YaBB.pl?num=1191974198

{frame = (frame+1) % numFrames;}

{frameThree = (frameThree+1) % numFramesThree;}

{frameHat = (frameHat+1) % numFramesHat;}

{frameBhat = (frameBhat+1) % numFramesBhat;}

{frameTap = (frameTap+1) % numFramesTap;}

{frameKick = (frameKick+1) % numFramesKick;}

{frameBackspin = (frameBackspin+1) % numFramesBackspin;}

{frameRobot = (frameRobot+1) % numFramesRobot;}

{frameHump = (frameHump+1) % numFramesHump;}

opencv.copy(kinect.getDepthImage());

// image(opencv.image(), 0, 0);

// code ref: Kinect example - Blob detection

// opencv.threshold(180);

// Run the tracking analysis

tracker.track();

// Show the image

// tracker.display();

// code ref: Kinect example - Blob detection

Blob[] blobs = opencv.blobs(10, width*height/2, 100, true);

// fill(255, 200, 0);

// Let's draw the raw location

PVector v1 = tracker.getPos();

// Let's draw the "lerped" location

PVector v2 = tracker.getLerpedPos();

for( int i = 0; i < blobs.length; i++) {

// beginShape();

{image(imagesThree[frameThree], 0, 0); }

// code ref: Lynneal Santos

//left

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{image(imagesKick[frameKick], 0, 0);}

// left - trigger 'song'

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{ mjsong.play();

}

//right

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

{image(images[frame], 0, 0);}

//right - trigger 'yow

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > height/3) && (v1.y < (2*height)/3))

// code ref: http://www.kittymamuse.com/images/portfolio/interactive/processing/project2_final/applet/project2_final.pde

if (!yow.isPlaying()) {

yow.play();

yow.rewind();

}

// top

if ((v1.x > width/3) && (v1.x < (2*width)/3) &&

(v1.y > 0) && (v1.y < height/3))

{image(imagesHat[frameHat], 0, 0);}

// top - trigger 'eheh'

if ((v1.x > width/3) && (v1.x < (2*width)/3) &&

(v1.y > 0) && (v1.y < height/3))

if (!eheh.isPlaying()) {

eheh.play();

eheh.rewind();

}

// topright

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > 0) && (v1.y < height/3))

{image(imagesBackspin[frameBackspin], 0, 0);}

// topright - trigger 'scratch'

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > 0) && (v1.y < height/3))

if (!scratch.isPlaying()) {

scratch.play();

scratch.rewind();

}

// bottomright

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > (2*height)/3) && (v1.y < height))

{image(imagesRobot[frameRobot], 0, 0);}

// bottomright

if ((v1.x > (2*width)/3) && (v1.x < width) &&

(v1.y > (2*height)/3) && (v1.y < height))

if (!woo.isPlaying()) {

woo.play();

woo.rewind();

}

// topleft

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > 0) && (v1.y < height/3))

{image(imagesBhat[frameBhat], 0, 0);}

// bottom

if ((v1.x > width/3) && (v1.x < (2*width)/3) &&

(v1.y > (2*height)/3) && (v1.y < height))

{image(imagesTap[frameTap], 0, 0);}

// bottom - trigger 'tah'

if ((v1.x > width/3) && (v1.x < (2*width)/3) &&

(v1.y > (2*height)/3) && (v1.y < height))

if (!tah.isPlaying()) {

tah.play();

tah.rewind();

}

// bottomleft

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > (2*height)/3) && (v1.y < height))

{image(imagesHump[frameHump], 0, 0);}

// bottomleft - trigger 'cheer'

if ((v1.x > 0) && (v1.x < width/3) &&

(v1.y > (2*height)/3) && (v1.y < height))

if (!cheer.isPlaying()) {

cheer.play();

cheer.rewind();

}

for(int j = 0; j < blobs[i].points.length; j++) {

vertex(blobs[i].points[j].x, blobs[i].points[j].y);

}

// void keyPressed()

//{

// code ref: Minim example - trigger

// {

// if ( key == 'o' ) mjsong.trigger(); // Starts song

// }

//}

void stop()

{

// code ref: Minim example - trigger

wave.close();

mjsong.close();

yow.close();

// snip.close();

minim.stop();

super.stop();

}

..........................................................................

// code ref: Kinect example - Average Point Tracking

class KinectTracker {

// Size of kinect image

int kw = 640;

int kh = 480;

int threshold = 650;

// Raw location

PVector loc;

// Interpolated location

PVector lerpedLoc;

// Depth data

int[] depth;

PImage display;

KinectTracker() {

// kinect.start();

kinect.enableDepth(true);

// kinect.enableDepth(true);

// We could skip processing the grayscale image for efficiency

// but this example is just demonstrating everything

kinect.processDepthImage(true);

display = createImage(kw,kh,PConstants.RGB);

loc = new PVector(0,0);

lerpedLoc = new PVector(0,0);

}

void track() {

// Get the raw depth as array of integers

depth = kinect.getRawDepth();

// Being overly cautious here

if (depth == null) return;

float sumX = 0;

float sumY = 0;

float count = 0;

for(int x = 0; x < kw; x++) {

for(int y = 0; y < kh; y++) {

// Mirroring the image

int offset = kw-x-1+y*kw;

// Grabbing the raw depth

int rawDepth = depth[offset];

// Testing against threshold

if (rawDepth < threshold) {

sumX += x;

sumY += y;

count++;

}

// As long as we found something

if (count != 0) {

loc = new PVector(sumX/count,sumY/count);

}

// Interpolating the location, doing it arbitrarily for now

lerpedLoc.x = PApplet.lerp(lerpedLoc.x, loc.x, 0.3f);

lerpedLoc.y = PApplet.lerp(lerpedLoc.y, loc.y, 0.3f);

}

PVector getLerpedPos() {

return lerpedLoc;

}

PVector getPos() {

return loc;

}

void display() {

PImage img = kinect.getDepthImage();

// Being overly cautious here

if (depth == null || img == null) return;

// Going to rewrite the depth image to show which pixels are in threshold

// A lot of this is redundant, but this is just for demonstration purposes

display.loadPixels();

for(int x = 0; x < kw; x++) {

for(int y = 0; y < kh; y++) {

// mirroring image

int offset = kw-x-1+y*kw;

// Raw depth

int rawDepth = depth[offset];

int pix = x+y*display.width;

if (rawDepth < threshold) {

// A red color instead

display.pixels[pix] = color(150,50,50);

}

// if (threshold > rawDepth ) {

// A red color instead

// for(int i=0; i<numFramesThree; i++) { // Code for full animation

//String imageName = "mw_" + nf(i, 2) + ".jpg";

//imagesThree[i] = loadImage(imageName);

// }

// if (rawDepth < threshold) {

// A red color instead

// woof.trigger(); }

// if (rawDepth > threshold) {

// A red color instead

// scratch.trigger(); }

// if (threshold > rawDepth ) {

// A red color instead

// meow.trigger(); }

else {

display.pixels[pix] = img.pixels[offset];

}

display.updatePixels();

// Draw the image

image(display,0,0);

}

void quit() {

kinect.quit();

}

int getThreshold() {

return threshold;

}

void setThreshold(int t) {

threshold = t;

}

dackdel

Re: Problem triggering audio files with kinect!

1 year ago

looks great. going to try it out in a bit. also do you think its possible to replace the audio with video? maybe use something like the GSVideo library http://gsvideo.sourceforge.net/

dackdel

Re: Problem triggering audio files with kinect!

1 year ago

also i had tried to do something with the kinect and sound too. i dont know what yours does but here is a link to what i tried. https://forum.processing.org/topic/video-trigger-using-kinect-simpleopenni-library

doctorvor..

Re: Problem triggering audio files with kinect!

1 year ago

yeah I know you can trigger video with it, Im not sure how to do it... I was triggering audio files and frame animations with positions in Up Right Down Left and the diagonals in between... I was taking frames from video and using that instead of triggering a video file, I cant remember why I decided to use that option, I think I originally had difficulty with video files so went with frames....

Good luck, have fun with it... feel free to ask more questions, will help with what I can (I dont have a kinect camera at the moment, but if I get my hands on one will check out that link)

Top Reply