Video Step Sequencer in Processing

trepid

Here is a new monome app I've been working on. It's a work in progress, but fully useable at this point for performance.

It is written in processing. The basic premise is that I play in a band and envisioned playing with video projected behind us. Our musical style is flexible and there are improvised elements so playing to predetermined video creates a disjoint between the live performance and the predetermined visuals. I wanted something that could be reactive and unique to a given live performance.

I originally wrote processing to do beat detection and fft to use those as inputs to video switching and cuts. This was used for one show, but seemed awkward since it just kept playing throughout the set. After this performance I rewrote it to be smoother and faster and to be monome controlled so that it could be faded in and out and so that someone could control the video in addition to the "robot" I wrote to do the cutting and jumps.

Anyhow, lemme know what y'alls think.

Video demo:

Source:

import ddf.minim.analysis.*;
import ddf.minim.*;
import processing.video.*;
import oscP5.*;
import netP5.*;
import java.util.Arrays;
import java.io.File;
import java.io.FilenameFilter;

// audio input setup
Minim minim;
AudioInput audioin;

// monome setup
String prefix = "/toad";
int monomePort = 16629;
int monomeWidth = 8;
int monomeHeight = 8;

int[][]led = new int[monomeWidth][monomeHeight]; //keep track of whether lights are on/off

OscP5 osc;
NetAddress listenPort;
NetAddress serialoscPort;

// clip setup
BeatDetect beat;
Movie movie;
int current_movie_id = 0;
int movie_id = 0;
int beatcount = 0;
float horizontal_scaling = 0.0;

int current_movie_list = 0;

String [][] movieListArray = loadClips();
Movie movies[][] = new Movie[movieListArray.length][];

// fft setup
FFT    fft;
PImage fade;

// fade out value
int alphaVal = 255;

// feature toggles
int fading = 0;
int monkey = 0;
int fftvis = 0;

// overlay feature
int overlay = 0;
Movie oldmovie;

String [][] loadClips() {
  // autoload clips from folders 0-7, anything with extensions that minim understands, mp4, mpg and avi
  int num_of_folders = 8;
  ArrayList <String[]> movieClipArray = new ArrayList<String[]>();
  for (int j=0; j<num_of_folders; j++) {
    File [] files=new File("/Users/jwoods/Documents/Processing/JeffVis/"+Integer.toString(j)+"/").listFiles(new FilenameFilter() { 
               public boolean accept(File dir, String filename)
                    { return (filename.endsWith(".mp4") || filename.endsWith(".mpg") || filename.endsWith(".avi")); }
      } );
    String filenames[]=new String[files.length];
    for(int i=0; i<files.length; i++) {      
      filenames[i]=files[i].getAbsolutePath();
    }
    movieClipArray.add(filenames);
  }
  // convert the arraylist back to a 2d array of strings
  String[][] movieClipArrayList = new String[movieClipArray.size()][];
  for (int i = 0; i < movieClipArray.size(); i++) {
        movieClipArrayList[i] = movieClipArray.get(i);
  }
  println(Arrays.deepToString(movieClipArrayList));
  return movieClipArrayList;
}  

void setup() {

  //size(640, 480, P3D);
    size(1280, 800, P3D);

  // Monome setup
  osc = new OscP5(this, 8000);
  listenPort = new NetAddress("127.0.0.1", 8000);
  serialoscPort = new NetAddress("127.0.0.1", monomePort);

  OscMessage msg = new OscMessage("/sys/port");
  msg.add(8000);
  osc.send(msg, serialoscPort);

  msg = new OscMessage("/sys/prefix");
  msg.add(prefix);
  osc.send(msg, serialoscPort);

  msg = new OscMessage(prefix+"/grid/led/all");
  msg.add(0);
  osc.send(msg, serialoscPort);

  ledSet(7,0,1);
  ledSet(5,0,1);

  // start listening!
  minim = new Minim(this);
  audioin = minim.getLineIn();

  horizontal_scaling = (float)width/(float)audioin.bufferSize();

  // start the beat detect0r
  beat = new BeatDetect();

  // fill movie array with clips and kick em off
  for (int j = 0; j < movieListArray.length; j++) {
    movies[j] = new Movie[movieListArray[j].length];
    for (int i = 0; i < movieListArray[j].length; i++) {
      movies[j][i] = new Movie(this, movieListArray[j][i]);
      selectMovie(i);
    }
  }

  // kick it off on some random movie
  movie_id = (int)(random(movieListArray[current_movie_list].length));
  selectMovie(movie_id);
}

void selectMovieList(int list_id) {
  // stop all currently playing movies
  for (int i=0; i<movieListArray[current_movie_list].length; i++) {
     movies[current_movie_list][i].stop(); 
  }

  current_movie_list = list_id;

  // start all new currently playing movies
  for (int i=0; i<movieListArray[current_movie_list].length; i++) {
     movies[current_movie_list][i].loop();
     movies[current_movie_list][i].volume(0);
  }

  setLEDMovieList(list_id);
  movie_id = (int)(random(movieListArray[current_movie_list].length));
  selectMovie(movie_id);  
}  

void selectMovie(int movie_id) {
  movie = movies[current_movie_list][movie_id];
  movie.loop();
  movie.volume(0);
  current_movie_id = movie_id;
  setLEDMovie(movie_id);
  setLEDPlayhead();
}

void selectMovieLocation(int movie_id, float movie_time) {
  movies[current_movie_list][movie_id].jump(movie_time);
  movie = movies[current_movie_list][movie_id];
  movie.loop();
  movie.volume(0);  
  current_movie_id = movie_id;
  setLEDPlayhead();
}  

void oscEvent(OscMessage msg) {
// println(msg);
 if(msg.checkAddrPattern(prefix+"/grid/key")==true){
    int x = msg.get(0).intValue();
    int y = msg.get(1).intValue();
    int z = msg.get(2).intValue();
    monomePress(x,y,z);
 }
}

void monomePress(int x, int y, int z){
  if (z == 1){
    // if home row
    if (x == 7) {
      // *** MOVIE SET SELECTOR ***
      if (y < movieListArray.length) {
        selectMovieList(y);
      }
    } else if (x == 5) {
      // *** MOVIE SELECTOR ***
      if (y < movieListArray[current_movie_list].length) {
        selectMovie(y);
      }
    } else if (x <= 4 && x >= 1) {
      // *** MOVIE JUMPER ***
      int shiftx = (x-2)*-1+2;
      selectMovieLocation(current_movie_id, ceil(movies[current_movie_list][current_movie_id].duration()/32) * (((shiftx)*8)+y));
      setLEDPlayhead();
    } else {
      // row 0 - bottom row turn on and off functions and states
      int state = led[x][y];
      state = abs(1 - state); // turns 1 to 0, 0 to 1

      ledSet(x,y,state);

      // monkey on
      if (x == 0 && y == 0) {
        monkey = abs(1 - monkey);
      }

      // fftvis on
      if (x == 0 && y == 1) {
        fftvis = abs(1 - fftvis);
      }

      // overlay on
      if (x == 0 && y == 2) {
        if (overlay==0) {
          oldmovie = movies[current_movie_list][current_movie_id];
          oldmovie.loop();
        }
        overlay = abs(1 - overlay);
      }

      // fade on
      if (x == 0 && y == 7) {
        fading = abs(1 - fading);
      }            
    }
  }
}

void setLEDMovieList(int list_id) {
  for (int j=0; j <= 7; j++) {
    if (led[7][j] == 1 && j != list_id) {
      ledSet(7, j, 0);
    } else if (led[7][j] == 0 && j == list_id) {
      ledSet(7, j, 1);        
    }
  }
}

void setLEDMovie(int movie_id) {
  for (int j=0; j <= 7; j++) {
    if (led[5][j] == 1 && j != movie_id) {
      ledSet(5, j, 0);
    } else if (led[5][j] == 0 && j == movie_id) {
      ledSet(5, j, 1);        
    }
  }
}

void setLEDPlayhead() {
  int x = 0;
  int y = 0;
  int numLED = 0;

  numLED = (int)(floor((32*movie.time()/movie.duration())));
//  println (numLED);
  x = (int)(floor(numLED/8))+1;
  y = (int)(numLED%8);
  int shiftx = (x-2)*-1+3;

  // for the width of these rows, shut off all leds, except the active playhead one
  for (int i=1; i <=4; i++) {
    for (int j=0; j <= 7; j++) {
      if (led[i][j] == 1 && !((i == shiftx) && j == y)) {
        ledSet(i, j, 0);
      } else if (led[i][j] == 0 && i == shiftx && j == y) {
        ledSet(shiftx, y, 1);
      }
    }
  }
}  

void ledSet(int x, int y, int z){
  OscMessage msg = new OscMessage(prefix+"/grid/led/set");
  msg.add(x);
  msg.add(y);
  msg.add(z);
  osc.send(msg, serialoscPort);
  led[x][y] = z;
} 

void draw(){
  // monkey!
  beat.detect(audioin.mix);

  // display beat detection
  if (beat.isOnset()) {
    for (int i=0; i<=7; i++) {
      if (led[6][i] == 0) {        
        ledSet(6, i, 1);
      }
    }
  } else {
    for (int i=0; i<=6; i++) {
      if (led[6][i+1] == 1 && led[6][i] == 0) {
        ledSet(6, i, 1);
      } else if (led[6][i+1] == 0 && led[6][i] == 1) {
        ledSet(6, i, 0);
      }
    }      
    ledSet(6, 7, 0);
  }

  // video switcher monkey!
  if (monkey == 1) {

    if (beat.isOnset()) {
      beatcount++;
      // jitter control
      if (beatcount > 2) {
        beatcount = 0;
        // 1/10 times do something
        if (random(0,100) >=80) {

          // turn off overlay if it's already on
          if (overlay == 1) {
            overlay = 0;
          } else {
            // x percent of the time do overlay
            if (random(0,100) <= 20) {
              overlay = 1;
              oldmovie = movies[current_movie_list][current_movie_id];
              oldmovie.loop(); 
            }  
          }

          float random_action = random(0,100);
          // change movie
          if (random_action >= 70) {
            selectMovie((int)random(0, (movieListArray[current_movie_list].length-1)));
          // random jump
          } else if (random_action >= 40) {
            float t;
            t = random(10, movies[current_movie_list][current_movie_id].duration()-10);
            selectMovieLocation(current_movie_id, t);
          // loop
          } else if (random_action >= 0) {
            selectMovieLocation(current_movie_id, movie.time()-3); 
          }  
        }
      }  
    }
  }  

  if (movie.available()) {
    movie.read();
  }
  image (movie, 0, 0, width, height);
  setLEDPlayhead();
  if (overlay == 1) {
    tint(255,127);
    image (oldmovie, 20, 0, width, height);
  }

  // FADE!!!! //
  if (fading == 1 && alphaVal < 255) {
    alphaVal ++;
  } else if (fading == 0 && alphaVal > 0) {  
    alphaVal --;
  }

  fill (0, 0, 0, alphaVal);
  rect(0, 0, width, height);

}