Mad Lips

I recreated the Mad Libs from my youth for the video age, which I titled Mad Lips.

Mad Lips works like this:

• The application starts up and asks the user a question, such as “what do you do if your waitress brings the wrong order?”
• The program then prompts you to answer the question and records your reply.
• Once your response is recorded, the program plays a famous movie scene and, at the appropriate time, substitutes your lips reciting your response to the question with a key actor during the crucial moment in the scene.

The final version looks something like this:

and like this:

Technically, here’s what’s happening:

The program uses the Minim library to record the audio and the native Processing MovieMaker library to record the video. I’m also calling the OpenCV library to calculate face detection and then approximating the location of the mouth using some basic math, which is the only part of the video that’s recorded. Once the audio and video are recorded, it uses the GStreamer library to playback the movie clip and then inserts the recently recorded video in the middle, substituting the user’s lips for the actor’s at a pivotal scene.

Here’s the Processing code:

// Comp Cameras Final Project
// Aaron Uhrmacher
// May 2011

import codeanticode.gsvideo.*;
import hypermedia.video.*; 
import processing.video.*;
import java.awt.Rectangle;
import ddf.minim.*;

// AUDIO
Minim minim;
AudioInput in;
AudioRecorder recorder;
AudioPlayer player;

// VIDEO
GSMovie movie; 
GSMovie movie2;
OpenCV opencv;
Rectangle bestRect = new Rectangle(0, 0, 0, 0);

// VARIABLES
boolean playBack = false;
boolean recordingLoaded = false;
PImage baby;
int state = 5; 
PFont font;
int startingTime;
boolean movieNotStarted = true;

// recording movie: objects and variables
MovieMaker mm;
boolean readyToRecord = false;
boolean recordingStarted = false;
String title = "Baby";
int m, h;
int newFrame = 0;
int newFrame2 = 0;

void setup() {
  size(640, 352);
  baby =  loadImage("baby.png");
  opencv = new OpenCV( this );
  opencv.capture( 320, 240 );                   // open video stream
  opencv.cascade( OpenCV.CASCADE_FRONTALFACE_ALT );  // load detection description, here-> front face detection : "haarcascade_frontalface_alt.xml"

  font = loadFont("Apollo.vlw"); 
  textFont(font); 

  // VIDEO RECORDING VARIABLES
  m = minute();  
  h = hour();   
  println("Press 'E' to enable recording");

  // AUDIO
  minim = new Minim(this);
  in = minim.getLineIn(Minim.STEREO, 2048);
  recorder = minim.createRecorder(in, "data/myrecording.wav", true);

  movie = new GSMovie(this, "baby.mov"); 
  startingTime = millis() - 1000;
}

void movieEvent(GSMovie movie) { 
  movie.read();
}


public void stop() {
  opencv.stop();
  in.close();
  if ( player != null ) {
    player.close();
  }
  minim.stop();
  super.stop();
}

void draw() {

  if (state == 0 && movieNotStarted) {
    movie.play();
    movieNotStarted = false;
  }

  else if (state == 5) {
    background (0);
    fill (255);
    text ("Mad Lips", 20, 50); 
    text("What's your favorite", 20, 120);
    text("bumper sticker slogan?", 20, 150);
    text ("Press 'A' To Record Your Answer", 20, 250);
    if (key == 'a') {
      state = 1;
      println(state);
    }
  }

  else if (state == 0) {
    if (1 < movie.width && 1 < movie.height) {
      if (newFrame != movie.frame()) {

        if (movie.time() >=15.5) { // BEGIN MOVIE PLAYING
          movie.pause();
          movie.jump(457);
          playBack =true;
          state = 6;
          println("NEW STATE: " + state);
        }
      }
      image(movie, 0, 0, width, height);
    }
  }

  else if (state == 1) {
    if (!playBack) {
      opencv.read();
      // proceed detection
      Rectangle[] faces = opencv.detect( 1.2, 2, OpenCV.HAAR_DO_CANNY_PRUNING, 40, 40 );
      // display the image
      PImage frame = opencv.image(); // get the video
      if (faces.length > 0) {
        float lower_3 = (faces[0].height / 3) *2;
        float x_offset = (faces[0].width * 0.25);
        // the mouth PImage comes from the video, so it is essentially a mask
        PImage mouth = frame.get((int)(faces[0].x + x_offset), (int)(faces[0].y + lower_3), (int)(faces[0].width - x_offset), (int)(faces[0].height /3));  
        mouth.resize(110, 47); // resize the mouth video 
        image(mouth, 240, 205);  // place the mouth under the png image
      }

      image(baby, 0, 0);
    }
  }

  else if (state == 3 ) { 
    if (movie.available()) {
      //movie.pause();
      // movie.jump(294);
      //movie.read();
      movie.play();
      println("playing movie");
    }
    image(movie, 0, 0, width, height);
  }

  if (readyToRecord == true) {
    mm.addFrame();
  }

  if (state == 6) {
    if (playBack) {
      if (!recordingLoaded) {
        //  println("loading movie");
        movie2 = new GSMovie(this, "Jack.mov");
        player = minim.loadFile("data/myrecording.wav", 2048);
        recordingLoaded = true;
        movie2.play();
        player.play();
      }
      if (1 < movie2.width && 1 < movie2.height) {
        if (newFrame2 != movie2.frame()) {
          // println("ready to playback");
          // movie2.play();
          image(movie2, 0, 0, width, height);
          if (!movie2.isPlaying()) {
            state = 3;
            println("NEW STATE: " + state);
          }
        }
      }
    }
  }
}

void keyPressed() {
  if (key == 's') {        // start or stop the record head when the ’s’ is pressed
    if ( recordingStarted == true ) {
      if ( readyToRecord == true ) {
        readyToRecord = false;
        recorder.endRecord();
        minim.stop();
        println( "Cut" );
      }

      else if ( readyToRecord == false ) {
        readyToRecord = true;
        recorder.beginRecord();
        println( "Rolling" );
      }
    }
    else {
      println( "Press 'E' to enable recording" );
    }
  }
  if ( key == 'e' ) {          // create new movie for saving
    if ( recordingStarted == false ) {
      recordingStarted = true;
      // compression and frame rate options
      mm = new MovieMaker( this, width, height, "data/"+title+".mov", 15, MovieMaker.ANIMATION, MovieMaker.HIGH );
      println();
      println( "Press 'S' to start and stop the record head" );
      println( "Press 'X' to save the movie and exit the sketch" );
    }
  }
  if (key == 'x') {                      // Finish the movie if the ‘x’ is pressed
    if (recordingStarted == true) {
      mm.finish();
      player = recorder.save();
      if (readyToRecord == true) {

        println("Cut");
      }
      println("Print!");
      state = 0;
    }
  }
  if (key == 'p') {
    playBack = true;
  }
}