Android SpeechRecognition class within Processing

DanielJay

Hey all,

so I i've been digging through the Android API / stack exchange forum posts for days to find a way to avoid Eclipse, and implement the SpeechRecognition class within Processing. The perks of implementing the class directly as oppose to solely relying on the RecognizerIntent class is you have more control over the UI.

The issue I am facing is with the current sketch I have (pasted below), everything compiles, but when the class cant process speech (loud obscure sound), an error dialog pops up, halting the main stack of the window. (it also neglects to use the SpeechRecognition class. ). I want control to simply trigger the class as a continuous background process as I have seen referenced multiple times on stackexchange

suggestions? I have two chunks of code below; one is the working speechRecognition sketch, and the one below is my attempt to implement the SpeechRecognition class, where I get the error also followed below the code.

Thanks for your time!

**working code with graphical UI: **

/*

 Simple voice recognizer


 */
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.os.Bundle;
import android.speech.RecognizerIntent;

/************************************************************************

 --------------------------------  DATAS ---------------------------------

 *************************************************************************/
PFont androidFont;
String [] fontList;
int VOICE_RECOGNITION_REQUEST_CODE = 1234;

/************************************************************************

 --------------------------------  SETUP ---------------------------------

 *************************************************************************/
void setup() {
  orientation(LANDSCAPE);
  fontList = PFont.list();
  androidFont = createFont(fontList[0], 18, true);
  textFont(androidFont);


  PackageManager pm = getPackageManager();
  ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
  new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
  if (activities.size() != 0) {
   // text("il y a un recognizer!", 20, 60);
  } 
  else {
    //text("Recognizer not present", 20, 60);
  }
}

/************************************************************************

 --------------------------------  DRAW ---------------------------------

 *************************************************************************/

void draw() {
}
/************************************************************************

 --------------------------------  EVENTS ---------------------------------

 *************************************************************************/

void mousePressed() {
  startVoiceRecognitionActivity();
}

void startVoiceRecognitionActivity() {
  Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
  intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
  intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo");
  startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE);
}

void onActivityResult(int requestCode, int resultCode, Intent data) {
  if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) {
    background(0);
    // Fill the list view with the strings the recognizer thought it could have heard
    ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
    String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      text(s[0], 60, 20);
      //println(s[i]);
    }
  }

  super.onActivityResult(requestCode, resultCode, data);
}

and now my attempt throwing all kinds of mess my way:

import java.util.*;

import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.os.Bundle;
import android.speech.RecognizerIntent;
import android.speech.SpeechRecognizer;
import android.speech.RecognitionListener;
import   android.view.View;
import android.app.Activity;
import android.os.Handler;
import android.os.Message;
import android.widget.TextView;

import android.view.View.OnClickListener;
import android.widget.Button;
import java.util.ArrayList;
import android.util.Log;

/************************************************************************

 --------------------------------  DATAS ---------------------------------

 *************************************************************************/
PFont androidFont;
String [] fontList;
int VOICE_RECOGNITION_REQUEST_CODE = 1234;
Intent intent;
//RecognitionListner listenVal;
boolean mIsListening = false; 
final static int DIM = 20, DELAY = 1000;
int nextTimer, counter;
public SpeechRecognizer sr;

//voiceRecognitionTest.listener listenVal;
/************************************************************************

 --------------------------------  SETUP ---------------------------------

 *************************************************************************/
void setup() {
  orientation(LANDSCAPE);
  fontList = PFont.list();
  androidFont = createFont(fontList[0], 18, true);
  textFont(androidFont);
 //listenVal = listener.new voiceRecognitionTest();

  sr = SpeechRecognizer.createSpeechRecognizer(this);       
            sr.setRecognitionListener(new listener());
  /*
  PackageManager pm = getPackageManager();
   ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
   new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
   if (activities.size() != 0) {
   // text("il y a un recognizer!", 20, 60);
   } 
   else {
   //text("Recognizer not present", 20, 60);
   }

   */
}

/************************************************************************

 --------------------------------  DRAW ---------------------------------

 *************************************************************************/

void draw() {
}
/************************************************************************

 --------------------------------  EVENTS ---------------------------------

 *************************************************************************/

void mousePressed(){


                Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);        
                intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
                intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE,"voice.recognition.test");

                intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS,5); 
                     sr.startListening(intent);
                     println("11111111");

}


/*
void onActivityResult(int requestCode, int resultCode, Intent data) {

  println("result code = " + resultCode);


  if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK ) {
    background(0);



    // Fill the list view with the strings the recognizer thought it could have heard
    ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
    String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      //text(s[0], 60, 20);
      // println(s[0]);


      //recognizer.setRecognitionListener(listenVal);



  }
  }

  super.onActivityResult(requestCode, resultCode, data);
}

*/


/*************RECOGNITION LISTENER CLASS*************************/


  public class listener implements RecognitionListener          
   {
            public void onReadyForSpeech(Bundle params)
            {
                     println( "onReadyForSpeech");
            }
            public void onBeginningOfSpeech()
            {
                     println( "onBeginningOfSpeech");
            }
            public void onRmsChanged(float rmsdB)
            {
                     println( "onRmsChanged");
            }
            public void onBufferReceived(byte[] buffer)
            {
                     println( "onBufferReceived");
            }
            public void onEndOfSpeech()
            {
                     println( "onEndofSpeech");
            }
            public void onError(int error)
            {
                     println( "error " +  error);
                   //  mText.setText("error " + error);
            }
            public void onResults(Bundle results)                   
            {
                     String str = new String();
                     println( "onResults " + results);
                     ArrayList data = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
                     for (int i = 0; i < data.size(); i++)
                     {
                               println("result " + data.get(i));
                               str += data.get(i);
                     }
                    // mText.setText("results: "+String.valueOf(data.size()));        
            }
            public void onPartialResults(Bundle partialResults)
            {
                     println( "onPartialResults");
            }
            public void onEvent(int eventType, Bundle params)
            {
                     println( "onEvent " + eventType);
            }
   }

**and the following error from my attempt : **

calsign

Your error message is fairly descriptive; you must create the SpeechRecognizer from the application's main thread. Try this:

//Android's native function for running code on the UI thread
runOnUiThread(new Runnable() {
  public void run() {
    //Code to run on the UI thread
    sr = SpeechRecognizer.createSpeechRecognizer(this);
  }
});

DanielJay

i've attempted your suggestions as follows:

    void mousePressed() {




      runOnUiThread(new Runnable() {



      public void run() {
        //Code to run on the UI thread



   recognizer = SpeechRecognizer.createSpeechRecognizer(this);



    }
    });





}

I get a "cannot find symbol" error despite declaring it globally. Oh, and I changed the variable name from "sr"t o "recognizer" for my own sake

calsign

Now, you are trying to instantiate the SpeechRecognizer with this which, inside run(), refers to the anonymous class, not the PApplet Activity. This seems like it is getting increasingly complicated... but this should work:

void mousePressed() {
  runOnUiThread(new Runnable() {
    @ Override
    public void run() {
      //Initialize the recognizer on the UI thread
      initRecognizer();
    }
  });
}

void initRecognizer() {
  recognizer = SpeechRecognizer.createSpeechRecognizer(this);
}

DanielJay

Hey calsign,

thanks for the help thus far. Same error, with a "FATAL EXCEPTION : main". Here is my revised code:

import java.util.*;

import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.os.Bundle;
import android.speech.RecognizerIntent;
import android.speech.SpeechRecognizer;
import   android.app.Activity;
import android.speech.RecognitionListener;

/************************************************************************

 --------------------------------  DATAS ---------------------------------

 *************************************************************************/
PFont androidFont;
String [] fontList;
int VOICE_RECOGNITION_REQUEST_CODE = 1234;
Intent intent;
listener listenVal;
boolean mIsListening = false; 
final static int DIM = 20, DELAY = 1000;
int nextTimer, counter;
   SpeechRecognizer recognizer;




/************************************************************************

 --------------------------------  SETUP ---------------------------------

 *************************************************************************/
void setup() {
  orientation(LANDSCAPE);
  fontList = PFont.list();
  androidFont = createFont(fontList[0], 18, true);
  textFont(androidFont);
    listenVal = new listener();
/*
  PackageManager pm = getPackageManager();
  ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
  new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
  if (activities.size() != 0) {
    // text("il y a un recognizer!", 20, 60);
  } 
  else {
    //text("Recognizer not present", 20, 60);
  }

  */



}

/************************************************************************

 --------------------------------  DRAW ---------------------------------

 *************************************************************************/

void draw() {






}
/************************************************************************

 --------------------------------  EVENTS ---------------------------------

 *************************************************************************/

void mousePressed() {



   runOnUiThread(new Runnable() {


 @Override
  public void run() {
    //Code to run on the UI thread



      recognizer.startListening(intent);

initRecognizer();

}
});


}

 void initRecognizer() {  


  recognizer = SpeechRecognizer.createSpeechRecognizer(this);

}


     //   listenVal.startVoiceRecognitionActivity();









void onActivityResult(int requestCode, int resultCode, Intent data) {
   recognizer.setRecognitionListener(new listener());



   // recognizer.setRecognitionListener(listenVal);
      intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
  intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
  intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo");
  startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE);


if (!mIsListening)
{
    mIsListening = true;        
    recognizer.startListening(intent);
}  






  if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) {
    background(0);



    // Fill the list view with the strings the recognizer thought it could have heard
    ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
    String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      //text(s[0], 60, 20);
      // println(s[0]);



    }
  }


  super.onActivityResult(requestCode, resultCode, data);
}




/*************RECOGNITION LISTENER CLASS*************************/


class listener implements RecognitionListener {



void startVoiceRecognitionActivity() {



  intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
  intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
  intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo");
  startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE);


}


 // @Override  i have enabled these but for formatting in this code, it wont work
    public void onBeginningOfSpeech() {
    println("onBeginningOfSpeech");
  }

//  @Override
    public void onBufferReceived(byte[] buffer) {
    println( "onBufferReceived = " + buffer);
  }

  //@Override
    public void onEndOfSpeech() {
    println( "onEndOfSpeech");
  }

 // @Override
    public void onError(int error) {

    println( "onError = " + error);

  if(error==8){


  recognizer.startListening(intent);
}  

  }

//  @Override
    public void onEvent(int eventType, Bundle params) {
    println( "onEvent = "+ eventType);
  }

  //@Override
    public void onPartialResults(Bundle partialResults) {
      mIsListening = false;
    println("onPartialResults = " + partialResults);
  }

  //@Override
    public void onReadyForSpeech(Bundle params) {
      mIsListening = false;
    println("onReadyForSpeech = " + params);


  }





  //@Override
    public void onResults(Bundle results) {
ArrayList<String> matches = results.getStringArrayList(
        recognizer.RESULTS_RECOGNITION);
        String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      text(s[0], 60, 20);
    //println("results = " + s[0]);
  }
    }

  //@Override
    public void onRmsChanged(float rmsdB) {
    //println( "onRmsChanged");
  }

}

calsign

Is the stack trace the same? You may be trying to access the SpeechRecognizer's methods from non-UI threads in other places. If the stack trace is exactly the same as it was before, then there is something else going wrong.

DanielJay

Hey Calsign,

I actually decided to incorporate your suggestion in the original code I posted that was troublesome (since thats what you've probably been gearing your efforts towards :-w sorry! ) It compiles, but I am left with the "error code 9".

edit: error = sketch permissions for record audio + internet which im enabling. Will update soon.

Anyways, here is the revised code:

import java.util.*;

import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.os.Bundle;
import android.speech.RecognizerIntent;
import android.speech.SpeechRecognizer;
import android.speech.RecognitionListener;
import   android.view.View;
import android.app.Activity;
import android.os.Handler;
import android.os.Message;
import android.widget.TextView;

import android.view.View.OnClickListener;
import android.widget.Button;
import java.util.ArrayList;
import android.util.Log;

/************************************************************************

 --------------------------------  DATAS ---------------------------------

 *************************************************************************/
PFont androidFont;
String [] fontList;
int VOICE_RECOGNITION_REQUEST_CODE = 1234;
Intent intent;
//RecognitionListner listenVal;
boolean mIsListening = false; 
final static int DIM = 20, DELAY = 1000;
int nextTimer, counter;
public SpeechRecognizer sr;

//voiceRecognitionTest.listener listenVal;
/************************************************************************

 --------------------------------  SETUP ---------------------------------

 *************************************************************************/
void setup() {
  orientation(LANDSCAPE);
  fontList = PFont.list();
  androidFont = createFont(fontList[0], 18, true);
  textFont(androidFont);
 //listenVal = listener.new voiceRecognitionTest();

 // sr = SpeechRecognizer.createSpeechRecognizer(this);       
  /*
  PackageManager pm = getPackageManager();
   ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
   new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
   if (activities.size() != 0) {
   // text("il y a un recognizer!", 20, 60);
   } 
   else {
   //text("Recognizer not present", 20, 60);
   }

   */
}

/************************************************************************

 --------------------------------  DRAW ---------------------------------

 *************************************************************************/

void draw() {
}
/************************************************************************

 --------------------------------  EVENTS ---------------------------------

 *************************************************************************/

void mousePressed(){
   runOnUiThread(new Runnable() {
    @ Override
    public void run() {
      //Initialize the recognizer on the UI thread
      initRecognizer();
    }
  });

}



void initRecognizer() {


 sr = SpeechRecognizer.createSpeechRecognizer(this);
          sr.setRecognitionListener(new listener());

                Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);        
                intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
                intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE,"voice.recognition.test");

                intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS,5); 
                     sr.startListening(intent);


}



/*
void onActivityResult(int requestCode, int resultCode, Intent data) {

  println("result code = " + resultCode);


  if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK ) {
    background(0);



    // Fill the list view with the strings the recognizer thought it could have heard
    ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
    String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      //text(s[0], 60, 20);
      // println(s[0]);


      //recognizer.setRecognitionListener(listenVal);



  }
  }

  super.onActivityResult(requestCode, resultCode, data);
}

*/


/*************RECOGNITION LISTENER CLASS*************************/


  public class listener implements RecognitionListener          
   {
            public void onReadyForSpeech(Bundle params)
            {
                     println( "onReadyForSpeech");
            }
            public void onBeginningOfSpeech()
            {
                     println( "onBeginningOfSpeech");
            }
            public void onRmsChanged(float rmsdB)
            {
                     println( "onRmsChanged");
            }
            public void onBufferReceived(byte[] buffer)
            {
                     println( "onBufferReceived");
            }
            public void onEndOfSpeech()
            {
                     println( "onEndofSpeech");
            }
            public void onError(int error)
            {
                     println( "error " +  error);
                   //  mText.setText("error " + error);
            }
            public void onResults(Bundle results)                   
            {
                     String str = new String();
                     println( "onResults " + results);
                     ArrayList data = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
                     for (int i = 0; i < data.size(); i++)
                     {
                               println("result " + data.get(i));
                               str += data.get(i);
                     }
                    // mText.setText("results: "+String.valueOf(data.size()));        
            }
            public void onPartialResults(Bundle partialResults)
            {
                     println( "onPartialResults");
            }
            public void onEvent(int eventType, Bundle params)
            {
                     println( "onEvent " + eventType);
            }
   }

DanielJay

You are amazing Calsign! Solved! I'd buy you a beer/beverage of preference if I could!

mcspud

Yes he is, check out his apps!

DanielJay

References?