Android SpeechRecognition class within Processing

edited February 2014 in Android Mode

Hey all,

so I i've been digging through the Android API / stack exchange forum posts for days to find a way to avoid Eclipse, and implement the SpeechRecognition class within Processing. The perks of implementing the class directly as oppose to solely relying on the RecognizerIntent class is you have more control over the UI.

The issue I am facing is with the current sketch I have (pasted below), everything compiles, but when the class cant process speech (loud obscure sound), an error dialog pops up, halting the main stack of the window. (it also neglects to use the SpeechRecognition class. ). I want control to simply trigger the class as a continuous background process as I have seen referenced multiple times on stackexchange

suggestions? I have two chunks of code below; one is the working speechRecognition sketch, and the one below is my attempt to implement the SpeechRecognition class, where I get the error also followed below the code.

Thanks for your time!

**working code with graphical UI: **

/*

 Simple voice recognizer


 */
import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.os.Bundle;
import android.speech.RecognizerIntent;

/************************************************************************

 --------------------------------  DATAS ---------------------------------

 *************************************************************************/
PFont androidFont;
String [] fontList;
int VOICE_RECOGNITION_REQUEST_CODE = 1234;

/************************************************************************

 --------------------------------  SETUP ---------------------------------

 *************************************************************************/
void setup() {
  orientation(LANDSCAPE);
  fontList = PFont.list();
  androidFont = createFont(fontList[0], 18, true);
  textFont(androidFont);


  PackageManager pm = getPackageManager();
  ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
  new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
  if (activities.size() != 0) {
   // text("il y a un recognizer!", 20, 60);
  } 
  else {
    //text("Recognizer not present", 20, 60);
  }
}

/************************************************************************

 --------------------------------  DRAW ---------------------------------

 *************************************************************************/

void draw() {
}
/************************************************************************

 --------------------------------  EVENTS ---------------------------------

 *************************************************************************/

void mousePressed() {
  startVoiceRecognitionActivity();
}

void startVoiceRecognitionActivity() {
  Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
  intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
  intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo");
  startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE);
}

void onActivityResult(int requestCode, int resultCode, Intent data) {
  if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) {
    background(0);
    // Fill the list view with the strings the recognizer thought it could have heard
    ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
    String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      text(s[0], 60, 20);
      //println(s[i]);
    }
  }

  super.onActivityResult(requestCode, resultCode, data);
}

and now my attempt throwing all kinds of mess my way:

import java.util.*;

import android.content.Intent;
import android.content.pm.PackageManager;
import android.content.pm.ResolveInfo;
import android.os.Bundle;
import android.speech.RecognizerIntent;
import android.speech.SpeechRecognizer;
import android.speech.RecognitionListener;
import   android.view.View;
import android.app.Activity;
import android.os.Handler;
import android.os.Message;
import android.widget.TextView;

import android.view.View.OnClickListener;
import android.widget.Button;
import java.util.ArrayList;
import android.util.Log;

/************************************************************************

 --------------------------------  DATAS ---------------------------------

 *************************************************************************/
PFont androidFont;
String [] fontList;
int VOICE_RECOGNITION_REQUEST_CODE = 1234;
Intent intent;
//RecognitionListner listenVal;
boolean mIsListening = false; 
final static int DIM = 20, DELAY = 1000;
int nextTimer, counter;
public SpeechRecognizer sr;

//voiceRecognitionTest.listener listenVal;
/************************************************************************

 --------------------------------  SETUP ---------------------------------

 *************************************************************************/
void setup() {
  orientation(LANDSCAPE);
  fontList = PFont.list();
  androidFont = createFont(fontList[0], 18, true);
  textFont(androidFont);
 //listenVal = listener.new voiceRecognitionTest();

  sr = SpeechRecognizer.createSpeechRecognizer(this);       
            sr.setRecognitionListener(new listener());
  /*
  PackageManager pm = getPackageManager();
   ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
   new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
   if (activities.size() != 0) {
   // text("il y a un recognizer!", 20, 60);
   } 
   else {
   //text("Recognizer not present", 20, 60);
   }

   */
}

/************************************************************************

 --------------------------------  DRAW ---------------------------------

 *************************************************************************/

void draw() {
}
/************************************************************************

 --------------------------------  EVENTS ---------------------------------

 *************************************************************************/

void mousePressed(){


                Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);        
                intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
                intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE,"voice.recognition.test");

                intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS,5); 
                     sr.startListening(intent);
                     println("11111111");

}


/*
void onActivityResult(int requestCode, int resultCode, Intent data) {

  println("result code = " + resultCode);


  if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK ) {
    background(0);



    // Fill the list view with the strings the recognizer thought it could have heard
    ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
    String s[] = (String[]) matches.toArray(new String[matches.size()]);
    fill(255);
    for (int i=0; i<s.length; i++) {
      //text(s[0], 60, 20);
      // println(s[0]);


      //recognizer.setRecognitionListener(listenVal);



  }
  }

  super.onActivityResult(requestCode, resultCode, data);
}

*/


/*************RECOGNITION LISTENER CLASS*************************/


  public class listener implements RecognitionListener          
   {
            public void onReadyForSpeech(Bundle params)
            {
                     println( "onReadyForSpeech");
            }
            public void onBeginningOfSpeech()
            {
                     println( "onBeginningOfSpeech");
            }
            public void onRmsChanged(float rmsdB)
            {
                     println( "onRmsChanged");
            }
            public void onBufferReceived(byte[] buffer)
            {
                     println( "onBufferReceived");
            }
            public void onEndOfSpeech()
            {
                     println( "onEndofSpeech");
            }
            public void onError(int error)
            {
                     println( "error " +  error);
                   //  mText.setText("error " + error);
            }
            public void onResults(Bundle results)                   
            {
                     String str = new String();
                     println( "onResults " + results);
                     ArrayList data = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
                     for (int i = 0; i < data.size(); i++)
                     {
                               println("result " + data.get(i));
                               str += data.get(i);
                     }
                    // mText.setText("results: "+String.valueOf(data.size()));        
            }
            public void onPartialResults(Bundle partialResults)
            {
                     println( "onPartialResults");
            }
            public void onEvent(int eventType, Bundle params)
            {
                     println( "onEvent " + eventType);
            }
   }

**and the following error from my attempt : **

image alt textScreen Shot 2014-02-01 at 1.21.13 AM

Answers

  • Your error message is fairly descriptive; you must create the SpeechRecognizer from the application's main thread. Try this:

    //Android's native function for running code on the UI thread
    runOnUiThread(new Runnable() {
      public void run() {
        //Code to run on the UI thread
        sr = SpeechRecognizer.createSpeechRecognizer(this);
      }
    });
    
  • edited February 2014

    i've attempted your suggestions as follows:

        void mousePressed() {
    
    
    
    
          runOnUiThread(new Runnable() {
    
    
    
          public void run() {
            //Code to run on the UI thread
    
    
    
       recognizer = SpeechRecognizer.createSpeechRecognizer(this);
    
    
    
        }
        });
    
    
    
    
    
    }
    

    I get a "cannot find symbol" error despite declaring it globally. Oh, and I changed the variable name from "sr"t o "recognizer" for my own sake

  • Answer ✓

    Now, you are trying to instantiate the SpeechRecognizer with this which, inside run(), refers to the anonymous class, not the PApplet Activity. This seems like it is getting increasingly complicated... but this should work:

    void mousePressed() {
      runOnUiThread(new Runnable() {
        @ Override
        public void run() {
          //Initialize the recognizer on the UI thread
          initRecognizer();
        }
      });
    }
    
    void initRecognizer() {
      recognizer = SpeechRecognizer.createSpeechRecognizer(this);
    }
    
  • edited February 2014

    Hey calsign,

    thanks for the help thus far. Same error, with a "FATAL EXCEPTION : main". Here is my revised code:

    import java.util.*;
    
    import android.content.Intent;
    import android.content.pm.PackageManager;
    import android.content.pm.ResolveInfo;
    import android.os.Bundle;
    import android.speech.RecognizerIntent;
    import android.speech.SpeechRecognizer;
    import   android.app.Activity;
    import android.speech.RecognitionListener;
    
    /************************************************************************
    
     --------------------------------  DATAS ---------------------------------
    
     *************************************************************************/
    PFont androidFont;
    String [] fontList;
    int VOICE_RECOGNITION_REQUEST_CODE = 1234;
    Intent intent;
    listener listenVal;
    boolean mIsListening = false; 
    final static int DIM = 20, DELAY = 1000;
    int nextTimer, counter;
       SpeechRecognizer recognizer;
    
    
    
    
    /************************************************************************
    
     --------------------------------  SETUP ---------------------------------
    
     *************************************************************************/
    void setup() {
      orientation(LANDSCAPE);
      fontList = PFont.list();
      androidFont = createFont(fontList[0], 18, true);
      textFont(androidFont);
        listenVal = new listener();
    /*
      PackageManager pm = getPackageManager();
      ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
      new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
      if (activities.size() != 0) {
        // text("il y a un recognizer!", 20, 60);
      } 
      else {
        //text("Recognizer not present", 20, 60);
      }
    
      */
    
    
    
    }
    
    /************************************************************************
    
     --------------------------------  DRAW ---------------------------------
    
     *************************************************************************/
    
    void draw() {
    
    
    
    
    
    
    }
    /************************************************************************
    
     --------------------------------  EVENTS ---------------------------------
    
     *************************************************************************/
    
    void mousePressed() {
    
    
    
       runOnUiThread(new Runnable() {
    
    
     @Override
      public void run() {
        //Code to run on the UI thread
    
    
    
          recognizer.startListening(intent);
    
    initRecognizer();
    
    }
    });
    
    
    }
    
     void initRecognizer() {  
    
    
      recognizer = SpeechRecognizer.createSpeechRecognizer(this);
    
    }
    
    
         //   listenVal.startVoiceRecognitionActivity();
    
    
    
    
    
    
    
    
    
    void onActivityResult(int requestCode, int resultCode, Intent data) {
       recognizer.setRecognitionListener(new listener());
    
    
    
       // recognizer.setRecognitionListener(listenVal);
          intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
      intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
      intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo");
      startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE);
    
    
    if (!mIsListening)
    {
        mIsListening = true;        
        recognizer.startListening(intent);
    }  
    
    
    
    
    
    
      if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) {
        background(0);
    
    
    
        // Fill the list view with the strings the recognizer thought it could have heard
        ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
        String s[] = (String[]) matches.toArray(new String[matches.size()]);
        fill(255);
        for (int i=0; i<s.length; i++) {
          //text(s[0], 60, 20);
          // println(s[0]);
    
    
    
        }
      }
    
    
      super.onActivityResult(requestCode, resultCode, data);
    }
    
    
    
    
    /*************RECOGNITION LISTENER CLASS*************************/
    
    
    class listener implements RecognitionListener {
    
    
    
    void startVoiceRecognitionActivity() {
    
    
    
      intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
      intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
      intent.putExtra(RecognizerIntent.EXTRA_PROMPT, "Speech recognition demo");
      startActivityForResult(intent, VOICE_RECOGNITION_REQUEST_CODE);
    
    
    }
    
    
     // @Override  i have enabled these but for formatting in this code, it wont work
        public void onBeginningOfSpeech() {
        println("onBeginningOfSpeech");
      }
    
    //  @Override
        public void onBufferReceived(byte[] buffer) {
        println( "onBufferReceived = " + buffer);
      }
    
      //@Override
        public void onEndOfSpeech() {
        println( "onEndOfSpeech");
      }
    
     // @Override
        public void onError(int error) {
    
        println( "onError = " + error);
    
      if(error==8){
    
    
      recognizer.startListening(intent);
    }  
    
      }
    
    //  @Override
        public void onEvent(int eventType, Bundle params) {
        println( "onEvent = "+ eventType);
      }
    
      //@Override
        public void onPartialResults(Bundle partialResults) {
          mIsListening = false;
        println("onPartialResults = " + partialResults);
      }
    
      //@Override
        public void onReadyForSpeech(Bundle params) {
          mIsListening = false;
        println("onReadyForSpeech = " + params);
    
    
      }
    
    
    
    
    
      //@Override
        public void onResults(Bundle results) {
    ArrayList<String> matches = results.getStringArrayList(
            recognizer.RESULTS_RECOGNITION);
            String s[] = (String[]) matches.toArray(new String[matches.size()]);
        fill(255);
        for (int i=0; i<s.length; i++) {
          text(s[0], 60, 20);
        //println("results = " + s[0]);
      }
        }
    
      //@Override
        public void onRmsChanged(float rmsdB) {
        //println( "onRmsChanged");
      }
    
    }
    
  • Answer ✓

    Is the stack trace the same? You may be trying to access the SpeechRecognizer's methods from non-UI threads in other places. If the stack trace is exactly the same as it was before, then there is something else going wrong.

  • edited February 2014

    Hey Calsign,

    I actually decided to incorporate your suggestion in the original code I posted that was troublesome (since thats what you've probably been gearing your efforts towards :-w sorry! ) It compiles, but I am left with the "error code 9".

    edit: error = sketch permissions for record audio + internet which im enabling. Will update soon.

    Anyways, here is the revised code:

    import java.util.*;
    
    import android.content.Intent;
    import android.content.pm.PackageManager;
    import android.content.pm.ResolveInfo;
    import android.os.Bundle;
    import android.speech.RecognizerIntent;
    import android.speech.SpeechRecognizer;
    import android.speech.RecognitionListener;
    import   android.view.View;
    import android.app.Activity;
    import android.os.Handler;
    import android.os.Message;
    import android.widget.TextView;
    
    import android.view.View.OnClickListener;
    import android.widget.Button;
    import java.util.ArrayList;
    import android.util.Log;
    
    /************************************************************************
    
     --------------------------------  DATAS ---------------------------------
    
     *************************************************************************/
    PFont androidFont;
    String [] fontList;
    int VOICE_RECOGNITION_REQUEST_CODE = 1234;
    Intent intent;
    //RecognitionListner listenVal;
    boolean mIsListening = false; 
    final static int DIM = 20, DELAY = 1000;
    int nextTimer, counter;
    public SpeechRecognizer sr;
    
    //voiceRecognitionTest.listener listenVal;
    /************************************************************************
    
     --------------------------------  SETUP ---------------------------------
    
     *************************************************************************/
    void setup() {
      orientation(LANDSCAPE);
      fontList = PFont.list();
      androidFont = createFont(fontList[0], 18, true);
      textFont(androidFont);
     //listenVal = listener.new voiceRecognitionTest();
    
     // sr = SpeechRecognizer.createSpeechRecognizer(this);       
      /*
      PackageManager pm = getPackageManager();
       ArrayList<ResolveInfo> activities = (ArrayList<ResolveInfo>)pm.queryIntentActivities(
       new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH), 0);
       if (activities.size() != 0) {
       // text("il y a un recognizer!", 20, 60);
       } 
       else {
       //text("Recognizer not present", 20, 60);
       }
    
       */
    }
    
    /************************************************************************
    
     --------------------------------  DRAW ---------------------------------
    
     *************************************************************************/
    
    void draw() {
    }
    /************************************************************************
    
     --------------------------------  EVENTS ---------------------------------
    
     *************************************************************************/
    
    void mousePressed(){
       runOnUiThread(new Runnable() {
        @ Override
        public void run() {
          //Initialize the recognizer on the UI thread
          initRecognizer();
        }
      });
    
    }
    
    
    
    void initRecognizer() {
    
    
     sr = SpeechRecognizer.createSpeechRecognizer(this);
              sr.setRecognitionListener(new listener());
    
                    Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);        
                    intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
                    intent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE,"voice.recognition.test");
    
                    intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS,5); 
                         sr.startListening(intent);
    
    
    }
    
    
    
    /*
    void onActivityResult(int requestCode, int resultCode, Intent data) {
    
      println("result code = " + resultCode);
    
    
      if (requestCode == VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK ) {
        background(0);
    
    
    
        // Fill the list view with the strings the recognizer thought it could have heard
        ArrayList<String>  matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
        String s[] = (String[]) matches.toArray(new String[matches.size()]);
        fill(255);
        for (int i=0; i<s.length; i++) {
          //text(s[0], 60, 20);
          // println(s[0]);
    
    
          //recognizer.setRecognitionListener(listenVal);
    
    
    
      }
      }
    
      super.onActivityResult(requestCode, resultCode, data);
    }
    
    */
    
    
    /*************RECOGNITION LISTENER CLASS*************************/
    
    
      public class listener implements RecognitionListener          
       {
                public void onReadyForSpeech(Bundle params)
                {
                         println( "onReadyForSpeech");
                }
                public void onBeginningOfSpeech()
                {
                         println( "onBeginningOfSpeech");
                }
                public void onRmsChanged(float rmsdB)
                {
                         println( "onRmsChanged");
                }
                public void onBufferReceived(byte[] buffer)
                {
                         println( "onBufferReceived");
                }
                public void onEndOfSpeech()
                {
                         println( "onEndofSpeech");
                }
                public void onError(int error)
                {
                         println( "error " +  error);
                       //  mText.setText("error " + error);
                }
                public void onResults(Bundle results)                   
                {
                         String str = new String();
                         println( "onResults " + results);
                         ArrayList data = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
                         for (int i = 0; i < data.size(); i++)
                         {
                                   println("result " + data.get(i));
                                   str += data.get(i);
                         }
                        // mText.setText("results: "+String.valueOf(data.size()));        
                }
                public void onPartialResults(Bundle partialResults)
                {
                         println( "onPartialResults");
                }
                public void onEvent(int eventType, Bundle params)
                {
                         println( "onEvent " + eventType);
                }
       }
    
  • You are amazing Calsign! Solved! I'd buy you a beer/beverage of preference if I could!

  • Yes he is, check out his apps!

  • References?

Sign In or Register to comment.