diff options
Diffstat (limited to 'java/src/com/android/inputmethod/deprecated/voice')
9 files changed, 2022 insertions, 0 deletions
diff --git a/java/src/com/android/inputmethod/deprecated/voice/FieldContext.java b/java/src/com/android/inputmethod/deprecated/voice/FieldContext.java new file mode 100644 index 000000000..3c79cc218 --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/FieldContext.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import android.os.Bundle; +import android.util.Log; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.InputConnection; + +/** + * Represents information about a given text field, which can be passed + * to the speech recognizer as context information. + */ +public class FieldContext { + private static final boolean DBG = false; + + static final String LABEL = "label"; + static final String HINT = "hint"; + static final String PACKAGE_NAME = "packageName"; + static final String FIELD_ID = "fieldId"; + static final String FIELD_NAME = "fieldName"; + static final String SINGLE_LINE = "singleLine"; + static final String INPUT_TYPE = "inputType"; + static final String IME_OPTIONS = "imeOptions"; + static final String SELECTED_LANGUAGE = "selectedLanguage"; + static final String ENABLED_LANGUAGES = "enabledLanguages"; + + Bundle mFieldInfo; + + public FieldContext(InputConnection conn, EditorInfo info, + String selectedLanguage, String[] enabledLanguages) { + mFieldInfo = new Bundle(); + addEditorInfoToBundle(info, mFieldInfo); + addInputConnectionToBundle(conn, mFieldInfo); + addLanguageInfoToBundle(selectedLanguage, enabledLanguages, mFieldInfo); + if (DBG) Log.i("FieldContext", "Bundle = " + mFieldInfo.toString()); + } + + private static String safeToString(Object o) { + if (o == null) { + return ""; + } + return o.toString(); + } + + private static void addEditorInfoToBundle(EditorInfo info, Bundle bundle) { + if (info == null) { + return; + } + + bundle.putString(LABEL, safeToString(info.label)); + bundle.putString(HINT, safeToString(info.hintText)); + bundle.putString(PACKAGE_NAME, safeToString(info.packageName)); + bundle.putInt(FIELD_ID, info.fieldId); + bundle.putString(FIELD_NAME, safeToString(info.fieldName)); + bundle.putInt(INPUT_TYPE, info.inputType); + bundle.putInt(IME_OPTIONS, info.imeOptions); + } + + @SuppressWarnings("static-access") + private static void addInputConnectionToBundle( + InputConnection conn, Bundle bundle) { + if (conn == null) { + return; + } + + ExtractedText et = conn.getExtractedText(new ExtractedTextRequest(), 0); + if (et == null) { + return; + } + bundle.putBoolean(SINGLE_LINE, (et.flags & et.FLAG_SINGLE_LINE) > 0); + } + + private static void addLanguageInfoToBundle( + String selectedLanguage, String[] enabledLanguages, Bundle bundle) { + bundle.putString(SELECTED_LANGUAGE, selectedLanguage); + bundle.putStringArray(ENABLED_LANGUAGES, enabledLanguages); + } + + public Bundle getBundle() { + return mFieldInfo; + } + + @Override + public String toString() { + return mFieldInfo.toString(); + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/Hints.java b/java/src/com/android/inputmethod/deprecated/voice/Hints.java new file mode 100644 index 000000000..06b234381 --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/Hints.java @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.SharedPreferencesCompat; + +import android.content.ContentResolver; +import android.content.Context; +import android.content.SharedPreferences; +import android.view.inputmethod.InputConnection; + +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; + +/** + * Logic to determine when to display hints on usage to the user. + */ +public class Hints { + public interface Display { + public void showHint(int viewResource); + } + + private static final String PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN = + "voice_hint_num_unique_days_shown"; + private static final String PREF_VOICE_HINT_LAST_TIME_SHOWN = + "voice_hint_last_time_shown"; + private static final String PREF_VOICE_INPUT_LAST_TIME_USED = + "voice_input_last_time_used"; + private static final String PREF_VOICE_PUNCTUATION_HINT_VIEW_COUNT = + "voice_punctuation_hint_view_count"; + private static final int DEFAULT_SWIPE_HINT_MAX_DAYS_TO_SHOW = 7; + private static final int DEFAULT_PUNCTUATION_HINT_MAX_DISPLAYS = 7; + + private final Context mContext; + private final SharedPreferences mPrefs; + private final Display mDisplay; + private boolean mVoiceResultContainedPunctuation; + private int mSwipeHintMaxDaysToShow; + private int mPunctuationHintMaxDisplays; + + // Only show punctuation hint if voice result did not contain punctuation. + static final Map<CharSequence, String> SPEAKABLE_PUNCTUATION + = new HashMap<CharSequence, String>(); + static { + SPEAKABLE_PUNCTUATION.put(",", "comma"); + SPEAKABLE_PUNCTUATION.put(".", "period"); + SPEAKABLE_PUNCTUATION.put("?", "question mark"); + } + + public Hints(Context context, SharedPreferences prefs, Display display) { + mContext = context; + mPrefs = prefs; + mDisplay = display; + + ContentResolver cr = mContext.getContentResolver(); + mSwipeHintMaxDaysToShow = SettingsUtil.getSettingsInt( + cr, + SettingsUtil.LATIN_IME_VOICE_INPUT_SWIPE_HINT_MAX_DAYS, + DEFAULT_SWIPE_HINT_MAX_DAYS_TO_SHOW); + mPunctuationHintMaxDisplays = SettingsUtil.getSettingsInt( + cr, + SettingsUtil.LATIN_IME_VOICE_INPUT_PUNCTUATION_HINT_MAX_DISPLAYS, + DEFAULT_PUNCTUATION_HINT_MAX_DISPLAYS); + } + + public boolean showSwipeHintIfNecessary(boolean fieldRecommended) { + if (fieldRecommended && shouldShowSwipeHint()) { + showHint(R.layout.voice_swipe_hint); + return true; + } + + return false; + } + + public boolean showPunctuationHintIfNecessary(InputConnection ic) { + if (!mVoiceResultContainedPunctuation + && ic != null + && getAndIncrementPref(PREF_VOICE_PUNCTUATION_HINT_VIEW_COUNT) + < mPunctuationHintMaxDisplays) { + CharSequence charBeforeCursor = ic.getTextBeforeCursor(1, 0); + if (SPEAKABLE_PUNCTUATION.containsKey(charBeforeCursor)) { + showHint(R.layout.voice_punctuation_hint); + return true; + } + } + + return false; + } + + public void registerVoiceResult(String text) { + // Update the current time as the last time voice input was used. + SharedPreferences.Editor editor = mPrefs.edit(); + editor.putLong(PREF_VOICE_INPUT_LAST_TIME_USED, System.currentTimeMillis()); + SharedPreferencesCompat.apply(editor); + + mVoiceResultContainedPunctuation = false; + for (CharSequence s : SPEAKABLE_PUNCTUATION.keySet()) { + if (text.indexOf(s.toString()) >= 0) { + mVoiceResultContainedPunctuation = true; + break; + } + } + } + + private boolean shouldShowSwipeHint() { + final SharedPreferences prefs = mPrefs; + + int numUniqueDaysShown = prefs.getInt(PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN, 0); + + // If we've already shown the hint for enough days, we'll return false. + if (numUniqueDaysShown < mSwipeHintMaxDaysToShow) { + + long lastTimeVoiceWasUsed = prefs.getLong(PREF_VOICE_INPUT_LAST_TIME_USED, 0); + + // If the user has used voice today, we'll return false. (We don't show the hint on + // any day that the user has already used voice.) + if (!isFromToday(lastTimeVoiceWasUsed)) { + return true; + } + } + + return false; + } + + /** + * Determines whether the provided time is from some time today (i.e., this day, month, + * and year). + */ + private boolean isFromToday(long timeInMillis) { + if (timeInMillis == 0) return false; + + Calendar today = Calendar.getInstance(); + today.setTimeInMillis(System.currentTimeMillis()); + + Calendar timestamp = Calendar.getInstance(); + timestamp.setTimeInMillis(timeInMillis); + + return (today.get(Calendar.YEAR) == timestamp.get(Calendar.YEAR) && + today.get(Calendar.DAY_OF_MONTH) == timestamp.get(Calendar.DAY_OF_MONTH) && + today.get(Calendar.MONTH) == timestamp.get(Calendar.MONTH)); + } + + private void showHint(int hintViewResource) { + final SharedPreferences prefs = mPrefs; + + int numUniqueDaysShown = prefs.getInt(PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN, 0); + long lastTimeHintWasShown = prefs.getLong(PREF_VOICE_HINT_LAST_TIME_SHOWN, 0); + + // If this is the first time the hint is being shown today, increase the saved values + // to represent that. We don't need to increase the last time the hint was shown unless + // it is a different day from the current value. + if (!isFromToday(lastTimeHintWasShown)) { + SharedPreferences.Editor editor = prefs.edit(); + editor.putInt(PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN, numUniqueDaysShown + 1); + editor.putLong(PREF_VOICE_HINT_LAST_TIME_SHOWN, System.currentTimeMillis()); + SharedPreferencesCompat.apply(editor); + } + + if (mDisplay != null) { + mDisplay.showHint(hintViewResource); + } + } + + private int getAndIncrementPref(String pref) { + final SharedPreferences prefs = mPrefs; + int value = prefs.getInt(pref, 0); + SharedPreferences.Editor editor = prefs.edit(); + editor.putInt(pref, value + 1); + SharedPreferencesCompat.apply(editor); + return value; + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/RecognitionView.java b/java/src/com/android/inputmethod/deprecated/voice/RecognitionView.java new file mode 100644 index 000000000..dcb826e8f --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/RecognitionView.java @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.SubtypeSwitcher; + +import android.content.Context; +import android.content.res.Resources; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.CornerPathEffect; +import android.graphics.Paint; +import android.graphics.Path; +import android.graphics.PathEffect; +import android.graphics.drawable.Drawable; +import android.os.Handler; +import android.util.Log; +import android.view.LayoutInflater; +import android.view.View; +import android.view.View.OnClickListener; +import android.widget.Button; +import android.widget.ImageView; +import android.widget.ProgressBar; +import android.widget.TextView; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; +import java.util.Locale; + +/** + * The user interface for the "Speak now" and "working" states. + * Displays a recognition dialog (with waveform, voice meter, etc.), + * plays beeps, shows errors, etc. + */ +public class RecognitionView { + private static final String TAG = "RecognitionView"; + + private Handler mUiHandler; // Reference to UI thread + private View mView; + private Context mContext; + + private TextView mText; + private ImageView mImage; + private View mProgress; + private SoundIndicator mSoundIndicator; + private TextView mLanguage; + private Button mButton; + + private Drawable mInitializing; + private Drawable mError; + + private static final int INIT = 0; + private static final int LISTENING = 1; + private static final int WORKING = 2; + private static final int READY = 3; + + private int mState = INIT; + + private final View mPopupLayout; + + private final Drawable mListeningBorder; + private final Drawable mWorkingBorder; + private final Drawable mErrorBorder; + + public RecognitionView(Context context, OnClickListener clickListener) { + mUiHandler = new Handler(); + + LayoutInflater inflater = (LayoutInflater) context.getSystemService( + Context.LAYOUT_INFLATER_SERVICE); + + mView = inflater.inflate(R.layout.recognition_status, null); + + mPopupLayout= mView.findViewById(R.id.popup_layout); + + // Pre-load volume level images + Resources r = context.getResources(); + + mListeningBorder = r.getDrawable(R.drawable.vs_dialog_red); + mWorkingBorder = r.getDrawable(R.drawable.vs_dialog_blue); + mErrorBorder = r.getDrawable(R.drawable.vs_dialog_yellow); + + mInitializing = r.getDrawable(R.drawable.mic_slash); + mError = r.getDrawable(R.drawable.caution); + + mImage = (ImageView) mView.findViewById(R.id.image); + mProgress = mView.findViewById(R.id.progress); + mSoundIndicator = (SoundIndicator) mView.findViewById(R.id.sound_indicator); + + mButton = (Button) mView.findViewById(R.id.button); + mButton.setOnClickListener(clickListener); + mText = (TextView) mView.findViewById(R.id.text); + mLanguage = (TextView) mView.findViewById(R.id.language); + + mContext = context; + } + + public View getView() { + return mView; + } + + public void restoreState() { + mUiHandler.post(new Runnable() { + @Override + public void run() { + // Restart the spinner + if (mState == WORKING) { + ((ProgressBar) mProgress).setIndeterminate(false); + ((ProgressBar) mProgress).setIndeterminate(true); + } + } + }); + } + + public void showInitializing() { + mUiHandler.post(new Runnable() { + @Override + public void run() { + mState = INIT; + prepareDialog(mContext.getText(R.string.voice_initializing), mInitializing, + mContext.getText(R.string.cancel)); + } + }); + } + + public void showListening() { + Log.d(TAG, "#showListening"); + mUiHandler.post(new Runnable() { + @Override + public void run() { + mState = LISTENING; + prepareDialog(mContext.getText(R.string.voice_listening), null, + mContext.getText(R.string.cancel)); + } + }); + } + + public void updateVoiceMeter(float rmsdB) { + mSoundIndicator.setRmsdB(rmsdB); + } + + public void showError(final String message) { + mUiHandler.post(new Runnable() { + @Override + public void run() { + mState = READY; + prepareDialog(message, mError, mContext.getText(R.string.ok)); + } + }); + } + + public void showWorking( + final ByteArrayOutputStream waveBuffer, + final int speechStartPosition, + final int speechEndPosition) { + mUiHandler.post(new Runnable() { + @Override + public void run() { + mState = WORKING; + prepareDialog(mContext.getText(R.string.voice_working), null, mContext + .getText(R.string.cancel)); + final ShortBuffer buf = ByteBuffer.wrap(waveBuffer.toByteArray()).order( + ByteOrder.nativeOrder()).asShortBuffer(); + buf.position(0); + waveBuffer.reset(); + showWave(buf, speechStartPosition / 2, speechEndPosition / 2); + } + }); + } + + private void prepareDialog(CharSequence text, Drawable image, + CharSequence btnTxt) { + + /* + * The mic of INIT and of LISTENING has to be displayed in the same position. To accomplish + * that, some text visibility are not set as GONE but as INVISIBLE. + */ + switch (mState) { + case INIT: + mText.setVisibility(View.INVISIBLE); + + mProgress.setVisibility(View.GONE); + + mImage.setVisibility(View.VISIBLE); + mImage.setImageResource(R.drawable.mic_slash); + + mSoundIndicator.setVisibility(View.GONE); + mSoundIndicator.stop(); + + mLanguage.setVisibility(View.INVISIBLE); + + mPopupLayout.setBackgroundDrawable(mListeningBorder); + break; + case LISTENING: + mText.setVisibility(View.VISIBLE); + mText.setText(text); + + mProgress.setVisibility(View.GONE); + + mImage.setVisibility(View.GONE); + + mSoundIndicator.setVisibility(View.VISIBLE); + mSoundIndicator.start(); + + Locale locale = SubtypeSwitcher.getInstance().getInputLocale(); + + mLanguage.setVisibility(View.VISIBLE); + mLanguage.setText(SubtypeSwitcher.getFullDisplayName(locale, true)); + + mPopupLayout.setBackgroundDrawable(mListeningBorder); + break; + case WORKING: + + mText.setVisibility(View.VISIBLE); + mText.setText(text); + + mProgress.setVisibility(View.VISIBLE); + + mImage.setVisibility(View.VISIBLE); + + mSoundIndicator.setVisibility(View.GONE); + mSoundIndicator.stop(); + + mLanguage.setVisibility(View.GONE); + + mPopupLayout.setBackgroundDrawable(mWorkingBorder); + break; + case READY: + mText.setVisibility(View.VISIBLE); + mText.setText(text); + + mProgress.setVisibility(View.GONE); + + mImage.setVisibility(View.VISIBLE); + mImage.setImageResource(R.drawable.caution); + + mSoundIndicator.setVisibility(View.GONE); + mSoundIndicator.stop(); + + mLanguage.setVisibility(View.GONE); + + mPopupLayout.setBackgroundDrawable(mErrorBorder); + break; + default: + Log.w(TAG, "Unknown state " + mState); + } + mPopupLayout.requestLayout(); + mButton.setText(btnTxt); + } + + /** + * @return an average abs of the specified buffer. + */ + private static int getAverageAbs(ShortBuffer buffer, int start, int i, int npw) { + int from = start + i * npw; + int end = from + npw; + int total = 0; + for (int x = from; x < end; x++) { + total += Math.abs(buffer.get(x)); + } + return total / npw; + } + + + /** + * Shows waveform of input audio. + * + * Copied from version in VoiceSearch's RecognitionActivity. + * + * TODO: adjust stroke width based on the size of data. + * TODO: use dip rather than pixels. + */ + private void showWave(ShortBuffer waveBuffer, int startPosition, int endPosition) { + final int w = ((View) mImage.getParent()).getWidth(); + final int h = ((View) mImage.getParent()).getHeight(); + if (w <= 0 || h <= 0) { + // view is not visible this time. Skip drawing. + return; + } + final Bitmap b = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); + final Canvas c = new Canvas(b); + final Paint paint = new Paint(); + paint.setColor(0xFFFFFFFF); // 0xAARRGGBB + paint.setAntiAlias(true); + paint.setStyle(Paint.Style.STROKE); + paint.setAlpha(80); + + final PathEffect effect = new CornerPathEffect(3); + paint.setPathEffect(effect); + + final int numSamples = waveBuffer.remaining(); + int endIndex; + if (endPosition == 0) { + endIndex = numSamples; + } else { + endIndex = Math.min(endPosition, numSamples); + } + + int startIndex = startPosition - 2000; // include 250ms before speech + if (startIndex < 0) { + startIndex = 0; + } + final int numSamplePerWave = 200; // 8KHz 25ms = 200 samples + final float scale = 10.0f / 65536.0f; + + final int count = (endIndex - startIndex) / numSamplePerWave; + final float deltaX = 1.0f * w / count; + int yMax = h / 2; + Path path = new Path(); + c.translate(0, yMax); + float x = 0; + path.moveTo(x, 0); + for (int i = 0; i < count; i++) { + final int avabs = getAverageAbs(waveBuffer, startIndex, i , numSamplePerWave); + int sign = ( (i & 01) == 0) ? -1 : 1; + final float y = Math.min(yMax, avabs * h * scale) * sign; + path.lineTo(x, y); + x += deltaX; + path.lineTo(x, y); + } + if (deltaX > 4) { + paint.setStrokeWidth(2); + } else { + paint.setStrokeWidth(Math.max(0, (int) (deltaX -.05))); + } + c.drawPath(path, paint); + mImage.setImageBitmap(b); + } + + public void finish() { + mUiHandler.post(new Runnable() { + @Override + public void run() { + mSoundIndicator.stop(); + } + }); + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/SettingsUtil.java b/java/src/com/android/inputmethod/deprecated/voice/SettingsUtil.java new file mode 100644 index 000000000..855a09a1d --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/SettingsUtil.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import android.content.ContentResolver; +import android.provider.Settings; + +/** + * Utility for retrieving settings from Settings.Secure. + */ +public class SettingsUtil { + /** + * A whitespace-separated list of supported locales for voice input from the keyboard. + */ + public static final String LATIN_IME_VOICE_INPUT_SUPPORTED_LOCALES = + "latin_ime_voice_input_supported_locales"; + + /** + * A whitespace-separated list of recommended app packages for voice input from the + * keyboard. + */ + public static final String LATIN_IME_VOICE_INPUT_RECOMMENDED_PACKAGES = + "latin_ime_voice_input_recommended_packages"; + + /** + * The maximum number of unique days to show the swipe hint for voice input. + */ + public static final String LATIN_IME_VOICE_INPUT_SWIPE_HINT_MAX_DAYS = + "latin_ime_voice_input_swipe_hint_max_days"; + + /** + * The maximum number of times to show the punctuation hint for voice input. + */ + public static final String LATIN_IME_VOICE_INPUT_PUNCTUATION_HINT_MAX_DISPLAYS = + "latin_ime_voice_input_punctuation_hint_max_displays"; + + /** + * Endpointer parameters for voice input from the keyboard. + */ + public static final String LATIN_IME_SPEECH_MINIMUM_LENGTH_MILLIS = + "latin_ime_speech_minimum_length_millis"; + public static final String LATIN_IME_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS = + "latin_ime_speech_input_complete_silence_length_millis"; + public static final String LATIN_IME_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS = + "latin_ime_speech_input_possibly_complete_silence_length_millis"; + + /** + * Min and max volume levels that can be displayed on the "speak now" screen. + */ + public static final String LATIN_IME_MIN_MICROPHONE_LEVEL = + "latin_ime_min_microphone_level"; + public static final String LATIN_IME_MAX_MICROPHONE_LEVEL = + "latin_ime_max_microphone_level"; + + /** + * The number of sentence-level alternates to request of the server. + */ + public static final String LATIN_IME_MAX_VOICE_RESULTS = "latin_ime_max_voice_results"; + + /** + * Get a string-valued setting. + * + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if none can be found + * @return The value of the setting, or defaultValue if it couldn't be found + */ + public static String getSettingsString(ContentResolver cr, String key, String defaultValue) { + String result = Settings.Secure.getString(cr, key); + return (result == null) ? defaultValue : result; + } + + /** + * Get an int-valued setting. + * + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if the setting couldn't be found or parsed + * @return The value of the setting, or defaultValue if it couldn't be found or parsed + */ + public static int getSettingsInt(ContentResolver cr, String key, int defaultValue) { + return Settings.Secure.getInt(cr, key, defaultValue); + } + + /** + * Get a float-valued setting. + * + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if the setting couldn't be found or parsed + * @return The value of the setting, or defaultValue if it couldn't be found or parsed + */ + public static float getSettingsFloat(ContentResolver cr, String key, float defaultValue) { + return Settings.Secure.getFloat(cr, key, defaultValue); + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/SoundIndicator.java b/java/src/com/android/inputmethod/deprecated/voice/SoundIndicator.java new file mode 100644 index 000000000..25b314085 --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/SoundIndicator.java @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Bitmap.Config; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.PorterDuff; +import android.graphics.PorterDuffXfermode; +import android.graphics.Rect; +import android.graphics.drawable.BitmapDrawable; +import android.graphics.drawable.Drawable; +import android.os.Handler; +import android.util.AttributeSet; +import android.widget.ImageView; + +import com.android.inputmethod.latin.R; + +/** + * A widget which shows the volume of audio using a microphone icon + */ +public class SoundIndicator extends ImageView { + @SuppressWarnings("unused") + private static final String TAG = "SoundIndicator"; + + private static final float UP_SMOOTHING_FACTOR = 0.9f; + private static final float DOWN_SMOOTHING_FACTOR = 0.4f; + + private static final float AUDIO_METER_MIN_DB = 7.0f; + private static final float AUDIO_METER_DB_RANGE = 20.0f; + + private static final long FRAME_DELAY = 50; + + private Bitmap mDrawingBuffer; + private Canvas mBufferCanvas; + private Bitmap mEdgeBitmap; + private float mLevel = 0.0f; + private Drawable mFrontDrawable; + private Paint mClearPaint; + private Paint mMultPaint; + private int mEdgeBitmapOffset; + + private Handler mHandler; + + private Runnable mDrawFrame = new Runnable() { + public void run() { + invalidate(); + mHandler.postDelayed(mDrawFrame, FRAME_DELAY); + } + }; + + public SoundIndicator(Context context) { + this(context, null); + } + + public SoundIndicator(Context context, AttributeSet attrs) { + super(context, attrs); + + mFrontDrawable = getDrawable(); + BitmapDrawable edgeDrawable = + (BitmapDrawable) context.getResources().getDrawable(R.drawable.vs_popup_mic_edge); + mEdgeBitmap = edgeDrawable.getBitmap(); + mEdgeBitmapOffset = mEdgeBitmap.getHeight() / 2; + + mDrawingBuffer = + Bitmap.createBitmap(mFrontDrawable.getIntrinsicWidth(), + mFrontDrawable.getIntrinsicHeight(), Config.ARGB_8888); + + mBufferCanvas = new Canvas(mDrawingBuffer); + + // Initialize Paints. + mClearPaint = new Paint(); + mClearPaint.setXfermode(new PorterDuffXfermode(PorterDuff.Mode.CLEAR)); + + mMultPaint = new Paint(); + mMultPaint.setXfermode(new PorterDuffXfermode(PorterDuff.Mode.MULTIPLY)); + + mHandler = new Handler(); + } + + @Override + public void onDraw(Canvas canvas) { + //super.onDraw(canvas); + + float w = getWidth(); + float h = getHeight(); + + // Clear the buffer canvas + mBufferCanvas.drawRect(0, 0, w, h, mClearPaint); + + // Set its clip so we don't draw the front image all the way to the top + Rect clip = new Rect(0, + (int) ((1.0 - mLevel) * (h + mEdgeBitmapOffset)) - mEdgeBitmapOffset, + (int) w, + (int) h); + + mBufferCanvas.save(); + mBufferCanvas.clipRect(clip); + + // Draw the front image + mFrontDrawable.setBounds(new Rect(0, 0, (int) w, (int) h)); + mFrontDrawable.draw(mBufferCanvas); + + mBufferCanvas.restore(); + + // Draw the edge image on top of the buffer image with a multiply mode + mBufferCanvas.drawBitmap(mEdgeBitmap, 0, clip.top, mMultPaint); + + // Draw the buffer image (on top of the background image) + canvas.drawBitmap(mDrawingBuffer, 0, 0, null); + } + + /** + * Sets the sound level + * + * @param rmsdB The level of the sound, in dB. + */ + public void setRmsdB(float rmsdB) { + float level = ((rmsdB - AUDIO_METER_MIN_DB) / AUDIO_METER_DB_RANGE); + + level = Math.min(Math.max(0.0f, level), 1.0f); + + // We smooth towards the new level + if (level > mLevel) { + mLevel = (level - mLevel) * UP_SMOOTHING_FACTOR + mLevel; + } else { + mLevel = (level - mLevel) * DOWN_SMOOTHING_FACTOR + mLevel; + } + invalidate(); + } + + public void start() { + mHandler.post(mDrawFrame); + } + + public void stop() { + mHandler.removeCallbacks(mDrawFrame); + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/VoiceInput.java b/java/src/com/android/inputmethod/deprecated/voice/VoiceInput.java new file mode 100644 index 000000000..b718ebbb7 --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/VoiceInput.java @@ -0,0 +1,685 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import com.android.inputmethod.latin.EditingUtils; +import com.android.inputmethod.latin.LatinImeLogger; +import com.android.inputmethod.latin.R; + +import android.content.ContentResolver; +import android.content.Context; +import android.content.Intent; +import android.content.res.Configuration; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.Message; +import android.os.Parcelable; +import android.speech.RecognitionListener; +import android.speech.RecognizerIntent; +import android.speech.SpeechRecognizer; +import android.util.Log; +import android.view.View; +import android.view.View.OnClickListener; +import android.view.inputmethod.InputConnection; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Speech recognition input, including both user interface and a background + * process to stream audio to the network recognizer. This class supplies a + * View (getView()), which it updates as recognition occurs. The user of this + * class is responsible for making the view visible to the user, as well as + * handling various events returned through UiListener. + */ +public class VoiceInput implements OnClickListener { + private static final String TAG = "VoiceInput"; + private static final String EXTRA_RECOGNITION_CONTEXT = + "android.speech.extras.RECOGNITION_CONTEXT"; + private static final String EXTRA_CALLING_PACKAGE = "calling_package"; + private static final String EXTRA_ALTERNATES = "android.speech.extra.ALTERNATES"; + private static final int MAX_ALT_LIST_LENGTH = 6; + private static boolean DBG = LatinImeLogger.sDBG; + + private static final String DEFAULT_RECOMMENDED_PACKAGES = + "com.android.mms " + + "com.google.android.gm " + + "com.google.android.talk " + + "com.google.android.apps.googlevoice " + + "com.android.email " + + "com.android.browser "; + + // WARNING! Before enabling this, fix the problem with calling getExtractedText() in + // landscape view. It causes Extracted text updates to be rejected due to a token mismatch + public static boolean ENABLE_WORD_CORRECTIONS = true; + + // Dummy word suggestion which means "delete current word" + public static final String DELETE_SYMBOL = " \u00D7 "; // times symbol + + private Whitelist mRecommendedList; + private Whitelist mBlacklist; + + private VoiceInputLogger mLogger; + + // Names of a few extras defined in VoiceSearch's RecognitionController + // Note, the version of voicesearch that shipped in Froyo returns the raw + // RecognitionClientAlternates protocol buffer under the key "alternates", + // so a VS market update must be installed on Froyo devices in order to see + // alternatives. + private static final String ALTERNATES_BUNDLE = "alternates_bundle"; + + // This is copied from the VoiceSearch app. + @SuppressWarnings("unused") + private static final class AlternatesBundleKeys { + public static final String ALTERNATES = "alternates"; + public static final String CONFIDENCE = "confidence"; + public static final String LENGTH = "length"; + public static final String MAX_SPAN_LENGTH = "max_span_length"; + public static final String SPANS = "spans"; + public static final String SPAN_KEY_DELIMITER = ":"; + public static final String START = "start"; + public static final String TEXT = "text"; + } + + // Names of a few intent extras defined in VoiceSearch's RecognitionService. + // These let us tweak the endpointer parameters. + private static final String EXTRA_SPEECH_MINIMUM_LENGTH_MILLIS = + "android.speech.extras.SPEECH_INPUT_MINIMUM_LENGTH_MILLIS"; + private static final String EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS = + "android.speech.extras.SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS"; + private static final String EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS = + "android.speech.extras.SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS"; + + // The usual endpointer default value for input complete silence length is 0.5 seconds, + // but that's used for things like voice search. For dictation-like voice input like this, + // we go with a more liberal value of 1 second. This value will only be used if a value + // is not provided from Gservices. + private static final String INPUT_COMPLETE_SILENCE_LENGTH_DEFAULT_VALUE_MILLIS = "1000"; + + // Used to record part of that state for logging purposes. + public static final int DEFAULT = 0; + public static final int LISTENING = 1; + public static final int WORKING = 2; + public static final int ERROR = 3; + + private int mAfterVoiceInputDeleteCount = 0; + private int mAfterVoiceInputInsertCount = 0; + private int mAfterVoiceInputInsertPunctuationCount = 0; + private int mAfterVoiceInputCursorPos = 0; + private int mAfterVoiceInputSelectionSpan = 0; + + private int mState = DEFAULT; + + private final static int MSG_RESET = 1; + + private final Handler mHandler = new Handler() { + @Override + public void handleMessage(Message msg) { + if (msg.what == MSG_RESET) { + mState = DEFAULT; + mRecognitionView.finish(); + mUiListener.onCancelVoice(); + } + } + }; + + /** + * Events relating to the recognition UI. You must implement these. + */ + public interface UiListener { + + /** + * @param recognitionResults a set of transcripts for what the user + * spoke, sorted by likelihood. + */ + public void onVoiceResults( + List<String> recognitionResults, + Map<String, List<CharSequence>> alternatives); + + /** + * Called when the user cancels speech recognition. + */ + public void onCancelVoice(); + } + + private SpeechRecognizer mSpeechRecognizer; + private RecognitionListener mRecognitionListener; + private RecognitionView mRecognitionView; + private UiListener mUiListener; + private Context mContext; + + /** + * @param context the service or activity in which we're running. + * @param uiHandler object to receive events from VoiceInput. + */ + public VoiceInput(Context context, UiListener uiHandler) { + mLogger = VoiceInputLogger.getLogger(context); + mRecognitionListener = new ImeRecognitionListener(); + mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(context); + mSpeechRecognizer.setRecognitionListener(mRecognitionListener); + mUiListener = uiHandler; + mContext = context; + newView(); + + String recommendedPackages = SettingsUtil.getSettingsString( + context.getContentResolver(), + SettingsUtil.LATIN_IME_VOICE_INPUT_RECOMMENDED_PACKAGES, + DEFAULT_RECOMMENDED_PACKAGES); + + mRecommendedList = new Whitelist(); + for (String recommendedPackage : recommendedPackages.split("\\s+")) { + mRecommendedList.addApp(recommendedPackage); + } + + mBlacklist = new Whitelist(); + mBlacklist.addApp("com.google.android.setupwizard"); + } + + public void setCursorPos(int pos) { + mAfterVoiceInputCursorPos = pos; + } + + public int getCursorPos() { + return mAfterVoiceInputCursorPos; + } + + public void setSelectionSpan(int span) { + mAfterVoiceInputSelectionSpan = span; + } + + public int getSelectionSpan() { + return mAfterVoiceInputSelectionSpan; + } + + public void incrementTextModificationDeleteCount(int count){ + mAfterVoiceInputDeleteCount += count; + // Send up intents for other text modification types + if (mAfterVoiceInputInsertCount > 0) { + logTextModifiedByTypingInsertion(mAfterVoiceInputInsertCount); + mAfterVoiceInputInsertCount = 0; + } + if (mAfterVoiceInputInsertPunctuationCount > 0) { + logTextModifiedByTypingInsertionPunctuation(mAfterVoiceInputInsertPunctuationCount); + mAfterVoiceInputInsertPunctuationCount = 0; + } + + } + + public void incrementTextModificationInsertCount(int count){ + mAfterVoiceInputInsertCount += count; + if (mAfterVoiceInputSelectionSpan > 0) { + // If text was highlighted before inserting the char, count this as + // a delete. + mAfterVoiceInputDeleteCount += mAfterVoiceInputSelectionSpan; + } + // Send up intents for other text modification types + if (mAfterVoiceInputDeleteCount > 0) { + logTextModifiedByTypingDeletion(mAfterVoiceInputDeleteCount); + mAfterVoiceInputDeleteCount = 0; + } + if (mAfterVoiceInputInsertPunctuationCount > 0) { + logTextModifiedByTypingInsertionPunctuation(mAfterVoiceInputInsertPunctuationCount); + mAfterVoiceInputInsertPunctuationCount = 0; + } + } + + public void incrementTextModificationInsertPunctuationCount(int count){ + mAfterVoiceInputInsertPunctuationCount += count; + if (mAfterVoiceInputSelectionSpan > 0) { + // If text was highlighted before inserting the char, count this as + // a delete. + mAfterVoiceInputDeleteCount += mAfterVoiceInputSelectionSpan; + } + // Send up intents for aggregated non-punctuation insertions + if (mAfterVoiceInputDeleteCount > 0) { + logTextModifiedByTypingDeletion(mAfterVoiceInputDeleteCount); + mAfterVoiceInputDeleteCount = 0; + } + if (mAfterVoiceInputInsertCount > 0) { + logTextModifiedByTypingInsertion(mAfterVoiceInputInsertCount); + mAfterVoiceInputInsertCount = 0; + } + } + + public void flushAllTextModificationCounters() { + if (mAfterVoiceInputInsertCount > 0) { + logTextModifiedByTypingInsertion(mAfterVoiceInputInsertCount); + mAfterVoiceInputInsertCount = 0; + } + if (mAfterVoiceInputDeleteCount > 0) { + logTextModifiedByTypingDeletion(mAfterVoiceInputDeleteCount); + mAfterVoiceInputDeleteCount = 0; + } + if (mAfterVoiceInputInsertPunctuationCount > 0) { + logTextModifiedByTypingInsertionPunctuation(mAfterVoiceInputInsertPunctuationCount); + mAfterVoiceInputInsertPunctuationCount = 0; + } + } + + /** + * The configuration of the IME changed and may have caused the views to be layed out + * again. Restore the state of the recognition view. + */ + public void onConfigurationChanged(Configuration configuration) { + mRecognitionView.restoreState(); + mRecognitionView.getView().dispatchConfigurationChanged(configuration); + } + + /** + * @return true if field is blacklisted for voice + */ + public boolean isBlacklistedField(FieldContext context) { + return mBlacklist.matches(context); + } + + /** + * Used to decide whether to show voice input hints for this field, etc. + * + * @return true if field is recommended for voice + */ + public boolean isRecommendedField(FieldContext context) { + return mRecommendedList.matches(context); + } + + /** + * Start listening for speech from the user. This will grab the microphone + * and start updating the view provided by getView(). It is the caller's + * responsibility to ensure that the view is visible to the user at this stage. + * + * @param context the same FieldContext supplied to voiceIsEnabled() + * @param swipe whether this voice input was started by swipe, for logging purposes + */ + public void startListening(FieldContext context, boolean swipe) { + if (DBG) { + Log.d(TAG, "startListening: " + context); + } + + if (mState != DEFAULT) { + Log.w(TAG, "startListening in the wrong status " + mState); + } + + // If everything works ok, the voice input should be already in the correct state. As this + // class can be called by third-party, we call reset just to be on the safe side. + reset(); + + Locale locale = Locale.getDefault(); + String localeString = locale.getLanguage() + "-" + locale.getCountry(); + + mLogger.start(localeString, swipe); + + mState = LISTENING; + + mRecognitionView.showInitializing(); + startListeningAfterInitialization(context); + } + + /** + * Called only when the recognition manager's initialization completed + * + * @param context context with which {@link #startListening(FieldContext, boolean)} was executed + */ + private void startListeningAfterInitialization(FieldContext context) { + Intent intent = makeIntent(); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, ""); + intent.putExtra(EXTRA_RECOGNITION_CONTEXT, context.getBundle()); + intent.putExtra(EXTRA_CALLING_PACKAGE, "VoiceIME"); + intent.putExtra(EXTRA_ALTERNATES, true); + intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, + SettingsUtil.getSettingsInt( + mContext.getContentResolver(), + SettingsUtil.LATIN_IME_MAX_VOICE_RESULTS, + 1)); + // Get endpointer params from Gservices. + // TODO: Consider caching these values for improved performance on slower devices. + final ContentResolver cr = mContext.getContentResolver(); + putEndpointerExtra( + cr, + intent, + SettingsUtil.LATIN_IME_SPEECH_MINIMUM_LENGTH_MILLIS, + EXTRA_SPEECH_MINIMUM_LENGTH_MILLIS, + null /* rely on endpointer default */); + putEndpointerExtra( + cr, + intent, + SettingsUtil.LATIN_IME_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, + EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, + INPUT_COMPLETE_SILENCE_LENGTH_DEFAULT_VALUE_MILLIS + /* our default value is different from the endpointer's */); + putEndpointerExtra( + cr, + intent, + SettingsUtil. + LATIN_IME_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, + EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, + null /* rely on endpointer default */); + + mSpeechRecognizer.startListening(intent); + } + + /** + * Gets the value of the provided Gservices key, attempts to parse it into a long, + * and if successful, puts the long value as an extra in the provided intent. + */ + private void putEndpointerExtra(ContentResolver cr, Intent i, + String gservicesKey, String intentExtraKey, String defaultValue) { + long l = -1; + String s = SettingsUtil.getSettingsString(cr, gservicesKey, defaultValue); + if (s != null) { + try { + l = Long.valueOf(s); + } catch (NumberFormatException e) { + Log.e(TAG, "could not parse value for " + gservicesKey + ": " + s); + } + } + + if (l != -1) i.putExtra(intentExtraKey, l); + } + + public void destroy() { + mSpeechRecognizer.destroy(); + } + + /** + * Creates a new instance of the view that is returned by {@link #getView()} + * Clients should use this when a previously returned view is stuck in a + * layout that is being thrown away and a new one is need to show to the + * user. + */ + public void newView() { + mRecognitionView = new RecognitionView(mContext, this); + } + + /** + * @return a view that shows the recognition flow--e.g., "Speak now" and + * "working" dialogs. + */ + public View getView() { + return mRecognitionView.getView(); + } + + /** + * Handle the cancel button. + */ + @Override + public void onClick(View view) { + switch(view.getId()) { + case R.id.button: + cancel(); + break; + } + } + + public void logTextModifiedByTypingInsertion(int length) { + mLogger.textModifiedByTypingInsertion(length); + } + + public void logTextModifiedByTypingInsertionPunctuation(int length) { + mLogger.textModifiedByTypingInsertionPunctuation(length); + } + + public void logTextModifiedByTypingDeletion(int length) { + mLogger.textModifiedByTypingDeletion(length); + } + + public void logTextModifiedByChooseSuggestion(String suggestion, int index, + String wordSeparators, InputConnection ic) { + String wordToBeReplaced = EditingUtils.getWordAtCursor(ic, wordSeparators); + // If we enable phrase-based alternatives, only send up the first word + // in suggestion and wordToBeReplaced. + mLogger.textModifiedByChooseSuggestion(suggestion.length(), wordToBeReplaced.length(), + index, wordToBeReplaced, suggestion); + } + + public void logKeyboardWarningDialogShown() { + mLogger.keyboardWarningDialogShown(); + } + + public void logKeyboardWarningDialogDismissed() { + mLogger.keyboardWarningDialogDismissed(); + } + + public void logKeyboardWarningDialogOk() { + mLogger.keyboardWarningDialogOk(); + } + + public void logKeyboardWarningDialogCancel() { + mLogger.keyboardWarningDialogCancel(); + } + + public void logSwipeHintDisplayed() { + mLogger.swipeHintDisplayed(); + } + + public void logPunctuationHintDisplayed() { + mLogger.punctuationHintDisplayed(); + } + + public void logVoiceInputDelivered(int length) { + mLogger.voiceInputDelivered(length); + } + + public void logInputEnded() { + mLogger.inputEnded(); + } + + public void flushLogs() { + mLogger.flush(); + } + + private static Intent makeIntent() { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + + // On Cupcake, use VoiceIMEHelper since VoiceSearch doesn't support. + // On Donut, always use VoiceSearch, since VoiceIMEHelper and + // VoiceSearch may conflict. + if (Build.VERSION.RELEASE.equals("1.5")) { + intent = intent.setClassName( + "com.google.android.voiceservice", + "com.google.android.voiceservice.IMERecognitionService"); + } else { + intent = intent.setClassName( + "com.google.android.voicesearch", + "com.google.android.voicesearch.RecognitionService"); + } + + return intent; + } + + /** + * Reset the current voice recognition. + */ + public void reset() { + if (mState != DEFAULT) { + mState = DEFAULT; + + // Remove all pending tasks (e.g., timers to cancel voice input) + mHandler.removeMessages(MSG_RESET); + + mSpeechRecognizer.cancel(); + mRecognitionView.finish(); + } + } + + /** + * Cancel in-progress speech recognition. + */ + public void cancel() { + switch (mState) { + case LISTENING: + mLogger.cancelDuringListening(); + break; + case WORKING: + mLogger.cancelDuringWorking(); + break; + case ERROR: + mLogger.cancelDuringError(); + break; + } + + reset(); + mUiListener.onCancelVoice(); + } + + private int getErrorStringId(int errorType, boolean endpointed) { + switch (errorType) { + // We use CLIENT_ERROR to signify that voice search is not available on the device. + case SpeechRecognizer.ERROR_CLIENT: + return R.string.voice_not_installed; + case SpeechRecognizer.ERROR_NETWORK: + return R.string.voice_network_error; + case SpeechRecognizer.ERROR_NETWORK_TIMEOUT: + return endpointed ? + R.string.voice_network_error : R.string.voice_too_much_speech; + case SpeechRecognizer.ERROR_AUDIO: + return R.string.voice_audio_error; + case SpeechRecognizer.ERROR_SERVER: + return R.string.voice_server_error; + case SpeechRecognizer.ERROR_SPEECH_TIMEOUT: + return R.string.voice_speech_timeout; + case SpeechRecognizer.ERROR_NO_MATCH: + return R.string.voice_no_match; + default: return R.string.voice_error; + } + } + + private void onError(int errorType, boolean endpointed) { + Log.i(TAG, "error " + errorType); + mLogger.error(errorType); + onError(mContext.getString(getErrorStringId(errorType, endpointed))); + } + + private void onError(String error) { + mState = ERROR; + mRecognitionView.showError(error); + // Wait a couple seconds and then automatically dismiss message. + mHandler.sendMessageDelayed(Message.obtain(mHandler, MSG_RESET), 2000); + } + + private class ImeRecognitionListener implements RecognitionListener { + // Waveform data + final ByteArrayOutputStream mWaveBuffer = new ByteArrayOutputStream(); + int mSpeechStart; + private boolean mEndpointed = false; + + @Override + public void onReadyForSpeech(Bundle noiseParams) { + mRecognitionView.showListening(); + } + + @Override + public void onBeginningOfSpeech() { + mEndpointed = false; + mSpeechStart = mWaveBuffer.size(); + } + + @Override + public void onRmsChanged(float rmsdB) { + mRecognitionView.updateVoiceMeter(rmsdB); + } + + @Override + public void onBufferReceived(byte[] buf) { + try { + mWaveBuffer.write(buf); + } catch (IOException e) { + // ignore. + } + } + + @Override + public void onEndOfSpeech() { + mEndpointed = true; + mState = WORKING; + mRecognitionView.showWorking(mWaveBuffer, mSpeechStart, mWaveBuffer.size()); + } + + @Override + public void onError(int errorType) { + mState = ERROR; + VoiceInput.this.onError(errorType, mEndpointed); + } + + @Override + public void onResults(Bundle resultsBundle) { + List<String> results = resultsBundle + .getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION); + // VS Market update is needed for IME froyo clients to access the alternatesBundle + // TODO: verify this. + Bundle alternatesBundle = resultsBundle.getBundle(ALTERNATES_BUNDLE); + mState = DEFAULT; + + final Map<String, List<CharSequence>> alternatives = + new HashMap<String, List<CharSequence>>(); + + if (ENABLE_WORD_CORRECTIONS && alternatesBundle != null && results.size() > 0) { + // Use the top recognition result to map each alternative's start:length to a word. + String[] words = results.get(0).split(" "); + Bundle spansBundle = alternatesBundle.getBundle(AlternatesBundleKeys.SPANS); + for (String key : spansBundle.keySet()) { + // Get the word for which these alternates correspond to. + Bundle spanBundle = spansBundle.getBundle(key); + int start = spanBundle.getInt(AlternatesBundleKeys.START); + int length = spanBundle.getInt(AlternatesBundleKeys.LENGTH); + // Only keep single-word based alternatives. + if (length == 1 && start < words.length) { + // Get the alternatives associated with the span. + // If a word appears twice in a recognition result, + // concatenate the alternatives for the word. + List<CharSequence> altList = alternatives.get(words[start]); + if (altList == null) { + altList = new ArrayList<CharSequence>(); + alternatives.put(words[start], altList); + } + Parcelable[] alternatesArr = spanBundle + .getParcelableArray(AlternatesBundleKeys.ALTERNATES); + for (int j = 0; j < alternatesArr.length && + altList.size() < MAX_ALT_LIST_LENGTH; j++) { + Bundle alternateBundle = (Bundle) alternatesArr[j]; + String alternate = alternateBundle.getString(AlternatesBundleKeys.TEXT); + // Don't allow duplicates in the alternates list. + if (!altList.contains(alternate)) { + altList.add(alternate); + } + } + } + } + } + + if (results.size() > 5) { + results = results.subList(0, 5); + } + mUiListener.onVoiceResults(results, alternatives); + mRecognitionView.finish(); + } + + @Override + public void onPartialResults(final Bundle partialResults) { + // currently - do nothing + } + + @Override + public void onEvent(int eventType, Bundle params) { + // do nothing - reserved for events that might be added in the future + } + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/VoiceInputLogger.java b/java/src/com/android/inputmethod/deprecated/voice/VoiceInputLogger.java new file mode 100644 index 000000000..22e8207bf --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/VoiceInputLogger.java @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import com.android.common.speech.LoggingEvents; +import com.android.inputmethod.deprecated.compat.VoiceInputLoggerCompatUtils; + +import android.content.Context; +import android.content.Intent; + +/** + * Provides the logging facility for voice input events. This fires broadcasts back to + * the voice search app which then logs on our behalf. + * + * Note that debug console logging does not occur in this class. If you want to + * see console output of these logging events, there is a boolean switch to turn + * on on the VoiceSearch side. + */ +public class VoiceInputLogger { + @SuppressWarnings("unused") + private static final String TAG = VoiceInputLogger.class.getSimpleName(); + + private static VoiceInputLogger sVoiceInputLogger; + + private final Context mContext; + + // The base intent used to form all broadcast intents to the logger + // in VoiceSearch. + private final Intent mBaseIntent; + + // This flag is used to indicate when there are voice events that + // need to be flushed. + private boolean mHasLoggingInfo = false; + + /** + * Returns the singleton of the logger. + * + * @param contextHint a hint context used when creating the logger instance. + * Ignored if the singleton instance already exists. + */ + public static synchronized VoiceInputLogger getLogger(Context contextHint) { + if (sVoiceInputLogger == null) { + sVoiceInputLogger = new VoiceInputLogger(contextHint); + } + return sVoiceInputLogger; + } + + public VoiceInputLogger(Context context) { + mContext = context; + + mBaseIntent = new Intent(LoggingEvents.ACTION_LOG_EVENT); + mBaseIntent.putExtra(LoggingEvents.EXTRA_APP_NAME, LoggingEvents.VoiceIme.APP_NAME); + } + + private Intent newLoggingBroadcast(int event) { + Intent i = new Intent(mBaseIntent); + i.putExtra(LoggingEvents.EXTRA_EVENT, event); + return i; + } + + public void flush() { + if (hasLoggingInfo()) { + Intent i = new Intent(mBaseIntent); + i.putExtra(LoggingEvents.EXTRA_FLUSH, true); + mContext.sendBroadcast(i); + setHasLoggingInfo(false); + } + } + + public void keyboardWarningDialogShown() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_SHOWN)); + } + + public void keyboardWarningDialogDismissed() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_DISMISSED)); + } + + public void keyboardWarningDialogOk() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_OK)); + } + + public void keyboardWarningDialogCancel() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_CANCEL)); + } + + public void settingsWarningDialogShown() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_SHOWN)); + } + + public void settingsWarningDialogDismissed() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_DISMISSED)); + } + + public void settingsWarningDialogOk() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_OK)); + } + + public void settingsWarningDialogCancel() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_CANCEL)); + } + + public void swipeHintDisplayed() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.SWIPE_HINT_DISPLAYED)); + } + + public void cancelDuringListening() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.CANCEL_DURING_LISTENING)); + } + + public void cancelDuringWorking() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.CANCEL_DURING_WORKING)); + } + + public void cancelDuringError() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.CANCEL_DURING_ERROR)); + } + + public void punctuationHintDisplayed() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.PUNCTUATION_HINT_DISPLAYED)); + } + + public void error(int code) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.ERROR); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_ERROR_CODE, code); + mContext.sendBroadcast(i); + } + + public void start(String locale, boolean swipe) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.START); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_START_LOCALE, locale); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_START_SWIPE, swipe); + i.putExtra(LoggingEvents.EXTRA_TIMESTAMP, System.currentTimeMillis()); + mContext.sendBroadcast(i); + } + + public void voiceInputDelivered(int length) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.VOICE_INPUT_DELIVERED); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_LENGTH, length); + mContext.sendBroadcast(i); + } + + public void textModifiedByTypingInsertion(int length) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.TEXT_MODIFIED); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_LENGTH, length); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_TYPE, + LoggingEvents.VoiceIme.TEXT_MODIFIED_TYPE_TYPING_INSERTION); + mContext.sendBroadcast(i); + } + + public void textModifiedByTypingInsertionPunctuation(int length) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.TEXT_MODIFIED); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_LENGTH, length); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_TYPE, + LoggingEvents.VoiceIme.TEXT_MODIFIED_TYPE_TYPING_INSERTION_PUNCTUATION); + mContext.sendBroadcast(i); + } + + public void textModifiedByTypingDeletion(int length) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.TEXT_MODIFIED); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_LENGTH, length); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_TYPE, + LoggingEvents.VoiceIme.TEXT_MODIFIED_TYPE_TYPING_DELETION); + + mContext.sendBroadcast(i); + } + + + public void textModifiedByChooseSuggestion(int suggestionLength, int replacedPhraseLength, + int index, String before, String after) { + setHasLoggingInfo(true); + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.TEXT_MODIFIED); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_LENGTH, suggestionLength); + i.putExtra(VoiceInputLoggerCompatUtils.EXTRA_TEXT_REPLACED_LENGTH, replacedPhraseLength); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_TEXT_MODIFIED_TYPE, + LoggingEvents.VoiceIme.TEXT_MODIFIED_TYPE_CHOOSE_SUGGESTION); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_N_BEST_CHOOSE_INDEX, index); + i.putExtra(VoiceInputLoggerCompatUtils.EXTRA_BEFORE_N_BEST_CHOOSE, before); + i.putExtra(VoiceInputLoggerCompatUtils.EXTRA_AFTER_N_BEST_CHOOSE, after); + mContext.sendBroadcast(i); + } + + public void inputEnded() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.INPUT_ENDED)); + } + + public void voiceInputSettingEnabled() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.VOICE_INPUT_SETTING_ENABLED)); + } + + public void voiceInputSettingDisabled() { + setHasLoggingInfo(true); + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.VOICE_INPUT_SETTING_DISABLED)); + } + + private void setHasLoggingInfo(boolean hasLoggingInfo) { + mHasLoggingInfo = hasLoggingInfo; + // If applications that call UserHappinessSignals.userAcceptedImeText + // make that call after VoiceInputLogger.flush() calls this method with false, we + // will lose those happiness signals. For example, consider the gmail sequence: + // 1. compose message + // 2. speak message into message field + // 3. type subject into subject field + // 4. press send + // We will NOT get the signal that the user accepted the voice inputted message text + // because when the user tapped on the subject field, the ime's flush will be triggered + // and the hasLoggingInfo will be then set to false. So by the time the user hits send + // we have essentially forgotten about any voice input. + // However the following (more common) use case is properly logged + // 1. compose message + // 2. type subject in subject field + // 3. speak message in message field + // 4. press send + VoiceInputLoggerCompatUtils.setHasVoiceLoggingInfoCompat(hasLoggingInfo); + } + + private boolean hasLoggingInfo(){ + return mHasLoggingInfo; + } + +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/WaveformImage.java b/java/src/com/android/inputmethod/deprecated/voice/WaveformImage.java new file mode 100644 index 000000000..8ed279f42 --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/WaveformImage.java @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2008-2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Paint; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; + +/** + * Utility class to draw a waveform into a bitmap, given a byte array + * that represents the waveform as a sequence of 16-bit integers. + * Adapted from RecognitionActivity.java. + */ +public class WaveformImage { + private static final int SAMPLING_RATE = 8000; + + private WaveformImage() { + // Intentional empty constructor. + } + + public static Bitmap drawWaveform( + ByteArrayOutputStream waveBuffer, int w, int h, int start, int end) { + final Bitmap b = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); + final Canvas c = new Canvas(b); + final Paint paint = new Paint(); + paint.setColor(0xFFFFFFFF); // 0xRRGGBBAA + paint.setAntiAlias(true); + paint.setStrokeWidth(0); + + final ShortBuffer buf = ByteBuffer + .wrap(waveBuffer.toByteArray()) + .order(ByteOrder.nativeOrder()) + .asShortBuffer(); + buf.position(0); + + final int numSamples = waveBuffer.size() / 2; + final int delay = (SAMPLING_RATE * 100 / 1000); + int endIndex = end / 2 + delay; + if (end == 0 || endIndex >= numSamples) { + endIndex = numSamples; + } + int index = start / 2 - delay; + if (index < 0) { + index = 0; + } + final int size = endIndex - index; + int numSamplePerPixel = 32; + int delta = size / (numSamplePerPixel * w); + if (delta == 0) { + numSamplePerPixel = size / w; + delta = 1; + } + + final float scale = 3.5f / 65536.0f; + // do one less column to make sure we won't read past + // the buffer. + try { + for (int i = 0; i < w - 1 ; i++) { + final float x = i; + for (int j = 0; j < numSamplePerPixel; j++) { + final short s = buf.get(index); + final float y = (h / 2) - (s * h * scale); + c.drawPoint(x, y, paint); + index += delta; + } + } + } catch (IndexOutOfBoundsException e) { + // this can happen, but we don't care + } + + return b; + } +} diff --git a/java/src/com/android/inputmethod/deprecated/voice/Whitelist.java b/java/src/com/android/inputmethod/deprecated/voice/Whitelist.java new file mode 100644 index 000000000..6c5f52ae2 --- /dev/null +++ b/java/src/com/android/inputmethod/deprecated/voice/Whitelist.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.deprecated.voice; + +import android.os.Bundle; + +import java.util.ArrayList; +import java.util.List; + +/** + * A set of text fields where speech has been explicitly enabled. + */ +public class Whitelist { + private List<Bundle> mConditions; + + public Whitelist() { + mConditions = new ArrayList<Bundle>(); + } + + public Whitelist(List<Bundle> conditions) { + this.mConditions = conditions; + } + + public void addApp(String app) { + Bundle bundle = new Bundle(); + bundle.putString("packageName", app); + mConditions.add(bundle); + } + + /** + * @return true if the field is a member of the whitelist. + */ + public boolean matches(FieldContext context) { + for (Bundle condition : mConditions) { + if (matches(condition, context.getBundle())) { + return true; + } + } + return false; + } + + /** + * @return true of all values in condition are matched by a value + * in target. + */ + private boolean matches(Bundle condition, Bundle target) { + for (String key : condition.keySet()) { + if (!condition.getString(key).equals(target.getString(key))) { + return false; + } + } + return true; + } +} |