diff options
Diffstat (limited to 'src')
20 files changed, 2770 insertions, 78 deletions
diff --git a/src/com/android/inputmethod/latin/CandidateView.java b/src/com/android/inputmethod/latin/CandidateView.java index f397363c3..a31714eaf 100755 --- a/src/com/android/inputmethod/latin/CandidateView.java +++ b/src/com/android/inputmethod/latin/CandidateView.java @@ -113,7 +113,7 @@ public class CandidateView extends View { public CandidateView(Context context, AttributeSet attrs) { super(context, attrs); mSelectionHighlight = context.getResources().getDrawable( - com.android.internal.R.drawable.list_selector_background_pressed); + R.drawable.list_selector_background_pressed); LayoutInflater inflate = (LayoutInflater) context diff --git a/src/com/android/inputmethod/latin/Hints.java b/src/com/android/inputmethod/latin/Hints.java new file mode 100644 index 000000000..109d3f08e --- /dev/null +++ b/src/com/android/inputmethod/latin/Hints.java @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.latin; + +import com.android.inputmethod.voice.GoogleSettingsUtil; + +import android.content.ContentResolver; +import android.content.Context; +import android.content.SharedPreferences; +import android.preference.PreferenceManager; +import android.view.inputmethod.InputConnection; + +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; + +/** + * Logic to determine when to display hints on usage to the user. + */ +public class Hints { + public interface Display { + public void showHint(int viewResource); + } + + private static final String TAG = "Hints"; + private static final String PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN = + "voice_hint_num_unique_days_shown"; + private static final String PREF_VOICE_HINT_LAST_TIME_SHOWN = + "voice_hint_last_time_shown"; + private static final String PREF_VOICE_INPUT_LAST_TIME_USED = + "voice_input_last_time_used"; + private static final String PREF_VOICE_PUNCTUATION_HINT_VIEW_COUNT = + "voice_punctuation_hint_view_count"; + private static final int DEFAULT_SWIPE_HINT_MAX_DAYS_TO_SHOW = 7; + private static final int DEFAULT_PUNCTUATION_HINT_MAX_DISPLAYS = 7; + + private Context mContext; + private Display mDisplay; + private boolean mVoiceResultContainedPunctuation; + private int mSwipeHintMaxDaysToShow; + private int mPunctuationHintMaxDisplays; + + // Only show punctuation hint if voice result did not contain punctuation. + static final Map<CharSequence, String> SPEAKABLE_PUNCTUATION + = new HashMap<CharSequence, String>(); + static { + SPEAKABLE_PUNCTUATION.put(",", "comma"); + SPEAKABLE_PUNCTUATION.put(".", "period"); + SPEAKABLE_PUNCTUATION.put("?", "question mark"); + } + + public Hints(Context context, Display display) { + mContext = context; + mDisplay = display; + + ContentResolver cr = mContext.getContentResolver(); + mSwipeHintMaxDaysToShow = GoogleSettingsUtil.getGservicesInt( + cr, + GoogleSettingsUtil.LATIN_IME_VOICE_INPUT_SWIPE_HINT_MAX_DAYS, + DEFAULT_SWIPE_HINT_MAX_DAYS_TO_SHOW); + mPunctuationHintMaxDisplays = GoogleSettingsUtil.getGservicesInt( + cr, + GoogleSettingsUtil.LATIN_IME_VOICE_INPUT_PUNCTUATION_HINT_MAX_DISPLAYS, + DEFAULT_PUNCTUATION_HINT_MAX_DISPLAYS); + } + + public boolean showSwipeHintIfNecessary(boolean fieldRecommended) { + if (fieldRecommended && shouldShowSwipeHint()) { + showHint(R.layout.voice_swipe_hint); + return true; + } + + return false; + } + + public boolean showPunctuationHintIfNecessary(InputConnection ic) { + if (!mVoiceResultContainedPunctuation + && ic != null + && getAndIncrementPref(PREF_VOICE_PUNCTUATION_HINT_VIEW_COUNT) + < mPunctuationHintMaxDisplays) { + CharSequence charBeforeCursor = ic.getTextBeforeCursor(1, 0); + if (SPEAKABLE_PUNCTUATION.containsKey(charBeforeCursor)) { + showHint(R.layout.voice_punctuation_hint); + return true; + } + } + + return false; + } + + public void registerVoiceResult(String text) { + // Update the current time as the last time voice input was used. + SharedPreferences.Editor editor = + PreferenceManager.getDefaultSharedPreferences(mContext).edit(); + editor.putLong(PREF_VOICE_INPUT_LAST_TIME_USED, System.currentTimeMillis()); + editor.commit(); + + mVoiceResultContainedPunctuation = false; + for (CharSequence s : SPEAKABLE_PUNCTUATION.keySet()) { + if (text.indexOf(s.toString()) >= 0) { + mVoiceResultContainedPunctuation = true; + break; + } + } + } + + private boolean shouldShowSwipeHint() { + SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(mContext); + + int numUniqueDaysShown = sp.getInt(PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN, 0); + + // If we've already shown the hint for enough days, we'll return false. + if (numUniqueDaysShown < mSwipeHintMaxDaysToShow) { + + long lastTimeVoiceWasUsed = sp.getLong(PREF_VOICE_INPUT_LAST_TIME_USED, 0); + + // If the user has used voice today, we'll return false. (We don't show the hint on + // any day that the user has already used voice.) + if (!isFromToday(lastTimeVoiceWasUsed)) { + return true; + } + } + + return false; + } + + /** + * Determines whether the provided time is from some time today (i.e., this day, month, + * and year). + */ + private boolean isFromToday(long timeInMillis) { + if (timeInMillis == 0) return false; + + Calendar today = Calendar.getInstance(); + today.setTimeInMillis(System.currentTimeMillis()); + + Calendar timestamp = Calendar.getInstance(); + timestamp.setTimeInMillis(timeInMillis); + + return (today.get(Calendar.YEAR) == timestamp.get(Calendar.YEAR) && + today.get(Calendar.DAY_OF_MONTH) == timestamp.get(Calendar.DAY_OF_MONTH) && + today.get(Calendar.MONTH) == timestamp.get(Calendar.MONTH)); + } + + private void showHint(int hintViewResource) { + SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(mContext); + + int numUniqueDaysShown = sp.getInt(PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN, 0); + long lastTimeHintWasShown = sp.getLong(PREF_VOICE_HINT_LAST_TIME_SHOWN, 0); + + // If this is the first time the hint is being shown today, increase the saved values + // to represent that. We don't need to increase the last time the hint was shown unless + // it is a different day from the current value. + if (!isFromToday(lastTimeHintWasShown)) { + SharedPreferences.Editor editor = sp.edit(); + editor.putInt(PREF_VOICE_HINT_NUM_UNIQUE_DAYS_SHOWN, numUniqueDaysShown + 1); + editor.putLong(PREF_VOICE_HINT_LAST_TIME_SHOWN, System.currentTimeMillis()); + editor.commit(); + } + + if (mDisplay != null) { + mDisplay.showHint(hintViewResource); + } + } + + private int getAndIncrementPref(String pref) { + SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(mContext); + int value = sp.getInt(pref, 0); + SharedPreferences.Editor editor = sp.edit(); + editor.putInt(pref, value + 1); + editor.commit(); + return value; + } +} diff --git a/src/com/android/inputmethod/latin/KeyboardSwitcher.java b/src/com/android/inputmethod/latin/KeyboardSwitcher.java index 03b008e34..aa52c0381 100644 --- a/src/com/android/inputmethod/latin/KeyboardSwitcher.java +++ b/src/com/android/inputmethod/latin/KeyboardSwitcher.java @@ -48,6 +48,13 @@ public class KeyboardSwitcher { private static final int SYMBOLS_MODE_STATE_SYMBOL = 2; LatinKeyboardView mInputView; + private static final int[] ALPHABET_MODES = { + KEYBOARDMODE_NORMAL, + KEYBOARDMODE_URL, + KEYBOARDMODE_EMAIL, + KEYBOARDMODE_IM}; + + //LatinIME mContext; Context mContext; InputMethodService mInputMethodService; @@ -56,11 +63,17 @@ public class KeyboardSwitcher { private KeyboardId mCurrentId; private Map<KeyboardId, LatinKeyboard> mKeyboards; - - private int mMode; + + /** + * Maps keyboard mode to the equivalent mode with voice. + */ + private Map<Integer, Integer> mModeToVoice; + + private int mMode; /** One of the MODE_XXX values */ private int mImeOptions; private int mTextMode = MODE_TEXT_QWERTY; private boolean mIsSymbols; + private boolean mHasVoice; private boolean mPreferSymbols; private int mSymbolsModeState = SYMBOLS_MODE_STATE_NONE; @@ -73,6 +86,11 @@ public class KeyboardSwitcher { mKeyboards = new HashMap<KeyboardId, LatinKeyboard>(); mSymbolsId = new KeyboardId(R.xml.kbd_symbols); mSymbolsShiftedId = new KeyboardId(R.xml.kbd_symbols_shift); + mModeToVoice = new HashMap<Integer, Integer>(); + mModeToVoice.put(R.id.mode_normal, R.id.mode_normal_voice); + mModeToVoice.put(R.id.mode_url, R.id.mode_url_voice); + mModeToVoice.put(R.id.mode_email, R.id.mode_email_voice); + mModeToVoice.put(R.id.mode_im, R.id.mode_im_voice); mInputMethodService = ims; } @@ -110,12 +128,12 @@ public class KeyboardSwitcher { */ private static class KeyboardId { public int mXml; - public int mMode; + public int mKeyboardMode; /** A KEYBOARDMODE_XXX value */ public boolean mEnableShiftLock; public KeyboardId(int xml, int mode, boolean enableShiftLock) { this.mXml = xml; - this.mMode = mode; + this.mKeyboardMode = mode; this.mEnableShiftLock = enableShiftLock; } @@ -128,27 +146,40 @@ public class KeyboardSwitcher { } public boolean equals(KeyboardId other) { - return other.mXml == this.mXml && other.mMode == this.mMode; + return other.mXml == this.mXml + && other.mKeyboardMode == this.mKeyboardMode + && other.mEnableShiftLock == this.mEnableShiftLock; } public int hashCode() { - return (mXml + 1) * (mMode + 1) * (mEnableShiftLock ? 2 : 1); + return (mXml + 1) * (mKeyboardMode + 1) * (mEnableShiftLock ? 2 : 1); } } - void setKeyboardMode(int mode, int imeOptions) { + void setVoiceMode(boolean enableVoice) { + setKeyboardMode(mMode, mImeOptions, enableVoice, mIsSymbols); + } + + void setKeyboardMode(int mode, int imeOptions, boolean enableVoice) { mSymbolsModeState = SYMBOLS_MODE_STATE_NONE; mPreferSymbols = mode == MODE_SYMBOLS; - setKeyboardMode(mode == MODE_SYMBOLS ? MODE_TEXT : mode, imeOptions, + setKeyboardMode(mode == MODE_SYMBOLS ? MODE_TEXT : mode, imeOptions, enableVoice, mPreferSymbols); } - void setKeyboardMode(int mode, int imeOptions, boolean isSymbols) { + void setKeyboardMode(int mode, int imeOptions, + boolean enableVoice, boolean isSymbols) { mMode = mode; mImeOptions = imeOptions; + mHasVoice = enableVoice; mIsSymbols = isSymbols; + mInputView.setPreviewEnabled(true); KeyboardId id = getKeyboardId(mode, imeOptions, isSymbols); + + if (enableVoice && mModeToVoice.containsKey(id.mKeyboardMode)) { + id.mKeyboardMode = mModeToVoice.get(id.mKeyboardMode); + } LatinKeyboard keyboard = getKeyboard(id); if (mode == MODE_PHONE) { @@ -166,7 +197,6 @@ public class KeyboardSwitcher { keyboard.setShifted(false); keyboard.setShiftLocked(keyboard.isShiftLocked()); keyboard.setImeOptions(mContext.getResources(), mMode, imeOptions); - } private LatinKeyboard getKeyboard(KeyboardId id) { @@ -177,11 +207,16 @@ public class KeyboardSwitcher { conf.locale = mInputLocale; orig.updateConfiguration(conf, null); LatinKeyboard keyboard = new LatinKeyboard( - mContext, id.mXml, id.mMode); - if (id.mMode == KEYBOARDMODE_NORMAL - || id.mMode == KEYBOARDMODE_URL - || id.mMode == KEYBOARDMODE_IM - || id.mMode == KEYBOARDMODE_EMAIL) { + mContext, id.mXml, id.mKeyboardMode); + if (id.mKeyboardMode == KEYBOARDMODE_NORMAL + || id.mKeyboardMode == KEYBOARDMODE_URL + || id.mKeyboardMode == KEYBOARDMODE_IM + || id.mKeyboardMode == KEYBOARDMODE_EMAIL + || id.mKeyboardMode == R.id.mode_normal_voice + || id.mKeyboardMode == R.id.mode_url_voice + || id.mKeyboardMode == R.id.mode_im_voice + || id.mKeyboardMode == R.id.mode_email_voice + ) { keyboard.setExtension(R.xml.kbd_extension); } @@ -241,7 +276,7 @@ public class KeyboardSwitcher { mTextMode = position; } if (isTextMode()) { - setKeyboardMode(MODE_TEXT, mImeOptions); + setKeyboardMode(MODE_TEXT, mImeOptions, mHasVoice); } } @@ -250,11 +285,13 @@ public class KeyboardSwitcher { } boolean isAlphabetMode() { - KeyboardId current = mCurrentId; - return current.mMode == KEYBOARDMODE_NORMAL - || current.mMode == KEYBOARDMODE_URL - || current.mMode == KEYBOARDMODE_EMAIL - || current.mMode == KEYBOARDMODE_IM; + int currentMode = mCurrentId.mKeyboardMode; + for (Integer mode : ALPHABET_MODES) { + if (currentMode == mode || currentMode == mModeToVoice.get(mode)) { + return true; + } + } + return false; } void toggleShift() { @@ -278,7 +315,7 @@ public class KeyboardSwitcher { } void toggleSymbols() { - setKeyboardMode(mMode, mImeOptions, !mIsSymbols); + setKeyboardMode(mMode, mImeOptions, mHasVoice, !mIsSymbols); if (mIsSymbols && !mPreferSymbols) { mSymbolsModeState = SYMBOLS_MODE_STATE_BEGIN; } else { diff --git a/src/com/android/inputmethod/latin/LatinIME.java b/src/com/android/inputmethod/latin/LatinIME.java index 98f47c2c6..cbf3a4a52 100644 --- a/src/com/android/inputmethod/latin/LatinIME.java +++ b/src/com/android/inputmethod/latin/LatinIME.java @@ -16,13 +16,7 @@ package com.android.inputmethod.latin; -import java.io.FileDescriptor; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; - -import com.android.inputmethod.latin.UserDictionary; +import com.google.android.collect.Lists; import android.app.AlertDialog; import android.backup.BackupManager; @@ -53,23 +47,42 @@ import android.util.Log; import android.util.PrintWriterPrinter; import android.util.Printer; import android.view.KeyEvent; +import android.view.LayoutInflater; import android.view.View; import android.view.Window; import android.view.WindowManager; import android.view.inputmethod.CompletionInfo; import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.ExtractedTextRequest; import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputMethodManager; +import com.android.inputmethod.voice.EditingUtil; +import com.android.inputmethod.voice.FieldContext; +import com.android.inputmethod.voice.GoogleSettingsUtil; +import com.android.inputmethod.voice.VoiceInput; + +import java.io.FileDescriptor; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + /** * Input method implementation for Qwerty'ish keyboard. */ -public class LatinIME extends InputMethodService +public class LatinIME extends InputMethodService implements KeyboardView.OnKeyboardActionListener, - SharedPreferences.OnSharedPreferenceChangeListener { - + VoiceInput.UiListener, + SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = "LatinIME"; static final boolean DEBUG = false; static final boolean TRACE = false; + static final boolean VOICE_INSTALLED = true; + static final boolean ENABLE_VOICE_BUTTON = true; private static final String PREF_VIBRATE_ON = "vibrate_on"; private static final String PREF_SOUND_ON = "sound_on"; @@ -77,12 +90,53 @@ public class LatinIME extends InputMethodService private static final String PREF_QUICK_FIXES = "quick_fixes"; private static final String PREF_SHOW_SUGGESTIONS = "show_suggestions"; private static final String PREF_AUTO_COMPLETE = "auto_complete"; + private static final String PREF_ENABLE_VOICE = "enable_voice_input"; + private static final String PREF_VOICE_SERVER_URL = "voice_server_url"; + + // Whether or not the user has used voice input before (and thus, whether to show the + // first-run warning dialog or not). + private static final String PREF_HAS_USED_VOICE_INPUT = "has_used_voice_input"; + + // Whether or not the user has used voice input from an unsupported locale UI before. + // For example, the user has a Chinese UI but activates voice input. + private static final String PREF_HAS_USED_VOICE_INPUT_UNSUPPORTED_LOCALE = + "has_used_voice_input_unsupported_locale"; + + // A list of locales which are supported by default for voice input, unless we get a + // different list from Gservices. + public static final String DEFAULT_VOICE_INPUT_SUPPORTED_LOCALES = + "en " + + "en_US " + + "en_GB " + + "en_AU " + + "en_CA " + + "en_IE " + + "en_IN " + + "en_NZ " + + "en_SG " + + "en_ZA "; + + // The private IME option used to indicate that no microphone should be shown for a + // given text field. For instance this is specified by the search dialog when the + // dialog is already showing a voice search button. + private static final String IME_OPTION_NO_MICROPHONE = "nm"; + public static final String PREF_SELECTED_LANGUAGES = "selected_languages"; public static final String PREF_INPUT_LANGUAGE = "input_language"; private static final int MSG_UPDATE_SUGGESTIONS = 0; private static final int MSG_START_TUTORIAL = 1; private static final int MSG_UPDATE_SHIFT_STATE = 2; + private static final int MSG_VOICE_RESULTS = 3; + private static final int MSG_START_LISTENING_AFTER_SWIPE = 4; + + // If we detect a swipe gesture within N ms of typing, then swipe is + // ignored, since it may in fact be two key presses in quick succession. + private static final long MIN_MILLIS_AFTER_TYPING_BEFORE_SWIPE = 1000; + + // If we detect a swipe gesture, and the user types N ms later, cancel the + // swipe since it was probably a false trigger. + private static final long MIN_MILLIS_AFTER_SWIPE_TO_WAIT_FOR_TYPING = 500; // How many continuous deletes at which to start deleting at a higher speed. private static final int DELETE_ACCELERATE_AT = 20; @@ -102,7 +156,7 @@ public class LatinIME extends InputMethodService // Contextual menu positions private static final int POS_SETTINGS = 0; private static final int POS_METHOD = 1; - + private LatinKeyboardView mInputView; private CandidateViewContainer mCandidateViewContainer; private CandidateView mCandidateView; @@ -110,6 +164,7 @@ public class LatinIME extends InputMethodService private CompletionInfo[] mCompletions; private AlertDialog mOptionsDialog; + private AlertDialog mVoiceWarningDialog; KeyboardSwitcher mKeyboardSwitcher; @@ -117,6 +172,8 @@ public class LatinIME extends InputMethodService private ContactsDictionary mContactsDictionary; private ExpandableDictionary mAutoDictionary; + private Hints mHints; + Resources mResources; private String mLocale; @@ -125,6 +182,13 @@ public class LatinIME extends InputMethodService private WordComposer mWord = new WordComposer(); private int mCommittedLength; private boolean mPredicting; + private boolean mRecognizing; + private boolean mAfterVoiceInput; + private boolean mImmediatelyAfterVoiceInput; + private boolean mShowingVoiceSuggestions; + private boolean mImmediatelyAfterVoiceSuggestions; + private boolean mVoiceInputHighlighted; + private boolean mEnableVoiceButton; private CharSequence mBestWord; private boolean mPredictionOn; private boolean mCompletionOn; @@ -133,14 +197,22 @@ public class LatinIME extends InputMethodService private boolean mAutoCorrectEnabled; private boolean mAutoCorrectOn; private boolean mCapsLock; + private boolean mPasswordText; + private boolean mEmailText; private boolean mVibrateOn; private boolean mSoundOn; private boolean mAutoCap; private boolean mQuickFixes; + private boolean mHasUsedVoiceInput; + private boolean mHasUsedVoiceInputUnsupportedLocale; + private boolean mLocaleSupportedForVoiceInput; private boolean mShowSuggestions; + private boolean mSuggestionShouldReplaceCurrentWord; + private boolean mIsShowingHint; private int mCorrectionMode; + private boolean mEnableVoice = true; private int mOrientation; - + // Indicates whether the suggestion strip is to be on in landscape private boolean mJustAccepted; private CharSequence mJustRevertedSeparator; @@ -159,6 +231,17 @@ public class LatinIME extends InputMethodService private String mWordSeparators; private String mSentenceSeparators; + private VoiceInput mVoiceInput; + private VoiceResults mVoiceResults = new VoiceResults(); + private long mSwipeTriggerTimeMillis; + + // For each word, a list of potential replacements, usually from voice. + private Map<String, List<CharSequence>> mWordToSuggestions = new HashMap(); + + private class VoiceResults { + List<String> candidates; + Map<String, List<CharSequence>> alternatives; + } private int mCurrentInputLocale = 0; private String mInputLanguage; private String[] mSelectedLanguageArray; @@ -186,6 +269,13 @@ public class LatinIME extends InputMethodService case MSG_UPDATE_SHIFT_STATE: updateShiftKeyState(getCurrentInputEditorInfo()); break; + case MSG_VOICE_RESULTS: + handleVoiceResults(); + break; + case MSG_START_LISTENING_AFTER_SWIPE: + if (mLastKeyTime < mSwipeTriggerTimeMillis) { + startListening(true); + } } } }; @@ -212,6 +302,19 @@ public class LatinIME extends InputMethodService // register to receive ringer mode changes for silent mode IntentFilter filter = new IntentFilter(AudioManager.RINGER_MODE_CHANGED_ACTION); registerReceiver(mReceiver, filter); + if (VOICE_INSTALLED) { + mVoiceInput = new VoiceInput(this, this); + mHints = new Hints(this, new Hints.Display() { + public void showHint(int viewResource) { + LayoutInflater inflater = (LayoutInflater) getSystemService( + Context.LAYOUT_INFLATER_SERVICE); + View view = inflater.inflate(viewResource, null); + setCandidatesView(view); + setCandidatesViewShown(true); + mIsShowingHint = true; + } + }); + } PreferenceManager.getDefaultSharedPreferences(this) .registerOnSharedPreferenceChangeListener(this); } @@ -228,7 +331,6 @@ public class LatinIME extends InputMethodService mSuggest.close(); } mSuggest = new Suggest(this, R.raw.main); - if (mUserDictionary != null) mUserDictionary.close(); mUserDictionary = new UserDictionary(this); if (mContactsDictionary == null) { mContactsDictionary = new ContactsDictionary(this); @@ -248,10 +350,14 @@ public class LatinIME extends InputMethodService orig.updateConfiguration(conf, orig.getDisplayMetrics()); } - @Override public void onDestroy() { + @Override + public void onDestroy() { mUserDictionary.close(); mContactsDictionary.close(); unregisterReceiver(mReceiver); + if (VOICE_INSTALLED) { + mVoiceInput.destroy(); + } super.onDestroy(); } @@ -262,7 +368,9 @@ public class LatinIME extends InputMethodService } // If orientation changed while predicting, commit the change if (conf.orientation != mOrientation) { - commitTyped(getCurrentInputConnection()); + InputConnection ic = getCurrentInputConnection(); + commitTyped(ic); + if (ic != null) ic.finishComposingText(); // For voice input mOrientation = conf.orientation; } reloadKeyboards(); @@ -276,11 +384,28 @@ public class LatinIME extends InputMethodService mKeyboardSwitcher.setInputView(mInputView); mKeyboardSwitcher.makeKeyboards(true); mInputView.setOnKeyboardActionListener(this); - mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_TEXT, 0); + mKeyboardSwitcher.setKeyboardMode( + KeyboardSwitcher.MODE_TEXT, 0, + shouldShowVoiceButton(makeFieldContext(), getCurrentInputEditorInfo())); return mInputView; } @Override + public void onInitializeInterface() { + // Create a new view associated with voice input if the old + // view is stuck in another layout (e.g. if switching from + // portrait to landscape while speaking) + // NOTE: This must be done here because for some reason + // onCreateInputView isn't called after an orientation change while + // speech rec is in progress. + if (mVoiceInput != null && mVoiceInput.getView().getParent() != null) { + mVoiceInput.newView(); + } + + super.onInitializeInterface(); + } + + @Override public View onCreateCandidatesView() { mKeyboardSwitcher.makeKeyboards(true); mCandidateViewContainer = (CandidateViewContainer) getLayoutInflater().inflate( @@ -308,32 +433,54 @@ public class LatinIME extends InputMethodService TextEntryState.newSession(this); + // Most such things we decide below in the switch statement, but we need to know + // now whether this is a password text field, because we need to know now (before + // the switch statement) whether we want to enable the voice button. + mPasswordText = false; + int variation = attribute.inputType & EditorInfo.TYPE_MASK_VARIATION; + if (variation == EditorInfo.TYPE_TEXT_VARIATION_PASSWORD || + variation == EditorInfo.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD) { + mPasswordText = true; + } + + mEnableVoiceButton = shouldShowVoiceButton(makeFieldContext(), attribute); + + mAfterVoiceInput = false; + mImmediatelyAfterVoiceInput = false; + mShowingVoiceSuggestions = false; + mImmediatelyAfterVoiceSuggestions = false; + mVoiceInputHighlighted = false; + boolean disableAutoCorrect = false; + mWordToSuggestions.clear(); mInputTypeNoAutoCorrect = false; mPredictionOn = false; mCompletionOn = false; mCompletions = null; mCapsLock = false; - switch (attribute.inputType&EditorInfo.TYPE_MASK_CLASS) { + mEmailText = false; + switch (attribute.inputType & EditorInfo.TYPE_MASK_CLASS) { case EditorInfo.TYPE_CLASS_NUMBER: case EditorInfo.TYPE_CLASS_DATETIME: mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_SYMBOLS, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); break; case EditorInfo.TYPE_CLASS_PHONE: mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_PHONE, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); break; case EditorInfo.TYPE_CLASS_TEXT: mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_TEXT, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); //startPrediction(); mPredictionOn = true; // Make sure that passwords are not displayed in candidate view - int variation = attribute.inputType & EditorInfo.TYPE_MASK_VARIATION; if (variation == EditorInfo.TYPE_TEXT_VARIATION_PASSWORD || variation == EditorInfo.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD ) { mPredictionOn = false; } + if (variation == EditorInfo.TYPE_TEXT_VARIATION_EMAIL_ADDRESS) { + mEmailText = true; + } if (variation == EditorInfo.TYPE_TEXT_VARIATION_EMAIL_ADDRESS || variation == EditorInfo.TYPE_TEXT_VARIATION_PERSON_NAME) { mAutoSpace = false; @@ -343,14 +490,14 @@ public class LatinIME extends InputMethodService if (variation == EditorInfo.TYPE_TEXT_VARIATION_EMAIL_ADDRESS) { mPredictionOn = false; mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_EMAIL, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); } else if (variation == EditorInfo.TYPE_TEXT_VARIATION_URI) { mPredictionOn = false; mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_URL, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); } else if (variation == EditorInfo.TYPE_TEXT_VARIATION_SHORT_MESSAGE) { mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_IM, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); } else if (variation == EditorInfo.TYPE_TEXT_VARIATION_FILTER) { mPredictionOn = false; } else if (variation == EditorInfo.TYPE_TEXT_VARIATION_WEB_EDIT_TEXT) { @@ -379,17 +526,25 @@ public class LatinIME extends InputMethodService break; default: mKeyboardSwitcher.setKeyboardMode(KeyboardSwitcher.MODE_TEXT, - attribute.imeOptions); + attribute.imeOptions, mEnableVoiceButton); updateShiftKeyState(attribute); } mInputView.closing(); mComposing.setLength(0); mPredicting = false; mDeleteCount = 0; - setCandidatesViewShown(false); - if (mCandidateView != null) mCandidateView.setSuggestions(null, false, false, false); loadSettings(); + setCandidatesViewShown(false); + setSuggestions(null, false, false, false); + + // Override auto correct + if (disableAutoCorrect) { + mAutoCorrectOn = false; + if (mCorrectionMode == Suggest.CORRECTION_FULL) { + mCorrectionMode = Suggest.CORRECTION_BASIC; + } + } // If the dictionary is not big enough, don't auto correct mHasDictionary = mSuggest.hasMainDictionary(); @@ -404,10 +559,31 @@ public class LatinIME extends InputMethodService @Override public void onFinishInput() { super.onFinishInput(); - + + if (mAfterVoiceInput) mVoiceInput.logInputEnded(); + + mVoiceInput.flushLogs(); + if (mInputView != null) { mInputView.closing(); } + if (VOICE_INSTALLED & mRecognizing) { + mVoiceInput.cancel(); + } + } + + @Override + public void onUpdateExtractedText(int token, ExtractedText text) { + super.onUpdateExtractedText(token, text); + InputConnection ic = getCurrentInputConnection(); + if (!mImmediatelyAfterVoiceInput && mAfterVoiceInput && ic != null) { + mVoiceInput.logTextModified(); + + if (mHints.showPunctuationHintIfNecessary(ic)) { + mVoiceInput.logPunctuationHintDisplayed(); + } + } + mImmediatelyAfterVoiceInput = false; } @Override @@ -416,10 +592,22 @@ public class LatinIME extends InputMethodService int candidatesStart, int candidatesEnd) { super.onUpdateSelection(oldSelStart, oldSelEnd, newSelStart, newSelEnd, candidatesStart, candidatesEnd); + + if (DEBUG) { + Log.i(TAG, "onUpdateSelection: oss=" + oldSelStart + + ", ose=" + oldSelEnd + + ", nss=" + newSelStart + + ", nse=" + newSelEnd + + ", cs=" + candidatesStart + + ", ce=" + candidatesEnd); + } + + mSuggestionShouldReplaceCurrentWord = false; // If the current selection in the text view changes, we should // clear whatever candidate text we have. - if (mComposing.length() > 0 && mPredicting && (newSelStart != candidatesEnd - || newSelEnd != candidatesEnd)) { + if ((((mComposing.length() > 0 && mPredicting) || mVoiceInputHighlighted) + && (newSelStart != candidatesEnd + || newSelEnd != candidatesEnd))) { mComposing.setLength(0); mPredicting = false; updateSuggestions(); @@ -428,25 +616,58 @@ public class LatinIME extends InputMethodService if (ic != null) { ic.finishComposingText(); } + mVoiceInputHighlighted = false; } else if (!mPredicting && !mJustAccepted && TextEntryState.getState() == TextEntryState.STATE_ACCEPTED_DEFAULT) { TextEntryState.reset(); } mJustAccepted = false; postUpdateShiftKeyState(); + + if (VOICE_INSTALLED) { + if (mShowingVoiceSuggestions) { + if (mImmediatelyAfterVoiceSuggestions) { + mImmediatelyAfterVoiceSuggestions = false; + } else { + updateSuggestions(); + mShowingVoiceSuggestions = false; + } + } + if (VoiceInput.ENABLE_WORD_CORRECTIONS) { + // If we have alternatives for the current word, then show them. + String word = EditingUtil.getWordAtCursor( + getCurrentInputConnection(), getWordSeparators()); + if (word != null && mWordToSuggestions.containsKey(word.trim())) { + mSuggestionShouldReplaceCurrentWord = true; + final List<CharSequence> suggestions = mWordToSuggestions.get(word.trim()); + + setSuggestions(suggestions, false, true, true); + setCandidatesViewShown(true); + } + } + } } @Override public void hideWindow() { + if (mAfterVoiceInput) mVoiceInput.logInputEnded(); if (TRACE) Debug.stopMethodTracing(); if (mOptionsDialog != null && mOptionsDialog.isShowing()) { mOptionsDialog.dismiss(); mOptionsDialog = null; } + if (mVoiceWarningDialog != null && mVoiceWarningDialog.isShowing()) { + mVoiceInput.logKeyboardWarningDialogDismissed(); + mVoiceWarningDialog.dismiss(); + mVoiceWarningDialog = null; + } if (mTutorial != null) { mTutorial.close(); mTutorial = null; } + if (VOICE_INSTALLED & mRecognizing) { + mVoiceInput.cancel(); + } super.hideWindow(); TextEntryState.endSession(); } @@ -462,7 +683,7 @@ public class LatinIME extends InputMethodService if (mCompletionOn) { mCompletions = completions; if (completions == null) { - mCandidateView.setSuggestions(null, false, false, false); + setSuggestions(null, false, false, false); return; } @@ -472,7 +693,7 @@ public class LatinIME extends InputMethodService if (ci != null) stringList.add(ci.getText()); } //CharSequence typedWord = mWord.getTypedWord(); - mCandidateView.setSuggestions(stringList, true, true, true); + setSuggestions(stringList, true, true, true); mBestWord = null; setCandidatesViewShown(isCandidateStripVisible() || mCompletionOn); } @@ -546,6 +767,20 @@ public class LatinIME extends InputMethodService return super.onKeyUp(keyCode, event); } + private void revertVoiceInput() { + InputConnection ic = getCurrentInputConnection(); + if (ic != null) ic.commitText("", 1); + updateSuggestions(); + mVoiceInputHighlighted = false; + } + + private void commitVoiceInput() { + InputConnection ic = getCurrentInputConnection(); + if (ic != null) ic.finishComposingText(); + updateSuggestions(); + mVoiceInputHighlighted = false; + } + private void reloadKeyboards() { if (mKeyboardSwitcher == null) { mKeyboardSwitcher = new KeyboardSwitcher(this, this); @@ -670,6 +905,11 @@ public class LatinIME extends InputMethodService case Keyboard.KEYCODE_MODE_CHANGE: changeKeyboardMode(); break; + case LatinKeyboardView.KEYCODE_VOICE: + if (VOICE_INSTALLED) { + startListening(false /* was a button press, was not a swipe */); + } + break; default: if (isWordSeparator(primaryCode)) { handleSeparator(primaryCode); @@ -698,6 +938,10 @@ public class LatinIME extends InputMethodService } private void handleBackspace() { + if (VOICE_INSTALLED && mVoiceInputHighlighted) { + revertVoiceInput(); + return; + } boolean deleteChar = false; InputConnection ic = getCurrentInputConnection(); if (ic == null) return; @@ -743,6 +987,9 @@ public class LatinIME extends InputMethodService } private void handleCharacter(int primaryCode, int[] keyCodes) { + if (VOICE_INSTALLED && mVoiceInputHighlighted) { + commitVoiceInput(); + } if (isAlphabet(primaryCode) && isPredictionOn() && !isCursorTouchingWord()) { if (!mPredicting) { mPredicting = true; @@ -778,6 +1025,9 @@ public class LatinIME extends InputMethodService } private void handleSeparator(int primaryCode) { + if (VOICE_INSTALLED && mVoiceInputHighlighted) { + commitVoiceInput(); + } boolean pickedDefault = false; // Handle separator InputConnection ic = getCurrentInputConnection(); @@ -816,9 +1066,12 @@ public class LatinIME extends InputMethodService ic.endBatchEdit(); } } - + private void handleClose() { commitTyped(getCurrentInputConnection()); + if (VOICE_INSTALLED & mRecognizing) { + mVoiceInput.cancel(); + } requestHideSelf(0); mInputView.closing(); TextEntryState.endSession(); @@ -852,14 +1105,205 @@ public class LatinIME extends InputMethodService return isPredictionOn() && mShowSuggestions; } + public void onCancelVoice() { + if (mRecognizing) { + switchToKeyboardView(); + } + } + + private void switchToKeyboardView() { + mHandler.post(new Runnable() { + public void run() { + mRecognizing = false; + if (mInputView != null) { + setInputView(mInputView); + } + updateInputViewShown(); + }}); + } + + private void switchToRecognitionStatusView() { + mHandler.post(new Runnable() { + public void run() { + mRecognizing = true; + setInputView(mVoiceInput.getView()); + updateInputViewShown(); + }}); + } + + private void startListening(boolean swipe) { + if (!mHasUsedVoiceInput || + (!mLocaleSupportedForVoiceInput && !mHasUsedVoiceInputUnsupportedLocale)) { + // Calls reallyStartListening if user clicks OK, does nothing if user clicks Cancel. + showVoiceWarningDialog(swipe); + } else { + reallyStartListening(swipe); + } + } + + private void reallyStartListening(boolean swipe) { + if (!mHasUsedVoiceInput) { + // The user has started a voice input, so remember that in the + // future (so we don't show the warning dialog after the first run). + SharedPreferences.Editor editor = + PreferenceManager.getDefaultSharedPreferences(this).edit(); + editor.putBoolean(PREF_HAS_USED_VOICE_INPUT, true); + editor.commit(); + mHasUsedVoiceInput = true; + } + + if (!mLocaleSupportedForVoiceInput && !mHasUsedVoiceInputUnsupportedLocale) { + // The user has started a voice input from an unsupported locale, so remember that + // in the future (so we don't show the warning dialog the next time they do this). + SharedPreferences.Editor editor = + PreferenceManager.getDefaultSharedPreferences(this).edit(); + editor.putBoolean(PREF_HAS_USED_VOICE_INPUT_UNSUPPORTED_LOCALE, true); + editor.commit(); + mHasUsedVoiceInputUnsupportedLocale = true; + } + + // Clear N-best suggestions + setSuggestions(null, false, false, true); + + FieldContext context = new FieldContext( + getCurrentInputConnection(), getCurrentInputEditorInfo()); + mVoiceInput.startListening(context, swipe); + switchToRecognitionStatusView(); + } + + private void showVoiceWarningDialog(final boolean swipe) { + AlertDialog.Builder builder = new AlertDialog.Builder(this); + builder.setCancelable(true); + builder.setIcon(R.drawable.ic_mic_dialog); + builder.setPositiveButton(android.R.string.ok, new DialogInterface.OnClickListener() { + public void onClick(DialogInterface dialog, int whichButton) { + mVoiceInput.logKeyboardWarningDialogOk(); + reallyStartListening(swipe); + } + }); + builder.setNegativeButton(android.R.string.cancel, new DialogInterface.OnClickListener() { + public void onClick(DialogInterface dialog, int whichButton) { + mVoiceInput.logKeyboardWarningDialogCancel(); + } + }); + + if (mLocaleSupportedForVoiceInput) { + String message = getString(R.string.voice_warning_may_not_understand) + "\n\n" + + getString(R.string.voice_warning_how_to_turn_off); + builder.setMessage(message); + } else { + String message = getString(R.string.voice_warning_locale_not_supported) + "\n\n" + + getString(R.string.voice_warning_may_not_understand) + "\n\n" + + getString(R.string.voice_warning_how_to_turn_off); + builder.setMessage(message); + } + + builder.setTitle(R.string.voice_warning_title); + mVoiceWarningDialog = builder.create(); + + Window window = mVoiceWarningDialog.getWindow(); + WindowManager.LayoutParams lp = window.getAttributes(); + lp.token = mInputView.getWindowToken(); + lp.type = WindowManager.LayoutParams.TYPE_APPLICATION_ATTACHED_DIALOG; + window.setAttributes(lp); + window.addFlags(WindowManager.LayoutParams.FLAG_ALT_FOCUSABLE_IM); + mVoiceInput.logKeyboardWarningDialogShown(); + mVoiceWarningDialog.show(); + } + + public void onVoiceResults(List<String> candidates, + Map<String, List<CharSequence>> alternatives) { + if (!mRecognizing) { + return; + } + mVoiceResults.candidates = candidates; + mVoiceResults.alternatives = alternatives; + mHandler.sendMessage(mHandler.obtainMessage(MSG_VOICE_RESULTS)); + } + + private void handleVoiceResults() { + mAfterVoiceInput = true; + mImmediatelyAfterVoiceInput = true; + + InputConnection ic = getCurrentInputConnection(); + if (!isFullscreenMode()) { + // Start listening for updates to the text from typing, etc. + if (ic != null) { + ExtractedTextRequest req = new ExtractedTextRequest(); + ic.getExtractedText(req, InputConnection.GET_EXTRACTED_TEXT_MONITOR); + } + } + + vibrate(); + switchToKeyboardView(); + + final List<CharSequence> nBest = new ArrayList<CharSequence>(); + boolean capitalizeFirstWord = preferCapitalization() + || (mKeyboardSwitcher.isAlphabetMode() && mInputView.isShifted()); + for (String c : mVoiceResults.candidates) { + if (capitalizeFirstWord) { + c = Character.toUpperCase(c.charAt(0)) + c.substring(1, c.length()); + } + nBest.add(c); + } + + if (nBest.size() == 0) { + return; + } + + String bestResult = nBest.get(0).toString(); + + mVoiceInput.logVoiceInputDelivered(); + + mHints.registerVoiceResult(bestResult); + + if (ic != null) ic.beginBatchEdit(); // To avoid extra updates on committing older text + + commitTyped(ic); + EditingUtil.appendText(ic, bestResult); + + if (ic != null) ic.endBatchEdit(); + + // Show N-Best alternates, if there is more than one choice. + if (nBest.size() > 1) { + mImmediatelyAfterVoiceSuggestions = true; + mShowingVoiceSuggestions = true; + setSuggestions(nBest.subList(1, nBest.size()), false, true, true); + setCandidatesViewShown(true); + } + mVoiceInputHighlighted = true; + mWordToSuggestions.putAll(mVoiceResults.alternatives); + + } + + private void setSuggestions( + List<CharSequence> suggestions, + boolean completions, + + boolean typedWordValid, + boolean haveMinimalSuggestion) { + + if (mIsShowingHint) { + setCandidatesView(mCandidateViewContainer); + mIsShowingHint = false; + } + + if (mCandidateView != null) { + mCandidateView.setSuggestions( + suggestions, completions, typedWordValid, haveMinimalSuggestion); + } + } + private void updateSuggestions() { + mSuggestionShouldReplaceCurrentWord = false; + // Check if we have a suggestion engine attached. - if (mSuggest == null || !isPredictionOn()) { + if ((mSuggest == null || !isPredictionOn()) && !mVoiceInputHighlighted) { return; } - + if (!mPredicting) { - mCandidateView.setSuggestions(null, false, false, false); + setSuggestions(null, false, false, false); return; } @@ -876,7 +1320,7 @@ public class LatinIME extends InputMethodService // Don't auto-correct words with multiple capital letter correctionAvailable &= !mWord.isMostlyCaps(); - mCandidateView.setSuggestions(stringList, false, typedWordValid, correctionAvailable); + setSuggestions(stringList, false, typedWordValid, correctionAvailable); if (stringList.size() > 0) { if (correctionAvailable && !typedWordValid && stringList.size() > 1) { mBestWord = stringList.get(1); @@ -903,6 +1347,8 @@ public class LatinIME extends InputMethodService } public void pickSuggestionManually(int index, CharSequence suggestion) { + if (mAfterVoiceInput && mShowingVoiceSuggestions) mVoiceInput.logNBestChoose(index); + if (mCompletionOn && mCompletions != null && index >= 0 && index < mCompletions.length) { CompletionInfo ci = mCompletions[index]; @@ -937,7 +1383,12 @@ public class LatinIME extends InputMethodService } InputConnection ic = getCurrentInputConnection(); if (ic != null) { - ic.commitText(suggestion, 1); + if (mSuggestionShouldReplaceCurrentWord) { + EditingUtil.deleteWordAtCursor(ic, getWordSeparators()); + } + if (!VoiceInput.DELETE_SYMBOL.equals(suggestion)) { + ic.commitText(suggestion, 1); + } } // Add the word to the auto dictionary if it's not a known word if (mAutoDictionary.isValidWord(suggestion) || !mSuggest.isValidWord(suggestion)) { @@ -945,9 +1396,7 @@ public class LatinIME extends InputMethodService } mPredicting = false; mCommittedLength = suggestion.length(); - if (mCandidateView != null) { - mCandidateView.setSuggestions(null, false, false, false); - } + setSuggestions(null, false, false, false); updateShiftKeyState(getCurrentInputEditorInfo()); } @@ -1016,6 +1465,11 @@ public class LatinIME extends InputMethodService } public void swipeRight() { + if (userHasNotTypedRecently() && VOICE_INSTALLED && mEnableVoice && + fieldCanDoVoice(makeFieldContext())) { + startListening(true /* was a swipe */); + } + if (LatinKeyboardView.DEBUG_AUTO_PLAY) { ClipboardManager cm = ((ClipboardManager)getSystemService(CLIPBOARD_SERVICE)); CharSequence text = cm.getText(); @@ -1035,7 +1489,7 @@ public class LatinIME extends InputMethodService int currentKeyboardMode = mKeyboardSwitcher.getKeyboardMode(); reloadKeyboards(); mKeyboardSwitcher.makeKeyboards(true); - mKeyboardSwitcher.setKeyboardMode(currentKeyboardMode, 0); + mKeyboardSwitcher.setKeyboardMode(currentKeyboardMode, 0, mEnableVoiceButton); initSuggest(mInputLanguage); persistInputLanguage(mInputLanguage); updateShiftKeyState(getCurrentInputEditorInfo()); @@ -1068,7 +1522,30 @@ public class LatinIME extends InputMethodService public void onRelease(int primaryCode) { //vibrate(); } + + private FieldContext makeFieldContext() { + return new FieldContext(getCurrentInputConnection(), getCurrentInputEditorInfo()); + } + + private boolean fieldCanDoVoice(FieldContext fieldContext) { + return !mPasswordText + && mVoiceInput != null + && !mVoiceInput.isBlacklistedField(fieldContext); + } + + private boolean fieldIsRecommendedForVoice(FieldContext fieldContext) { + // TODO: Move this logic into the VoiceInput method. + return !mPasswordText && !mEmailText && mVoiceInput.isRecommendedField(fieldContext); + } + private boolean shouldShowVoiceButton(FieldContext fieldContext, EditorInfo attribute) { + return ENABLE_VOICE_BUTTON + && mEnableVoice + && fieldCanDoVoice(fieldContext) + && !(attribute != null && attribute.privateImeOptions != null + && attribute.privateImeOptions.equals(IME_OPTION_NO_MICROPHONE)); + } + // receive ringer mode changes to detect silent mode private BroadcastReceiver mReceiver = new BroadcastReceiver() { @Override @@ -1087,6 +1564,26 @@ public class LatinIME extends InputMethodService } } + private boolean userHasNotTypedRecently() { + return (SystemClock.uptimeMillis() - mLastKeyTime) + > MIN_MILLIS_AFTER_TYPING_BEFORE_SWIPE; + } + + /* + * Only trigger a swipe action if the user hasn't typed X millis before + * now, and if they don't type Y millis after the swipe is detected. This + * delays the onset of the swipe action by Y millis. + */ + private void conservativelyTriggerSwipeAction(final Runnable action) { + if (userHasNotTypedRecently()) { + mSwipeTriggerTimeMillis = System.currentTimeMillis(); + mHandler.sendMessageDelayed( + mHandler.obtainMessage(MSG_START_LISTENING_AFTER_SWIPE), + MIN_MILLIS_AFTER_SWIPE_TO_WAIT_FOR_TYPING); + } + } + + private void playKeyClick(int primaryCode) { // if mAudioManager is null, we don't have the ringer state yet // mAudioManager will be set by updateRingerMode @@ -1162,10 +1659,14 @@ public class LatinIME extends InputMethodService } } - private void launchSettings() { + protected void launchSettings() { + launchSettings(LatinIMESettings.class); + } + + protected void launchSettings(Class settingsClass) { handleClose(); Intent intent = new Intent(); - intent.setClass(LatinIME.this, LatinIMESettings.class); + intent.setClass(LatinIME.this, settingsClass); intent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK); startActivity(intent); } @@ -1177,10 +1678,37 @@ public class LatinIME extends InputMethodService mSoundOn = sp.getBoolean(PREF_SOUND_ON, false); mAutoCap = sp.getBoolean(PREF_AUTO_CAP, true); mQuickFixes = sp.getBoolean(PREF_QUICK_FIXES, true); + mHasUsedVoiceInput = sp.getBoolean(PREF_HAS_USED_VOICE_INPUT, false); + mHasUsedVoiceInputUnsupportedLocale = + sp.getBoolean(PREF_HAS_USED_VOICE_INPUT_UNSUPPORTED_LOCALE, false); + + // Get the current list of supported locales and check the current locale against that + // list. We cache this value so as not to check it every time the user starts a voice + // input. Because this method is called by onStartInputView, this should mean that as + // long as the locale doesn't change while the user is keeping the IME open, the + // value should never be stale. + String supportedLocalesString = GoogleSettingsUtil.getGservicesString( + getContentResolver(), + GoogleSettingsUtil.LATIN_IME_VOICE_INPUT_SUPPORTED_LOCALES, + DEFAULT_VOICE_INPUT_SUPPORTED_LOCALES); + ArrayList<String> voiceInputSupportedLocales = + Lists.newArrayList(supportedLocalesString.split("\\s+")); + + mLocaleSupportedForVoiceInput = voiceInputSupportedLocales.contains(mLocale); + // If there is no auto text data, then quickfix is forced to "on", so that the other options // will continue to work + if (AutoText.getSize(mInputView) < 1) mQuickFixes = true; mShowSuggestions = sp.getBoolean(PREF_SHOW_SUGGESTIONS, true) & mQuickFixes; + + if (VOICE_INSTALLED) { + boolean enableVoice = sp.getBoolean(PREF_ENABLE_VOICE, true); + if (enableVoice != mEnableVoice && mKeyboardSwitcher != null) { + mKeyboardSwitcher.setVoiceMode(enableVoice); + } + mEnableVoice = enableVoice; + } mAutoCorrectEnabled = sp.getBoolean(PREF_AUTO_COMPLETE, mResources.getBoolean(R.bool.enable_autocorrect)) & mShowSuggestions; updateCorrectionMode(); @@ -1219,7 +1747,7 @@ public class LatinIME extends InputMethodService builder.setIcon(R.drawable.ic_dialog_keyboard); builder.setNegativeButton(android.R.string.cancel, null); CharSequence itemSettings = getString(R.string.english_ime_settings); - CharSequence itemInputMethod = getString(com.android.internal.R.string.inputMethod); + CharSequence itemInputMethod = getString(R.string.inputMethod); builder.setItems(new CharSequence[] { itemSettings, itemInputMethod}, new DialogInterface.OnClickListener() { @@ -1327,6 +1855,3 @@ public class LatinIME extends InputMethodService } } } - - - diff --git a/src/com/android/inputmethod/latin/LatinIMEBackupAgent.java b/src/com/android/inputmethod/latin/LatinIMEBackupAgent.java index c454f120e..b6a800ebd 100644 --- a/src/com/android/inputmethod/latin/LatinIMEBackupAgent.java +++ b/src/com/android/inputmethod/latin/LatinIMEBackupAgent.java @@ -26,6 +26,6 @@ public class LatinIMEBackupAgent extends BackupHelperAgent { public void onCreate() { addHelper("shared_pref", new SharedPreferencesBackupHelper(this, - "com.android.inputmethod.latin_preferences")); + getPackageName() + "_preferences")); } } diff --git a/src/com/android/inputmethod/latin/LatinIMESettings.java b/src/com/android/inputmethod/latin/LatinIMESettings.java index c8ea309e3..4c221b905 100644 --- a/src/com/android/inputmethod/latin/LatinIMESettings.java +++ b/src/com/android/inputmethod/latin/LatinIMESettings.java @@ -16,23 +16,53 @@ package com.android.inputmethod.latin; +import com.google.android.collect.Lists; + +import android.app.AlertDialog; +import android.app.Dialog; import android.backup.BackupManager; +import android.content.DialogInterface; import android.content.SharedPreferences; import android.os.Bundle; import android.preference.CheckBoxPreference; +import android.preference.ListPreference; +import android.preference.Preference; import android.preference.PreferenceActivity; import android.preference.PreferenceGroup; +import android.preference.Preference.OnPreferenceClickListener; import android.text.AutoText; +import android.util.Log; + +import com.android.inputmethod.voice.GoogleSettingsUtil; +import com.android.inputmethod.voice.VoiceInput; +import com.android.inputmethod.voice.VoiceInputLogger; + +import java.util.ArrayList; +import java.util.Locale; public class LatinIMESettings extends PreferenceActivity - implements SharedPreferences.OnSharedPreferenceChangeListener { + implements SharedPreferences.OnSharedPreferenceChangeListener, + OnPreferenceClickListener, + DialogInterface.OnDismissListener { private static final String QUICK_FIXES_KEY = "quick_fixes"; private static final String SHOW_SUGGESTIONS_KEY = "show_suggestions"; private static final String PREDICTION_SETTINGS_KEY = "prediction_settings"; + private static final String VOICE_SETTINGS_KEY = "enable_voice_input"; + private static final String VOICE_SERVER_KEY = "voice_server_url"; + + private static final String TAG = "LatinIMESettings"; + + // Dialog ids + private static final int VOICE_INPUT_CONFIRM_DIALOG = 0; private CheckBoxPreference mQuickFixes; private CheckBoxPreference mShowSuggestions; + private CheckBoxPreference mVoicePreference; + + private VoiceInputLogger mLogger; + + private boolean mOkClicked = false; @Override protected void onCreate(Bundle icicle) { @@ -40,8 +70,16 @@ public class LatinIMESettings extends PreferenceActivity addPreferencesFromResource(R.xml.prefs); mQuickFixes = (CheckBoxPreference) findPreference(QUICK_FIXES_KEY); mShowSuggestions = (CheckBoxPreference) findPreference(SHOW_SUGGESTIONS_KEY); - getPreferenceManager().getSharedPreferences().registerOnSharedPreferenceChangeListener( - this); + mVoicePreference = (CheckBoxPreference) findPreference(VOICE_SETTINGS_KEY); + + SharedPreferences prefs = getPreferenceManager().getSharedPreferences(); + prefs.registerOnSharedPreferenceChangeListener(this); + + mVoicePreference.setOnPreferenceClickListener(this); + mVoicePreference.setChecked(prefs.getBoolean( + VOICE_SETTINGS_KEY, getResources().getBoolean(R.bool.voice_input_default))); + + mLogger = VoiceInputLogger.getLogger(this); } @Override @@ -50,10 +88,17 @@ public class LatinIMESettings extends PreferenceActivity int autoTextSize = AutoText.getSize(getListView()); if (autoTextSize < 1) { ((PreferenceGroup) findPreference(PREDICTION_SETTINGS_KEY)) - .removePreference(mQuickFixes); + .removePreference(mQuickFixes); } else { mShowSuggestions.setDependency(QUICK_FIXES_KEY); } + if (!LatinIME.VOICE_INSTALLED + || !VoiceInput.voiceIsAvailable(this)) { + getPreferenceScreen().removePreference(mVoicePreference); + } + + mVoicePreference.setChecked( + getPreferenceManager().getSharedPreferences().getBoolean(VOICE_SETTINGS_KEY, true)); } @Override @@ -67,4 +112,91 @@ public class LatinIMESettings extends PreferenceActivity String key) { (new BackupManager(this)).dataChanged(); } + + public boolean onPreferenceClick(Preference preference) { + if (preference == mVoicePreference) { + if (mVoicePreference.isChecked()) { + mOkClicked = false; + showDialog(VOICE_INPUT_CONFIRM_DIALOG); + } else { + updateVoicePreference(); + } + } + return false; + } + + @Override + protected Dialog onCreateDialog(int id) { + switch (id) { + case VOICE_INPUT_CONFIRM_DIALOG: + DialogInterface.OnClickListener listener = new DialogInterface.OnClickListener() { + public void onClick(DialogInterface dialog, int whichButton) { + if (whichButton == DialogInterface.BUTTON_NEGATIVE) { + mVoicePreference.setChecked(false); + mLogger.settingsWarningDialogCancel(); + } else if (whichButton == DialogInterface.BUTTON_POSITIVE) { + mOkClicked = true; + mLogger.settingsWarningDialogOk(); + } + updateVoicePreference(); + } + }; + AlertDialog.Builder builder = new AlertDialog.Builder(this) + .setTitle(R.string.voice_warning_title) + .setPositiveButton(android.R.string.ok, listener) + .setNegativeButton(android.R.string.cancel, listener); + + // Get the current list of supported locales and check the current locale against + // that list, to decide whether to put a warning that voice input will not work in + // the current language as part of the pop-up confirmation dialog. + String supportedLocalesString = GoogleSettingsUtil.getGservicesString( + getContentResolver(), + GoogleSettingsUtil.LATIN_IME_VOICE_INPUT_SUPPORTED_LOCALES, + LatinIME.DEFAULT_VOICE_INPUT_SUPPORTED_LOCALES); + ArrayList<String> voiceInputSupportedLocales = + Lists.newArrayList(supportedLocalesString.split("\\s+")); + boolean localeSupported = voiceInputSupportedLocales.contains( + Locale.getDefault().toString()); + + if (localeSupported) { + String message = getString(R.string.voice_warning_may_not_understand) + "\n\n" + + getString(R.string.voice_hint_dialog_message); + builder.setMessage(message); + } else { + String message = getString(R.string.voice_warning_locale_not_supported) + + "\n\n" + getString(R.string.voice_warning_may_not_understand) + "\n\n" + + getString(R.string.voice_hint_dialog_message); + builder.setMessage(message); + } + + AlertDialog dialog = builder.create(); + dialog.setOnDismissListener(this); + mLogger.settingsWarningDialogShown(); + return dialog; + default: + Log.e(TAG, "unknown dialog " + id); + return null; + } + } + + public void onDismiss(DialogInterface dialog) { + mLogger.settingsWarningDialogDismissed(); + if (!mOkClicked) { + // This assumes that onPreferenceClick gets called first, and this if the user + // agreed after the warning, we set the mOkClicked value to true. + mVoicePreference.setChecked(false); + } + } + + private void updateVoicePreference() { + SharedPreferences.Editor editor = getPreferenceManager().getSharedPreferences().edit(); + boolean isChecked = mVoicePreference.isChecked(); + if (isChecked) { + mLogger.voiceInputSettingEnabled(); + } else { + mLogger.voiceInputSettingDisabled(); + } + editor.putBoolean(VOICE_SETTINGS_KEY, isChecked); + editor.commit(); + } } diff --git a/src/com/android/inputmethod/latin/LatinKeyboardView.java b/src/com/android/inputmethod/latin/LatinKeyboardView.java index 163d824e0..ea9ccf0b6 100644 --- a/src/com/android/inputmethod/latin/LatinKeyboardView.java +++ b/src/com/android/inputmethod/latin/LatinKeyboardView.java @@ -35,8 +35,8 @@ public class LatinKeyboardView extends KeyboardView { static final int KEYCODE_OPTIONS = -100; static final int KEYCODE_SHIFT_LONGPRESS = -101; - static final int KEYCODE_F1 = -102; - + static final int KEYCODE_VOICE = -102; + static final int KEYCODE_F1 = -103; private Keyboard mPhoneKeyboard; public LatinKeyboardView(Context context, AttributeSet attrs) { diff --git a/src/com/android/inputmethod/voice/EditingUtil.java b/src/com/android/inputmethod/voice/EditingUtil.java new file mode 100644 index 000000000..6316d8ccf --- /dev/null +++ b/src/com/android/inputmethod/voice/EditingUtil.java @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.InputConnection; + +/** + * Utility methods to deal with editing text through an InputConnection. + */ +public class EditingUtil { + private EditingUtil() {}; + + /** + * Append newText to the text field represented by connection. + * The new text becomes selected. + */ + public static void appendText(InputConnection connection, String newText) { + if (connection == null) { + return; + } + + // Commit the composing text + connection.finishComposingText(); + + // Add a space if the field already has text. + CharSequence charBeforeCursor = connection.getTextBeforeCursor(1, 0); + if (charBeforeCursor != null + && !charBeforeCursor.equals(" ") + && (charBeforeCursor.length() > 0)) { + newText = " " + newText; + } + + connection.setComposingText(newText, 1); + } + + private static int getCursorPosition(InputConnection connection) { + ExtractedText extracted = connection.getExtractedText( + new ExtractedTextRequest(), 0); + if (extracted == null) { + return -1; + } + return extracted.startOffset + extracted.selectionStart; + } + + private static int getSelectionEnd(InputConnection connection) { + ExtractedText extracted = connection.getExtractedText( + new ExtractedTextRequest(), 0); + if (extracted == null) { + return -1; + } + return extracted.startOffset + extracted.selectionEnd; + } + + /** + * @param connection connection to the current text field. + * @param sep characters which may separate words + * @return the word that surrounds the cursor, including up to one trailing + * separator. For example, if the field contains "he|llo world", where | + * represents the cursor, then "hello " will be returned. + */ + public static String getWordAtCursor( + InputConnection connection, String separators) { + Range range = getWordRangeAtCursor(connection, separators); + return (range == null) ? null : range.word; + } + + /** + * Removes the word surrounding the cursor. Parameters are identical to + * getWordAtCursor. + */ + public static void deleteWordAtCursor( + InputConnection connection, String separators) { + + Range range = getWordRangeAtCursor(connection, separators); + if (range == null) return; + + connection.finishComposingText(); + // Move cursor to beginning of word, to avoid crash when cursor is outside + // of valid range after deleting text. + int newCursor = getCursorPosition(connection) - range.charsBefore; + connection.setSelection(newCursor, newCursor); + connection.deleteSurroundingText(0, range.charsBefore + range.charsAfter); + } + + /** + * Represents a range of text, relative to the current cursor position. + */ + private static class Range { + /** Characters before selection start */ + int charsBefore; + + /** + * Characters after selection start, including one trailing word + * separator. + */ + int charsAfter; + + /** The actual characters that make up a word */ + String word; + + public Range(int charsBefore, int charsAfter, String word) { + if (charsBefore < 0 || charsAfter < 0) { + throw new IndexOutOfBoundsException(); + } + this.charsBefore = charsBefore; + this.charsAfter = charsAfter; + this.word = word; + } + } + + private static Range getWordRangeAtCursor( + InputConnection connection, String sep) { + if (connection == null || sep == null) { + return null; + } + CharSequence before = connection.getTextBeforeCursor(1000, 0); + CharSequence after = connection.getTextAfterCursor(1000, 0); + if (before == null || after == null) { + return null; + } + + // Find first word separator before the cursor + int start = before.length(); + while (--start > 0 && !isWhitespace(before.charAt(start - 1), sep)); + + // Find last word separator after the cursor + int end = -1; + while (++end < after.length() && !isWhitespace(after.charAt(end), sep)); + if (end < after.length() - 1) { + end++; // Include trailing space, if it exists, in word + } + + int cursor = getCursorPosition(connection); + if (start >= 0 && cursor + end <= after.length() + before.length()) { + String word = before.toString().substring(start, before.length()) + + after.toString().substring(0, end); + return new Range(before.length() - start, end, word); + } + + return null; + } + + private static boolean isWhitespace(int code, String whitespace) { + return whitespace.contains(String.valueOf((char) code)); + } +} diff --git a/src/com/android/inputmethod/voice/FieldContext.java b/src/com/android/inputmethod/voice/FieldContext.java new file mode 100644 index 000000000..0578af732 --- /dev/null +++ b/src/com/android/inputmethod/voice/FieldContext.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.os.Bundle; +import android.util.Log; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.ExtractedText; +import android.view.inputmethod.ExtractedTextRequest; +import android.view.inputmethod.InputConnection; + +/** + * Represents information about a given text field, which can be passed + * to the speech recognizer as context information. + */ +public class FieldContext { + static final String LABEL = "label"; + static final String HINT = "hint"; + static final String PACKAGE_NAME = "packageName"; + static final String FIELD_ID = "fieldId"; + static final String FIELD_NAME = "fieldName"; + static final String SINGLE_LINE = "singleLine"; + static final String INPUT_TYPE = "inputType"; + static final String IME_OPTIONS = "imeOptions"; + + Bundle mFieldInfo; + + public FieldContext(InputConnection conn, EditorInfo info) { + this.mFieldInfo = new Bundle(); + addEditorInfoToBundle(info, mFieldInfo); + addInputConnectionToBundle(conn, mFieldInfo); + Log.i("FieldContext", "Bundle = " + mFieldInfo.toString()); + } + + private static String safeToString(Object o) { + if (o == null) { + return ""; + } + return o.toString(); + } + + private static void addEditorInfoToBundle(EditorInfo info, Bundle bundle) { + if (info == null) { + return; + } + + + bundle.putString(LABEL, safeToString(info.label)); + bundle.putString(HINT, safeToString(info.hintText)); + bundle.putString(PACKAGE_NAME, safeToString(info.packageName)); + bundle.putInt(FIELD_ID, info.fieldId); + bundle.putString(FIELD_NAME, safeToString(info.fieldName)); + bundle.putInt(INPUT_TYPE, info.inputType); + bundle.putInt(IME_OPTIONS, info.imeOptions); + } + + private static void addInputConnectionToBundle( + InputConnection conn, Bundle bundle) { + if (conn == null) { + return; + } + + ExtractedText et = conn.getExtractedText(new ExtractedTextRequest(), 0); + if (et == null) { + return; + } + bundle.putBoolean(SINGLE_LINE, (et.flags & et.FLAG_SINGLE_LINE) > 0); + } + + public Bundle getBundle() { + return mFieldInfo; + } + + public String toString() { + return mFieldInfo.toString(); + } +} diff --git a/src/com/android/inputmethod/voice/GoogleSettingsUtil.java b/src/com/android/inputmethod/voice/GoogleSettingsUtil.java new file mode 100644 index 000000000..d238579ba --- /dev/null +++ b/src/com/android/inputmethod/voice/GoogleSettingsUtil.java @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.content.ContentResolver; +import android.database.Cursor; +import android.net.Uri; +import android.util.Log; + +/** + * Utility for getting Google-specific settings from GoogleSettings.Partner or + * Gservices. Retrieving such settings may fail on a non-Google Experience + * Device (GED) + */ +public class GoogleSettingsUtil { + /** + * A whitespace-separated list of supported locales for voice input from the keyboard. + */ + public static final String LATIN_IME_VOICE_INPUT_SUPPORTED_LOCALES = + "latin_ime_voice_input_supported_locales"; + + /** + * A whitespace-separated list of recommended app packages for voice input from the + * keyboard. + */ + public static final String LATIN_IME_VOICE_INPUT_RECOMMENDED_PACKAGES = + "latin_ime_voice_input_recommended_packages"; + + /** + * The maximum number of unique days to show the swipe hint for voice input. + */ + public static final String LATIN_IME_VOICE_INPUT_SWIPE_HINT_MAX_DAYS = + "latin_ime_voice_input_swipe_hint_max_days"; + + /** + * The maximum number of times to show the punctuation hint for voice input. + */ + public static final String LATIN_IME_VOICE_INPUT_PUNCTUATION_HINT_MAX_DISPLAYS = + "latin_ime_voice_input_punctuation_hint_max_displays"; + + /** + * Endpointer parameters for voice input from the keyboard. + */ + public static final String LATIN_IME_SPEECH_MINIMUM_LENGTH_MILLIS = + "latin_ime_speech_minimum_length_millis"; + public static final String LATIN_IME_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS = + "latin_ime_speech_input_complete_silence_length_millis"; + public static final String LATIN_IME_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS = + "latin_ime_speech_input_possibly_complete_silence_length_millis"; + + /** + * Min and max volume levels that can be displayed on the "speak now" screen. + */ + public static final String LATIN_IME_MIN_MICROPHONE_LEVEL = + "latin_ime_min_microphone_level"; + public static final String LATIN_IME_MAX_MICROPHONE_LEVEL = + "latin_ime_max_microphone_level"; + + /** + * The number of sentence-level alternates to request of the server. + */ + public static final String LATIN_IME_MAX_VOICE_RESULTS = "latin_ime_max_voice_results"; + + /** + * Uri to use to access gservices settings + */ + private static final Uri GSERVICES_URI = Uri.parse("content://settings/gservices"); + + private static final String TAG = GoogleSettingsUtil.class.getSimpleName(); + + private static final boolean DBG = false; + + /** + * Safely query for a Gservices string setting, which may not be available if this + * is not a Google Experience Device. + * + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if none can be found + * @return The value of the setting, or defaultValue if it couldn't be found + */ + public static String getGservicesString(ContentResolver cr, String key, String defaultValue) { + return getSettingString(GSERVICES_URI, cr, key, defaultValue); + } + + /** + * Safely query for a Gservices int setting, which may not be available if this + * is not a Google Experience Device. + * + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if the setting couldn't be found or parsed + * @return The value of the setting, or defaultValue if it couldn't be found or parsed + */ + public static int getGservicesInt(ContentResolver cr, String key, int defaultValue) { + try { + return Integer.parseInt(getGservicesString(cr, key, String.valueOf(defaultValue))); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Safely query for a Gservices float setting, which may not be available if this + * is not a Google Experience Device. + * + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if the setting couldn't be found or parsed + * @return The value of the setting, or defaultValue if it couldn't be found or parsed + */ + public static float getGservicesFloat(ContentResolver cr, String key, float defaultValue) { + try { + return Float.parseFloat(getGservicesString(cr, key, String.valueOf(defaultValue))); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * A safe way to query for a setting on both Google Experience and + * non-Google Experience devices, (code adapted from maps application + * examples) + * + * @param uri The uri to provide to the content resolver + * @param cr The content resolver to use + * @param key The setting to look up + * @param defaultValue The default value to use if none can be found + * @return The value of the setting, or defaultValue if it couldn't be found + */ + private static String getSettingString(Uri uri, ContentResolver cr, String key, + String defaultValue) { + String value = null; + + Cursor cursor = null; + try { + cursor = cr.query(uri, new String[] { + "value" + }, "name='" + key + "'", null, null); + if ((cursor != null) && cursor.moveToFirst()) { + value = cursor.getString(cursor.getColumnIndexOrThrow("value")); + } + } catch (Throwable t) { + // This happens because we're probably running a non Type 1 aka + // Google Experience device which doesn't have the Google libraries. + if (DBG) { + Log.d(TAG, "Error getting setting from " + uri + " for key " + key + ": " + t); + } + } finally { + if (cursor != null) { + cursor.close(); + } + } + + if (DBG && value == null) { + Log.i(TAG, "no setting found from " + uri + " for key " + key + ", returning default"); + } + + return (value != null) ? value : defaultValue; + } +} diff --git a/src/com/android/inputmethod/voice/LatinIMEWithVoice.java b/src/com/android/inputmethod/voice/LatinIMEWithVoice.java new file mode 100644 index 000000000..ccbf5b6bc --- /dev/null +++ b/src/com/android/inputmethod/voice/LatinIMEWithVoice.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.content.Intent; + +import com.android.inputmethod.latin.LatinIME; + +public class LatinIMEWithVoice extends LatinIME { + @Override + protected void launchSettings() { + launchSettings(LatinIMEWithVoiceSettings.class); + } +} diff --git a/src/com/android/inputmethod/voice/LatinIMEWithVoiceSettings.java b/src/com/android/inputmethod/voice/LatinIMEWithVoiceSettings.java new file mode 100644 index 000000000..13a58e14d --- /dev/null +++ b/src/com/android/inputmethod/voice/LatinIMEWithVoiceSettings.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import com.android.inputmethod.latin.LatinIMESettings; + +public class LatinIMEWithVoiceSettings extends LatinIMESettings {} diff --git a/src/com/android/inputmethod/voice/LoggingEvents.java b/src/com/android/inputmethod/voice/LoggingEvents.java new file mode 100644 index 000000000..b63229186 --- /dev/null +++ b/src/com/android/inputmethod/voice/LoggingEvents.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +/** + * Logging event constants used for Voice Search and VoiceIME. These are the keys and values of + * extras to be specified in logging broadcast intents to the {@link LoggingReceiver}. + * + * This class is duplicated between VoiceSearch and LatinIME. Please keep both versions + * in sync. + */ +public class LoggingEvents { + // The name of the broadcast intent for logging. + public static final String ACTION_LOG_EVENT = "com.google.android.voicesearch.LOG_EVENT"; + + // The extra key used for the name of the app being logged. + public static final String EXTRA_APP_NAME = "app_name"; + + // The extra key used for the event value. The possible event values depend on the + // app being logged for, and are defined in the subclasses below. + public static final String EXTRA_EVENT = "extra_event"; + + // The extra key used (with a boolean value of 'true') as a way to trigger a flush + // of the log events to the server. + public static final String EXTRA_FLUSH = "flush"; + + /** + * Logging event constants for VoiceIME. Below are the extra values for + * {@link LoggingEvents#EXTRA_EVENT}, clustered with keys to additional extras + * for some events that need to be included as additional fields in the event. + */ + public class VoiceIme { + // The app name to be used for logging VoiceIME events. + public static final String APP_NAME = "voiceime"; + + public static final int KEYBOARD_WARNING_DIALOG_SHOWN = 0; + + public static final int KEYBOARD_WARNING_DIALOG_DISMISSED = 1; + + public static final int KEYBOARD_WARNING_DIALOG_OK = 2; + + public static final int KEYBOARD_WARNING_DIALOG_CANCEL = 3; + + public static final int SETTINGS_WARNING_DIALOG_SHOWN = 4; + + public static final int SETTINGS_WARNING_DIALOG_DISMISSED = 5; + + public static final int SETTINGS_WARNING_DIALOG_OK = 6; + + public static final int SETTINGS_WARNING_DIALOG_CANCEL = 7; + + public static final int SWIPE_HINT_DISPLAYED = 8; + + public static final int PUNCTUATION_HINT_DISPLAYED = 9; + + public static final int CANCEL_DURING_LISTENING = 10; + + public static final int CANCEL_DURING_WORKING = 11; + + public static final int CANCEL_DURING_ERROR = 12; + + public static final int ERROR = 13; + public static final String EXTRA_ERROR_CODE = "code"; // value should be int + + public static final int START = 14; + public static final String EXTRA_START_LOCALE = "locale"; // value should be String + public static final String EXTRA_START_SWIPE = "swipe"; // value should be boolean + + public static final int VOICE_INPUT_DELIVERED = 15; + + public static final int N_BEST_CHOOSE = 16; + public static final String EXTRA_N_BEST_CHOOSE_INDEX = "index"; // value should be int + + public static final int TEXT_MODIFIED = 17; + + public static final int INPUT_ENDED = 18; + + public static final int VOICE_INPUT_SETTING_ENABLED = 19; + + public static final int VOICE_INPUT_SETTING_DISABLED = 20; + } +} diff --git a/src/com/android/inputmethod/voice/RecognitionView.java b/src/com/android/inputmethod/voice/RecognitionView.java new file mode 100644 index 000000000..97acb1152 --- /dev/null +++ b/src/com/android/inputmethod/voice/RecognitionView.java @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.content.ContentResolver; +import android.content.Context; +import android.content.res.Resources; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.CornerPathEffect; +import android.graphics.Paint; +import android.graphics.Path; +import android.graphics.PathEffect; +import android.graphics.drawable.Drawable; +import android.os.Handler; +import android.util.TypedValue; +import android.view.LayoutInflater; +import android.view.View; +import android.view.View.OnClickListener; +import android.view.ViewGroup.MarginLayoutParams; +import android.widget.ImageView; +import android.widget.TextView; + +import com.android.inputmethod.latin.R; +import com.android.inputmethod.voice.GoogleSettingsUtil; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * The user interface for the "Speak now" and "working" states. + * Displays a recognition dialog (with waveform, voice meter, etc.), + * plays beeps, shows errors, etc. + */ +public class RecognitionView { + private static final String TAG = "RecognitionView"; + + // If there's a significant delay between starting up voice search and the + // onset of audio recording, show the "initializing" screen first. If not, + // jump directly to the "speak now" screen to avoid flashing "initializing" + // quickly. + private static final boolean EXPECT_RECORDING_DELAY = true; + + private Handler mUiHandler; // Reference to UI thread + private View mView; + private Context mContext; + + private ImageView mImage; + private TextView mText; + private View mButton; + private TextView mButtonText; + private View mProgress; + + private Drawable mInitializing; + private Drawable mError; + private List<Drawable> mSpeakNow; + + private float mVolume = 0.0f; + private int mLevel = 0; + + private enum State {LISTENING, WORKING, READY} + private State mState = State.READY; + + private float mMinMicrophoneLevel; + private float mMaxMicrophoneLevel; + + /** Updates the microphone icon to show user their volume.*/ + private Runnable mUpdateVolumeRunnable = new Runnable() { + public void run() { + if (mState != State.LISTENING) { + return; + } + + final float min = mMinMicrophoneLevel; + final float max = mMaxMicrophoneLevel; + final int maxLevel = mSpeakNow.size() - 1; + + int index = (int) ((mVolume - min) / (max - min) * maxLevel); + final int level = Math.min(Math.max(0, index), maxLevel); + + if (level != mLevel) { + mImage.setImageDrawable(mSpeakNow.get(level)); + mLevel = level; + } + mUiHandler.postDelayed(mUpdateVolumeRunnable, 50); + } + }; + + public RecognitionView(Context context, OnClickListener clickListener) { + mUiHandler = new Handler(); + + LayoutInflater inflater = (LayoutInflater) context.getSystemService( + Context.LAYOUT_INFLATER_SERVICE); + mView = inflater.inflate(R.layout.recognition_status, null); + + ContentResolver cr = context.getContentResolver(); + mMinMicrophoneLevel = GoogleSettingsUtil.getGservicesFloat( + cr, GoogleSettingsUtil.LATIN_IME_MIN_MICROPHONE_LEVEL, 15.f); + mMaxMicrophoneLevel = GoogleSettingsUtil.getGservicesFloat( + cr, GoogleSettingsUtil.LATIN_IME_MAX_MICROPHONE_LEVEL, 30.f); + + // Pre-load volume level images + Resources r = context.getResources(); + + mSpeakNow = new ArrayList<Drawable>(); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level0)); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level1)); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level2)); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level3)); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level4)); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level5)); + mSpeakNow.add(r.getDrawable(R.drawable.speak_now_level6)); + + mInitializing = r.getDrawable(R.drawable.mic_slash); + mError = r.getDrawable(R.drawable.caution); + + mImage = (ImageView) mView.findViewById(R.id.image); + mButton = mView.findViewById(R.id.button); + mButton.setOnClickListener(clickListener); + mText = (TextView) mView.findViewById(R.id.text); + mButtonText = (TextView) mView.findViewById(R.id.button_text); + mProgress = mView.findViewById(R.id.progress); + + mContext = context; + } + + public View getView() { + return mView; + } + + public void showInitializing() { + mUiHandler.post(new Runnable() { + public void run() { + mText.setText(R.string.voice_initializing); + mImage.setImageDrawable(mInitializing); + mButtonText.setText(mContext.getText(R.string.cancel)); + } + }); + } + + public void showStartState() { + if (EXPECT_RECORDING_DELAY) { + showInitializing(); + } else { + showListening(); + } + } + + public void showListening() { + mState = State.LISTENING; + mUiHandler.post(new Runnable() { + public void run() { + mText.setText(R.string.voice_listening); + mImage.setImageDrawable(mSpeakNow.get(0)); + mButtonText.setText(mContext.getText(R.string.cancel)); + } + }); + mUiHandler.postDelayed(mUpdateVolumeRunnable, 50); + } + + public void updateVoiceMeter(final float rmsdB) { + mVolume = rmsdB; + } + + public void showError(final String message) { + mState = State.READY; + mUiHandler.post(new Runnable() { + public void run() { + exitWorking(); + mText.setText(message); + mImage.setImageDrawable(mError); + mButtonText.setText(mContext.getText(R.string.ok)); + } + }); + } + + public void showWorking( + final ByteArrayOutputStream waveBuffer, + final int speechStartPosition, + final int speechEndPosition) { + + mState = State.WORKING; + + mUiHandler.post(new Runnable() { + public void run() { + mText.setText(R.string.voice_working); + mImage.setVisibility(View.GONE); + mProgress.setVisibility(View.VISIBLE); + final ShortBuffer buf = ByteBuffer.wrap(waveBuffer.toByteArray()) + .order(ByteOrder.nativeOrder()).asShortBuffer(); + buf.position(0); + waveBuffer.reset(); + showWave(buf, speechStartPosition / 2, speechEndPosition / 2); + } + }); + } + + /** + * @return an average abs of the specified buffer. + */ + private static int getAverageAbs(ShortBuffer buffer, int start, int i, int npw) { + int from = start + i * npw; + int end = from + npw; + int total = 0; + for (int x = from; x < end; x++) { + total += Math.abs(buffer.get(x)); + } + return total / npw; + } + + + /** + * Shows waveform of input audio. + * + * Copied from version in VoiceSearch's RecognitionActivity. + * + * TODO: adjust stroke width based on the size of data. + * TODO: use dip rather than pixels. + */ + private void showWave(ShortBuffer waveBuffer, int startPosition, int endPosition) { + final int w = ((View) mImage.getParent()).getWidth(); + final int h = mImage.getHeight(); + if (w <= 0 || h <= 0) { + // view is not visible this time. Skip drawing. + return; + } + final Bitmap b = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); + final Canvas c = new Canvas(b); + final Paint paint = new Paint(); + paint.setColor(0xFFFFFFFF); // 0xAARRGGBB + paint.setAntiAlias(true); + paint.setStyle(Paint.Style.STROKE); + paint.setAlpha(0x90); + + final PathEffect effect = new CornerPathEffect(3); + paint.setPathEffect(effect); + + final int numSamples = waveBuffer.remaining(); + int endIndex; + if (endPosition == 0) { + endIndex = numSamples; + } else { + endIndex = Math.min(endPosition, numSamples); + } + + int startIndex = startPosition - 2000; // include 250ms before speech + if (startIndex < 0) { + startIndex = 0; + } + final int numSamplePerWave = 200; // 8KHz 25ms = 200 samples + final float scale = 10.0f / 65536.0f; + + final int count = (endIndex - startIndex) / numSamplePerWave; + final float deltaX = 1.0f * w / count; + int yMax = h / 2 - 10; + Path path = new Path(); + c.translate(0, yMax); + float x = 0; + path.moveTo(x, 0); + yMax -= 10; + for (int i = 0; i < count; i++) { + final int avabs = getAverageAbs(waveBuffer, startIndex, i , numSamplePerWave); + int sign = ( (i & 01) == 0) ? -1 : 1; + final float y = Math.min(yMax, avabs * h * scale) * sign; + path.lineTo(x, y); + x += deltaX; + path.lineTo(x, y); + } + if (deltaX > 4) { + paint.setStrokeWidth(3); + } else { + paint.setStrokeWidth(Math.max(1, (int) (deltaX -.05))); + } + c.drawPath(path, paint); + mImage.setImageBitmap(b); + mImage.setVisibility(View.VISIBLE); + MarginLayoutParams mProgressParams = (MarginLayoutParams)mProgress.getLayoutParams(); + mProgressParams.topMargin = (int) TypedValue.applyDimension(TypedValue.COMPLEX_UNIT_DIP, + -h / 2 - 18, mContext.getResources().getDisplayMetrics()); + + // Tweak the padding manually to fill out the whole view horizontally. + // TODO: Do this in the xml layout instead. + ((View) mImage.getParent()).setPadding(4, ((View) mImage.getParent()).getPaddingTop(), 3, + ((View) mImage.getParent()).getPaddingBottom()); + mProgress.setLayoutParams(mProgressParams); + } + + + public void finish() { + mState = State.READY; + mUiHandler.post(new Runnable() { + public void run() { + exitWorking(); + } + }); + showStartState(); + } + + private void exitWorking() { + mProgress.setVisibility(View.GONE); + mImage.setVisibility(View.VISIBLE); + } +} diff --git a/src/com/android/inputmethod/voice/VoiceInput.java b/src/com/android/inputmethod/voice/VoiceInput.java new file mode 100644 index 000000000..2f45b654a --- /dev/null +++ b/src/com/android/inputmethod/voice/VoiceInput.java @@ -0,0 +1,551 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.content.ComponentName; +import android.content.ContentResolver; +import android.content.Context; +import android.content.Intent; +import android.os.Build; +import android.os.Bundle; +import android.os.IBinder; +import android.os.RemoteException; +import android.util.Log; +import android.speech.IRecognitionListener; +import android.speech.RecognitionServiceUtil; +import android.speech.RecognizerIntent; +import android.speech.RecognitionResult; +import android.view.View; +import android.view.View.OnClickListener; +import com.android.inputmethod.latin.R; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Speech recognition input, including both user interface and a background + * process to stream audio to the network recognizer. This class supplies a + * View (getView()), which it updates as recognition occurs. The user of this + * class is responsible for making the view visible to the user, as well as + * handling various events returned through UiListener. + */ +public class VoiceInput implements OnClickListener { + private static final String TAG = "VoiceInput"; + private static final String EXTRA_RECOGNITION_CONTEXT = + "android.speech.extras.RECOGNITION_CONTEXT"; + private static final String EXTRA_CALLING_PACKAGE = "calling_package"; + + private static final String DEFAULT_RECOMMENDED_PACKAGES = + "com.android.mms " + + "com.google.android.gm " + + "com.google.android.talk " + + "com.google.android.apps.googlevoice " + + "com.android.email " + + "com.android.browser "; + + // WARNING! Before enabling this, fix the problem with calling getExtractedText() in + // landscape view. It causes Extracted text updates to be rejected due to a token mismatch + public static boolean ENABLE_WORD_CORRECTIONS = false; + + private static Boolean sVoiceIsAvailable = null; + + // Dummy word suggestion which means "delete current word" + public static final String DELETE_SYMBOL = " \u00D7 "; // times symbol + + private Whitelist mRecommendedList; + private Whitelist mBlacklist; + + private VoiceInputLogger mLogger; + + // Names of a few intent extras defined in VoiceSearch's RecognitionService. + // These let us tweak the endpointer parameters. + private static final String EXTRA_SPEECH_MINIMUM_LENGTH_MILLIS = + "android.speech.extras.SPEECH_INPUT_MINIMUM_LENGTH_MILLIS"; + private static final String EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS = + "android.speech.extras.SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS"; + private static final String EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS = + "android.speech.extras.SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS"; + + // The usual endpointer default value for input complete silence length is 0.5 seconds, + // but that's used for things like voice search. For dictation-like voice input like this, + // we go with a more liberal value of 1 second. This value will only be used if a value + // is not provided from Gservices. + private static final String INPUT_COMPLETE_SILENCE_LENGTH_DEFAULT_VALUE_MILLIS = "1000"; + + // Used to record part of that state for logging purposes. + public static final int DEFAULT = 0; + public static final int LISTENING = 1; + public static final int WORKING = 2; + public static final int ERROR = 3; + + private int mState = DEFAULT; + + /** + * Events relating to the recognition UI. You must implement these. + */ + public interface UiListener { + + /** + * @param recognitionResults a set of transcripts for what the user + * spoke, sorted by likelihood. + */ + public void onVoiceResults( + List<String> recognitionResults, + Map<String, List<CharSequence>> alternatives); + + /** + * Called when the user cancels speech recognition. + */ + public void onCancelVoice(); + } + + private RecognitionServiceUtil.Connection mRecognitionConnection; + private IRecognitionListener mRecognitionListener; + private RecognitionView mRecognitionView; + private UiListener mUiListener; + private Context mContext; + private ScheduledThreadPoolExecutor mExecutor; + + /** + * @param context the service or activity in which we're runing. + * @param uiHandler object to receive events from VoiceInput. + */ + public VoiceInput(Context context, UiListener uiHandler) { + mLogger = VoiceInputLogger.getLogger(context); + mRecognitionListener = new IMERecognitionListener(); + mRecognitionConnection = new RecognitionServiceUtil.Connection() { + public synchronized void onServiceConnected( + ComponentName name, IBinder service) { + super.onServiceConnected(name, service); + } + }; + mUiListener = uiHandler; + mContext = context; + newView(); + + String recommendedPackages = GoogleSettingsUtil.getGservicesString( + context.getContentResolver(), + GoogleSettingsUtil.LATIN_IME_VOICE_INPUT_RECOMMENDED_PACKAGES, + DEFAULT_RECOMMENDED_PACKAGES); + + mRecommendedList = new Whitelist(); + for (String recommendedPackage : recommendedPackages.split("\\s+")) { + mRecommendedList.addApp(recommendedPackage); + } + + mBlacklist = new Whitelist(); + mBlacklist.addApp("com.android.setupwizard"); + + mExecutor = new ScheduledThreadPoolExecutor(1); + bindIfNecessary(); + } + + /** + * @return true if field is blacklisted for voice + */ + public boolean isBlacklistedField(FieldContext context) { + return mBlacklist.matches(context); + } + + /** + * Used to decide whether to show voice input hints for this field, etc. + * + * @return true if field is recommended for voice + */ + public boolean isRecommendedField(FieldContext context) { + return mRecommendedList.matches(context); + } + + /** + * @return true if the speech service is available on the platform. + */ + public static boolean voiceIsAvailable(Context context) { + if (sVoiceIsAvailable != null) { + return sVoiceIsAvailable; + } + + RecognitionServiceUtil.Connection recognitionConnection = + new RecognitionServiceUtil.Connection(); + boolean bound = context.bindService( + makeIntent(), recognitionConnection, Context.BIND_AUTO_CREATE); + context.unbindService(recognitionConnection); + sVoiceIsAvailable = bound; + return bound; + } + + /** + * Start listening for speech from the user. This will grab the microphone + * and start updating the view provided by getView(). It is the caller's + * responsibility to ensure that the view is visible to the user at this stage. + * + * @param context the same FieldContext supplied to voiceIsEnabled() + * @param swipe whether this voice input was started by swipe, for logging purposes + */ + public void startListening(FieldContext context, boolean swipe) { + mState = DEFAULT; + + Locale locale = Locale.getDefault(); + String localeString = locale.getLanguage() + "-" + locale.getCountry(); + + mLogger.start(localeString, swipe); + + mState = LISTENING; + + if (mRecognitionConnection.mService == null) { + mRecognitionView.showInitializing(); + } else { + mRecognitionView.showStartState(); + } + + if (!bindIfNecessary()) { + mState = ERROR; + + // We use CLIENT_ERROR to signify voice search is not available on the device. + onError(RecognitionResult.CLIENT_ERROR, false); + cancel(); + } + + if (mRecognitionConnection.mService != null) { + try { + Intent intent = makeIntent(); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, ""); + intent.putExtra(EXTRA_RECOGNITION_CONTEXT, context.getBundle()); + intent.putExtra(EXTRA_CALLING_PACKAGE, "VoiceIME"); + intent.putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, + GoogleSettingsUtil.getGservicesInt( + mContext.getContentResolver(), + GoogleSettingsUtil.LATIN_IME_MAX_VOICE_RESULTS, + 1)); + + // Get endpointer params from Gservices. + // TODO: Consider caching these values for improved performance on slower devices. + ContentResolver cr = mContext.getContentResolver(); + putEndpointerExtra( + cr, + intent, + GoogleSettingsUtil.LATIN_IME_SPEECH_MINIMUM_LENGTH_MILLIS, + EXTRA_SPEECH_MINIMUM_LENGTH_MILLIS, + null /* rely on endpointer default */); + putEndpointerExtra( + cr, + intent, + GoogleSettingsUtil.LATIN_IME_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, + EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, + INPUT_COMPLETE_SILENCE_LENGTH_DEFAULT_VALUE_MILLIS + /* our default value is different from the endpointer's */); + putEndpointerExtra( + cr, + intent, + GoogleSettingsUtil. + LATIN_IME_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, + EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, + null /* rely on endpointer default */); + + mRecognitionConnection.mService.startListening( + intent, mRecognitionListener); + } catch (RemoteException e) { + Log.e(TAG, "Could not start listening", e); + onError(-1 /* no specific error, just show default error */, false); + } + } + } + + /** + * Gets the value of the provided Gservices key, attempts to parse it into a long, + * and if successful, puts the long value as an extra in the provided intent. + */ + private void putEndpointerExtra(ContentResolver cr, Intent i, + String gservicesKey, String intentExtraKey, String defaultValue) { + long l = -1; + String s = GoogleSettingsUtil.getGservicesString(cr, gservicesKey, defaultValue); + if (s != null) { + try { + l = Long.valueOf(s); + } catch (NumberFormatException e) { + Log.e(TAG, "could not parse value for " + gservicesKey + ": " + s); + } + } + + if (l != -1) i.putExtra(intentExtraKey, l); + } + + public void destroy() { + if (mRecognitionConnection.mService != null) { + //mContext.unbindService(mRecognitionConnection); + } + } + + /** + * Creates a new instance of the view that is returned by {@link #getView()} + * Clients should use this when a previously returned view is stuck in a + * layout that is being thrown away and a new one is need to show to the + * user. + */ + public void newView() { + mRecognitionView = new RecognitionView(mContext, this); + } + + /** + * @return a view that shows the recognition flow--e.g., "Speak now" and + * "working" dialogs. + */ + public View getView() { + return mRecognitionView.getView(); + } + + /** + * Handle the cancel button. + */ + public void onClick(View view) { + switch(view.getId()) { + case R.id.button: + cancel(); + break; + } + } + + public void logTextModified() { + mLogger.textModified(); + } + + public void logKeyboardWarningDialogShown() { + mLogger.keyboardWarningDialogShown(); + } + + public void logKeyboardWarningDialogDismissed() { + mLogger.keyboardWarningDialogDismissed(); + } + + public void logKeyboardWarningDialogOk() { + mLogger.keyboardWarningDialogOk(); + } + + public void logKeyboardWarningDialogCancel() { + mLogger.keyboardWarningDialogCancel(); + } + + public void logSwipeHintDisplayed() { + mLogger.swipeHintDisplayed(); + } + + public void logPunctuationHintDisplayed() { + mLogger.punctuationHintDisplayed(); + } + + public void logVoiceInputDelivered() { + mLogger.voiceInputDelivered(); + } + + public void logNBestChoose(int index) { + mLogger.nBestChoose(index); + } + + public void logInputEnded() { + mLogger.inputEnded(); + } + + public void flushLogs() { + mLogger.flush(); + } + + private static Intent makeIntent() { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + + // On Cupcake, use VoiceIMEHelper since VoiceSearch doesn't support. + // On Donut, always use VoiceSearch, since VoiceIMEHelper and + // VoiceSearch may conflict. + if (Build.VERSION.RELEASE.equals("1.5")) { + intent = intent.setClassName( + "com.google.android.voiceservice", + "com.google.android.voiceservice.IMERecognitionService"); + } else { + intent = intent.setClassName( + "com.google.android.voicesearch", + "com.google.android.voicesearch.RecognitionService"); + } + + return intent; + } + + /** + * Bind to the recognition service if necessary. + * @return true if we are bound or binding to the service, false if + * the recognition service is unavailable. + */ + private boolean bindIfNecessary() { + if (mRecognitionConnection.mService != null) { + return true; + } + return mContext.bindService( + makeIntent(), mRecognitionConnection, Context.BIND_AUTO_CREATE); + } + + /** + * Cancel in-progress speech recognition. + */ + public void cancel() { + switch (mState) { + case LISTENING: + mLogger.cancelDuringListening(); + break; + case WORKING: + mLogger.cancelDuringWorking(); + break; + case ERROR: + mLogger.cancelDuringError(); + break; + } + mState = DEFAULT; + + // Remove all pending tasks (e.g., timers to cancel voice input) + for (Runnable runnable : mExecutor.getQueue()) { + mExecutor.remove(runnable); + } + + if (mRecognitionConnection.mService != null) { + try { + mRecognitionConnection.mService.cancel(); + } catch (RemoteException e) { + Log.e(TAG, "Exception on cancel", e); + } + } + mUiListener.onCancelVoice(); + mRecognitionView.finish(); + } + + private int getErrorStringId(int errorType, boolean endpointed) { + switch (errorType) { + // We use CLIENT_ERROR to signify that voice search is not available on the device. + case RecognitionResult.CLIENT_ERROR: + return R.string.voice_not_installed; + case RecognitionResult.NETWORK_ERROR: + return R.string.voice_network_error; + case RecognitionResult.NETWORK_TIMEOUT: + return endpointed ? + R.string.voice_network_error : R.string.voice_too_much_speech; + case RecognitionResult.AUDIO_ERROR: + return R.string.voice_audio_error; + case RecognitionResult.SERVER_ERROR: + return R.string.voice_server_error; + case RecognitionResult.SPEECH_TIMEOUT: + return R.string.voice_speech_timeout; + case RecognitionResult.NO_MATCH: + return R.string.voice_no_match; + default: return R.string.voice_error; + } + } + + private void onError(int errorType, boolean endpointed) { + Log.i(TAG, "error " + errorType); + mLogger.error(errorType); + onError(mContext.getString(getErrorStringId(errorType, endpointed))); + } + + private void onError(String error) { + mState = ERROR; + mRecognitionView.showError(error); + // Wait a couple seconds and then automatically dismiss message. + mExecutor.schedule(new Runnable() { + public void run() { + cancel(); + }}, 2000, TimeUnit.MILLISECONDS); + } + + private class IMERecognitionListener extends IRecognitionListener.Stub { + // Waveform data + final ByteArrayOutputStream mWaveBuffer = new ByteArrayOutputStream(); + int mSpeechStart; + private boolean mEndpointed = false; + + public void onReadyForSpeech(Bundle noiseParams) { + mRecognitionView.showListening(); + } + + public void onBeginningOfSpeech() { + mEndpointed = false; + mSpeechStart = mWaveBuffer.size(); + } + + public void onRmsChanged(float rmsdB) { + mRecognitionView.updateVoiceMeter(rmsdB); + } + + public void onBufferReceived(byte[] buf) { + try { + mWaveBuffer.write(buf); + } catch (IOException e) {} + } + + public void onEndOfSpeech() { + mEndpointed = true; + mState = WORKING; + mRecognitionView.showWorking(mWaveBuffer, mSpeechStart, mWaveBuffer.size()); + } + + public void onError(int errorType) { + mState = ERROR; + VoiceInput.this.onError(errorType, mEndpointed); + } + + public void onResults(List<RecognitionResult> results, long token) { + mState = DEFAULT; + List<String> resultsAsText = new ArrayList<String>(); + for (RecognitionResult result : results) { + resultsAsText.add(result.mText); + } + + Map<String, List<CharSequence>> alternatives = + new HashMap<String, List<CharSequence>>(); + if (resultsAsText.size() >= 2 && ENABLE_WORD_CORRECTIONS) { + String[][] words = new String[resultsAsText.size()][]; + for (int i = 0; i < words.length; i++) { + words[i] = resultsAsText.get(i).split(" "); + } + + for (int key = 0; key < words[0].length; key++) { + alternatives.put(words[0][key], new ArrayList<CharSequence>()); + for (int alt = 1; alt < words.length; alt++) { + int keyBegin = key * words[alt].length / words[0].length; + int keyEnd = (key + 1) * words[alt].length / words[0].length; + + for (int i = keyBegin; i < Math.min(words[alt].length, keyEnd); i++) { + List<CharSequence> altList = alternatives.get(words[0][key]); + if (!altList.contains(words[alt][i]) && altList.size() < 6) { + altList.add(words[alt][i]); + } + } + } + } + } + + if (resultsAsText.size() > 5) { + resultsAsText = resultsAsText.subList(0, 5); + } + mUiListener.onVoiceResults(resultsAsText, alternatives); + mRecognitionView.finish(); + + destroy(); + } + } +} diff --git a/src/com/android/inputmethod/voice/VoiceInputLogger.java b/src/com/android/inputmethod/voice/VoiceInputLogger.java new file mode 100644 index 000000000..07d4d1c8c --- /dev/null +++ b/src/com/android/inputmethod/voice/VoiceInputLogger.java @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.content.Context; +import android.content.Intent; + +/** + * Provides the logging facility for voice input events. This fires broadcasts back to + * the voice search app which then logs on our behalf. + * + * Note that debug console logging does not occur in this class. If you want to + * see console output of these logging events, there is a boolean switch to turn + * on on the VoiceSearch side. + */ +public class VoiceInputLogger { + private static final String TAG = VoiceInputLogger.class.getSimpleName(); + + private static VoiceInputLogger sVoiceInputLogger; + + private final Context mContext; + + // The base intent used to form all broadcast intents to the logger + // in VoiceSearch. + private final Intent mBaseIntent; + + /** + * Returns the singleton of the logger. + * + * @param contextHint a hint context used when creating the logger instance. + * Ignored if the singleton instance already exists. + */ + public static synchronized VoiceInputLogger getLogger(Context contextHint) { + if (sVoiceInputLogger == null) { + sVoiceInputLogger = new VoiceInputLogger(contextHint); + } + return sVoiceInputLogger; + } + + public VoiceInputLogger(Context context) { + mContext = context; + + mBaseIntent = new Intent(LoggingEvents.ACTION_LOG_EVENT); + mBaseIntent.putExtra(LoggingEvents.EXTRA_APP_NAME, LoggingEvents.VoiceIme.APP_NAME); + } + + private Intent newLoggingBroadcast(int event) { + Intent i = new Intent(mBaseIntent); + i.putExtra(LoggingEvents.EXTRA_EVENT, event); + return i; + } + + public void flush() { + Intent i = new Intent(mBaseIntent); + i.putExtra(LoggingEvents.EXTRA_FLUSH, true); + mContext.sendBroadcast(i); + } + + public void keyboardWarningDialogShown() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_SHOWN)); + } + + public void keyboardWarningDialogDismissed() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_DISMISSED)); + } + + public void keyboardWarningDialogOk() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_OK)); + } + + public void keyboardWarningDialogCancel() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.KEYBOARD_WARNING_DIALOG_CANCEL)); + } + + public void settingsWarningDialogShown() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_SHOWN)); + } + + public void settingsWarningDialogDismissed() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_DISMISSED)); + } + + public void settingsWarningDialogOk() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_OK)); + } + + public void settingsWarningDialogCancel() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.SETTINGS_WARNING_DIALOG_CANCEL)); + } + + public void swipeHintDisplayed() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.SWIPE_HINT_DISPLAYED)); + } + + public void cancelDuringListening() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.CANCEL_DURING_LISTENING)); + } + + public void cancelDuringWorking() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.CANCEL_DURING_WORKING)); + } + + public void cancelDuringError() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.CANCEL_DURING_ERROR)); + } + + public void punctuationHintDisplayed() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.PUNCTUATION_HINT_DISPLAYED)); + } + + public void error(int code) { + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.ERROR); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_ERROR_CODE, code); + mContext.sendBroadcast(i); + } + + public void start(String locale, boolean swipe) { + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.START); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_START_LOCALE, locale); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_START_SWIPE, swipe); + mContext.sendBroadcast(i); + } + + public void voiceInputDelivered() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.VOICE_INPUT_DELIVERED)); + } + + public void textModified() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.TEXT_MODIFIED)); + } + + public void nBestChoose(int index) { + Intent i = newLoggingBroadcast(LoggingEvents.VoiceIme.N_BEST_CHOOSE); + i.putExtra(LoggingEvents.VoiceIme.EXTRA_N_BEST_CHOOSE_INDEX, index); + mContext.sendBroadcast(i); + } + + public void inputEnded() { + mContext.sendBroadcast(newLoggingBroadcast(LoggingEvents.VoiceIme.INPUT_ENDED)); + } + + public void voiceInputSettingEnabled() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.VOICE_INPUT_SETTING_ENABLED)); + } + + public void voiceInputSettingDisabled() { + mContext.sendBroadcast(newLoggingBroadcast( + LoggingEvents.VoiceIme.VOICE_INPUT_SETTING_DISABLED)); + } +} diff --git a/src/com/android/inputmethod/voice/WaveformImage.java b/src/com/android/inputmethod/voice/WaveformImage.java new file mode 100644 index 000000000..08d87c8f3 --- /dev/null +++ b/src/com/android/inputmethod/voice/WaveformImage.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2008-2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Paint; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; + +/** + * Utility class to draw a waveform into a bitmap, given a byte array + * that represents the waveform as a sequence of 16-bit integers. + * Adapted from RecognitionActivity.java. + */ +public class WaveformImage { + private static final int SAMPLING_RATE = 8000; + + private WaveformImage() {} + + public static Bitmap drawWaveform( + ByteArrayOutputStream waveBuffer, int w, int h, int start, int end) { + final Bitmap b = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888); + final Canvas c = new Canvas(b); + final Paint paint = new Paint(); + paint.setColor(0xFFFFFFFF); // 0xRRGGBBAA + paint.setAntiAlias(true); + paint.setStrokeWidth(0); + + final ShortBuffer buf = ByteBuffer + .wrap(waveBuffer.toByteArray()) + .order(ByteOrder.nativeOrder()) + .asShortBuffer(); + buf.position(0); + + final int numSamples = waveBuffer.size() / 2; + final int delay = (SAMPLING_RATE * 100 / 1000); + int endIndex = end / 2 + delay; + if (end == 0 || endIndex >= numSamples) { + endIndex = numSamples; + } + int index = start / 2 - delay; + if (index < 0) { + index = 0; + } + final int size = endIndex - index; + int numSamplePerPixel = 32; + int delta = size / (numSamplePerPixel * w); + if (delta == 0) { + numSamplePerPixel = size / w; + delta = 1; + } + + final float scale = 3.5f / 65536.0f; + // do one less column to make sure we won't read past + // the buffer. + try { + for (int i = 0; i < w - 1 ; i++) { + final float x = i; + for (int j = 0; j < numSamplePerPixel; j++) { + final short s = buf.get(index); + final float y = (h / 2) - (s * h * scale); + c.drawPoint(x, y, paint); + index += delta; + } + } + } catch (IndexOutOfBoundsException e) { + // this can happen, but we don't care + } + + return b; + } +} diff --git a/src/com/android/inputmethod/voice/Whitelist.java b/src/com/android/inputmethod/voice/Whitelist.java new file mode 100644 index 000000000..167b688ca --- /dev/null +++ b/src/com/android/inputmethod/voice/Whitelist.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.voice; + +import android.os.Bundle; +import java.util.ArrayList; +import java.util.List; + +/** + * A set of text fields where speech has been explicitly enabled. + */ +public class Whitelist { + private List<Bundle> mConditions; + + public Whitelist() { + mConditions = new ArrayList<Bundle>(); + } + + public Whitelist(List<Bundle> conditions) { + this.mConditions = conditions; + } + + public void addApp(String app) { + Bundle bundle = new Bundle(); + bundle.putString("packageName", app); + mConditions.add(bundle); + } + + /** + * @return true if the field is a member of the whitelist. + */ + public boolean matches(FieldContext context) { + for (Bundle condition : mConditions) { + if (matches(condition, context.getBundle())) { + return true; + } + } + return false; + } + + /** + * @return true of all values in condition are matched by a value + * in target. + */ + private boolean matches(Bundle condition, Bundle target) { + for (String key : condition.keySet()) { + if (!condition.getString(key).equals(target.getString(key))) { + return false; + } + } + return true; + } +} diff --git a/src/com/google/android/voicesearch/LatinIMEWithVoice.java b/src/com/google/android/voicesearch/LatinIMEWithVoice.java new file mode 100644 index 000000000..8a339d14a --- /dev/null +++ b/src/com/google/android/voicesearch/LatinIMEWithVoice.java @@ -0,0 +1,29 @@ +/* + * + * Copyright (C) 2009 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +package com.google.android.voicesearch; + +import android.content.Intent; + +import com.android.inputmethod.latin.LatinIME; + +public class LatinIMEWithVoice extends LatinIME { + @Override + protected void launchSettings() { + launchSettings(LatinIMEWithVoiceSettings.class); + } +} diff --git a/src/com/google/android/voicesearch/LatinIMEWithVoiceSettings.java b/src/com/google/android/voicesearch/LatinIMEWithVoiceSettings.java new file mode 100644 index 000000000..a53cebfd9 --- /dev/null +++ b/src/com/google/android/voicesearch/LatinIMEWithVoiceSettings.java @@ -0,0 +1,5 @@ +package com.google.android.voicesearch; + +import com.android.inputmethod.latin.LatinIMESettings; + +public class LatinIMEWithVoiceSettings extends LatinIMESettings {} |