diff options
53 files changed, 1410 insertions, 2014 deletions
diff --git a/common/src/com/android/inputmethod/latin/common/Constants.java b/common/src/com/android/inputmethod/latin/common/Constants.java index baa859268..03abb0fdd 100644 --- a/common/src/com/android/inputmethod/latin/common/Constants.java +++ b/common/src/com/android/inputmethod/latin/common/Constants.java @@ -21,6 +21,7 @@ import com.android.inputmethod.annotations.UsedForTesting; import javax.annotation.Nonnull; public final class Constants { + public static final class Color { /** * The alpha value for fully opaque. diff --git a/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java b/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java index 5489de135..205d6487a 100644 --- a/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java +++ b/java-overridable/src/com/android/inputmethod/latin/DictionaryFacilitatorProvider.java @@ -16,8 +16,6 @@ package com.android.inputmethod.latin; -import android.content.Context; - /** * Factory for instantiating DictionaryFacilitator objects. */ @@ -25,8 +23,4 @@ public class DictionaryFacilitatorProvider { public static DictionaryFacilitator newDictionaryFacilitator() { return new DictionaryFacilitatorImpl(); } - - public static DictionaryFacilitator newDictionaryFacilitator(final Context context) { - return new DictionaryFacilitatorImpl(context); - } } diff --git a/java-overridable/src/com/android/inputmethod/latin/define/DecoderSpecificConstants.java b/java-overridable/src/com/android/inputmethod/latin/define/DecoderSpecificConstants.java index eb4a3eb59..4789e208d 100644 --- a/java-overridable/src/com/android/inputmethod/latin/define/DecoderSpecificConstants.java +++ b/java-overridable/src/com/android/inputmethod/latin/define/DecoderSpecificConstants.java @@ -27,4 +27,7 @@ public class DecoderSpecificConstants { // (MAX_PREV_WORD_COUNT_FOR_N_GRAM + 1)-gram is supported in Java side. Needs to modify // MAX_PREV_WORD_COUNT_FOR_N_GRAM in native/jni/src/defines.h for suggestions. public static final int MAX_PREV_WORD_COUNT_FOR_N_GRAM = 3; + + public static final String DECODER_DICT_SUFFIX = ""; + } diff --git a/java/res/layout/user_dictionary_add_word_fullscreen.xml b/java/res/layout/user_dictionary_add_word_fullscreen.xml index 9bcb189b4..cbdfba67d 100644 --- a/java/res/layout/user_dictionary_add_word_fullscreen.xml +++ b/java/res/layout/user_dictionary_add_word_fullscreen.xml @@ -44,26 +44,6 @@ android:columnCount="2" > <TextView - android:id="@+id/user_dictionary_add_shortcut_label" - style="?android:attr/textAppearanceSmall" - android:layout_width="wrap_content" - android:layout_height="wrap_content" - android:layout_gravity="start|center_vertical" - android:text="@string/user_dict_settings_add_shortcut_option_name" /> - - <EditText - android:id="@+id/user_dictionary_add_shortcut" - android:layout_width="wrap_content" - android:layout_gravity="fill_horizontal|center_vertical" - android:layout_marginBottom="8dip" - android:layout_marginStart="8dip" - android:layout_marginTop="8dip" - android:hint="@string/user_dict_settings_add_shortcut_hint" - android:imeOptions="flagNoFullscreen" - android:inputType="textNoSuggestions" - android:maxLength="@integer/config_user_dictionary_max_word_length" /> - - <TextView android:id="@+id/user_dictionary_add_locale_label" style="?android:attr/textAppearanceSmall" android:layout_width="wrap_content" diff --git a/java/src/com/android/inputmethod/compat/UserDictionaryCompatUtils.java b/java/src/com/android/inputmethod/compat/UserDictionaryCompatUtils.java deleted file mode 100644 index b78c357ab..000000000 --- a/java/src/com/android/inputmethod/compat/UserDictionaryCompatUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.compat; - -import android.annotation.TargetApi; -import android.content.Context; -import android.os.Build; -import android.provider.UserDictionary; - -import java.util.Locale; - -public final class UserDictionaryCompatUtils { - @SuppressWarnings("deprecation") - public static void addWord(final Context context, final String word, - final int freq, final String shortcut, final Locale locale) { - if (BuildCompatUtils.EFFECTIVE_SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) { - addWordWithShortcut(context, word, freq, shortcut, locale); - return; - } - // Fall back to the pre-JellyBean method. - final Locale currentLocale = context.getResources().getConfiguration().locale; - final int localeType = currentLocale.equals(locale) - ? UserDictionary.Words.LOCALE_TYPE_CURRENT : UserDictionary.Words.LOCALE_TYPE_ALL; - UserDictionary.Words.addWord(context, word, freq, localeType); - } - - // {@link UserDictionary.Words#addWord(Context,String,int,String,Locale)} was introduced - // in API level 16 (Build.VERSION_CODES.JELLY_BEAN). - @TargetApi(Build.VERSION_CODES.JELLY_BEAN) - private static void addWordWithShortcut(final Context context, final String word, - final int freq, final String shortcut, final Locale locale) { - UserDictionary.Words.addWord(context, word, freq, shortcut, locale); - } -} diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index 46476e29e..fca5ecbc0 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -40,13 +40,11 @@ import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.settings.SettingsValues; import com.android.inputmethod.latin.utils.CapsModeUtils; import com.android.inputmethod.latin.utils.LanguageOnSpacebarUtils; -import com.android.inputmethod.latin.utils.NetworkConnectivityUtils; import com.android.inputmethod.latin.utils.RecapitalizeStatus; import com.android.inputmethod.latin.utils.ResourceUtils; import com.android.inputmethod.latin.utils.ScriptUtils; -public final class KeyboardSwitcher implements KeyboardState.SwitchActions, - NetworkConnectivityUtils.NetworkStateChangeListener { +public final class KeyboardSwitcher implements KeyboardState.SwitchActions { private static final String TAG = KeyboardSwitcher.class.getSimpleName(); private InputView mCurrentInputView; @@ -412,15 +410,6 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions, return mCurrentInputView; } - // {@link NetworkConnectivityUtils.NetworkStateChangeListener#onNetworkStateChanged(boolean)}. - @Override - public void onNetworkStateChanged() { - if (mKeyboardView == null) { - return; - } - mKeyboardView.updateShortcutKey(mRichImm.isShortcutImeReady()); - } - public int getKeyboardShiftMode() { final Keyboard keyboard = getKeyboard(); if (keyboard == null) { diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java index ab8b7515d..ce4bb7454 100644 --- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java @@ -694,25 +694,25 @@ public final class MainKeyboardView extends KeyboardView implements DrawingProxy } @Override - public boolean onTouchEvent(final MotionEvent me) { + public boolean onTouchEvent(final MotionEvent event) { if (getKeyboard() == null) { return false; } if (mNonDistinctMultitouchHelper != null) { - if (me.getPointerCount() > 1 && mTimerHandler.isInKeyRepeat()) { + if (event.getPointerCount() > 1 && mTimerHandler.isInKeyRepeat()) { // Key repeating timer will be canceled if 2 or more keys are in action. mTimerHandler.cancelKeyRepeatTimers(); } // Non distinct multitouch screen support - mNonDistinctMultitouchHelper.processMotionEvent(me, mKeyDetector); + mNonDistinctMultitouchHelper.processMotionEvent(event, mKeyDetector); return true; } - return processMotionEvent(me); + return processMotionEvent(event); } - public boolean processMotionEvent(final MotionEvent me) { - final int index = me.getActionIndex(); - final int id = me.getPointerId(index); + public boolean processMotionEvent(final MotionEvent event) { + final int index = event.getActionIndex(); + final int id = event.getPointerId(index); final PointerTracker tracker = PointerTracker.getPointerTracker(id); // When a more keys panel is showing, we should ignore other fingers' single touch events // other than the finger that is showing the more keys panel. @@ -720,7 +720,7 @@ public final class MainKeyboardView extends KeyboardView implements DrawingProxy && PointerTracker.getActivePointerTrackerCount() == 1) { return true; } - tracker.processMotionEvent(me, mKeyDetector); + tracker.processMotionEvent(event, mKeyDetector); return true; } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 7e4d66583..7ec964d83 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -74,7 +74,7 @@ public final class BinaryDictionary extends Dictionary { private static final int FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX = 0; private static final int FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX = 1; private static final int FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX = 2; - private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; + private static final int FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX = 3; // DEPRECATED private static final int FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX = 4; // Format to get probability and historical info from native side via getWordPropertyNative(). @@ -410,11 +410,9 @@ public final class BinaryDictionary extends Dictionary { outFlags[FORMAT_WORD_PROPERTY_IS_NOT_A_WORD_INDEX], outFlags[FORMAT_WORD_PROPERTY_IS_POSSIBLY_OFFENSIVE_INDEX], outFlags[FORMAT_WORD_PROPERTY_HAS_NGRAMS_INDEX], - outFlags[FORMAT_WORD_PROPERTY_HAS_SHORTCUTS_INDEX], outFlags[FORMAT_WORD_PROPERTY_IS_BEGINNING_OF_SENTENCE_INDEX], outProbabilityInfo, outNgramPrevWordsArray, outNgramPrevWordIsBeginningOfSentenceArray, - outNgramTargets, outNgramProbabilityInfo, outShortcutTargets, - outShortcutProbabilities); + outNgramTargets, outNgramProbabilityInfo); } public static class GetNextWordPropertyResult { @@ -442,19 +440,16 @@ public final class BinaryDictionary extends Dictionary { } // Add a unigram entry to binary dictionary with unigram attributes in native code. - public boolean addUnigramEntry(final String word, final int probability, - final String shortcutTarget, final int shortcutProbability, - final boolean isBeginningOfSentence, final boolean isNotAWord, - final boolean isPossiblyOffensive, final int timestamp) { + public boolean addUnigramEntry( + final String word, final int probability, final boolean isBeginningOfSentence, + final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { return false; } final int[] codePoints = StringUtils.toCodePointArray(word); - final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? - StringUtils.toCodePointArray(shortcutTarget) : null; - if (!addUnigramEntryNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, - shortcutProbability, isBeginningOfSentence, isNotAWord, isPossiblyOffensive, - timestamp)) { + if (!addUnigramEntryNative(mNativeDict, codePoints, probability, + null /* shortcutTargetCodePoints */, 0 /* shortcutProbability */, + isBeginningOfSentence, isNotAWord, isPossiblyOffensive, timestamp)) { return false; } mHasUpdated = true; diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index 22fd90795..ba0f9b807 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -16,23 +16,16 @@ package com.android.inputmethod.latin; -import android.content.ContentResolver; import android.content.Context; -import android.database.ContentObserver; -import android.database.Cursor; -import android.database.sqlite.SQLiteException; import android.net.Uri; -import android.os.SystemClock; -import android.provider.BaseColumns; import android.provider.ContactsContract; import android.provider.ContactsContract.Contacts; import android.util.Log; import com.android.inputmethod.annotations.ExternallyReferenced; -import com.android.inputmethod.latin.common.Constants; +import com.android.inputmethod.latin.ContactsManager.ContactsChangedListener; import com.android.inputmethod.latin.common.StringUtils; import com.android.inputmethod.latin.personalization.AccountUtils; -import com.android.inputmethod.latin.utils.ExecutorUtils; import java.io.File; import java.util.ArrayList; @@ -41,11 +34,8 @@ import java.util.Locale; import javax.annotation.Nullable; -public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { - - private static final String[] PROJECTION = {BaseColumns._ID, Contacts.DISPLAY_NAME}; - private static final String[] PROJECTION_ID_ONLY = {BaseColumns._ID}; - +public class ContactsBinaryDictionary extends ExpandableBinaryDictionary + implements ContactsChangedListener { private static final String TAG = ContactsBinaryDictionary.class.getSimpleName(); private static final String NAME = "contacts"; @@ -53,35 +43,18 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { private static final boolean DEBUG_DUMP = false; /** - * Frequency for contacts information into the dictionary - */ - private static final int FREQUENCY_FOR_CONTACTS = 40; - private static final int FREQUENCY_FOR_CONTACTS_BIGRAM = 90; - - /** The maximum number of contacts that this dictionary supports. */ - private static final int MAX_CONTACT_COUNT = 10000; - - private static final int INDEX_NAME = 1; - - /** The number of contacts in the most recent dictionary rebuild. */ - private int mContactCountAtLastRebuild = 0; - - /** The hash code of ArrayList of contacts names in the most recent dictionary rebuild. */ - private int mHashCodeAtLastRebuild = 0; - - private ContentObserver mObserver; - - /** * Whether to use "firstname lastname" in bigram predictions. */ private final boolean mUseFirstLastBigrams; + private final ContactsManager mContactsManager; protected ContactsBinaryDictionary(final Context context, final Locale locale, final File dictFile, final String name) { super(context, getDictName(name, locale, dictFile), locale, Dictionary.TYPE_CONTACTS, dictFile); - mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale); - registerObserver(context); + mUseFirstLastBigrams = ContactsDictionaryUtils.useFirstLastBigramsForLocale(locale); + mContactsManager = new ContactsManager(context); + mContactsManager.registerForUpdates(this /* listener */); reloadDictionaryIfRequired(); } @@ -92,34 +65,17 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { return new ContactsBinaryDictionary(context, locale, dictFile, dictNamePrefix + NAME); } - private synchronized void registerObserver(final Context context) { - if (mObserver != null) return; - ContentResolver cres = context.getContentResolver(); - cres.registerContentObserver(Contacts.CONTENT_URI, true, mObserver = - new ContentObserver(null) { - @Override - public void onChange(boolean self) { - ExecutorUtils.getExecutor("Check Contacts").execute(new Runnable() { - @Override - public void run() { - if (haveContentsChanged()) { - setNeedsToRecreate(); - } - } - }); - } - }); - } - @Override public synchronized void close() { - if (mObserver != null) { - mContext.getContentResolver().unregisterContentObserver(mObserver); - mObserver = null; - } + mContactsManager.close(); super.close(); } + /** + * Typically called whenever the dictionary is created for the first time or + * recreated when we think that there are updates to the dictionary. + * This is called asynchronously. + */ @Override public void loadInitialContentsLocked() { loadDeviceAccountsEmailAddressesLocked(); @@ -128,6 +84,9 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { loadDictionaryForUriLocked(Contacts.CONTENT_URI); } + /** + * Loads device accounts to the dictionary. + */ private void loadDeviceAccountsEmailAddressesLocked() { final List<String> accountVocabulary = AccountUtils.getDeviceAccountsEmailAddresses(mContext); @@ -139,80 +98,25 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { Log.d(TAG, "loadAccountVocabulary: " + word); } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, - 0 /* shortcutFreq */, false /* isNotAWord */, false /* isPossiblyOffensive */, + addUnigramLocked(word, ContactsDictionaryConstants.FREQUENCY_FOR_CONTACTS, + false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } } + /** + * Loads data within content providers to the dictionary. + */ private void loadDictionaryForUriLocked(final Uri uri) { - Cursor cursor = null; - try { - cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null); - if (null == cursor) { - return; - } - if (cursor.moveToFirst()) { - mContactCountAtLastRebuild = getContactCount(); - addWordsLocked(cursor); - } - } catch (final SQLiteException e) { - Log.e(TAG, "SQLiteException in the remote Contacts process.", e); - } catch (final IllegalStateException e) { - Log.e(TAG, "Contacts DB is having problems", e); - } finally { - if (null != cursor) { - cursor.close(); - } + final ArrayList<String> validNames = mContactsManager.getValidNames(uri); + for (final String name : validNames) { + addNameLocked(name); } - } - - private static boolean useFirstLastBigramsForLocale(final Locale locale) { - // TODO: Add firstname/lastname bigram rules for other languages. - if (locale != null && locale.getLanguage().equals(Locale.ENGLISH.getLanguage())) { - return true; + if (uri.equals(Contacts.CONTENT_URI)) { + // Since we were able to add content successfully, update the local + // state of the manager. + mContactsManager.updateLocalState(validNames); } - return false; - } - - private void addWordsLocked(final Cursor cursor) { - int count = 0; - final ArrayList<String> names = new ArrayList<>(); - while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) { - String name = cursor.getString(INDEX_NAME); - if (isValidName(name)) { - names.add(name); - addNameLocked(name); - ++count; - } else { - if (DEBUG_DUMP) { - Log.d(TAG, "Invalid name: " + name); - } - } - cursor.moveToNext(); - } - mHashCodeAtLastRebuild = names.hashCode(); - } - - private int getContactCount() { - // TODO: consider switching to a rawQuery("select count(*)...") on the database if - // performance is a bottleneck. - Cursor cursor = null; - try { - cursor = mContext.getContentResolver().query(Contacts.CONTENT_URI, PROJECTION_ID_ONLY, - null, null, null); - if (null == cursor) { - return 0; - } - return cursor.getCount(); - } catch (final SQLiteException e) { - Log.e(TAG, "SQLiteException in the remote Contacts process.", e); - } finally { - if (null != cursor) { - cursor.close(); - } - } - return 0; } /** @@ -225,7 +129,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { // TODO: Better tokenization for non-Latin writing systems for (int i = 0; i < len; i++) { if (Character.isLetter(name.codePointAt(i))) { - int end = getWordEndPosition(name, len, i); + int end = ContactsDictionaryUtils.getWordEndPosition(name, len, i); String word = name.substring(i, end); if (DEBUG_DUMP) { Log.d(TAG, "addName word = " + word); @@ -239,13 +143,15 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { Log.d(TAG, "addName " + name + ", " + word + ", " + ngramContext); } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, - null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, + addUnigramLocked(word, + ContactsDictionaryConstants.FREQUENCY_FOR_CONTACTS, false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); - if (!ngramContext.isValid() && mUseFirstLastBigrams) { + if (ngramContext.isValid() && mUseFirstLastBigrams) { runGCIfRequiredLocked(true /* mindsBlockByGC */); - addNgramEntryLocked(ngramContext, word, FREQUENCY_FOR_CONTACTS_BIGRAM, + addNgramEntryLocked(ngramContext, + word, + ContactsDictionaryConstants.FREQUENCY_FOR_CONTACTS_BIGRAM, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } ngramContext = ngramContext.getNextNgramContext( @@ -255,75 +161,8 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } } - /** - * Returns the index of the last letter in the word, starting from position startIndex. - */ - private static int getWordEndPosition(final String string, final int len, - final int startIndex) { - int end; - int cp = 0; - for (end = startIndex + 1; end < len; end += Character.charCount(cp)) { - cp = string.codePointAt(end); - if (!(cp == Constants.CODE_DASH || cp == Constants.CODE_SINGLE_QUOTE - || Character.isLetter(cp))) { - break; - } - } - return end; - } - - boolean haveContentsChanged() { - final long startTime = SystemClock.uptimeMillis(); - final int contactCount = getContactCount(); - if (contactCount > MAX_CONTACT_COUNT) { - // If there are too many contacts then return false. In this rare case it is impossible - // to include all of them anyways and the cost of rebuilding the dictionary is too high. - // TODO: Sort and check only the MAX_CONTACT_COUNT most recent contacts? - return false; - } - if (contactCount != mContactCountAtLastRebuild) { - if (DEBUG) { - Log.d(TAG, "Contact count changed: " + mContactCountAtLastRebuild + " to " - + contactCount); - } - return true; - } - // Check all contacts since it's not possible to find out which names have changed. - // This is needed because it's possible to receive extraneous onChange events even when no - // name has changed. - final Cursor cursor = mContext.getContentResolver().query(Contacts.CONTENT_URI, PROJECTION, - null, null, null); - if (null == cursor) { - return false; - } - final ArrayList<String> names = new ArrayList<>(); - try { - if (cursor.moveToFirst()) { - while (!cursor.isAfterLast()) { - String name = cursor.getString(INDEX_NAME); - if (isValidName(name)) { - names.add(name); - } - cursor.moveToNext(); - } - } - if (names.hashCode() != mHashCodeAtLastRebuild) { - return true; - } - } finally { - cursor.close(); - } - if (DEBUG) { - Log.d(TAG, "No contacts changed. (runtime = " + (SystemClock.uptimeMillis() - startTime) - + " ms)"); - } - return false; - } - - private static boolean isValidName(final String name) { - if (name != null && -1 == name.indexOf(Constants.CODE_COMMERCIAL_AT)) { - return true; - } - return false; + @Override + public void onContactsChange() { + setNeedsToRecreate(); } } diff --git a/java/src/com/android/inputmethod/latin/ContactsContentObserver.java b/java/src/com/android/inputmethod/latin/ContactsContentObserver.java new file mode 100644 index 000000000..019d17d56 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/ContactsContentObserver.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.content.ContentResolver; +import android.content.Context; +import android.database.ContentObserver; +import android.os.SystemClock; +import android.provider.ContactsContract.Contacts; +import android.util.Log; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.ContactsManager.ContactsChangedListener; +import com.android.inputmethod.latin.utils.ExecutorUtils; + +import java.util.ArrayList; +import java.util.concurrent.ExecutorService; + +/** + * A content observer that listens to updates to content provider {@link Contacts.CONTENT_URI}. + */ +// TODO:add test +public class ContactsContentObserver { + private static final String TAG = ContactsContentObserver.class.getSimpleName(); + private static final boolean DEBUG = false; + + private ContentObserver mObserver; + + private final Context mContext; + private final ContactsManager mManager; + + public ContactsContentObserver(final ContactsManager manager, final Context context) { + mManager = manager; + mContext = context; + } + + public void registerObserver(final ContactsChangedListener listener) { + if (DEBUG) { + Log.d(TAG, "Registered Contacts Content Observer"); + } + mObserver = new ContentObserver(null /* handler */) { + @Override + public void onChange(boolean self) { + getBgExecutor().execute(new Runnable() { + @Override + public void run() { + if (haveContentsChanged()) { + if (DEBUG) { + Log.d(TAG, "Contacts have changed; notifying listeners"); + } + listener.onContactsChange(); + } + } + }); + } + }; + final ContentResolver contentResolver = mContext.getContentResolver(); + contentResolver.registerContentObserver(Contacts.CONTENT_URI, true, mObserver); + } + + @UsedForTesting + private ExecutorService getBgExecutor() { + return ExecutorUtils.getExecutor("Check Contacts"); + } + + private boolean haveContentsChanged() { + final long startTime = SystemClock.uptimeMillis(); + final int contactCount = mManager.getContactCount(); + if (contactCount > ContactsDictionaryConstants.MAX_CONTACT_COUNT) { + // If there are too many contacts then return false. In this rare case it is impossible + // to include all of them anyways and the cost of rebuilding the dictionary is too high. + // TODO: Sort and check only the MAX_CONTACT_COUNT most recent contacts? + return false; + } + if (contactCount != mManager.getContactCountAtLastRebuild()) { + if (DEBUG) { + Log.d(TAG, "Contact count changed: " + mManager.getContactCountAtLastRebuild() + + " to " + contactCount); + } + return true; + } + final ArrayList<String> names = mManager.getValidNames(Contacts.CONTENT_URI); + if (names.hashCode() != mManager.getHashCodeAtLastRebuild()) { + return true; + } + if (DEBUG) { + Log.d(TAG, "No contacts changed. (runtime = " + (SystemClock.uptimeMillis() - startTime) + + " ms)"); + } + return false; + } + + public void unregister() { + mContext.getContentResolver().unregisterContentObserver(mObserver); + } +} diff --git a/java/src/com/android/inputmethod/latin/ContactsDictionaryConstants.java b/java/src/com/android/inputmethod/latin/ContactsDictionaryConstants.java new file mode 100644 index 000000000..8d8faca58 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/ContactsDictionaryConstants.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.provider.BaseColumns; +import android.provider.ContactsContract.Contacts; + +/** + * Constants related to Contacts Content Provider. + */ +public class ContactsDictionaryConstants { + /** + * Projections for {@link Contacts.CONTENT_URI} + */ + public static final String[] PROJECTION = { BaseColumns._ID, Contacts.DISPLAY_NAME }; + public static final String[] PROJECTION_ID_ONLY = { BaseColumns._ID }; + + /** + * Frequency for contacts information into the dictionary + */ + public static final int FREQUENCY_FOR_CONTACTS = 40; + public static final int FREQUENCY_FOR_CONTACTS_BIGRAM = 90; + + /** + * The maximum number of contacts that this dictionary supports. + */ + public static final int MAX_CONTACT_COUNT = 10000; + + /** + * Index of the column for 'name' in content providers: + * Contacts & ContactsContract.Profile. + */ + public static final int NAME_INDEX = 1; +} diff --git a/java/src/com/android/inputmethod/latin/ContactsDictionaryUtils.java b/java/src/com/android/inputmethod/latin/ContactsDictionaryUtils.java new file mode 100644 index 000000000..b77388434 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/ContactsDictionaryUtils.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import com.android.inputmethod.latin.common.Constants; + +import java.util.Locale; + +/** + * Utility methods related contacts dictionary. + */ +public class ContactsDictionaryUtils { + + /** + * Returns the index of the last letter in the word, starting from position startIndex. + */ + public static int getWordEndPosition(final String string, final int len, + final int startIndex) { + int end; + int cp = 0; + for (end = startIndex + 1; end < len; end += Character.charCount(cp)) { + cp = string.codePointAt(end); + if (cp != Constants.CODE_DASH && cp != Constants.CODE_SINGLE_QUOTE + && !Character.isLetter(cp)) { + break; + } + } + return end; + } + + /** + * Returns true if the locale supports using first name and last name as bigrams. + */ + public static boolean useFirstLastBigramsForLocale(final Locale locale) { + // TODO: Add firstname/lastname bigram rules for other languages. + if (locale != null && locale.getLanguage().equals(Locale.ENGLISH.getLanguage())) { + return true; + } + return false; + } +} diff --git a/java/src/com/android/inputmethod/latin/ContactsManager.java b/java/src/com/android/inputmethod/latin/ContactsManager.java new file mode 100644 index 000000000..dc5abd955 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/ContactsManager.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.content.Context; +import android.database.Cursor; +import android.database.sqlite.SQLiteException; +import android.net.Uri; +import android.provider.ContactsContract.Contacts; +import android.util.Log; + +import com.android.inputmethod.latin.common.Constants; + +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Manages all interactions with Contacts DB. + * + * The manager provides an API for listening to meaning full updates by keeping a + * measure of the current state of the content provider. + */ +// TODO:Add test +public class ContactsManager { + private static final String TAG = ContactsManager.class.getSimpleName(); + private static final boolean DEBUG = false; + + /** + * Interface to implement for classes interested in getting notified for updates + * to Contacts content provider. + */ + public static interface ContactsChangedListener { + public void onContactsChange(); + } + + /** + * The number of contacts observed in the most recent instance of + * contacts content provider. + */ + private AtomicInteger mContactCountAtLastRebuild = new AtomicInteger(0); + + /** + * The hash code of list of valid contacts names in the most recent dictionary + * rebuild. + */ + private AtomicInteger mHashCodeAtLastRebuild = new AtomicInteger(0); + + private final Context mContext; + private final ContactsContentObserver mObserver; + + public ContactsManager(final Context context) { + mContext = context; + mObserver = new ContactsContentObserver(this /* ContactsManager */, context); + } + + // TODO: This was synchronized in previous version. Why? + public void registerForUpdates(final ContactsChangedListener listener) { + mObserver.registerObserver(listener); + } + + public int getContactCountAtLastRebuild() { + return mContactCountAtLastRebuild.get(); + } + + public int getHashCodeAtLastRebuild() { + return mHashCodeAtLastRebuild.get(); + } + + /** + * Returns all the valid names in the Contacts DB. Callers should also + * call {@link #updateLocalState(ArrayList)} after they are done with result + * so that the manager can cache local state for determining updates. + */ + public ArrayList<String> getValidNames(final Uri uri) { + final ArrayList<String> names = new ArrayList<>(); + // Check all contacts since it's not possible to find out which names have changed. + // This is needed because it's possible to receive extraneous onChange events even when no + // name has changed. + final Cursor cursor = mContext.getContentResolver().query(uri, + ContactsDictionaryConstants.PROJECTION, null, null, null); + if (cursor != null) { + try { + if (cursor.moveToFirst()) { + while (!cursor.isAfterLast()) { + final String name = cursor.getString( + ContactsDictionaryConstants.NAME_INDEX); + if (isValidName(name)) { + names.add(name); + } + cursor.moveToNext(); + } + } + } finally { + cursor.close(); + } + } + return names; + } + + /** + * Returns the number of contacts in contacts content provider. + */ + public int getContactCount() { + // TODO: consider switching to a rawQuery("select count(*)...") on the database if + // performance is a bottleneck. + Cursor cursor = null; + try { + cursor = mContext.getContentResolver().query(Contacts.CONTENT_URI, + ContactsDictionaryConstants.PROJECTION_ID_ONLY, null, null, null); + if (null == cursor) { + return 0; + } + return cursor.getCount(); + } catch (final SQLiteException e) { + Log.e(TAG, "SQLiteException in the remote Contacts process.", e); + } finally { + if (null != cursor) { + cursor.close(); + } + } + return 0; + } + + private static boolean isValidName(final String name) { + if (name != null && -1 == name.indexOf(Constants.CODE_COMMERCIAL_AT)) { + return true; + } + return false; + } + + /** + * Updates the local state of the manager. This should be called when the callers + * are done with all the updates of the content provider successfully. + */ + public void updateLocalState(final ArrayList<String> names) { + mContactCountAtLastRebuild.set(getContactCount()); + mHashCodeAtLastRebuild.set(names.hashCode()); + } + + /** + * Performs any necessary cleanup. + */ + public void close() { + mObserver.unregister(); + } +} diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index 3e4cda47a..c22dc287c 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -18,7 +18,6 @@ package com.android.inputmethod.latin; import android.content.Context; import android.util.Pair; -import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.KeyboardLayout; @@ -28,7 +27,6 @@ import com.android.inputmethod.latin.utils.SuggestionResults; import java.io.File; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -90,8 +88,6 @@ public interface DictionaryFacilitator { void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable); } - void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes); - // TODO: remove this, it's confusing with seamless multiple language switching void setIsMonolingualUser(final boolean isMonolingualUser); @@ -175,4 +171,12 @@ public interface DictionaryFacilitator { void dumpDictionaryForDebug(final String dictName); ArrayList<Pair<String, DictionaryStats>> getStatsOfEnabledSubDicts(); + + void addOrIncrementTerm(String fileName, + String finalWordToBeAdded, + NgramContext ngramContext, + int increment, + int timeStampInSeconds); + + void clearLanguageModel(String filePath); } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java index dd34faef8..3d76751ce 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorImpl.java @@ -20,7 +20,6 @@ import android.content.Context; import android.text.TextUtils; import android.util.Log; import android.util.Pair; -import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.KeyboardLayout; @@ -29,9 +28,6 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.common.Constants; import com.android.inputmethod.latin.personalization.UserHistoryDictionary; import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; -import com.android.inputmethod.latin.utils.DistracterFilter; -import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions; -import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.SuggestionResults; @@ -42,7 +38,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -76,7 +71,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { private volatile CountDownLatch mLatchForWaitingLoadingMainDictionaries = new CountDownLatch(0); // To synchronize assigning mDictionaryGroup to ensure closing dictionaries. private final Object mLock = new Object(); - private final DistracterFilter mDistracterFilter; public static final Map<String, Class<? extends ExpandableBinaryDictionary>> DICT_TYPE_TO_CLASS = new HashMap<>(); @@ -233,15 +227,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { } public DictionaryFacilitatorImpl() { - mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER; - } - - public DictionaryFacilitatorImpl(final Context context) { - mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context); - } - - public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { - mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); } // TODO: remove this, it's confusing with seamless multiple language switching @@ -545,7 +530,6 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { dictionaryGroup.closeDict(dictType); } } - mDistracterFilter.close(); } @UsedForTesting @@ -659,9 +643,7 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { // We don't add words with 0-frequency (assuming they would be profanity etc.). final boolean isValid = maxFreq > 0; UserHistoryDictionary.addToDictionary(userHistoryDictionary, ngramContext, secondWord, - isValid, timeStampInSeconds, - new DistracterFilterCheckingIsInDictionary( - mDistracterFilter, userHistoryDictionary)); + isValid, timeStampInSeconds); } private void removeWord(final String dictName, final String word) { @@ -764,10 +746,12 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { return maxFreq; } + @Override public int getFrequency(final String word) { return getFrequencyInternal(word, false /* isGettingMaxFrequencyOfExactMatches */); } + @Override public int getMaxFrequencyOfExactMatches(final String word) { return getFrequencyInternal(word, true /* isGettingMaxFrequencyOfExactMatches */); } @@ -811,4 +795,18 @@ public class DictionaryFacilitatorImpl implements DictionaryFacilitator { } return statsOfEnabledSubDicts; } + + @Override + public void addOrIncrementTerm(String fileName, + String word, + NgramContext ngramContext, + int increment, + int timeStampInSeconds) { + // Do nothing. + } + + @Override + public void clearLanguageModel(String filePath) { + // Do nothing. + } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 87d46e226..8c780027b 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -31,7 +31,6 @@ import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CombinedFormatUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.WordInputEventForPersonalization; @@ -40,7 +39,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -57,7 +55,6 @@ import javax.annotation.Nullable; * * A class that extends this abstract class must have a static factory method named * getDictionary(Context context, Locale locale, File dictFile, String dictNamePrefix) - * @see DictionaryFacilitator#getSubDict(String,Context,Locale,File,String) */ abstract public class ExpandableBinaryDictionary extends Dictionary { private static final boolean DEBUG = false; @@ -172,33 +169,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static void asyncExecuteTaskWithLock(final Lock lock, final String executorName, final Runnable task) { - asyncPreCheckAndExecuteTaskWithLock(lock, null /* preCheckTask */, executorName, task); - } - - private void asyncPreCheckAndExecuteTaskWithWriteLock( - final Callable<Boolean> preCheckTask, final Runnable task) { - asyncPreCheckAndExecuteTaskWithLock(mLock.writeLock(), preCheckTask, - mDictName /* executorName */, task); - - } - - // Execute task with lock when the result of preCheckTask is true or preCheckTask is null. - private static void asyncPreCheckAndExecuteTaskWithLock(final Lock lock, - final Callable<Boolean> preCheckTask, final String executorName, final Runnable task) { - final String tag = TAG; ExecutorUtils.getExecutor(executorName).execute(new Runnable() { @Override public void run() { - if (preCheckTask != null) { - try { - if (!preCheckTask.call().booleanValue()) { - return; - } - } catch (final Exception e) { - Log.e(tag, "The pre check task throws an exception.", e); - return; - } - } lock.lock(); try { task.run(); @@ -305,17 +278,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - private void updateDictionaryWithWriteLockIfWordIsNotADistracter( - @Nonnull final Runnable updateTask, - @Nonnull final String word, @Nonnull final DistracterFilter distracterFilter) { + private void updateDictionaryWithWriteLock(@Nonnull final Runnable updateTask) { reloadDictionaryIfRequired(); - final Callable<Boolean> preCheckTask = new Callable<Boolean>() { - @Override - public Boolean call() throws Exception { - return !distracterFilter.isDistracterToWordsInDictionaries( - NgramContext.EMPTY_PREV_WORDS_INFO, word, mLocale); - } - }; final Runnable task = new Runnable() { @Override public void run() { @@ -326,29 +290,25 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { updateTask.run(); } }; - asyncPreCheckAndExecuteTaskWithWriteLock(preCheckTask, task); + asyncExecuteTaskWithWriteLock(task); } /** * Adds unigram information of a word to the dictionary. May overwrite an existing entry. */ - public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency, - final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, - final boolean isPossiblyOffensive, final int timestamp, - @Nonnull final DistracterFilter distracterFilter) { - updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + public void addUnigramEntry(final String word, final int frequency, + final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { + updateDictionaryWithWriteLock(new Runnable() { @Override public void run() { - addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, - isNotAWord, isPossiblyOffensive, timestamp); + addUnigramLocked(word, frequency, isNotAWord, isPossiblyOffensive, timestamp); } - }, word, distracterFilter); + }); } protected void addUnigramLocked(final String word, final int frequency, - final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, - final boolean isPossiblyOffensive, final int timestamp) { - if (!mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq, + final boolean isNotAWord, final boolean isPossiblyOffensive, final int timestamp) { + if (!mBinaryDictionary.addUnigramEntry(word, frequency, false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive, timestamp)) { Log.e(TAG, "Cannot add unigram entry. word: " + word); } @@ -405,37 +365,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Dynamically remove the n-gram entry in the dictionary. - */ - @UsedForTesting - public void removeNgramDynamically(@Nonnull final NgramContext ngramContext, - final String word) { - reloadDictionaryIfRequired(); - asyncExecuteTaskWithWriteLock(new Runnable() { - @Override - public void run() { - final BinaryDictionary binaryDictionary = getBinaryDictionary(); - if (binaryDictionary == null) { - return; - } - runGCIfRequiredLocked(true /* mindsBlockByGC */); - if (!binaryDictionary.removeNgramEntry(ngramContext, word)) { - if (DEBUG) { - Log.i(TAG, "Cannot remove n-gram entry."); - Log.i(TAG, " NgramContext: " + ngramContext + ", word: " + word); - } - } - } - }); - } - - /** - * Update dictionary for the word with the ngramContext if the word is not a distracter. + * Update dictionary for the word with the ngramContext. */ - public void updateEntriesForWordWithCheckingDistracter(@Nonnull final NgramContext ngramContext, - final String word, final boolean isValidWord, final int count, final int timestamp, - @Nonnull final DistracterFilter distracterFilter) { - updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + public void updateEntriesForWord(@Nonnull final NgramContext ngramContext, + final String word, final boolean isValidWord, final int count, final int timestamp) { + updateDictionaryWithWriteLock(new Runnable() { @Override public void run() { final BinaryDictionary binaryDictionary = getBinaryDictionary(); @@ -446,20 +380,29 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { isValidWord, count, timestamp)) { if (DEBUG) { Log.e(TAG, "Cannot update counter. word: " + word - + " context: "+ ngramContext.toString()); + + " context: " + ngramContext.toString()); } } } - }, word, distracterFilter); + }); } + /** + * Used by Sketch. + * {@see https://cs.corp.google.com/#android/vendor/unbundled_google/packages/LatinIMEGoogle/tools/sketch/ime-simulator/src/com/android/inputmethod/sketch/imesimulator/ImeSimulator.java&q=updateEntriesForInputEventsCallback&l=286} + */ + @UsedForTesting public interface UpdateEntriesForInputEventsCallback { public void onFinished(); } /** * Dynamically update entries according to input events. + * + * Used by Sketch. + * {@see https://cs.corp.google.com/#android/vendor/unbundled_google/packages/LatinIMEGoogle/tools/sketch/ime-simulator/src/com/android/inputmethod/sketch/imesimulator/ImeSimulator.java&q=updateEntriesForInputEventsCallback&l=286} */ + @UsedForTesting public void updateEntriesForInputEvents( @Nonnull final ArrayList<WordInputEventForPersonalization> inputEvents, final UpdateEntriesForInputEventsCallback callback) { @@ -571,11 +514,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } - protected boolean isValidNgramLocked(final NgramContext ngramContext, final String word) { - if (mBinaryDictionary == null) return false; - return mBinaryDictionary.isValidNgram(ngramContext, word); - } - /** * Loads the current binary dictionary from internal storage. Assumes the dictionary file * exists. @@ -589,6 +527,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { Thread.sleep(15000); Log.w(TAG, "End stress in loading"); } catch (InterruptedException e) { + Log.w("Interrupted while loading: " + mDictName, e); } } final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; @@ -653,7 +592,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Reloads the dictionary. Access is controlled on a per dictionary file basis. */ - private final void asyncReloadDictionary() { + private void asyncReloadDictionary() { final AtomicBoolean isReloading = mIsReloading; if (!isReloading.compareAndSet(false, true)) { return; diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 74ef6481a..a1ab5c090 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -87,7 +87,6 @@ import com.android.inputmethod.latin.utils.ImportantNoticeUtils; import com.android.inputmethod.latin.utils.IntentUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LeakGuardHandlerWrapper; -import com.android.inputmethod.latin.utils.NetworkConnectivityUtils; import com.android.inputmethod.latin.utils.StatsUtils; import com.android.inputmethod.latin.utils.StatsUtilsManager; import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; @@ -128,7 +127,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Settings mSettings; private final DictionaryFacilitator mDictionaryFacilitator = - DictionaryFacilitatorProvider.newDictionaryFacilitator(this /* context */); + DictionaryFacilitatorProvider.newDictionaryFacilitator(); final InputLogic mInputLogic = new InputLogic(this /* LatinIME */, this /* SuggestionStripViewAccessor */, mDictionaryFacilitator); // We expect to have only one decoder in almost all cases, hence the default capacity of 1. @@ -565,8 +564,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen loadSettings(); resetDictionaryFacilitatorIfNecessary(); - NetworkConnectivityUtils.onCreate(this /* context */, mKeyboardSwitcher /* listener */); - // Register to receive ringer mode change. final IntentFilter filter = new IntentFilter(); filter.addAction(AudioManager.RINGER_MODE_CHANGED_ACTION); @@ -608,8 +605,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!mHandler.hasPendingReopenDictionaries()) { resetDictionaryFacilitator(locales); } - mDictionaryFacilitator.updateEnabledSubtypes(mRichImm.getMyEnabledInputMethodSubtypeList( - true /* allowsImplicitlySelectedSubtypes */)); refreshPersonalizationDictionarySession(currentSettingsValues); resetDictionaryFacilitatorIfNecessary(); mStatsUtilsManager.onLoadSettings(this /* context */, currentSettingsValues, @@ -705,7 +700,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen public void onDestroy() { mDictionaryFacilitator.closeDictionaries(); mSettings.onDestroy(); - NetworkConnectivityUtils.onDestroy(this /* context */); unregisterReceiver(mRingerModeChangeReceiver); unregisterReceiver(mDictionaryPackInstallReceiver); unregisterReceiver(mDictionaryDumpBroadcastReceiver); @@ -719,7 +713,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen unregisterReceiver(mDictionaryPackInstallReceiver); unregisterReceiver(mDictionaryDumpBroadcastReceiver); unregisterReceiver(mRingerModeChangeReceiver); - NetworkConnectivityUtils.onDestroy(this /* context */); mInputLogic.recycle(); } diff --git a/java/src/com/android/inputmethod/latin/NgramContext.java b/java/src/com/android/inputmethod/latin/NgramContext.java index 86155e0be..53bec6e59 100644 --- a/java/src/com/android/inputmethod/latin/NgramContext.java +++ b/java/src/com/android/inputmethod/latin/NgramContext.java @@ -108,7 +108,9 @@ public class NgramContext { mPrevWordsCount = prevWordsInfo.length; } - // Create next prevWordsInfo using current prevWordsInfo. + /** + * Create next prevWordsInfo using current prevWordsInfo. + */ @Nonnull public NgramContext getNextNgramContext(final WordInfo wordInfo) { final int nextPrevWordCount = Math.min( diff --git a/java/src/com/android/inputmethod/latin/RichInputMethodManager.java b/java/src/com/android/inputmethod/latin/RichInputMethodManager.java index 64a7cf347..c8e0b93bf 100644 --- a/java/src/com/android/inputmethod/latin/RichInputMethodManager.java +++ b/java/src/com/android/inputmethod/latin/RichInputMethodManager.java @@ -17,7 +17,6 @@ package com.android.inputmethod.latin; import static com.android.inputmethod.latin.common.Constants.Subtype.KEYBOARD_MODE; -import static com.android.inputmethod.latin.common.Constants.Subtype.ExtraValue.REQ_NETWORK_CONNECTIVITY; import android.content.Context; import android.content.SharedPreferences; @@ -36,7 +35,6 @@ import com.android.inputmethod.compat.InputMethodManagerCompatWrapper; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils; import com.android.inputmethod.latin.utils.LanguageOnSpacebarUtils; -import com.android.inputmethod.latin.utils.NetworkConnectivityUtils; import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; import java.util.Collections; @@ -288,24 +286,20 @@ public class RichInputMethodManager { } public boolean checkIfSubtypeBelongsToThisImeAndEnabled(final InputMethodSubtype subtype) { - return checkIfSubtypeBelongsToImeAndEnabled(getInputMethodInfoOfThisIme(), subtype); + return checkIfSubtypeBelongsToList(subtype, + getEnabledInputMethodSubtypeList( + getInputMethodInfoOfThisIme(), + true /* allowsImplicitlySelectedSubtypes */)); } public boolean checkIfSubtypeBelongsToThisImeAndImplicitlyEnabled( final InputMethodSubtype subtype) { final boolean subtypeEnabled = checkIfSubtypeBelongsToThisImeAndEnabled(subtype); - final boolean subtypeExplicitlyEnabled = checkIfSubtypeBelongsToList( - subtype, getMyEnabledInputMethodSubtypeList( - false /* allowsImplicitlySelectedSubtypes */)); + final boolean subtypeExplicitlyEnabled = checkIfSubtypeBelongsToList(subtype, + getMyEnabledInputMethodSubtypeList(false /* allowsImplicitlySelectedSubtypes */)); return subtypeEnabled && !subtypeExplicitlyEnabled; } - public boolean checkIfSubtypeBelongsToImeAndEnabled(final InputMethodInfo imi, - final InputMethodSubtype subtype) { - return checkIfSubtypeBelongsToList(subtype, getEnabledInputMethodSubtypeList(imi, - true /* allowsImplicitlySelectedSubtypes */)); - } - private static boolean checkIfSubtypeBelongsToList(final InputMethodSubtype subtype, final List<InputMethodSubtype> subtypes) { return getSubtypeIndexInList(subtype, subtypes) != INDEX_NOT_FOUND; @@ -564,16 +558,6 @@ public class RichInputMethodManager { }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR); } - public boolean isShortcutImeEnabled() { - if (mShortcutInputMethodInfo == null) { - return false; - } - if (mShortcutSubtype == null) { - return true; - } - return checkIfSubtypeBelongsToImeAndEnabled(mShortcutInputMethodInfo, mShortcutSubtype); - } - public boolean isShortcutImeReady() { if (mShortcutInputMethodInfo == null) { return false; @@ -581,9 +565,6 @@ public class RichInputMethodManager { if (mShortcutSubtype == null) { return true; } - if (mShortcutSubtype.containsExtraValueKey(REQ_NETWORK_CONNECTIVITY)) { - return NetworkConnectivityUtils.isNetworkConnected(); - } return true; } } diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index 1ed210377..fe24ccfc2 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -22,7 +22,6 @@ import android.database.ContentObserver; import android.database.Cursor; import android.database.sqlite.SQLiteException; import android.net.Uri; -import android.os.Build; import android.provider.UserDictionary.Words; import android.text.TextUtils; import android.util.Log; @@ -47,19 +46,8 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { private static final String USER_DICTIONARY_ALL_LANGUAGES = ""; private static final int HISTORICAL_DEFAULT_USER_DICTIONARY_FREQUENCY = 250; private static final int LATINIME_DEFAULT_USER_DICTIONARY_FREQUENCY = 160; - // Shortcut frequency is 0~15, with 15 = whitelist. We don't want user dictionary entries - // to auto-correct, so we set this to the highest frequency that won't, i.e. 14. - private static final int USER_DICT_SHORTCUT_FREQUENCY = 14; - - private static final String[] PROJECTION_QUERY_WITH_SHORTCUT = new String[] { - Words.WORD, - Words.SHORTCUT, - Words.FREQUENCY, - }; - private static final String[] PROJECTION_QUERY_WITHOUT_SHORTCUT = new String[] { - Words.WORD, - Words.FREQUENCY, - }; + + private static final String[] PROJECTION_QUERY = new String[] {Words.WORD, Words.FREQUENCY}; private static final String NAME = "userunigram"; @@ -171,20 +159,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { requestArguments = localeElements; } final String requestString = request.toString(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN) { - try { - addWordsFromProjectionLocked(PROJECTION_QUERY_WITH_SHORTCUT, requestString, - requestArguments); - } catch (IllegalArgumentException e) { - // This may happen on some non-compliant devices where the declared API is JB+ but - // the SHORTCUT column is not present for some reason. - addWordsFromProjectionLocked(PROJECTION_QUERY_WITHOUT_SHORTCUT, requestString, - requestArguments); - } - } else { - addWordsFromProjectionLocked(PROJECTION_QUERY_WITHOUT_SHORTCUT, requestString, - requestArguments); - } + addWordsFromProjectionLocked(PROJECTION_QUERY, requestString, requestArguments); } private void addWordsFromProjectionLocked(final String[] query, String request, @@ -219,31 +194,20 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } private void addWordsLocked(final Cursor cursor) { - final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; if (cursor == null) return; if (cursor.moveToFirst()) { final int indexWord = cursor.getColumnIndex(Words.WORD); - final int indexShortcut = hasShortcutColumn ? cursor.getColumnIndex(Words.SHORTCUT) : 0; final int indexFrequency = cursor.getColumnIndex(Words.FREQUENCY); while (!cursor.isAfterLast()) { final String word = cursor.getString(indexWord); - final String shortcut = hasShortcutColumn ? cursor.getString(indexShortcut) : null; final int frequency = cursor.getInt(indexFrequency); final int adjustedFrequency = scaleFrequencyFromDefaultToLatinIme(frequency); // Safeguard against adding really long words. if (word.length() <= MAX_WORD_LENGTH) { runGCIfRequiredLocked(true /* mindsBlockByGC */); - addUnigramLocked(word, adjustedFrequency, null /* shortcutTarget */, - 0 /* shortcutFreq */, false /* isNotAWord */, + addUnigramLocked(word, adjustedFrequency, false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); - if (null != shortcut && shortcut.length() <= MAX_WORD_LENGTH) { - runGCIfRequiredLocked(true /* mindsBlockByGC */); - addUnigramLocked(shortcut, adjustedFrequency, word, - USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */, - false /* isPossiblyOffensive */, - BinaryDictionary.NOT_A_VALID_TIMESTAMP); - } } cursor.moveToNext(); } diff --git a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java index 388d57816..264e75710 100644 --- a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java +++ b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java @@ -37,13 +37,11 @@ import javax.annotation.Nullable; public final class WordProperty implements Comparable<WordProperty> { public final String mWord; public final ProbabilityInfo mProbabilityInfo; - public final ArrayList<WeightedString> mShortcutTargets; public final ArrayList<NgramProperty> mNgrams; // TODO: Support mIsBeginningOfSentence. public final boolean mIsBeginningOfSentence; public final boolean mIsNotAWord; public final boolean mIsPossiblyOffensive; - public final boolean mHasShortcuts; public final boolean mHasNgrams; private int mHashCode = 0; @@ -51,12 +49,10 @@ public final class WordProperty implements Comparable<WordProperty> { // TODO: Support n-gram. @UsedForTesting public WordProperty(final String word, final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, @Nullable final ArrayList<WeightedString> bigrams, final boolean isNotAWord, final boolean isPossiblyOffensive) { mWord = word; mProbabilityInfo = probabilityInfo; - mShortcutTargets = shortcutTargets; if (null == bigrams) { mNgrams = null; } else { @@ -70,7 +66,6 @@ public final class WordProperty implements Comparable<WordProperty> { mIsNotAWord = isNotAWord; mIsPossiblyOffensive = isPossiblyOffensive; mHasNgrams = bigrams != null && !bigrams.isEmpty(); - mHasShortcuts = shortcutTargets != null && !shortcutTargets.isEmpty(); } private static ProbabilityInfo createProbabilityInfoFromArray(final int[] probabilityInfo) { @@ -84,21 +79,17 @@ public final class WordProperty implements Comparable<WordProperty> { // Construct word property using information from native code. // This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY. public WordProperty(final int[] codePoints, final boolean isNotAWord, - final boolean isPossiblyOffensive, final boolean hasBigram, final boolean hasShortcuts, + final boolean isPossiblyOffensive, final boolean hasBigram, final boolean isBeginningOfSentence, final int[] probabilityInfo, final ArrayList<int[][]> ngramPrevWordsArray, final ArrayList<boolean[]> ngramPrevWordIsBeginningOfSentenceArray, - final ArrayList<int[]> ngramTargets, final ArrayList<int[]> ngramProbabilityInfo, - final ArrayList<int[]> shortcutTargets, - final ArrayList<Integer> shortcutProbabilities) { + final ArrayList<int[]> ngramTargets, final ArrayList<int[]> ngramProbabilityInfo) { mWord = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo); - mShortcutTargets = new ArrayList<>(); final ArrayList<NgramProperty> ngrams = new ArrayList<>(); mIsBeginningOfSentence = isBeginningOfSentence; mIsNotAWord = isNotAWord; mIsPossiblyOffensive = isPossiblyOffensive; - mHasShortcuts = hasShortcuts; mHasNgrams = hasBigram; final int relatedNgramCount = ngramTargets.size(); @@ -121,14 +112,6 @@ public final class WordProperty implements Comparable<WordProperty> { ngrams.add(new NgramProperty(ngramTarget, ngramContext)); } mNgrams = ngrams.isEmpty() ? null : ngrams; - - final int shortcutTargetCount = shortcutTargets.size(); - for (int i = 0; i < shortcutTargetCount; i++) { - final String shortcutTargetString = - StringUtils.getStringFromNullTerminatedCodePointArray(shortcutTargets.get(i)); - mShortcutTargets.add( - new WeightedString(shortcutTargetString, shortcutProbabilities.get(i))); - } } // TODO: Remove @@ -154,7 +137,6 @@ public final class WordProperty implements Comparable<WordProperty> { return Arrays.hashCode(new Object[] { word.mWord, word.mProbabilityInfo, - word.mShortcutTargets, word.mNgrams, word.mIsNotAWord, word.mIsPossiblyOffensive @@ -185,10 +167,10 @@ public final class WordProperty implements Comparable<WordProperty> { if (o == this) return true; if (!(o instanceof WordProperty)) return false; WordProperty w = (WordProperty)o; - return mProbabilityInfo.equals(w.mProbabilityInfo) && mWord.equals(w.mWord) - && mShortcutTargets.equals(w.mShortcutTargets) && equals(mNgrams, w.mNgrams) + return mProbabilityInfo.equals(w.mProbabilityInfo) + && mWord.equals(w.mWord) && equals(mNgrams, w.mNgrams) && mIsNotAWord == w.mIsNotAWord && mIsPossiblyOffensive == w.mIsPossiblyOffensive - && mHasNgrams == w.mHasNgrams && mHasShortcuts && w.mHasNgrams; + && mHasNgrams == w.mHasNgrams; } // TDOO: Have a utility method like java.util.Objects.equals. diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 1d75a3098..b6286b203 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -25,7 +25,6 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.define.DecoderSpecificConstants; import com.android.inputmethod.latin.define.ProductionFlags; -import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.util.Locale; @@ -94,15 +93,14 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas * @param word the word the user inputted * @param isValid whether the word is valid or not * @param timestamp the timestamp when the word has been inputted - * @param distracterFilter the filter to check whether the word is a distracter */ public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary, @Nonnull final NgramContext ngramContext, final String word, final boolean isValid, - final int timestamp, @Nonnull final DistracterFilter distracterFilter) { + final int timestamp) { if (word.length() > DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH) { return; } - userHistoryDictionary.updateEntriesForWordWithCheckingDistracter(ngramContext, word, - isValid, 1 /* count */, timestamp, distracterFilter); + userHistoryDictionary.updateEntriesForWord(ngramContext, word, + isValid, 1 /* count */, timestamp); } } diff --git a/java/src/com/android/inputmethod/latin/settings/PreferencesSettingsFragment.java b/java/src/com/android/inputmethod/latin/settings/PreferencesSettingsFragment.java index 975396d2d..d9858e61f 100644 --- a/java/src/com/android/inputmethod/latin/settings/PreferencesSettingsFragment.java +++ b/java/src/com/android/inputmethod/latin/settings/PreferencesSettingsFragment.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin.settings; import android.content.Context; import android.content.SharedPreferences; import android.content.res.Resources; +import android.os.Build; import android.os.Bundle; import android.preference.Preference; @@ -38,6 +39,10 @@ import com.android.inputmethod.latin.RichInputMethodManager; * - Voice input key */ public final class PreferencesSettingsFragment extends SubScreenFragment { + + private static final boolean VOICE_IME_ENABLED = + Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; + @Override public void onCreate(final Bundle icicle) { super.onCreate(icicle); @@ -72,11 +77,9 @@ public final class PreferencesSettingsFragment extends SubScreenFragment { final Preference voiceInputKeyOption = findPreference(Settings.PREF_VOICE_INPUT_KEY); if (voiceInputKeyOption != null) { RichInputMethodManager.getInstance().refreshSubtypeCaches(); - final boolean isShortcutImeEnabled = RichInputMethodManager.getInstance() - .isShortcutImeEnabled(); - voiceInputKeyOption.setEnabled(isShortcutImeEnabled); - voiceInputKeyOption.setSummary( - isShortcutImeEnabled ? null : getText(R.string.voice_input_disabled_summary)); + voiceInputKeyOption.setEnabled(VOICE_IME_ENABLED); + voiceInputKeyOption.setSummary(VOICE_IME_ENABLED + ? null : getText(R.string.voice_input_disabled_summary)); } } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 9a1bb7784..6224071ea 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -21,6 +21,7 @@ import android.content.SharedPreferences; import android.content.pm.PackageInfo; import android.content.res.Configuration; import android.content.res.Resources; +import android.os.Build; import android.util.Log; import android.view.inputmethod.EditorInfo; @@ -136,7 +137,7 @@ public class SettingsValues { DebugSettings.PREF_SLIDING_KEY_INPUT_PREVIEW, true); mShowsVoiceInputKey = needsToShowVoiceInputKey(prefs, res) && mInputAttributes.mShouldShowVoiceInputKey - && RichInputMethodManager.getInstance().isShortcutImeEnabled(); + && Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; final String autoCorrectionThresholdRawValue = prefs.getString( Settings.PREF_AUTO_CORRECTION_THRESHOLD, res.getString(R.string.auto_correction_threshold_mode_index_modest)); diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java index 02151522d..95293bf2f 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java @@ -24,6 +24,7 @@ import android.text.InputType; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputMethodSubtype; import android.view.textservice.SuggestionsInfo; +import android.util.Log; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.KeyboardId; @@ -52,6 +53,9 @@ import java.util.concurrent.Semaphore; */ public final class AndroidSpellCheckerService extends SpellCheckerService implements SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = AndroidSpellCheckerService.class.getSimpleName(); + private static final boolean DEBUG = false; + public static final String PREF_USE_CONTACTS_KEY = "pref_spellcheck_use_contacts"; private static final int SPELLCHECKER_DUMMY_KEYBOARD_WIDTH = 480; @@ -80,6 +84,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService public static final String SINGLE_QUOTE = "\u0027"; public static final String APOSTROPHE = "\u2019"; + private UserDictionaryLookup mUserDictionaryLookup; public AndroidSpellCheckerService() { super(); @@ -95,6 +100,24 @@ public final class AndroidSpellCheckerService extends SpellCheckerService final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); prefs.registerOnSharedPreferenceChangeListener(this); onSharedPreferenceChanged(prefs, PREF_USE_CONTACTS_KEY); + // Create a UserDictionaryLookup. It needs to be close()d and set to null in onDestroy. + if (mUserDictionaryLookup == null) { + if (DEBUG) { + Log.d(TAG, "Creating mUserDictionaryLookup in onCreate"); + } + mUserDictionaryLookup = new UserDictionaryLookup(this); + } else if (DEBUG) { + Log.d(TAG, "mUserDictionaryLookup already created before onCreate"); + } + } + + @Override public void onDestroy() { + if (DEBUG) { + Log.d(TAG, "Closing and dereferencing mUserDictionaryLookup in onDestroy"); + } + mUserDictionaryLookup.close(); + mUserDictionaryLookup = null; + super.onDestroy(); } public float getRecommendedThreshold() { @@ -150,6 +173,16 @@ public final class AndroidSpellCheckerService extends SpellCheckerService public boolean isValidWord(final Locale locale, final String word) { mSemaphore.acquireUninterruptibly(); try { + if (mUserDictionaryLookup.isValidWord(word, locale)) { + if (DEBUG) { + Log.d(TAG, "mUserDictionaryLookup.isValidWord(" + word + ")=true"); + } + return true; + } else { + if (DEBUG) { + Log.d(TAG, "mUserDictionaryLookup.isValidWord(" + word + ")=false"); + } + } DictionaryFacilitator dictionaryFacilitatorForLocale = mDictionaryFacilitatorCache.get(locale); return dictionaryFacilitatorForLocale.isValidSpellingWord(word); diff --git a/java/src/com/android/inputmethod/latin/spellcheck/UserDictionaryLookup.java b/java/src/com/android/inputmethod/latin/spellcheck/UserDictionaryLookup.java new file mode 100644 index 000000000..baff8f066 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/spellcheck/UserDictionaryLookup.java @@ -0,0 +1,430 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.spellcheck; + +import android.content.ContentResolver; +import android.content.Context; +import android.database.ContentObserver; +import android.database.Cursor; +import android.net.Uri; +import android.provider.UserDictionary; +import android.util.Log; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.common.LocaleUtils; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * UserDictionaryLookup provides the ability to lookup into the system-wide "Personal dictionary". + * + * Note, that the initial dictionary loading happens asynchronously so it is possible (hopefully + * rarely) that isValidWord is called before the initial load has started. + * + * The caller should explicitly call close() when the object is no longer needed, in order to + * release any resources and references to this object. A service should create this object in + * onCreate and close() it in onDestroy. + */ +public class UserDictionaryLookup implements Closeable { + private static final String TAG = UserDictionaryLookup.class.getSimpleName(); + + /** + * This guards the execution of any Log.d() logging, so that if false, they are not even + */ + private static final boolean DEBUG = false; + + /** + * To avoid loading too many dictionary entries in memory, we cap them at this number. If + * that number is exceeded, the lowest-frequency items will be dropped. Note, there is no + * explicit cap on the number of locales in every entry. + */ + private static final int MAX_NUM_ENTRIES = 1000; + + /** + * The delay (in milliseconds) to impose on reloads. Previously scheduled reloads will be + * cancelled if a new reload is scheduled before the delay expires. Thus, only the last + * reload in the series of frequent reloads will execute. + * + * Note, this value should be low enough to allow the "Add to dictionary" feature in the + * TextView correction (red underline) drop-down menu to work properly in the following case: + * + * 1. User types OOV (out-of-vocabulary) word. + * 2. The OOV is red-underlined. + * 3. User selects "Add to dictionary". The red underline disappears while the OOV is + * in a composing span. + * 4. The user taps space. The red underline should NOT reappear. If this value is very + * high and the user performs the space tap fast enough, the red underline may reappear. + */ + @UsedForTesting + static final int RELOAD_DELAY_MS = 200; + + private final ContentResolver mResolver; + + /** + * Executor on which to perform the initial load and subsequent reloads (after a delay). + */ + private final ScheduledExecutorService mLoadExecutor = + Executors.newSingleThreadScheduledExecutor(); + + /** + * Runnable that calls loadUserDictionary(). + */ + private class UserDictionaryLoader implements Runnable { + @Override + public void run() { + if (DEBUG) { + Log.d(TAG, "Executing (re)load"); + } + loadUserDictionary(); + } + } + private final UserDictionaryLoader mLoader = new UserDictionaryLoader(); + + /** + * Content observer for UserDictionary changes. It has the following properties: + * 1. It spawns off a UserDictionary reload in another thread, after some delay. + * 2. It cancels previously scheduled reloads, and only executes the latest. + * 3. It may be called multiple times quickly in succession (and is in fact called so + * when UserDictionary is edited through its settings UI, when sometimes multiple + * notifications are sent for the edited entry, but also for the entire UserDictionary). + */ + private class UserDictionaryContentObserver extends ContentObserver { + public UserDictionaryContentObserver() { + super(null); + } + + @Override + public boolean deliverSelfNotifications() { + return true; + } + + // Support pre-API16 platforms. + @Override + public void onChange(boolean selfChange) { + onChange(selfChange, null); + } + + @Override + public void onChange(boolean selfChange, Uri uri) { + if (DEBUG) { + Log.d(TAG, "Received content observer onChange notification for URI: " + uri); + } + // Cancel (but don't interrupt) any pending reloads (except the initial load). + if (mReloadFuture != null && !mReloadFuture.isCancelled() && + !mReloadFuture.isDone()) { + // Note, that if already cancelled or done, this will do nothing. + boolean isCancelled = mReloadFuture.cancel(false); + if (DEBUG) { + if (isCancelled) { + Log.d(TAG, "Successfully canceled previous reload request"); + } else { + Log.d(TAG, "Unable to cancel previous reload request"); + } + } + } + + if (DEBUG) { + Log.d(TAG, "Scheduling reload in " + RELOAD_DELAY_MS + " ms"); + } + + // Schedule a new reload after RELOAD_DELAY_MS. + mReloadFuture = mLoadExecutor.schedule(mLoader, RELOAD_DELAY_MS, TimeUnit.MILLISECONDS); + } + } + private final ContentObserver mObserver = new UserDictionaryContentObserver(); + + /** + * Indicates that a load is in progress, so no need for another. + */ + private AtomicBoolean mIsLoading = new AtomicBoolean(false); + + /** + * Indicates that this lookup object has been close()d. + */ + private AtomicBoolean mIsClosed = new AtomicBoolean(false); + + /** + * We store a map from a dictionary word to the set of locales it belongs + * in. We then iterate over the set of locales to find a match using + * LocaleUtils. + */ + private volatile HashMap<String, ArrayList<Locale>> mDictWords; + + /** + * The last-scheduled reload future. Saved in order to cancel a pending reload if a new one + * is coming. + */ + private volatile ScheduledFuture<?> mReloadFuture; + + /** + * @param context the context from which to obtain content resolver + */ + public UserDictionaryLookup(Context context) { + if (DEBUG) { + Log.d(TAG, "UserDictionaryLookup constructor with context: " + context); + } + + // Obtain a content resolver. + mResolver = context.getContentResolver(); + + // Schedule the initial load to run immediately. It's possible that the first call to + // isValidWord occurs before the dictionary has actually loaded, so it should not + // assume that the dictionary has been loaded. + mLoadExecutor.schedule(mLoader, 0, TimeUnit.MILLISECONDS); + + // Register the observer to be notified on changes to the UserDictionary and all individual + // items. + // + // If the user is interacting with the UserDictionary settings UI, or with the + // "Add to dictionary" drop-down option, duplicate notifications will be sent for the same + // edit: if a new entry is added, there is a notification for the entry itself, and + // separately for the entire dictionary. However, when used programmatically, + // only notifications for the specific edits are sent. Thus, the observer is registered to + // receive every possible notification, and instead has throttling logic to avoid doing too + // many reloads. + mResolver.registerContentObserver( + UserDictionary.Words.CONTENT_URI, true /* notifyForDescendents */, mObserver); + } + + /** + * To be called by the garbage collector in the off chance that the service did not clean up + * properly. Do not rely on this getting called, and make sure close() is called explicitly. + */ + @Override + public void finalize() throws Throwable { + try { + if (DEBUG) { + Log.d(TAG, "Finalize called, calling close()"); + } + close(); + } finally { + super.finalize(); + } + } + + /** + * Cleans up UserDictionaryLookup: shuts down any extra threads and unregisters the observer. + * + * It is safe, but not advised to call this multiple times, and isValidWord would continue to + * work, but no data will be reloaded any longer. + */ + @Override + public void close() { + if (DEBUG) { + Log.d(TAG, "Close called (no pun intended), cleaning up executor and observer"); + } + if (mIsClosed.compareAndSet(false, true)) { + // Shut down the load executor. + mLoadExecutor.shutdown(); + + // Unregister the content observer. + mResolver.unregisterContentObserver(mObserver); + } + } + + /** + * Returns true if the initial load has been performed. + * + * @return true if the initial load is successful + */ + @UsedForTesting + boolean isLoaded() { + return mDictWords != null; + } + + /** + * Determines if the given word is a valid word in the given locale based on the UserDictionary. + * It tries hard to find a match: for example, casing is ignored and if the word is present in a + * more general locale (e.g. en or all locales), and isValidWord is asking for a more specific + * locale (e.g. en_US), it will be considered a match. + * + * @param word the word to match + * @param locale the locale in which to match the word + * @return true iff the word has been matched for this locale in the UserDictionary. + */ + public boolean isValidWord( + final String word, final Locale locale) { + if (!isLoaded()) { + // This is a corner case in the event the initial load of UserDictionary has not + // been loaded. In that case, we assume the word is not a valid word in + // UserDictionary. + if (DEBUG) { + Log.d(TAG, "isValidWord invoked, but initial load not complete"); + } + return false; + } + + // Atomically obtain the current copy of mDictWords; + final HashMap<String, ArrayList<Locale>> dictWords = mDictWords; + + if (DEBUG) { + Log.d(TAG, "isValidWord invoked for word [" + word + + "] in locale " + locale); + } + // Lowercase the word using the given locale. Note, that dictionary + // words are lowercased using their locale, and theoretically the + // lowercasing between two matching locales may differ. For simplicity + // we ignore that possibility. + final String lowercased = word.toLowerCase(locale); + final ArrayList<Locale> dictLocales = dictWords.get(lowercased); + if (null == dictLocales) { + if (DEBUG) { + Log.d(TAG, "isValidWord=false, since there is no entry for " + + "lowercased word [" + lowercased + "]"); + } + return false; + } else { + if (DEBUG) { + Log.d(TAG, "isValidWord found an entry for lowercased word [" + lowercased + + "]; examining locales"); + } + // Iterate over the locales this word is in. + for (final Locale dictLocale : dictLocales) { + final int matchLevel = LocaleUtils.getMatchLevel(dictLocale.toString(), + locale.toString()); + if (DEBUG) { + Log.d(TAG, "matchLevel for dictLocale=" + dictLocale + ", locale=" + + locale + " is " + matchLevel); + } + if (LocaleUtils.isMatch(matchLevel)) { + if (DEBUG) { + Log.d(TAG, "isValidWord=true, since matchLevel " + matchLevel + + " is a match"); + } + return true; + } + if (DEBUG) { + Log.d(TAG, "matchLevel " + matchLevel + " is not a match"); + } + } + if (DEBUG) { + Log.d(TAG, "isValidWord=false, since none of the locales matched"); + } + return false; + } + } + + /** + * Loads the UserDictionary in the current thread. + * + * Only one reload can happen at a time. If already running, will exit quickly. + */ + private void loadUserDictionary() { + // Bail out if already in the process of loading. + if (!mIsLoading.compareAndSet(false, true)) { + if (DEBUG) { + Log.d(TAG, "Already in the process of loading UserDictionary, skipping"); + } + return; + } + if (DEBUG) { + Log.d(TAG, "Loading UserDictionary"); + } + HashMap<String, ArrayList<Locale>> dictWords = + new HashMap<String, ArrayList<Locale>>(); + // Load the UserDictionary. Request that items be returned in the default sort order + // for UserDictionary, which is by frequency. + Cursor cursor = mResolver.query(UserDictionary.Words.CONTENT_URI, + null, null, null, UserDictionary.Words.DEFAULT_SORT_ORDER); + if (null == cursor || cursor.getCount() < 1) { + if (DEBUG) { + Log.d(TAG, "No entries found in UserDictionary"); + } + } else { + // Iterate over the entries in the UserDictionary. Note, that iteration is in + // descending frequency by default. + while (dictWords.size() < MAX_NUM_ENTRIES && cursor.moveToNext()) { + // If there is no column for locale, skip this entry. An empty + // locale on the other hand will not be skipped. + final int dictLocaleIndex = cursor.getColumnIndex( + UserDictionary.Words.LOCALE); + if (dictLocaleIndex < 0) { + if (DEBUG) { + Log.d(TAG, "Encountered UserDictionary entry " + + "without LOCALE, skipping"); + } + continue; + } + // If there is no column for word, skip this entry. + final int dictWordIndex = cursor.getColumnIndex( + UserDictionary.Words.WORD); + if (dictWordIndex < 0) { + if (DEBUG) { + Log.d(TAG, "Encountered UserDictionary entry without " + + "WORD, skipping"); + } + continue; + } + // If the word is null, skip this entry. + final String rawDictWord = cursor.getString(dictWordIndex); + if (null == rawDictWord) { + if (DEBUG) { + Log.d(TAG, "Encountered null word"); + } + continue; + } + // If the locale is null, that's interpreted to mean all locales. Note, the special + // zz locale for an Alphabet (QWERTY) layout will not match any actual language. + String localeString = cursor.getString(dictLocaleIndex); + if (null == localeString) { + if (DEBUG) { + Log.d(TAG, "Encountered null locale for word [" + + rawDictWord + "], assuming all locales"); + } + // For purposes of LocaleUtils, an empty locale matches + // everything. + localeString = ""; + } + final Locale dictLocale = LocaleUtils.constructLocaleFromString( + localeString); + // Lowercase the word before storing it. + final String dictWord = rawDictWord.toLowerCase(dictLocale); + if (DEBUG) { + Log.d(TAG, "Incorporating UserDictionary word [" + dictWord + + "] for locale " + dictLocale); + } + // Check if there is an existing entry for this word. + ArrayList<Locale> dictLocales = dictWords.get(dictWord); + if (null == dictLocales) { + // If there is no entry for this word, create one. + if (DEBUG) { + Log.d(TAG, "Word [" + dictWord + + "] not seen for other locales, creating new entry"); + } + dictLocales = new ArrayList<Locale>(); + dictWords.put(dictWord, dictLocales); + } + // Append the locale to the list of locales this word is in. + dictLocales.add(dictLocale); + } + } + + // Atomically replace the copy of mDictWords. + mDictWords = dictWords; + + // Allow other calls to loadUserDictionary to execute now. + mIsLoading.set(false); + } +} diff --git a/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryAddWordContents.java b/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryAddWordContents.java index 69029c51e..cb615f3af 100644 --- a/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryAddWordContents.java +++ b/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryAddWordContents.java @@ -26,7 +26,6 @@ import android.text.TextUtils; import android.view.View; import android.widget.EditText; -import com.android.inputmethod.compat.UserDictionaryCompatUtils; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.common.LocaleUtils; @@ -47,10 +46,7 @@ import javax.annotation.Nullable; public class UserDictionaryAddWordContents { public static final String EXTRA_MODE = "mode"; public static final String EXTRA_WORD = "word"; - public static final String EXTRA_SHORTCUT = "shortcut"; public static final String EXTRA_LOCALE = "locale"; - public static final String EXTRA_ORIGINAL_WORD = "originalWord"; - public static final String EXTRA_ORIGINAL_SHORTCUT = "originalShortcut"; public static final int MODE_EDIT = 0; public static final int MODE_INSERT = 1; @@ -63,20 +59,12 @@ public class UserDictionaryAddWordContents { private final int mMode; // Either MODE_EDIT or MODE_INSERT private final EditText mWordEditText; - private final EditText mShortcutEditText; private String mLocale; private final String mOldWord; - private final String mOldShortcut; private String mSavedWord; - private String mSavedShortcut; /* package */ UserDictionaryAddWordContents(final View view, final Bundle args) { mWordEditText = (EditText)view.findViewById(R.id.user_dictionary_add_word_text); - mShortcutEditText = (EditText)view.findViewById(R.id.user_dictionary_add_shortcut); - if (!UserDictionarySettings.IS_SHORTCUT_API_SUPPORTED) { - mShortcutEditText.setVisibility(View.GONE); - view.findViewById(R.id.user_dictionary_add_shortcut_label).setVisibility(View.GONE); - } final String word = args.getString(EXTRA_WORD); if (null != word) { mWordEditText.setText(word); @@ -84,17 +72,6 @@ public class UserDictionaryAddWordContents { // it's too long to be edited. mWordEditText.setSelection(mWordEditText.getText().length()); } - final String shortcut; - if (UserDictionarySettings.IS_SHORTCUT_API_SUPPORTED) { - shortcut = args.getString(EXTRA_SHORTCUT); - if (null != shortcut && null != mShortcutEditText) { - mShortcutEditText.setText(shortcut); - } - mOldShortcut = args.getString(EXTRA_SHORTCUT); - } else { - shortcut = null; - mOldShortcut = null; - } mMode = args.getInt(EXTRA_MODE); // default return value for #getInt() is 0 = MODE_EDIT mOldWord = args.getString(EXTRA_WORD); updateLocale(args.getString(EXTRA_LOCALE)); @@ -103,10 +80,8 @@ public class UserDictionaryAddWordContents { /* package */ UserDictionaryAddWordContents(final View view, final UserDictionaryAddWordContents oldInstanceToBeEdited) { mWordEditText = (EditText)view.findViewById(R.id.user_dictionary_add_word_text); - mShortcutEditText = (EditText)view.findViewById(R.id.user_dictionary_add_shortcut); mMode = MODE_EDIT; mOldWord = oldInstanceToBeEdited.mSavedWord; - mOldShortcut = oldInstanceToBeEdited.mSavedShortcut; updateLocale(mLocale); } @@ -118,13 +93,6 @@ public class UserDictionaryAddWordContents { /* package */ void saveStateIntoBundle(final Bundle outState) { outState.putString(EXTRA_WORD, mWordEditText.getText().toString()); - outState.putString(EXTRA_ORIGINAL_WORD, mOldWord); - if (null != mShortcutEditText) { - outState.putString(EXTRA_SHORTCUT, mShortcutEditText.getText().toString()); - } - if (null != mOldShortcut) { - outState.putString(EXTRA_ORIGINAL_SHORTCUT, mOldShortcut); - } outState.putString(EXTRA_LOCALE, mLocale); } @@ -132,7 +100,7 @@ public class UserDictionaryAddWordContents { if (MODE_EDIT == mMode && !TextUtils.isEmpty(mOldWord)) { // Mode edit: remove the old entry. final ContentResolver resolver = context.getContentResolver(); - UserDictionarySettings.deleteWord(mOldWord, mOldShortcut, resolver); + UserDictionarySettings.deleteWord(mOldWord, resolver); } // If we are in add mode, nothing was added, so we don't need to do anything. } @@ -143,50 +111,31 @@ public class UserDictionaryAddWordContents { final ContentResolver resolver = context.getContentResolver(); if (MODE_EDIT == mMode && !TextUtils.isEmpty(mOldWord)) { // Mode edit: remove the old entry. - UserDictionarySettings.deleteWord(mOldWord, mOldShortcut, resolver); + UserDictionarySettings.deleteWord(mOldWord, resolver); } final String newWord = mWordEditText.getText().toString(); - final String newShortcut; - if (!UserDictionarySettings.IS_SHORTCUT_API_SUPPORTED) { - newShortcut = null; - } else if (null == mShortcutEditText) { - newShortcut = null; - } else { - final String tmpShortcut = mShortcutEditText.getText().toString(); - if (TextUtils.isEmpty(tmpShortcut)) { - newShortcut = null; - } else { - newShortcut = tmpShortcut; - } - } if (TextUtils.isEmpty(newWord)) { // If the word is somehow empty, don't insert it. return CODE_CANCEL; } mSavedWord = newWord; - mSavedShortcut = newShortcut; - // If there is no shortcut, and the word already exists in the database, then we - // should not insert, because either A. the word exists with no shortcut, in which - // case the exact same thing we want to insert is already there, or B. the word - // exists with at least one shortcut, in which case it has priority on our word. - if (TextUtils.isEmpty(newShortcut) && hasWord(newWord, context)) { + // If the word already exists in the database, then we should not insert. + if (hasWord(newWord, context)) { return CODE_ALREADY_PRESENT; } - // Disallow duplicates. If the same word with no shortcut is defined, remove it; if - // the same word with the same shortcut is defined, remove it; but we don't mind if - // there is the same word with a different, non-empty shortcut. - UserDictionarySettings.deleteWord(newWord, null, resolver); - if (!TextUtils.isEmpty(newShortcut)) { - // If newShortcut is empty we just deleted this, no need to do it again - UserDictionarySettings.deleteWord(newWord, newShortcut, resolver); - } + // Disallow duplicates. If the same word is defined, remove it. + UserDictionarySettings.deleteWord(newWord, resolver); // In this class we use the empty string to represent 'all locales' and mLocale cannot // be null. However the addWord method takes null to mean 'all locales'. - UserDictionaryCompatUtils.addWord(context, newWord.toString(), - FREQUENCY_FOR_USER_DICTIONARY_ADDS, newShortcut, TextUtils.isEmpty(mLocale) ? - null : LocaleUtils.constructLocaleFromString(mLocale)); + final Locale locale = TextUtils.isEmpty(mLocale) ? + null : LocaleUtils.constructLocaleFromString(mLocale); + final Locale currentLocale = context.getResources().getConfiguration().locale; + final boolean useCurrentLocale = currentLocale.equals(locale); + UserDictionary.Words.addWord(context, newWord.toString(), + FREQUENCY_FOR_USER_DICTIONARY_ADDS, null /* shortcut */, + useCurrentLocale ? Locale.getDefault() : null); return CODE_WORD_ADDED; } diff --git a/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryList.java b/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryList.java index 6254b54ff..57347ce8c 100644 --- a/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryList.java +++ b/java/src/com/android/inputmethod/latin/userdictionary/UserDictionaryList.java @@ -20,6 +20,7 @@ import android.app.Activity; import android.content.Context; import android.content.Intent; import android.database.Cursor; +import android.os.Build; import android.os.Bundle; import android.preference.Preference; import android.preference.PreferenceFragment; @@ -74,7 +75,7 @@ public class UserDictionaryList extends PreferenceFragment { } finally { cursor.close(); } - if (!UserDictionarySettings.IS_SHORTCUT_API_SUPPORTED) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) { // For ICS, we need to show "For all languages" in case that the keyboard locale // is different from the system locale localeSet.add(""); diff --git a/java/src/com/android/inputmethod/latin/userdictionary/UserDictionarySettings.java b/java/src/com/android/inputmethod/latin/userdictionary/UserDictionarySettings.java index fabd49f46..3ccd1b3e6 100644 --- a/java/src/com/android/inputmethod/latin/userdictionary/UserDictionarySettings.java +++ b/java/src/com/android/inputmethod/latin/userdictionary/UserDictionarySettings.java @@ -48,42 +48,18 @@ import java.util.Locale; public class UserDictionarySettings extends ListFragment { - public static final boolean IS_SHORTCUT_API_SUPPORTED = - Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; - - private static final String[] QUERY_PROJECTION_SHORTCUT_UNSUPPORTED = - { UserDictionary.Words._ID, UserDictionary.Words.WORD}; - private static final String[] QUERY_PROJECTION_SHORTCUT_SUPPORTED = - { UserDictionary.Words._ID, UserDictionary.Words.WORD, UserDictionary.Words.SHORTCUT}; - private static final String[] QUERY_PROJECTION = - IS_SHORTCUT_API_SUPPORTED ? - QUERY_PROJECTION_SHORTCUT_SUPPORTED : QUERY_PROJECTION_SHORTCUT_UNSUPPORTED; - - // The index of the shortcut in the above array. - private static final int INDEX_SHORTCUT = 2; - - private static final String[] ADAPTER_FROM_SHORTCUT_UNSUPPORTED = { - UserDictionary.Words.WORD, + private static final String[] QUERY_PROJECTION = { + UserDictionary.Words._ID, UserDictionary.Words.WORD }; - private static final String[] ADAPTER_FROM_SHORTCUT_SUPPORTED = { - UserDictionary.Words.WORD, UserDictionary.Words.SHORTCUT + private static final String[] ADAPTER_FROM = { + UserDictionary.Words.WORD, UserDictionary.Words.SHORTCUT }; - private static final String[] ADAPTER_FROM = IS_SHORTCUT_API_SUPPORTED ? - ADAPTER_FROM_SHORTCUT_SUPPORTED : ADAPTER_FROM_SHORTCUT_UNSUPPORTED; - - private static final int[] ADAPTER_TO_SHORTCUT_UNSUPPORTED = { + private static final int[] ADAPTER_TO = { android.R.id.text1, }; - private static final int[] ADAPTER_TO_SHORTCUT_SUPPORTED = { - android.R.id.text1, android.R.id.text2 - }; - - private static final int[] ADAPTER_TO = IS_SHORTCUT_API_SUPPORTED ? - ADAPTER_TO_SHORTCUT_SUPPORTED : ADAPTER_TO_SHORTCUT_UNSUPPORTED; - // Either the locale is empty (means the word is applicable to all locales) // or the word equals our current locale private static final String QUERY_SELECTION = @@ -91,13 +67,7 @@ public class UserDictionarySettings extends ListFragment { private static final String QUERY_SELECTION_ALL_LOCALES = UserDictionary.Words.LOCALE + " is null"; - private static final String DELETE_SELECTION_WITH_SHORTCUT = UserDictionary.Words.WORD - + "=? AND " + UserDictionary.Words.SHORTCUT + "=?"; - private static final String DELETE_SELECTION_WITHOUT_SHORTCUT = UserDictionary.Words.WORD - + "=? AND " + UserDictionary.Words.SHORTCUT + " is null OR " - + UserDictionary.Words.SHORTCUT + "=''"; - private static final String DELETE_SELECTION_SHORTCUT_UNSUPPORTED = - UserDictionary.Words.WORD + "=?"; + private static final String DELETE_SELECTION = UserDictionary.Words.WORD + "=?"; private static final int OPTIONS_MENU_ADD = Menu.FIRST; @@ -192,20 +162,18 @@ public class UserDictionarySettings extends ListFragment { @Override public void onListItemClick(ListView l, View v, int position, long id) { final String word = getWord(position); - final String shortcut = getShortcut(position); if (word != null) { - showAddOrEditDialog(word, shortcut); + showAddOrEditDialog(word); } } @Override public void onCreateOptionsMenu(Menu menu, MenuInflater inflater) { - if (!UserDictionarySettings.IS_SHORTCUT_API_SUPPORTED) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) { final Locale systemLocale = getResources().getConfiguration().locale; if (!TextUtils.isEmpty(mLocale) && !mLocale.equals(systemLocale.toString())) { // Hide the add button for ICS because it doesn't support specifying a locale - // for an entry. This new "locale"-aware API has been added in conjunction - // with the shortcut API. + // for an entry. return; } } @@ -219,7 +187,7 @@ public class UserDictionarySettings extends ListFragment { @Override public boolean onOptionsItemSelected(MenuItem item) { if (item.getItemId() == OPTIONS_MENU_ADD) { - showAddOrEditDialog(null, null); + showAddOrEditDialog(null); return true; } return false; @@ -228,18 +196,15 @@ public class UserDictionarySettings extends ListFragment { /** * Add or edit a word. If editingWord is null, it's an add; otherwise, it's an edit. * @param editingWord the word to edit, or null if it's an add. - * @param editingShortcut the shortcut for this entry, or null if none. */ - private void showAddOrEditDialog(final String editingWord, final String editingShortcut) { + private void showAddOrEditDialog(final String editingWord) { final Bundle args = new Bundle(); args.putInt(UserDictionaryAddWordContents.EXTRA_MODE, null == editingWord ? UserDictionaryAddWordContents.MODE_INSERT : UserDictionaryAddWordContents.MODE_EDIT); args.putString(UserDictionaryAddWordContents.EXTRA_WORD, editingWord); - args.putString(UserDictionaryAddWordContents.EXTRA_SHORTCUT, editingShortcut); args.putString(UserDictionaryAddWordContents.EXTRA_LOCALE, mLocale); - android.preference.PreferenceActivity pa = - (android.preference.PreferenceActivity)getActivity(); + getActivity(); } private String getWord(final int position) { @@ -252,31 +217,8 @@ public class UserDictionarySettings extends ListFragment { mCursor.getColumnIndexOrThrow(UserDictionary.Words.WORD)); } - private String getShortcut(final int position) { - if (!IS_SHORTCUT_API_SUPPORTED) return null; - if (null == mCursor) return null; - mCursor.moveToPosition(position); - // Handle a possible race-condition - if (mCursor.isAfterLast()) return null; - - return mCursor.getString( - mCursor.getColumnIndexOrThrow(UserDictionary.Words.SHORTCUT)); - } - - public static void deleteWord(final String word, final String shortcut, - final ContentResolver resolver) { - if (!IS_SHORTCUT_API_SUPPORTED) { - resolver.delete(UserDictionary.Words.CONTENT_URI, DELETE_SELECTION_SHORTCUT_UNSUPPORTED, - new String[] { word }); - } else if (TextUtils.isEmpty(shortcut)) { - resolver.delete( - UserDictionary.Words.CONTENT_URI, DELETE_SELECTION_WITHOUT_SHORTCUT, - new String[] { word }); - } else { - resolver.delete( - UserDictionary.Words.CONTENT_URI, DELETE_SELECTION_WITH_SHORTCUT, - new String[] { word, shortcut }); - } + public static void deleteWord(final String word, final ContentResolver resolver) { + resolver.delete(UserDictionary.Words.CONTENT_URI, DELETE_SELECTION, new String[] { word }); } private static class MyAdapter extends SimpleCursorAdapter implements SectionIndexer { @@ -286,22 +228,7 @@ public class UserDictionarySettings extends ListFragment { @Override public boolean setViewValue(final View v, final Cursor c, final int columnIndex) { - if (!IS_SHORTCUT_API_SUPPORTED) { - // just let SimpleCursorAdapter set the view values - return false; - } - if (columnIndex == INDEX_SHORTCUT) { - final String shortcut = c.getString(INDEX_SHORTCUT); - if (TextUtils.isEmpty(shortcut)) { - v.setVisibility(View.GONE); - } else { - ((TextView)v).setText(shortcut); - v.setVisibility(View.VISIBLE); - } - v.invalidate(); - return true; - } - + // just let SimpleCursorAdapter set the view values return false; } }; diff --git a/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java b/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java index 476c13406..5c0c4328f 100644 --- a/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java @@ -19,7 +19,6 @@ package com.android.inputmethod.latin.utils; import com.android.inputmethod.latin.makedict.DictionaryHeader; import com.android.inputmethod.latin.makedict.NgramProperty; import com.android.inputmethod.latin.makedict.ProbabilityInfo; -import com.android.inputmethod.latin.makedict.WeightedString; import com.android.inputmethod.latin.makedict.WordProperty; import java.util.HashMap; @@ -29,7 +28,6 @@ public class CombinedFormatUtils { public static final String BIGRAM_TAG = "bigram"; public static final String NGRAM_TAG = "ngram"; public static final String NGRAM_PREV_WORD_TAG = "prev_word"; - public static final String SHORTCUT_TAG = "shortcut"; public static final String PROBABILITY_TAG = "f"; public static final String HISTORICAL_INFO_TAG = "historicalInfo"; public static final String HISTORICAL_INFO_SEPARATOR = ":"; @@ -71,14 +69,6 @@ public class CombinedFormatUtils { builder.append("," + POSSIBLY_OFFENSIVE_TAG + "=" + TRUE_VALUE); } builder.append("\n"); - if (wordProperty.mHasShortcuts) { - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - builder.append(" " + SHORTCUT_TAG + "=" + shortcutTarget.mWord); - builder.append(","); - builder.append(formatProbabilityInfo(shortcutTarget.mProbabilityInfo)); - builder.append("\n"); - } - } if (wordProperty.mHasNgrams) { for (final NgramProperty ngramProperty : wordProperty.mNgrams) { builder.append(" " + NGRAM_TAG + "=" + ngramProperty.mTargetWord.mWord); diff --git a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java index e355b7e1f..2e9cc8845 100644 --- a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java @@ -51,6 +51,7 @@ public class DictionaryInfoUtils { private static final String RESOURCE_PACKAGE_NAME = R.class.getPackage().getName(); private static final String DEFAULT_MAIN_DICT = "main"; private static final String MAIN_DICT_PREFIX = "main_"; + private static final String DECODER_DICT_SUFFIX = DecoderSpecificConstants.DECODER_DICT_SUFFIX; // 6 digits - unicode is limited to 21 bits private static final int MAX_HEX_DIGITS_FOR_CODEPOINT = 6; @@ -267,8 +268,8 @@ public class DictionaryInfoUtils { int resId; // Try to find main_language_country dictionary. if (!locale.getCountry().isEmpty()) { - final String dictLanguageCountry = - MAIN_DICT_PREFIX + locale.toString().toLowerCase(Locale.ROOT); + final String dictLanguageCountry = MAIN_DICT_PREFIX + + locale.toString().toLowerCase(Locale.ROOT) + DECODER_DICT_SUFFIX; if ((resId = res.getIdentifier( dictLanguageCountry, "raw", RESOURCE_PACKAGE_NAME)) != 0) { return resId; @@ -276,7 +277,7 @@ public class DictionaryInfoUtils { } // Try to find main_language dictionary. - final String dictLanguage = MAIN_DICT_PREFIX + locale.getLanguage(); + final String dictLanguage = MAIN_DICT_PREFIX + locale.getLanguage() + DECODER_DICT_SUFFIX; if ((resId = res.getIdentifier(dictLanguage, "raw", RESOURCE_PACKAGE_NAME)) != 0) { return resId; } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java deleted file mode 100644 index 525212c96..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.NgramContext; - -import java.util.List; -import java.util.Locale; - -import javax.annotation.Nonnull; - -public interface DistracterFilter { - /** - * Determine whether a word is a distracter to words in dictionaries. - * - * @param ngramContext the n-gram context - * @param testedWord the word that will be tested to see whether it is a distracter to words - * in dictionaries. - * @param locale the locale of word. - * @return true if testedWord is a distracter, otherwise false. - */ - public boolean isDistracterToWordsInDictionaries(final NgramContext ngramContext, - final String testedWord, final Locale locale); - - @UsedForTesting - public int getWordHandlingType(final NgramContext ngramContext, final String testedWord, - final Locale locale); - - public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes); - - public void close(); - - public static final class HandlingType { - private final static int REQUIRE_NO_SPECIAL_HANDLINGS = 0x0; - private final static int SHOULD_BE_LOWER_CASED = 0x1; - private final static int SHOULD_BE_HANDLED_AS_OOV = 0x2; - - public static int getHandlingType(final boolean shouldBeLowerCased, final boolean isOov) { - int wordHandlingType = HandlingType.REQUIRE_NO_SPECIAL_HANDLINGS; - if (shouldBeLowerCased) { - wordHandlingType |= HandlingType.SHOULD_BE_LOWER_CASED; - } - if (isOov) { - wordHandlingType |= HandlingType.SHOULD_BE_HANDLED_AS_OOV; - } - return wordHandlingType; - } - - public static boolean shouldBeLowerCased(final int handlingType) { - return (handlingType & SHOULD_BE_LOWER_CASED) != 0; - } - - public static boolean shouldBeHandledAsOov(final int handlingType) { - return (handlingType & SHOULD_BE_HANDLED_AS_OOV) != 0; - } - } - - @Nonnull - public static final DistracterFilter EMPTY_DISTRACTER_FILTER = new DistracterFilter() { - @Override - public boolean isDistracterToWordsInDictionaries(NgramContext ngramContext, - String testedWord, Locale locale) { - return false; - } - - @Override - public int getWordHandlingType(final NgramContext ngramContext, - final String testedWord, final Locale locale) { - return HandlingType.REQUIRE_NO_SPECIAL_HANDLINGS; - } - - @Override - public void close() { - } - - @Override - public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) { - } - }; -} diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java deleted file mode 100644 index becf13fd9..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import android.content.Context; -import android.content.res.Resources; -import android.text.InputType; -import android.util.Log; -import android.util.LruCache; -import android.util.Pair; -import android.view.inputmethod.EditorInfo; -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.keyboard.KeyboardId; -import com.android.inputmethod.keyboard.KeyboardLayoutSet; -import com.android.inputmethod.latin.DictionaryFacilitator; -import com.android.inputmethod.latin.DictionaryFacilitatorLruCache; -import com.android.inputmethod.latin.NgramContext; -import com.android.inputmethod.latin.RichInputMethodSubtype; -import com.android.inputmethod.latin.SuggestedWords; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.common.StringUtils; -import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; - -/** - * This class is used to prevent distracters being added to personalization - * or user history dictionaries - */ -public class DistracterFilterCheckingExactMatchesAndSuggestions implements DistracterFilter { - private static final String TAG = - DistracterFilterCheckingExactMatchesAndSuggestions.class.getSimpleName(); - private static final boolean DEBUG = false; - - private static final int MAX_DISTRACTERS_CACHE_SIZE = 1024; - - private final Context mContext; - private final ConcurrentHashMap<Locale, InputMethodSubtype> mLocaleToSubtypeCache; - private final ConcurrentHashMap<Locale, Keyboard> mLocaleToKeyboardCache; - private final DictionaryFacilitatorLruCache mDictionaryFacilitatorLruCache; - // The key is a pair of a locale and a word. The value indicates the word is a distracter to - // words of the locale. - private final LruCache<Pair<Locale, String>, Boolean> mDistractersCache; - private final Object mLock = new Object(); - - // If the score of the top suggestion exceeds this value, the tested word (e.g., - // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to - // words in dictionary. The greater the threshold is, the less likely the tested word would - // become a distracter, which means the tested word will be more likely to be added to - // the dictionary. - private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 0.4f; - - /** - * Create a DistracterFilter instance. - * - * @param context the context. - */ - public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context) { - mContext = context; - mLocaleToSubtypeCache = new ConcurrentHashMap<>(); - mLocaleToKeyboardCache = new ConcurrentHashMap<>(); - mDictionaryFacilitatorLruCache = new DictionaryFacilitatorLruCache( - context, "" /* dictionaryNamePrefix */); - mDistractersCache = new LruCache<>(MAX_DISTRACTERS_CACHE_SIZE); - } - - @Override - public void close() { - mLocaleToSubtypeCache.clear(); - mLocaleToKeyboardCache.clear(); - mDictionaryFacilitatorLruCache.evictAll(); - // Don't clear mDistractersCache. - } - - @Override - public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { - final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>(); - if (enabledSubtypes != null) { - for (final InputMethodSubtype subtype : enabledSubtypes) { - final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); - if (newLocaleToSubtypeMap.containsKey(locale)) { - // Multiple subtypes are enabled for one locale. - // TODO: Investigate what we should do for this case. - continue; - } - newLocaleToSubtypeMap.put(locale, subtype); - } - } - if (mLocaleToSubtypeCache.equals(newLocaleToSubtypeMap)) { - // Enabled subtypes have not been changed. - return; - } - // Update subtype and keyboard map for locales that are in the current mapping. - for (final Locale locale: mLocaleToSubtypeCache.keySet()) { - if (newLocaleToSubtypeMap.containsKey(locale)) { - final InputMethodSubtype newSubtype = newLocaleToSubtypeMap.remove(locale); - if (newSubtype.equals(newLocaleToSubtypeMap.get(locale))) { - // Mapping has not been changed. - continue; - } - mLocaleToSubtypeCache.replace(locale, newSubtype); - } else { - mLocaleToSubtypeCache.remove(locale); - } - mLocaleToKeyboardCache.remove(locale); - } - // Add locales that are not in the current mapping. - mLocaleToSubtypeCache.putAll(newLocaleToSubtypeMap); - } - - private Keyboard getKeyboardForLocale(final Locale locale) { - final Keyboard cachedKeyboard = mLocaleToKeyboardCache.get(locale); - if (cachedKeyboard != null) { - return cachedKeyboard; - } - final InputMethodSubtype subtype = mLocaleToSubtypeCache.get(locale); - if (subtype == null) { - return null; - } - final EditorInfo editorInfo = new EditorInfo(); - editorInfo.inputType = InputType.TYPE_CLASS_TEXT; - final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( - mContext, editorInfo); - final Resources res = mContext.getResources(); - final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); - final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); - builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); - builder.setSubtype(new RichInputMethodSubtype(subtype)); - builder.setIsSpellChecker(false /* isSpellChecker */); - final KeyboardLayoutSet layoutSet = builder.build(); - final Keyboard newKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); - mLocaleToKeyboardCache.put(locale, newKeyboard); - return newKeyboard; - } - - /** - * Determine whether a word is a distracter to words in dictionaries. - * - * @param ngramContext the n-gram context. Not used for now. - * @param testedWord the word that will be tested to see whether it is a distracter to words - * in dictionaries. - * @param locale the locale of word. - * @return true if testedWord is a distracter, otherwise false. - */ - @Override - public boolean isDistracterToWordsInDictionaries(final NgramContext ngramContext, - final String testedWord, final Locale locale) { - if (locale == null) { - return false; - } - if (!mLocaleToSubtypeCache.containsKey(locale)) { - Log.e(TAG, "Locale " + locale + " is not enabled."); - // TODO: Investigate what we should do for disabled locales. - return false; - } - final DictionaryFacilitator dictionaryFacilitator = - mDictionaryFacilitatorLruCache.get(locale); - if (DEBUG) { - Log.d(TAG, "testedWord: " + testedWord); - } - final Pair<Locale, String> cacheKey = new Pair<>(locale, testedWord); - final Boolean isCachedDistracter = mDistractersCache.get(cacheKey); - if (isCachedDistracter != null && isCachedDistracter) { - if (DEBUG) { - Log.d(TAG, "isDistracter: true (cache hit)"); - } - return true; - } - - final boolean isDistracterCheckedByGetMaxFreqencyOfExactMatches = - checkDistracterUsingMaxFreqencyOfExactMatches(dictionaryFacilitator, testedWord); - if (isDistracterCheckedByGetMaxFreqencyOfExactMatches) { - // Add the pair of locale and word to the cache. - mDistractersCache.put(cacheKey, Boolean.TRUE); - return true; - } - if (dictionaryFacilitator.isValidSuggestionWord(testedWord)) { - // Valid word is not a distracter. - if (DEBUG) { - Log.d(TAG, "isDistracter: false (valid word)"); - } - return false; - } - - final Keyboard keyboard = getKeyboardForLocale(locale); - final boolean isDistracterCheckedByGetSuggestion = - checkDistracterUsingGetSuggestions(dictionaryFacilitator, keyboard, testedWord); - if (isDistracterCheckedByGetSuggestion) { - // Add the pair of locale and word to the cache. - mDistractersCache.put(cacheKey, Boolean.TRUE); - return true; - } - return false; - } - - private static boolean checkDistracterUsingMaxFreqencyOfExactMatches( - final DictionaryFacilitator dictionaryFacilitator, final String testedWord) { - // The tested word is a distracter when there is a word that is exact matched to the tested - // word and its probability is higher than the tested word's probability. - final int perfectMatchFreq = dictionaryFacilitator.getFrequency(testedWord); - final int exactMatchFreq = dictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord); - final boolean isDistracter = perfectMatchFreq < exactMatchFreq; - if (DEBUG) { - Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq); - Log.d(TAG, "exactMatchFreq: " + exactMatchFreq); - Log.d(TAG, "isDistracter: " + isDistracter); - } - return isDistracter; - } - - private boolean checkDistracterUsingGetSuggestions( - final DictionaryFacilitator dictionaryFacilitator, final Keyboard keyboard, - final String testedWord) { - if (keyboard == null) { - return false; - } - final SettingsValuesForSuggestion settingsValuesForSuggestion = - new SettingsValuesForSuggestion(false /* blockPotentiallyOffensive */, - false /* spaceAwareGestureEnabled */); - final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); - final String consideredWord = trailingSingleQuotesCount > 0 ? - testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : - testedWord; - final WordComposer composer = new WordComposer(); - final int[] codePoints = StringUtils.toCodePointArray(testedWord); - final int[] coordinates = keyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates); - final SuggestionResults suggestionResults; - synchronized (mLock) { - suggestionResults = dictionaryFacilitator.getSuggestionResults(composer, - NgramContext.EMPTY_PREV_WORDS_INFO, - keyboard.getProximityInfo().getNativeProximityInfo(), - settingsValuesForSuggestion, 0 /* sessionId */, - SuggestedWords.INPUT_STYLE_TYPING, - keyboard.getKeyboardLayout()); - } - if (suggestionResults.isEmpty()) { - return false; - } - final SuggestedWordInfo firstSuggestion = suggestionResults.first(); - final boolean isDistracter = suggestionExceedsDistracterThreshold( - firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); - if (DEBUG) { - Log.d(TAG, "isDistracter: " + isDistracter); - } - return isDistracter; - } - - private static boolean suggestionExceedsDistracterThreshold(final SuggestedWordInfo suggestion, - final String consideredWord, final float distracterThreshold) { - if (suggestion == null) { - return false; - } - final int suggestionScore = suggestion.mScore; - final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( - consideredWord, suggestion.mWord, suggestionScore); - if (DEBUG) { - Log.d(TAG, "normalizedScore: " + normalizedScore); - Log.d(TAG, "distracterThreshold: " + distracterThreshold); - } - if (normalizedScore > distracterThreshold) { - return true; - } - return false; - } - - private boolean shouldBeLowerCased(final NgramContext ngramContext, final String testedWord, - final Locale locale) { - final DictionaryFacilitator dictionaryFacilitator = - mDictionaryFacilitatorLruCache.get(locale); - if (dictionaryFacilitator.isValidSuggestionWord(testedWord)) { - return false; - } - final String lowerCaseWord = testedWord.toLowerCase(locale); - if (testedWord.equals(lowerCaseWord)) { - return false; - } - if (dictionaryFacilitator.isValidSuggestionWord(lowerCaseWord)) { - return true; - } - if (StringUtils.getCapitalizationType(testedWord) == StringUtils.CAPITALIZE_FIRST - && !ngramContext.isValid()) { - // TODO: Check beginning-of-sentence. - return true; - } - return false; - } - - @Override - public int getWordHandlingType(final NgramContext ngramContext, final String testedWord, - final Locale locale) { - // TODO: Use this method for user history dictionary. - if (testedWord == null|| locale == null) { - return HandlingType.getHandlingType(false /* shouldBeLowerCased */, false /* isOov */); - } - final boolean shouldBeLowerCased = shouldBeLowerCased(ngramContext, testedWord, locale); - final String caseModifiedWord = shouldBeLowerCased - ? testedWord.toLowerCase(locale) : testedWord; - final boolean isOov = !mDictionaryFacilitatorLruCache.get(locale).isValidSuggestionWord( - caseModifiedWord); - return HandlingType.getHandlingType(shouldBeLowerCased, isOov); - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java deleted file mode 100644 index 4c99fed9f..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.util.List; -import java.util.Locale; - -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.NgramContext; - -public class DistracterFilterCheckingIsInDictionary implements DistracterFilter { - private final DistracterFilter mDistracterFilter; - private final Dictionary mDictionary; - - public DistracterFilterCheckingIsInDictionary(final DistracterFilter distracterFilter, - final Dictionary dictionary) { - mDistracterFilter = distracterFilter; - mDictionary = dictionary; - } - - @Override - public boolean isDistracterToWordsInDictionaries(NgramContext ngramContext, - String testedWord, Locale locale) { - if (mDictionary.isInDictionary(testedWord)) { - // This filter treats entries that are already in the dictionary as non-distracters - // because they have passed the filtering in the past. - return false; - } - return mDistracterFilter.isDistracterToWordsInDictionaries( - ngramContext, testedWord, locale); - } - - @Override - public int getWordHandlingType(final NgramContext ngramContext, final String testedWord, - final Locale locale) { - return mDistracterFilter.getWordHandlingType(ngramContext, testedWord, locale); - } - - @Override - public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) { - // Do nothing. - } - - @Override - public void close() { - // Do nothing. - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/ExecutorUtils.java b/java/src/com/android/inputmethod/latin/utils/ExecutorUtils.java index e77f6fd40..50be16072 100644 --- a/java/src/com/android/inputmethod/latin/utils/ExecutorUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/ExecutorUtils.java @@ -21,13 +21,14 @@ import com.android.inputmethod.annotations.UsedForTesting; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; /** * Utilities to manage executors. */ public class ExecutorUtils { - static final ConcurrentHashMap<String, ExecutorService> sExecutorMap = + static final ConcurrentHashMap<String, ScheduledExecutorService> sExecutorMap = new ConcurrentHashMap<>(); private static class ThreadFactoryWithId implements ThreadFactory { @@ -46,13 +47,14 @@ public class ExecutorUtils { /** * Gets the executor for the given id. */ - public static ExecutorService getExecutor(final String id) { - ExecutorService executor = sExecutorMap.get(id); + public static ScheduledExecutorService getExecutor(final String id) { + ScheduledExecutorService executor = sExecutorMap.get(id); if (executor == null) { synchronized (sExecutorMap) { executor = sExecutorMap.get(id); if (executor == null) { - executor = Executors.newSingleThreadExecutor(new ThreadFactoryWithId(id)); + executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryWithId(id)); sExecutorMap.put(id, executor); } } @@ -66,7 +68,7 @@ public class ExecutorUtils { @UsedForTesting public static void shutdownAllExecutors() { synchronized (sExecutorMap) { - for (final ExecutorService executor : sExecutorMap.values()) { + for (final ScheduledExecutorService executor : sExecutorMap.values()) { executor.execute(new Runnable() { @Override public void run() { diff --git a/java/src/com/android/inputmethod/latin/utils/NetworkConnectivityUtils.java b/java/src/com/android/inputmethod/latin/utils/NetworkConnectivityUtils.java deleted file mode 100644 index 101c55067..000000000 --- a/java/src/com/android/inputmethod/latin/utils/NetworkConnectivityUtils.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.content.BroadcastReceiver; -import android.content.Context; -import android.content.Intent; -import android.content.IntentFilter; -import android.net.ConnectivityManager; -import android.net.NetworkInfo; - -import javax.annotation.Nonnull; - -/** - * This class keeps track of the network connectivity state by receiving the system intent - * {@link ConnectivityManager#CONNECTIVITY_ACTION}, and invokes an registered call back to notify - * changes of the network connectivity state. - */ -public final class NetworkConnectivityUtils { - private static NetworkConnectivityReceiver sNetworkConnectivityReceiver; - - public interface NetworkStateChangeListener { - /** - * Called when the network connectivity state has changed. - */ - public void onNetworkStateChanged(); - } - - private static class NetworkConnectivityReceiver extends BroadcastReceiver { - @Nonnull - private final NetworkStateChangeListener mListener; - private boolean mIsNetworkConnected; - - public NetworkConnectivityReceiver(@Nonnull final NetworkStateChangeListener listener, - final boolean isNetworkConnected) { - mListener = listener; - mIsNetworkConnected = isNetworkConnected; - } - - public synchronized boolean isNetworkConnected() { - return mIsNetworkConnected; - } - - @Override - public void onReceive(final Context context, final Intent intent) { - final String action = intent.getAction(); - if (action.equals(ConnectivityManager.CONNECTIVITY_ACTION)) { - final boolean noConnection = intent.getBooleanExtra( - ConnectivityManager.EXTRA_NO_CONNECTIVITY, false); - synchronized (this) { - mIsNetworkConnected = !noConnection; - } - mListener.onNetworkStateChanged(); - } - } - } - - private NetworkConnectivityUtils() { - // This utility class is not publicly instantiable. - } - - public static void onCreate(@Nonnull final Context context, - @Nonnull final NetworkStateChangeListener listener) { - final ConnectivityManager connectivityManager = - (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE); - final NetworkInfo info = connectivityManager.getActiveNetworkInfo(); - final boolean isNetworkConnected = (info != null && info.isConnected()); - - // Register {@link BroadcastReceiver} for the network connectivity state change. - final NetworkConnectivityReceiver receiver = new NetworkConnectivityReceiver( - listener, isNetworkConnected); - final IntentFilter filter = new IntentFilter(); - filter.addAction(ConnectivityManager.CONNECTIVITY_ACTION); - context.registerReceiver(receiver, filter); - - sNetworkConnectivityReceiver = receiver; - } - - public static void onDestroy(final Context context) { - context.unregisterReceiver(sNetworkConnectivityReceiver); - } - - public static boolean isNetworkConnected() { - final NetworkConnectivityReceiver receiver = sNetworkConnectivityReceiver; - return receiver != null && receiver.isNetworkConnected(); - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java b/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java index 0e244666d..4679f7814 100644 --- a/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java @@ -23,9 +23,10 @@ import java.util.TreeMap; * A class to help with handling different writing scripts. */ public class ScriptUtils { + // Used for hardware keyboards public static final int SCRIPT_UNKNOWN = -1; - // TODO: should we use ISO 15924 identifiers instead? + public static final int SCRIPT_ARABIC = 0; public static final int SCRIPT_ARMENIAN = 1; public static final int SCRIPT_BENGALI = 2; @@ -44,35 +45,31 @@ public class ScriptUtils { public static final int SCRIPT_TAMIL = 15; public static final int SCRIPT_TELUGU = 16; public static final int SCRIPT_THAI = 17; - public static final TreeMap<String, Integer> mSpellCheckerLanguageToScript; + + private static final TreeMap<String, Integer> mLanguageCodeToScriptCode; + static { - // List of the supported languages and their associated script. We won't check - // words written in another script than the selected script, because we know we - // don't have those in our dictionary so we will underline everything and we - // will never have any suggestions, so it makes no sense checking them, and this - // is done in {@link #shouldFilterOut}. Also, the script is used to choose which - // proximity to pass to the dictionary descent algorithm. - // IMPORTANT: this only contains languages - do not write countries in there. - // Only the language is searched from the map. - mSpellCheckerLanguageToScript = new TreeMap<>(); - mSpellCheckerLanguageToScript.put("cs", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("da", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("de", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("el", SCRIPT_GREEK); - mSpellCheckerLanguageToScript.put("en", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("es", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("fi", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("fr", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("hr", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("it", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("lt", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("lv", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("nb", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("nl", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("pt", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("sl", SCRIPT_LATIN); - mSpellCheckerLanguageToScript.put("ru", SCRIPT_CYRILLIC); + mLanguageCodeToScriptCode = new TreeMap<>(); + mLanguageCodeToScriptCode.put("", SCRIPT_LATIN); // default + mLanguageCodeToScriptCode.put("ar", SCRIPT_ARABIC); + mLanguageCodeToScriptCode.put("hy", SCRIPT_ARMENIAN); + mLanguageCodeToScriptCode.put("bn", SCRIPT_BENGALI); + mLanguageCodeToScriptCode.put("bg", SCRIPT_CYRILLIC); + mLanguageCodeToScriptCode.put("sr", SCRIPT_CYRILLIC); + mLanguageCodeToScriptCode.put("ru", SCRIPT_CYRILLIC); + mLanguageCodeToScriptCode.put("ka", SCRIPT_GEORGIAN); + mLanguageCodeToScriptCode.put("el", SCRIPT_GREEK); + mLanguageCodeToScriptCode.put("he", SCRIPT_HEBREW); + mLanguageCodeToScriptCode.put("km", SCRIPT_KHMER); + mLanguageCodeToScriptCode.put("lo", SCRIPT_LAO); + mLanguageCodeToScriptCode.put("ml", SCRIPT_MALAYALAM); + mLanguageCodeToScriptCode.put("my", SCRIPT_MYANMAR); + mLanguageCodeToScriptCode.put("si", SCRIPT_SINHALA); + mLanguageCodeToScriptCode.put("ta", SCRIPT_TAMIL); + mLanguageCodeToScriptCode.put("te", SCRIPT_TELUGU); + mLanguageCodeToScriptCode.put("th", SCRIPT_THAI); } + /* * Returns whether the code point is a letter that makes sense for the specified * locale for this spell checker. @@ -181,11 +178,17 @@ public class ScriptUtils { } } + /** + * @param locale spell checker locale + * @return internal Latin IME script code that maps to a language code + * {@see http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes} + */ public static int getScriptFromSpellCheckerLocale(final Locale locale) { - final Integer script = mSpellCheckerLanguageToScript.get(locale.getLanguage()); - if (null == script) { - throw new RuntimeException("We have been called with an unsupported language: \"" - + locale.getLanguage() + "\". Framework bug?"); + String language = locale.getLanguage(); + Integer script = mLanguageCodeToScriptCode.get(language); + if (script == null) { + // Default to Latin. + script = mLanguageCodeToScriptCode.get(""); } return script; } diff --git a/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java b/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java index e9a0e7a61..fc0a9cb6c 100644 --- a/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java +++ b/java/src/com/android/inputmethod/latin/utils/WordInputEventForPersonalization.java @@ -23,7 +23,6 @@ import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.common.StringUtils; import com.android.inputmethod.latin.define.DecoderSpecificConstants; import com.android.inputmethod.latin.settings.SpacingAndPunctuations; -import com.android.inputmethod.latin.utils.DistracterFilter.HandlingType; import java.util.ArrayList; import java.util.List; @@ -41,17 +40,15 @@ public final class WordInputEventForPersonalization { new int[DecoderSpecificConstants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; public final boolean[] mIsPrevWordBeginningOfSentenceArray = new boolean[DecoderSpecificConstants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; - public final boolean mIsValid; // Time stamp in seconds. public final int mTimestamp; @UsedForTesting public WordInputEventForPersonalization(final CharSequence targetWord, - final NgramContext ngramContext, final boolean isValid, final int timestamp) { + final NgramContext ngramContext, final int timestamp) { mTargetWord = StringUtils.toCodePointArray(targetWord); mPrevWordsCount = ngramContext.getPrevWordCount(); ngramContext.outputToArray(mPrevWordArray, mIsPrevWordBeginningOfSentenceArray); - mIsValid = isValid; mTimestamp = timestamp; } @@ -59,8 +56,7 @@ public final class WordInputEventForPersonalization { // objects. public static ArrayList<WordInputEventForPersonalization> createInputEventFrom( final List<String> tokens, final int timestamp, - final SpacingAndPunctuations spacingAndPunctuations, final Locale locale, - final DistracterFilter distracterFilter) { + final SpacingAndPunctuations spacingAndPunctuations, final Locale locale) { final ArrayList<WordInputEventForPersonalization> inputEvents = new ArrayList<>(); final int N = tokens.size(); NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; @@ -89,7 +85,7 @@ public final class WordInputEventForPersonalization { } final WordInputEventForPersonalization inputEvent = detectWhetherVaildWordOrNotAndGetInputEvent( - ngramContext, tempWord, timestamp, locale, distracterFilter); + ngramContext, tempWord, timestamp, locale); if (inputEvent == null) { continue; } @@ -101,19 +97,10 @@ public final class WordInputEventForPersonalization { private static WordInputEventForPersonalization detectWhetherVaildWordOrNotAndGetInputEvent( final NgramContext ngramContext, final String targetWord, final int timestamp, - final Locale locale, final DistracterFilter distracterFilter) { + final Locale locale) { if (locale == null) { return null; } - final int wordHandlingType = distracterFilter.getWordHandlingType(ngramContext, - targetWord, locale); - final String word = HandlingType.shouldBeLowerCased(wordHandlingType) ? - targetWord.toLowerCase(locale) : targetWord; - if (distracterFilter.isDistracterToWordsInDictionaries(ngramContext, targetWord, locale)) { - // The word is a distracter. - return null; - } - return new WordInputEventForPersonalization(word, ngramContext, - !HandlingType.shouldBeHandledAsOov(wordHandlingType), timestamp); + return new WordInputEventForPersonalization(targetWord, ngramContext, timestamp); } } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index cff489dd5..a51558e43 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -700,8 +700,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */, - Dictionary.NOT_A_PROBABILITY /* shortcutProbability */, + binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isPossiblyOffensive */, mCurrentTime); final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; @@ -782,8 +781,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { int prevWordCount = 0; for (int i = 0; i < inputEvents.length; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); - inputEvents[i] = new WordInputEventForPersonalization(word, ngramContext, - true /* isValid */, mCurrentTime); + inputEvents[i] = new WordInputEventForPersonalization(word, ngramContext, mCurrentTime); unigrams.add(word); if (prevWordCount >= 2) { final Pair<String, String> prevWordsPair = bigrams.get(bigrams.size() - 1); diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index a8f0d81af..161879841 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -182,8 +182,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { addUnigramWord(binaryDictionary, validLongWord, probability); addUnigramWord(binaryDictionary, invalidLongWord, probability); // Too long short cut. - binaryDictionary.addUnigramEntry("a", probability, invalidLongWord, - 10 /* shortcutProbability */, false /* isBeginningOfSentence */, + binaryDictionary.addUnigramEntry("a", probability, false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); addUnigramWord(binaryDictionary, "abc", probability); @@ -201,8 +200,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { - binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, - Dictionary.NOT_A_PROBABILITY /* shortcutProbability */, + binaryDictionary.addUnigramEntry(word, probability, false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); @@ -884,7 +882,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { final boolean isPossiblyOffensive = random.nextBoolean(); // TODO: Add tests for historical info. binaryDictionary.addUnigramEntry(word, unigramProbability, - null /* shortcutTarget */, Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { @@ -899,9 +896,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(isNotAWord, wordProperty.mIsNotAWord); assertEquals(isPossiblyOffensive, wordProperty.mIsPossiblyOffensive); assertEquals(false, wordProperty.mHasNgrams); - assertEquals(false, wordProperty.mHasShortcuts); assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability); - assertTrue(wordProperty.mShortcutTargets.isEmpty()); } for (int i = 0; i < BIGRAM_COUNT; i++) { @@ -1035,137 +1030,10 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertTrue(bigramSet.isEmpty()); } - public void testAddShortcuts() { - for (final int formatVersion : DICT_FORMAT_VERSIONS) { - testAddShortcuts(formatVersion); - } - } - - private void testAddShortcuts(final int formatVersion) { - final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); - - final int unigramProbability = 100; - final int shortcutProbability = 10; - binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", - shortcutProbability, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); - WordProperty wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(1, wordProperty.mShortcutTargets.size()); - assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); - assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability()); - final int updatedShortcutProbability = 2; - binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", - updatedShortcutProbability, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); - wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(1, wordProperty.mShortcutTargets.size()); - assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); - assertEquals(updatedShortcutProbability, - wordProperty.mShortcutTargets.get(0).getProbability()); - binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy", - shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isPossiblyOffensive */, 0 /* timestamp */); - final HashMap<String, Integer> shortcutTargets = new HashMap<>(); - shortcutTargets.put("zzz", updatedShortcutProbability); - shortcutTargets.put("yyy", shortcutProbability); - wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(2, wordProperty.mShortcutTargets.size()); - for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); - assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), - shortcutTarget.getProbability()); - shortcutTargets.remove(shortcutTarget.mWord); - } - shortcutTargets.put("zzz", updatedShortcutProbability); - shortcutTargets.put("yyy", shortcutProbability); - binaryDictionary.flushWithGC(); - wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(2, wordProperty.mShortcutTargets.size()); - for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); - assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), - shortcutTarget.getProbability()); - shortcutTargets.remove(shortcutTarget.mWord); - } - } - - public void testAddManyShortcuts() { - for (final int formatVersion : DICT_FORMAT_VERSIONS) { - testAddManyShortcuts(formatVersion); - } - } - - private void testAddManyShortcuts(final int formatVersion) { - final long seed = System.currentTimeMillis(); - final Random random = new Random(seed); - final int UNIGRAM_COUNT = 1000; - final int SHORTCUT_COUNT = 10000; - final int codePointSetSize = 20; - final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - - final ArrayList<String> words = new ArrayList<>(); - final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); - final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>(); - final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); - - for (int i = 0; i < UNIGRAM_COUNT; i++) { - final String word = CodePointUtils.generateWord(random, codePointSet); - final int unigramProbability = random.nextInt(0xFF); - addUnigramWord(binaryDictionary, word, unigramProbability); - words.add(word); - unigramProbabilities.put(word, unigramProbability); - if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDictionary.flushWithGC(); - } - } - for (int i = 0; i < SHORTCUT_COUNT; i++) { - final String shortcutTarget = CodePointUtils.generateWord(random, codePointSet); - final int shortcutProbability = random.nextInt(0xF); - final String word = words.get(random.nextInt(words.size())); - final int unigramProbability = unigramProbabilities.get(word); - binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget, - shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isPossiblyOffensive */, 0 /* timestamp */); - if (shortcutTargets.containsKey(word)) { - final HashMap<String, Integer> shortcutTargetsOfWord = shortcutTargets.get(word); - shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); - } else { - final HashMap<String, Integer> shortcutTargetsOfWord = new HashMap<>(); - shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); - shortcutTargets.put(word, shortcutTargetsOfWord); - } - if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDictionary.flushWithGC(); - } - } - - for (final String word : words) { - final WordProperty wordProperty = binaryDictionary.getWordProperty(word, - false /* isBeginningOfSentence */); - assertEquals((int)unigramProbabilities.get(word), - wordProperty.mProbabilityInfo.mProbability); - if (!shortcutTargets.containsKey(word)) { - // The word does not have shortcut targets. - continue; - } - assertEquals(shortcutTargets.get(word).size(), wordProperty.mShortcutTargets.size()); - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - final String targetCodePonts = shortcutTarget.mWord; - assertEquals((int)shortcutTargets.get(word).get(targetCodePonts), - shortcutTarget.getProbability()); - } - } - } - public void testPossiblyOffensiveAttributeMaintained() { final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(FormatSpec.VERSION403); - binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY, - false, true, true, 0); + binaryDictionary.addUnigramEntry("ddd", 100, false, true, true, 0); WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false); assertEquals(true, wordProperty.mIsPossiblyOffensive); } @@ -1184,11 +1052,11 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int bigramProbability = 150; addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability); final int shortcutProbability = 10; - binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability, + binaryDictionary.addUnigramEntry("ccc", unigramProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); - binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */, - Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */, + binaryDictionary.addUnigramEntry("ddd", unigramProbability, + false /* isBeginningOfSentence */, true /* isNotAWord */, true /* isPossiblyOffensive */, 0 /* timestamp */); binaryDictionary.addNgramEntry(NgramContext.BEGINNING_OF_SENTENCE, "aaa", bigramProbability, 0 /* timestamp */); @@ -1207,8 +1075,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); WordProperty wordProperty = binaryDictionary.getWordProperty("ccc", false /* isBeginningOfSentence */); - assertEquals(1, wordProperty.mShortcutTargets.size()); - assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord); wordProperty = binaryDictionary.getWordProperty("ddd", false /* isBeginningOfSentence */); assertTrue(wordProperty.mIsPossiblyOffensive); diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index 07d7c3225..6c6f62872 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -35,19 +35,19 @@ public class FusionDictionaryTests extends AndroidTestCase { FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions(new HashMap<String,String>())); - dict.add("abc", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("abc", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc")); - dict.add("aa", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("aa", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa")); - dict.add("babcd", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("babcd", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); - dict.add("bacde", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("bacde", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd")); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index d833b9736..39da9fcd6 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -57,15 +57,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private static final int UNIGRAM_FREQ = 10; private static final int BIGRAM_FREQ = 50; private static final int TOLERANCE_OF_BIGRAM_FREQ = 5; - private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50; - private static final int NUM_OF_SHORTCUTS = 5; private static final ArrayList<String> sWords = new ArrayList<>(); private static final ArrayList<String> sWordsWithVariousCodePoints = new ArrayList<>(); private static final SparseArray<List<Integer>> sEmptyBigrams = new SparseArray<>(); private static final SparseArray<List<Integer>> sStarBigrams = new SparseArray<>(); private static final SparseArray<List<Integer>> sChainBigrams = new SparseArray<>(); - private static final HashMap<String, List<String>> sShortcuts = new HashMap<>(); final Random mRandom; @@ -95,16 +92,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { for (int i = 1; i < maxBigrams; ++i) { sStarBigrams.get(0).add(i); } - - sShortcuts.clear(); - for (int i = 0; i < NUM_OF_NODES_HAVING_SHORTCUTS; ++i) { - final int from = Math.abs(mRandom.nextInt()) % sWords.size(); - sShortcuts.put(sWords.get(from), new ArrayList<String>()); - for (int j = 0; j < NUM_OF_SHORTCUTS; ++j) { - final int to = Math.abs(mRandom.nextInt()) % sWords.size(); - sShortcuts.get(sWords.get(from)).add(sWords.get(to)); - } - } } @Override @@ -142,17 +129,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { * Adds unigrams to the dictionary. */ private static void addUnigrams(final int number, final FusionDictionary dict, - final List<String> words, final HashMap<String, List<String>> shortcutMap) { + final List<String> words) { for (int i = 0; i < number; ++i) { final String word = words.get(i); final ArrayList<WeightedString> shortcuts = new ArrayList<>(); - if (shortcutMap != null && shortcutMap.containsKey(word)) { - for (final String shortcut : shortcutMap.get(word)) { - shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ)); - } - } - dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), - (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */, + dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), false /* isNotAWord */, false /* isPossiblyOffensive */); } } @@ -200,8 +181,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private static void checkDictionary(final FusionDictionary dict, final List<String> words, - final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap) { + final SparseArray<List<Integer>> bigrams) { assertNotNull(dict); // check unigram @@ -219,19 +199,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { assertNotNull(words.get(w1) + "," + words.get(w2), ptNode.getBigram(words.get(w2))); } } - - // check shortcut - if (shortcutMap != null) { - for (final Entry<String, List<String>> entry : shortcutMap.entrySet()) { - assertTrue(words.contains(entry.getKey())); - final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, - entry.getKey()); - for (final String word : entry.getValue()) { - assertNotNull("shortcut not found: " + entry.getKey() + ", " + word, - ptNode.getShortcut(word)); - } - } - } } private static String outputOptions(final int bufferType, @@ -244,8 +211,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Tests for readDictionaryBinary and writeDictionaryBinary private static long timeReadingAndCheckDict(final File file, final List<String> words, - final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap, final int bufferType) { + final SparseArray<List<Integer>> bigrams, final int bufferType) { long now, diff = -1; FusionDictionary dict = null; @@ -261,13 +227,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { Log.e(TAG, "Unsupported format", e); } - checkDictionary(dict, words, bigrams, shortcutMap); + checkDictionary(dict, words, bigrams); return diff; } // Tests for readDictionaryBinary and writeDictionaryBinary private String runReadAndWrite(final List<String> words, - final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcuts, + final SparseArray<List<Integer>> bigrams, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { @@ -278,12 +244,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, shortcuts); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, bigrams); - checkDictionary(dict, words, bigrams, shortcuts); + checkDictionary(dict, words, bigrams); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); + final long read = timeReadingAndCheckDict(file, words, bigrams, bufferType); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -291,20 +257,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private void runReadAndWriteTests(final List<String> results, final int bufferType, final FormatSpec.FormatOptions formatOptions) { - results.add(runReadAndWrite(sWords, sEmptyBigrams, null /* shortcuts */, bufferType, + results.add(runReadAndWrite(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram")); - results.add(runReadAndWrite(sWords, sChainBigrams, null /* shortcuts */, bufferType, + results.add(runReadAndWrite(sWords, sChainBigrams, bufferType, formatOptions, "chain")); - results.add(runReadAndWrite(sWords, sStarBigrams, null /* shortcuts */, bufferType, + results.add(runReadAndWrite(sWords, sStarBigrams, bufferType, formatOptions, "star")); - results.add(runReadAndWrite(sWords, sEmptyBigrams, sShortcuts, bufferType, formatOptions, + results.add(runReadAndWrite(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram with shortcuts")); - results.add(runReadAndWrite(sWords, sChainBigrams, sShortcuts, bufferType, formatOptions, + results.add(runReadAndWrite(sWords, sChainBigrams, bufferType, formatOptions, "chain with shortcuts")); - results.add(runReadAndWrite(sWords, sStarBigrams, sShortcuts, bufferType, formatOptions, + results.add(runReadAndWrite(sWords, sStarBigrams, bufferType, formatOptions, "star with shortcuts")); results.add(runReadAndWrite(sWordsWithVariousCodePoints, sEmptyBigrams, - null /* shortcuts */, bufferType, formatOptions, + bufferType, formatOptions, "unigram with various code points")); } @@ -326,7 +292,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { FormatSpec.MINIMUM_SUPPORTED_STATIC_VERSION); final FusionDictionary sourcedict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), sourcedict, words, null /* shortcutMap */); + addUnigrams(words.size(), sourcedict, words); dictEncoder.writeDictionary(sourcedict, formatOptions); // Read the dictionary @@ -472,7 +438,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, null /* shortcutMap */); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); @@ -482,7 +448,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // TODO: Abandon the Java code, and implement the v4 dictionary reading code in native. long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, !formatOptions.mHasTimestamp /* checkProbability */); - long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, + long fullReading = timeReadingAndCheckDict(file, words, bigrams, bufferType); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap @@ -567,7 +533,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); + addUnigrams(sWords.size(), dict, sWords); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); @@ -636,12 +602,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testVer2DictGetWordProperty() { final FormatOptions formatOptions = BinaryDictUtils.STATIC_OPTIONS; final ArrayList<String> words = sWords; - final HashMap<String, List<String>> shortcuts = sShortcuts; final String dictName = "testGetWordProperty"; final String dictVersion = Long.toString(System.currentTimeMillis()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, shortcuts); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, sEmptyBigrams); final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, getContext().getCacheDir()); @@ -655,30 +620,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { false /* isBeginningOfSentence */); assertEquals(word, wordProperty.mWord); assertEquals(UNIGRAM_FREQ, wordProperty.getProbability()); - if (shortcuts.containsKey(word)) { - assertEquals(shortcuts.get(word).size(), wordProperty.mShortcutTargets.size()); - final List<String> shortcutList = shortcuts.get(word); - assertTrue(wordProperty.mHasShortcuts); - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutList.contains(shortcutTarget.mWord)); - assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability()); - shortcutList.remove(shortcutTarget.mWord); - } - assertTrue(shortcutList.isEmpty()); - } } } public void testVer2DictIteration() { final FormatOptions formatOptions = BinaryDictUtils.STATIC_OPTIONS; final ArrayList<String> words = sWords; - final HashMap<String, List<String>> shortcuts = sShortcuts; final SparseArray<List<Integer>> bigrams = sEmptyBigrams; final String dictName = "testGetWordProperty"; final String dictVersion = Long.toString(System.currentTimeMillis()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, shortcuts); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, bigrams); final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, getContext().getCacheDir()); @@ -708,17 +661,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final String word0 = wordProperty.mWord; assertEquals(UNIGRAM_FREQ, wordProperty.mProbabilityInfo.mProbability); wordSet.remove(word0); - if (shortcuts.containsKey(word0)) { - assertEquals(shortcuts.get(word0).size(), wordProperty.mShortcutTargets.size()); - final List<String> shortcutList = shortcuts.get(word0); - assertNotNull(wordProperty.mShortcutTargets); - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutList.contains(shortcutTarget.mWord)); - assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability()); - shortcutList.remove(shortcutTarget.mWord); - } - assertTrue(shortcutList.isEmpty()); - } if (wordProperty.mHasNgrams) { for (final WeightedString bigramTarget : wordProperty.getBigrams()) { final String word1 = bigramTarget.mWord; diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index ce905c499..bd5136583 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -16,9 +16,7 @@ package com.android.inputmethod.latin.makedict; -import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -92,38 +90,6 @@ public class BinaryDictEncoderUtils { } /** - * Compute the size of a shortcut in bytes. - */ - private static int getShortcutSize(final WeightedString shortcut, - final HashMap<Integer, Integer> codePointToOneByteCodeMap) { - int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; - final String word = shortcut.mWord; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - size += CharEncoding.getCharSize(codePoint, codePointToOneByteCodeMap); - } - size += FormatSpec.PTNODE_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the size of a shortcut list in bytes. - * - * This is known in advance and does not change according to position in the file - * like address lists do. - */ - static int getShortcutListSize(final ArrayList<WeightedString> shortcutList, - final HashMap<Integer, Integer> codePointToOneByteCodeMap) { - if (null == shortcutList || shortcutList.isEmpty()) return 0; - int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - for (final WeightedString shortcut : shortcutList) { - size += getShortcutSize(shortcut, codePointToOneByteCodeMap); - } - return size; - } - - /** * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. * * @param ptNode the PtNode to compute the size of. @@ -137,8 +103,6 @@ public class BinaryDictEncoderUtils { size += FormatSpec.PTNODE_FREQUENCY_SIZE; } size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address - // TODO: Use codePointToOneByteCodeMap for shortcuts. - size += getShortcutListSize(ptNode.mShortcutTargets, null /* codePointToOneByteCodeMap */); if (null != ptNode.mBigrams) { size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE) @@ -241,27 +205,6 @@ public class BinaryDictEncoderUtils { } } - @UsedForTesting - static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value, - final int size) { - switch(size) { - case 4: - dictBuffer.put((byte) ((value >> 24) & 0xFF)); - /* fall through */ - case 3: - dictBuffer.put((byte) ((value >> 16) & 0xFF)); - /* fall through */ - case 2: - dictBuffer.put((byte) ((value >> 8) & 0xFF)); - /* fall through */ - case 1: - dictBuffer.put((byte) (value & 0xFF)); - break; - default: - /* nop */ - } - } - // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time @@ -391,9 +334,6 @@ public class BinaryDictEncoderUtils { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, nodeSize + size, ptNode.mChildren)); } - // TODO: Use codePointToOneByteCodeMap for shortcuts. - nodeSize += getShortcutListSize(ptNode.mShortcutTargets, - null /* codePointToOneByteCodeMap */); if (null != ptNode.mBigrams) { for (WeightedString bigram : ptNode.mBigrams) { final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, @@ -568,14 +508,13 @@ public class BinaryDictEncoderUtils { * @param hasMultipleChars whether the PtNode has multiple chars. * @param isTerminal whether the PtNode is terminal. * @param childrenAddressSize the size of a children address. - * @param hasShortcuts whether the PtNode has shortcuts. * @param hasBigrams whether the PtNode has bigrams. * @param isNotAWord whether the PtNode is not a word. * @param isPossiblyOffensive whether the PtNode is a possibly offensive entry. * @return the flags */ static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, - final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, + final int childrenAddressSize, final boolean hasBigrams, final boolean isNotAWord, final boolean isPossiblyOffensive) { byte flags = 0; if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; @@ -596,7 +535,6 @@ public class BinaryDictEncoderUtils { default: throw new RuntimeException("Node with a strange address"); } - if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; if (isPossiblyOffensive) flags |= FormatSpec.FLAG_IS_POSSIBLY_OFFENSIVE; @@ -606,7 +544,6 @@ public class BinaryDictEncoderUtils { /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset) { return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(), getByteSize(childrenOffset), - node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), node.mBigrams != null && !node.mBigrams.isEmpty(), node.mIsNotAWord, node.mIsPossiblyOffensive); } @@ -621,7 +558,7 @@ public class BinaryDictEncoderUtils { * @param word the second bigram, for debugging purposes * @return the flags */ - /* package */ static final int makeBigramFlags(final boolean more, final int offset, + /* package */ static int makeBigramFlags(final boolean more, final int offset, final int bigramFrequency, final int unigramFrequency, final String word) { int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); @@ -690,19 +627,7 @@ public class BinaryDictEncoderUtils { return discretizedFrequency > 0 ? discretizedFrequency : 0; } - /** - * Makes the flag value for a shortcut. - * - * @param more whether there are more attributes after this one. - * @param frequency the frequency of the attribute, 0..15 - * @return the flags - */ - static final int makeShortcutFlags(final boolean more, final int frequency) { - return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) - + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); - } - - /* package */ static final int getChildrenPosition(final PtNode ptNode, + /* package */ static int getChildrenPosition(final PtNode ptNode, final HashMap<Integer, Integer> codePointToOneByteCodeMap) { int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate + getNodeHeaderSize(ptNode, codePointToOneByteCodeMap); diff --git a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index a42f0a93d..52060bed2 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -82,7 +82,6 @@ public final class FusionDictionary implements Iterable<WordProperty> { public static final class PtNode { private static final int NOT_A_TERMINAL = -1; final int mChars[]; - ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; // null == mProbabilityInfo indicates this is not a terminal. ProbabilityInfo mProbabilityInfo; @@ -100,26 +99,23 @@ public final class FusionDictionary implements Iterable<WordProperty> { int mCachedAddressBeforeUpdate; // The address of this PtNode (before update) int mCachedAddressAfterUpdate; // The address of this PtNode (after update) - public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, - final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo, - final boolean isNotAWord, final boolean isPossiblyOffensive) { + public PtNode(final int[] chars, final ArrayList<WeightedString> bigrams, + final ProbabilityInfo probabilityInfo, final boolean isNotAWord, + final boolean isPossiblyOffensive) { mChars = chars; mProbabilityInfo = probabilityInfo; mTerminalId = probabilityInfo == null ? NOT_A_TERMINAL : probabilityInfo.mProbability; - mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = null; mIsNotAWord = isNotAWord; mIsPossiblyOffensive = isPossiblyOffensive; } - public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, - final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo, - final boolean isNotAWord, final boolean isPossiblyOffensive, - final PtNodeArray children) { + public PtNode(final int[] chars, final ArrayList<WeightedString> bigrams, + final ProbabilityInfo probabilityInfo, final boolean isNotAWord, + final boolean isPossiblyOffensive, final PtNodeArray children) { mChars = chars; mProbabilityInfo = probabilityInfo; - mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = children; mIsNotAWord = isNotAWord; @@ -153,14 +149,6 @@ public final class FusionDictionary implements Iterable<WordProperty> { return mIsPossiblyOffensive; } - public ArrayList<WeightedString> getShortcutTargets() { - // We don't want write permission to escape outside the package, so we return a copy - if (null == mShortcutTargets) return null; - final ArrayList<WeightedString> copyOfShortcutTargets = - new ArrayList<>(mShortcutTargets); - return copyOfShortcutTargets; - } - public ArrayList<WeightedString> getBigrams() { // We don't want write permission to escape outside the package, so we return a copy if (null == mBigrams) return null; @@ -191,24 +179,6 @@ public final class FusionDictionary implements Iterable<WordProperty> { } /** - * Gets the shortcut target for the given word. Returns null if the word is not in the - * shortcut list. - */ - public WeightedString getShortcut(final String word) { - // TODO: Don't do a linear search - if (mShortcutTargets != null) { - final int size = mShortcutTargets.size(); - for (int i = 0; i < size; ++i) { - WeightedString shortcut = mShortcutTargets.get(i); - if (shortcut.mWord.equals(word)) { - return shortcut; - } - } - } - return null; - } - - /** * Gets the bigram for the given word. * Returns null if the word is not in the bigrams list. */ @@ -232,27 +202,9 @@ public final class FusionDictionary implements Iterable<WordProperty> { * updated if they are higher than the existing ones. */ void update(final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final boolean isNotAWord, final boolean isPossiblyOffensive) { mProbabilityInfo = ProbabilityInfo.max(mProbabilityInfo, probabilityInfo); - if (shortcutTargets != null) { - if (mShortcutTargets == null) { - mShortcutTargets = shortcutTargets; - } else { - final int size = shortcutTargets.size(); - for (int i = 0; i < size; ++i) { - final WeightedString shortcut = shortcutTargets.get(i); - final WeightedString existingShortcut = getShortcut(shortcut.mWord); - if (existingShortcut == null) { - mShortcutTargets.add(shortcut); - } else { - existingShortcut.mProbabilityInfo = ProbabilityInfo.max( - existingShortcut.mProbabilityInfo, shortcut.mProbabilityInfo); - } - } - } - } if (bigrams != null) { if (mBigrams == null) { mBigrams = bigrams; @@ -312,19 +264,16 @@ public final class FusionDictionary implements Iterable<WordProperty> { * Helper method to add a word as a string. * * This method adds a word to the dictionary with the given frequency. Optional - * lists of bigrams and shortcuts can be passed here. For each word inside, + * lists of bigrams can be passed here. For each word inside, * they will be added to the dictionary as necessary. - * - * @param word the word to add. + * @param word the word to add. * @param probabilityInfo probability information of the word. - * @param shortcutTargets a list of shortcut targets for this word, or null. * @param isNotAWord true if this should not be considered a word (e.g. shortcut only) * @param isPossiblyOffensive true if this word is possibly offensive */ public void add(final String word, final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord, - final boolean isPossiblyOffensive) { - add(getCodePoints(word), probabilityInfo, shortcutTargets, isNotAWord, isPossiblyOffensive); + final boolean isNotAWord, final boolean isPossiblyOffensive) { + add(getCodePoints(word), probabilityInfo, isNotAWord, isPossiblyOffensive); } /** @@ -358,7 +307,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { if (ptNode0 != null) { final PtNode ptNode1 = findWordInTree(mRootNodeArray, word1); if (ptNode1 == null) { - add(getCodePoints(word1), new ProbabilityInfo(0), null, false /* isNotAWord */, + add(getCodePoints(word1), new ProbabilityInfo(0), false /* isNotAWord */, false /* isPossiblyOffensive */); // The PtNode for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been @@ -376,15 +325,12 @@ public final class FusionDictionary implements Iterable<WordProperty> { * * The shortcuts, if any, have to be in the dictionary already. If they aren't, * an exception is thrown. - * - * @param word the word, as an int array. + * @param word the word, as an int array. * @param probabilityInfo the probability information of the word. - * @param shortcutTargets an optional list of shortcut targets for this word (null if none). * @param isNotAWord true if this is not a word for spellcheking purposes (shortcut only or so) * @param isPossiblyOffensive true if this word is possibly offensive */ private void add(final int[] word, final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord, final boolean isPossiblyOffensive) { assert(probabilityInfo.mProbability <= FormatSpec.MAX_TERMINAL_FREQUENCY); if (word.length >= DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH) { @@ -414,7 +360,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { // No node at this point to accept the word. Create one. final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length), - shortcutTargets, null /* bigrams */, probabilityInfo, isNotAWord, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive); currentNodeArray.mData.add(insertionIndex, newPtNode); if (DBG) checkStack(currentNodeArray); @@ -425,14 +371,14 @@ public final class FusionDictionary implements Iterable<WordProperty> { // The new word is a prefix of an existing word, but the node on which it // should end already exists as is. Since the old PtNode was not a terminal, // make it one by filling in its frequency and other attributes - currentPtNode.update(probabilityInfo, shortcutTargets, null, isNotAWord, + currentPtNode.update(probabilityInfo, null, isNotAWord, isPossiblyOffensive); } else { // The new word matches the full old word and extends past it. // We only have to create a new node and add it to the end of this. final PtNode newNode = new PtNode( Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), - shortcutTargets, null /* bigrams */, probabilityInfo, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive); currentPtNode.mChildren = new PtNodeArray(); currentPtNode.mChildren.mData.add(newNode); @@ -441,7 +387,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { if (0 == differentCharIndex) { // Exact same word. Update the frequency if higher. This will also add the // new shortcuts to the existing shortcut list if it already exists. - currentPtNode.update(probabilityInfo, shortcutTargets, null, + currentPtNode.update(probabilityInfo, null, currentPtNode.mIsNotAWord && isNotAWord, currentPtNode.mIsPossiblyOffensive || isPossiblyOffensive); } else { @@ -450,7 +396,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { PtNodeArray newChildren = new PtNodeArray(); final PtNode newOldWord = new PtNode( Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex, - currentPtNode.mChars.length), currentPtNode.mShortcutTargets, + currentPtNode.mChars.length), currentPtNode.mBigrams, currentPtNode.mProbabilityInfo, currentPtNode.mIsNotAWord, currentPtNode.mIsPossiblyOffensive, currentPtNode.mChildren); @@ -460,17 +406,17 @@ public final class FusionDictionary implements Iterable<WordProperty> { if (charIndex + differentCharIndex >= word.length) { newParent = new PtNode( Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), - shortcutTargets, null /* bigrams */, probabilityInfo, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive, newChildren); } else { newParent = new PtNode( Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), - null /* shortcutTargets */, null /* bigrams */, - null /* probabilityInfo */, false /* isNotAWord */, - false /* isPossiblyOffensive */, newChildren); + null /* bigrams */, null /* probabilityInfo */, + false /* isNotAWord */, false /* isPossiblyOffensive */, + newChildren); final PtNode newWord = new PtNode(Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), - shortcutTargets, null /* bigrams */, probabilityInfo, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive); final int addIndex = word[charIndex + differentCharIndex] > currentPtNode.mChars[differentCharIndex] ? 1 : 0; @@ -532,7 +478,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { final ArrayList<PtNode> data = nodeArray.mData; final PtNode reference = new PtNode(new int[] { character }, - null /* shortcutTargets */, null /* bigrams */, null /* probabilityInfo */, + null /* bigrams */, null /* probabilityInfo */, false /* isNotAWord */, false /* isPossiblyOffensive */); int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR); return result >= 0 ? result : -result - 1; @@ -669,8 +615,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { } if (currentPtNode.isTerminal()) { return new WordProperty(mCurrentString.toString(), - currentPtNode.mProbabilityInfo, - currentPtNode.mShortcutTargets, currentPtNode.mBigrams, + currentPtNode.mProbabilityInfo, currentPtNode.mBigrams, currentPtNode.mIsNotAWord, currentPtNode.mIsPossiblyOffensive); } } else { diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java index 5c261a94d..7ee1df92b 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -285,7 +285,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder { // Insert unigrams into the fusion dictionary. for (final WordProperty wordProperty : wordProperties) { fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, - wordProperty.mShortcutTargets, wordProperty.mIsNotAWord, + wordProperty.mIsNotAWord, wordProperty.mIsPossiblyOffensive); } // Insert bigrams into the fusion dictionary. diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java index b52b8c485..c63b972eb 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java @@ -240,37 +240,6 @@ public class Ver2DictEncoder implements DictEncoder { } /** - * Write a shortcut attributes list to mBuffer. - * - * @param shortcuts the shortcut attributes list. - */ - private void writeShortcuts(final ArrayList<WeightedString> shortcuts, - final HashMap<Integer, Integer> codePointToOneByteCodeMap) { - if (null == shortcuts || shortcuts.isEmpty()) return; - - final int indexOfShortcutByteSize = mPosition; - mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), - target.getProbability()); - mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord, - codePointToOneByteCodeMap); - mPosition += shortcutShift; - } - final int shortcutByteSize = mPosition - indexOfShortcutByteSize; - if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { - throw new RuntimeException("Shortcut list too large"); - } - BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); - } - - /** * Write a bigram attributes list to mBuffer. * * @param bigrams the bigram attributes list. @@ -305,8 +274,6 @@ public class Ver2DictEncoder implements DictEncoder { writeCharacters(ptNode.mChars, ptNode.hasSeveralChars(), codePointToOneByteCodeMap); writeFrequency(ptNode.getProbability()); writeChildrenPosition(ptNode, codePointToOneByteCodeMap); - // TODO: Use codePointToOneByteCodeMap for shortcuts. - writeShortcuts(ptNode.mShortcutTargets, null /* codePointToOneByteCodeMap */); writeBigrams(ptNode.mBigrams, dict); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java index 7d858760e..dbf9b7ac2 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java @@ -16,9 +16,7 @@ package com.android.inputmethod.latin.makedict; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -49,7 +47,7 @@ public class Ver2DictEncoderTests extends AndroidTestCase { new FormatSpec.FormatOptions(FormatSpec.VERSION2); final FusionDictionary sourcedict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(sourcedict, words, null /* shortcutMap */); + addUnigrams(sourcedict, words); final CodePointTable codePointTable = Ver2DictEncoder.makeCodePointTable(sourcedict); // Check if mCodePointOccurrenceArray is correct @@ -73,17 +71,10 @@ public class Ver2DictEncoderTests extends AndroidTestCase { /** * Adds unigrams to the dictionary. */ - private static void addUnigrams(final FusionDictionary dict, final List<String> words, - final HashMap<String, List<String>> shortcutMap) { + private static void addUnigrams(final FusionDictionary dict, final List<String> words) { for (final String word : words) { - final ArrayList<WeightedString> shortcuts = new ArrayList<>(); - if (shortcutMap != null && shortcutMap.containsKey(word)) { - for (final String shortcut : shortcutMap.get(word)) { - shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ)); - } - } dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), - (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */, + false /* isNotAWord */, false /* isPossiblyOffensive */); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 63ea89c1d..746431dfa 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -84,7 +84,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder { // Insert unigrams into the fusion dictionary. for (final WordProperty wordProperty : wordProperties) { fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, - wordProperty.mShortcutTargets, wordProperty.mIsNotAWord, + wordProperty.mIsNotAWord, wordProperty.mIsPossiblyOffensive); } // Insert bigrams into the fusion dictionary. diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 1e4bd768c..6e7b37d54 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -74,26 +74,10 @@ public class Ver4DictEncoder implements DictEncoder { throw new IOException("Cannot create dictionary file"); } for (final WordProperty wordProperty : dict) { - // TODO: switch to addMultipleDictionaryEntries when they support shortcuts - if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) { - if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(), - null /* shortcutTarget */, 0 /* shortcutProbability */, - wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, - wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) { - MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord); - } - } else { - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - if (!binaryDict.addUnigramEntry(wordProperty.mWord, - wordProperty.getProbability(), - shortcutTarget.mWord, shortcutTarget.getProbability(), - wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, - wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) { - MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord - + ", shortcutTarget: " + shortcutTarget.mWord); - return; - } - } + if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(), + wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, + wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) { + MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord); } if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { if (!binaryDict.flushWithGC()) { diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 6dddc971b..e1260f608 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -24,7 +24,6 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.NgramContext.WordInfo; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.util.List; @@ -252,8 +251,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { random); NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; for (final String word : words) { - UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime, - DistracterFilter.EMPTY_DISTRACTER_FILTER); + UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime); ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); dict.waitAllTasksForTests(); assertTrue(dict.isInDictionary(word)); diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java index d394c0faa..94105aa12 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java @@ -21,7 +21,6 @@ import android.content.Context; import com.android.inputmethod.latin.NgramContext; import com.android.inputmethod.latin.NgramContext.WordInfo; import com.android.inputmethod.latin.common.FileUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; import java.io.File; import java.io.FilenameFilter; @@ -101,8 +100,7 @@ public class UserHistoryDictionaryTestsHelper { final List<String> words, final int timestamp) { NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO; for (final String word : words) { - UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp, - DistracterFilter.EMPTY_DISTRACTER_FILTER); + UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp); ngramContext = ngramContext.getNextNgramContext(new WordInfo(word)); } } diff --git a/tests/src/com/android/inputmethod/latin/spellcheck/UserDictionaryLookupTest.java b/tests/src/com/android/inputmethod/latin/spellcheck/UserDictionaryLookupTest.java new file mode 100644 index 000000000..e5c813942 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/spellcheck/UserDictionaryLookupTest.java @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.spellcheck; + +import android.annotation.SuppressLint; +import android.content.ContentResolver; +import android.database.Cursor; +import android.net.Uri; +import android.provider.UserDictionary; +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; +import android.util.Log; + +import java.util.HashSet; +import java.util.Locale; + +/** + * Unit tests for {@link UserDictionaryLookup}. + * + * Note, this test doesn't mock out the ContentResolver, in order to make sure UserDictionaryLookup + * works in a real setting. + */ +@SmallTest +public class UserDictionaryLookupTest extends AndroidTestCase { + private static final String TAG = UserDictionaryLookupTest.class.getSimpleName(); + + private ContentResolver mContentResolver; + private HashSet<Uri> mAddedBackup; + + @Override + protected void setUp() throws Exception { + super.setUp(); + mContentResolver = mContext.getContentResolver(); + mAddedBackup = new HashSet<Uri>(); + } + + @Override + protected void tearDown() throws Exception { + // Remove all entries added during this test. + for (Uri row : mAddedBackup) { + mContentResolver.delete(row, null, null); + } + mAddedBackup.clear(); + + super.tearDown(); + } + + /** + * Adds the given word to UserDictionary. + * + * @param word the word to add + * @param locale the locale of the word to add + * @param frequency the frequency of the word to add + * @return the Uri for the given word + */ + @SuppressLint("NewApi") + private Uri addWord(final String word, final Locale locale, int frequency) { + // Add the given word for the given locale. + UserDictionary.Words.addWord(mContext, word, frequency, null, locale); + // Obtain an Uri for the given word. + Cursor cursor = mContentResolver.query(UserDictionary.Words.CONTENT_URI, null, + UserDictionary.Words.WORD + "='" + word + "'", null, null); + assertTrue(cursor.moveToFirst()); + Uri uri = Uri.withAppendedPath(UserDictionary.Words.CONTENT_URI, + cursor.getString(cursor.getColumnIndex(UserDictionary.Words._ID))); + // Add the row to the backup for later clearing. + mAddedBackup.add(uri); + return uri; + } + + /** + * Deletes the entry for the given word from UserDictionary. + * + * @param uri the Uri for the word as returned by addWord + */ + private void deleteWord(Uri uri) { + // Remove the word from the backup so that it's not cleared again later. + mAddedBackup.remove(uri); + // Remove the word from UserDictionary. + mContentResolver.delete(uri, null, null); + } + + public void testExactLocaleMatch() { + Log.d(TAG, "testExactLocaleMatch"); + + // Insert "Foo" as capitalized in the UserDictionary under en_US locale. + addWord("Foo", Locale.US, 17); + + // Create the UserDictionaryLookup and wait until it's loaded. + UserDictionaryLookup lookup = new UserDictionaryLookup(mContext); + while (!lookup.isLoaded()) { + } + + // Any capitalization variation should match. + assertTrue(lookup.isValidWord("foo", Locale.US)); + assertTrue(lookup.isValidWord("Foo", Locale.US)); + assertTrue(lookup.isValidWord("FOO", Locale.US)); + // But similar looking words don't match. + assertFalse(lookup.isValidWord("fo", Locale.US)); + assertFalse(lookup.isValidWord("fop", Locale.US)); + assertFalse(lookup.isValidWord("fooo", Locale.US)); + // Other locales, including more general locales won't match. + assertFalse(lookup.isValidWord("foo", Locale.ENGLISH)); + assertFalse(lookup.isValidWord("foo", Locale.UK)); + assertFalse(lookup.isValidWord("foo", Locale.FRENCH)); + assertFalse(lookup.isValidWord("foo", new Locale(""))); + + lookup.close(); + } + + public void testSubLocaleMatch() { + Log.d(TAG, "testSubLocaleMatch"); + + // Insert "Foo" as capitalized in the UserDictionary under the en locale. + addWord("Foo", Locale.ENGLISH, 17); + + // Create the UserDictionaryLookup and wait until it's loaded. + UserDictionaryLookup lookup = new UserDictionaryLookup(mContext); + while (!lookup.isLoaded()) { + } + + // Any capitalization variation should match for both en and en_US. + assertTrue(lookup.isValidWord("foo", Locale.ENGLISH)); + assertTrue(lookup.isValidWord("foo", Locale.US)); + assertTrue(lookup.isValidWord("Foo", Locale.US)); + assertTrue(lookup.isValidWord("FOO", Locale.US)); + // But similar looking words don't match. + assertFalse(lookup.isValidWord("fo", Locale.US)); + assertFalse(lookup.isValidWord("fop", Locale.US)); + assertFalse(lookup.isValidWord("fooo", Locale.US)); + + lookup.close(); + } + + public void testAllLocalesMatch() { + Log.d(TAG, "testAllLocalesMatch"); + + // Insert "Foo" as capitalized in the UserDictionary under the all locales. + addWord("Foo", null, 17); + + // Create the UserDictionaryLookup and wait until it's loaded. + UserDictionaryLookup lookup = new UserDictionaryLookup(mContext); + while (!lookup.isLoaded()) { + } + + // Any capitalization variation should match for fr, en and en_US. + assertTrue(lookup.isValidWord("foo", new Locale(""))); + assertTrue(lookup.isValidWord("foo", Locale.FRENCH)); + assertTrue(lookup.isValidWord("foo", Locale.ENGLISH)); + assertTrue(lookup.isValidWord("foo", Locale.US)); + assertTrue(lookup.isValidWord("Foo", Locale.US)); + assertTrue(lookup.isValidWord("FOO", Locale.US)); + // But similar looking words don't match. + assertFalse(lookup.isValidWord("fo", Locale.US)); + assertFalse(lookup.isValidWord("fop", Locale.US)); + assertFalse(lookup.isValidWord("fooo", Locale.US)); + + lookup.close(); + } + + public void testMultipleLocalesMatch() { + Log.d(TAG, "testMultipleLocalesMatch"); + + // Insert "Foo" as capitalized in the UserDictionary under the en_US and en_CA and fr + // locales. + addWord("Foo", Locale.US, 17); + addWord("foO", Locale.CANADA, 17); + addWord("fOo", Locale.FRENCH, 17); + + // Create the UserDictionaryLookup and wait until it's loaded. + UserDictionaryLookup lookup = new UserDictionaryLookup(mContext); + while (!lookup.isLoaded()) { + } + + // Both en_CA and en_US match. + assertTrue(lookup.isValidWord("foo", Locale.CANADA)); + assertTrue(lookup.isValidWord("foo", Locale.US)); + assertTrue(lookup.isValidWord("foo", Locale.FRENCH)); + // Other locales, including more general locales won't match. + assertFalse(lookup.isValidWord("foo", Locale.ENGLISH)); + assertFalse(lookup.isValidWord("foo", Locale.UK)); + assertFalse(lookup.isValidWord("foo", new Locale(""))); + + lookup.close(); + } + + public void testReload() { + Log.d(TAG, "testReload"); + + // Insert "foo". + Uri uri = addWord("foo", Locale.US, 17); + + // Create the UserDictionaryLookup and wait until it's loaded. + UserDictionaryLookup lookup = new UserDictionaryLookup(mContext); + while (!lookup.isLoaded()) { + } + + // "foo" should match. + assertTrue(lookup.isValidWord("foo", Locale.US)); + + // "bar" shouldn't match. + assertFalse(lookup.isValidWord("bar", Locale.US)); + + // Now delete "foo" and add "bar". + deleteWord(uri); + addWord("bar", Locale.US, 18); + + // Wait a little bit before expecting a change. The time we wait should be greater than + // UserDictionaryLookup.RELOAD_DELAY_MS. + try { + Thread.sleep(UserDictionaryLookup.RELOAD_DELAY_MS + 1000); + } catch (InterruptedException e) { + } + + // Perform lookups again. Reload should have occured. + // + // "foo" should not match. + assertFalse(lookup.isValidWord("foo", Locale.US)); + + // "bar" should match. + assertTrue(lookup.isValidWord("bar", Locale.US)); + + lookup.close(); + } + + public void testClose() { + Log.d(TAG, "testClose"); + + // Insert "foo". + Uri uri = addWord("foo", Locale.US, 17); + + // Create the UserDictionaryLookup and wait until it's loaded. + UserDictionaryLookup lookup = new UserDictionaryLookup(mContext); + while (!lookup.isLoaded()) { + } + + // "foo" should match. + assertTrue(lookup.isValidWord("foo", Locale.US)); + + // "bar" shouldn't match. + assertFalse(lookup.isValidWord("bar", Locale.US)); + + // Now close (prevents further reloads). + lookup.close(); + + // Now delete "foo" and add "bar". + deleteWord(uri); + addWord("bar", Locale.US, 18); + + // Wait a little bit before expecting a change. The time we wait should be greater than + // UserDictionaryLookup.RELOAD_DELAY_MS. + try { + Thread.sleep(UserDictionaryLookup.RELOAD_DELAY_MS + 1000); + } catch (InterruptedException e) { + } + + // Perform lookups again. Reload should not have occurred. + // + // "foo" should stil match. + assertTrue(lookup.isValidWord("foo", Locale.US)); + + // "bar" should still not match. + assertFalse(lookup.isValidWord("bar", Locale.US)); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java deleted file mode 100644 index 8360d53fb..000000000 --- a/tests/src/com/android/inputmethod/latin/utils/DistracterFilterTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.util.ArrayList; -import java.util.Locale; - -import android.content.Context; -import android.test.AndroidTestCase; -import android.test.suitebuilder.annotation.LargeTest; -import android.view.inputmethod.InputMethodSubtype; - -import com.android.inputmethod.latin.NgramContext; -import com.android.inputmethod.latin.RichInputMethodManager; -import com.android.inputmethod.latin.utils.DistracterFilter.HandlingType; - -/** - * Unit test for DistracterFilter - */ -@LargeTest -public class DistracterFilterTest extends AndroidTestCase { - private DistracterFilterCheckingExactMatchesAndSuggestions mDistracterFilter; - - @Override - protected void setUp() throws Exception { - super.setUp(); - final Context context = getContext(); - mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context); - RichInputMethodManager.init(context); - final RichInputMethodManager richImm = RichInputMethodManager.getInstance(); - final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>(); - subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( - Locale.US.toString(), "qwerty")); - subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( - Locale.FRENCH.toString(), "azerty")); - subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( - Locale.GERMAN.toString(), "qwertz")); - mDistracterFilter.updateEnabledSubtypes(subtypes); - } - - @Override - protected void tearDown() { - mDistracterFilter.close(); - } - - public void testIsDistracterToWordsInDictionaries() { - final NgramContext EMPTY_PREV_WORDS_INFO = NgramContext.EMPTY_PREV_WORDS_INFO; - - final Locale localeEnUs = new Locale("en", "US"); - String typedWord; - - typedWord = "Bill"; - // For this test case, we consider "Bill" is a distracter to "bill". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "nOt"; - // For this test case, we consider "nOt" is a distracter to "not". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "youre"; - // For this test case, we consider "youre" is a distracter to "you're". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "Banana"; - // For this test case, we consider "Banana" is a distracter to "banana". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "orange"; - // For this test case, we consider "orange" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "Orange"; - // For this test case, we consider "Orange" is a distracter to "orange". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "café"; - // For this test case, we consider "café" is a distracter to "cafe". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "cafe"; - // For this test case, we consider "cafe" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "I'll"; - // For this test case, we consider "I'll" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "ill"; - // For this test case, we consider "ill" is a distracter to "I'll" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "asdfd"; - // For this test case, we consider "asdfd" is not a distracter to any word in dictionaries. - assertFalse( - mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thank"; - // For this test case, we consider "thank" is not a distracter to any other word - // in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thabk"; - // For this test case, we consider "thabk" is a distracter to "thank" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thanks"; - // For this test case, we consider "thanks" is not a distracter to any other word - // in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thabks"; - // For this test case, we consider "thabks" is a distracter to "thanks" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "think"; - // For this test case, we consider "think" is not a distracter to any other word - // in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "thibk"; - // For this test case, we consider "thibk" is a distracter to "think" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - typedWord = "tgis"; - // For this test case, we consider "tgis" is a distracter to "this" - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs)); - - final Locale localeDeDe = new Locale("de"); - - typedWord = "fUEr"; - // For this test case, we consider "fUEr" is a distracter to "für". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); - - typedWord = "fuer"; - // For this test case, we consider "fuer" is a distracter to "für". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); - - typedWord = "fur"; - // For this test case, we consider "fur" is a distracter to "für". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe)); - - final Locale localeFrFr = new Locale("fr"); - - typedWord = "a"; - // For this test case, we consider "a" is a distracter to "à". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "à"; - // For this test case, we consider "à" is not a distracter to any word in dictionaries. - assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "etre"; - // For this test case, we consider "etre" is a distracter to "être". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "États-unis"; - // For this test case, we consider "États-unis" is a distracter to "États-Unis". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - - typedWord = "ÉtatsUnis"; - // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis". - assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries( - EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr)); - } - - public void testGetWordHandlingType() { - final Locale localeEnUs = new Locale("en", "US"); - final NgramContext EMPTY_PREV_WORDS_INFO = NgramContext.EMPTY_PREV_WORDS_INFO; - int handlingType = 0; - - handlingType = mDistracterFilter.getWordHandlingType(EMPTY_PREV_WORDS_INFO, - "this", localeEnUs); - assertFalse(HandlingType.shouldBeLowerCased(handlingType)); - assertFalse(HandlingType.shouldBeHandledAsOov(handlingType)); - - handlingType = mDistracterFilter.getWordHandlingType(EMPTY_PREV_WORDS_INFO, - "This", localeEnUs); - assertTrue(HandlingType.shouldBeLowerCased(handlingType)); - assertFalse(HandlingType.shouldBeHandledAsOov(handlingType)); - - handlingType = mDistracterFilter.getWordHandlingType(EMPTY_PREV_WORDS_INFO, - "thibk", localeEnUs); - assertFalse(HandlingType.shouldBeLowerCased(handlingType)); - assertTrue(HandlingType.shouldBeHandledAsOov(handlingType)); - } -} |