diff options
38 files changed, 470 insertions, 297 deletions
diff --git a/dictionaries/de_wordlist.combined.gz b/dictionaries/de_wordlist.combined.gz Binary files differindex 400718df1..803211c01 100644 --- a/dictionaries/de_wordlist.combined.gz +++ b/dictionaries/de_wordlist.combined.gz diff --git a/dictionaries/en_GB_wordlist.combined.gz b/dictionaries/en_GB_wordlist.combined.gz Binary files differindex bf637e93d..1fa9b85ea 100644 --- a/dictionaries/en_GB_wordlist.combined.gz +++ b/dictionaries/en_GB_wordlist.combined.gz diff --git a/dictionaries/en_US_wordlist.combined.gz b/dictionaries/en_US_wordlist.combined.gz Binary files differindex 9ea04b146..2e039ff05 100644 --- a/dictionaries/en_US_wordlist.combined.gz +++ b/dictionaries/en_US_wordlist.combined.gz diff --git a/dictionaries/en_wordlist.combined.gz b/dictionaries/en_wordlist.combined.gz Binary files differindex 87a863330..e845346d6 100644 --- a/dictionaries/en_wordlist.combined.gz +++ b/dictionaries/en_wordlist.combined.gz diff --git a/dictionaries/es_wordlist.combined.gz b/dictionaries/es_wordlist.combined.gz Binary files differindex 8bb21027f..3391e64b4 100644 --- a/dictionaries/es_wordlist.combined.gz +++ b/dictionaries/es_wordlist.combined.gz diff --git a/dictionaries/ro_wordlist.combined.gz b/dictionaries/ro_wordlist.combined.gz Binary files differindex 829abd9b8..53078553c 100644 --- a/dictionaries/ro_wordlist.combined.gz +++ b/dictionaries/ro_wordlist.combined.gz diff --git a/dictionaries/ru_wordlist.combined.gz b/dictionaries/ru_wordlist.combined.gz Binary files differindex 7794f7bed..401ad08b0 100644 --- a/dictionaries/ru_wordlist.combined.gz +++ b/dictionaries/ru_wordlist.combined.gz diff --git a/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java b/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java index 9d7258de7..5738ea2b2 100644 --- a/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java +++ b/java-overridable/src/com/android/inputmethod/latin/define/ProductionFlags.java @@ -24,14 +24,6 @@ public final class ProductionFlags { public static final boolean IS_HARDWARE_KEYBOARD_SUPPORTED = false; /** - * When true, enable {@link InputMethodService#onUpdateCursorAnchorInfo} callback via - * {@link InputConnection#requestUpdateCursorAnchorInfo}. This flag has no effect in API - * Level 20 and prior. In general, this callback provides detailed positional information, - * even though an explicit support is required by the editor. - */ - public static final boolean ENABLE_CURSOR_ANCHOR_INFO_CALLBACK = true; - - /** * Include all suggestions from all dictionaries in {@link SuggestedWords#mRawSuggestions}. */ public static final boolean INCLUDE_RAW_SUGGESTIONS = false; diff --git a/java/AndroidManifest.xml b/java/AndroidManifest.xml index 25fa0f9e3..ee1cef6b5 100644 --- a/java/AndroidManifest.xml +++ b/java/AndroidManifest.xml @@ -89,8 +89,7 @@ <activity android:name=".settings.SettingsActivity" android:theme="@style/platformSettingsTheme" - android:label="@string/english_ime_settings" - android:uiOptions="splitActionBarWhenNarrow"> + android:label="@string/english_ime_settings"> <intent-filter> <action android:name="android.intent.action.MAIN" /> </intent-filter> diff --git a/java/res/raw/main_de.dict b/java/res/raw/main_de.dict Binary files differindex 3cbf7105d..45b288375 100644 --- a/java/res/raw/main_de.dict +++ b/java/res/raw/main_de.dict diff --git a/java/res/raw/main_en.dict b/java/res/raw/main_en.dict Binary files differindex d631d6fbc..5bbb85761 100644 --- a/java/res/raw/main_en.dict +++ b/java/res/raw/main_en.dict diff --git a/java/res/raw/main_es.dict b/java/res/raw/main_es.dict Binary files differindex 8c42fda8e..fae131850 100644 --- a/java/res/raw/main_es.dict +++ b/java/res/raw/main_es.dict diff --git a/java/res/raw/main_ru.dict b/java/res/raw/main_ru.dict Binary files differindex 9fdaf3148..8d1576987 100644 --- a/java/res/raw/main_ru.dict +++ b/java/res/raw/main_ru.dict diff --git a/java/res/raw/setup_welcome_image.png b/java/res/raw/setup_welcome_image.png Binary files differindex 98e731331..b72618114 100644 --- a/java/res/raw/setup_welcome_image.png +++ b/java/res/raw/setup_welcome_image.png diff --git a/java/res/raw/setup_welcome_video.mp4 b/java/res/raw/setup_welcome_video.mp4 Binary files differindex 224bf250c..09be565e4 100644 --- a/java/res/raw/setup_welcome_video.mp4 +++ b/java/res/raw/setup_welcome_video.mp4 diff --git a/java/src/com/android/inputmethod/compat/SuggestionSpanUtils.java b/java/src/com/android/inputmethod/compat/SuggestionSpanUtils.java index c33c01552..27db9b8e8 100644 --- a/java/src/com/android/inputmethod/compat/SuggestionSpanUtils.java +++ b/java/src/com/android/inputmethod/compat/SuggestionSpanUtils.java @@ -23,10 +23,11 @@ import android.text.Spanned; import android.text.TextUtils; import android.text.style.SuggestionSpan; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.SuggestedWords; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.define.DebugFlags; import com.android.inputmethod.latin.SuggestionSpanPickedNotificationReceiver; +import com.android.inputmethod.latin.define.DebugFlags; import java.lang.reflect.Field; import java.util.ArrayList; @@ -51,12 +52,14 @@ public final class SuggestionSpanUtils { // This utility class is not publicly instantiable. } + @UsedForTesting public static CharSequence getTextWithAutoCorrectionIndicatorUnderline( final Context context, final String text) { if (TextUtils.isEmpty(text) || OBJ_FLAG_AUTO_CORRECTION == null) { return text; } final Spannable spannable = new SpannableString(text); + // TODO: Set locale if it is feasible. final SuggestionSpan suggestionSpan = new SuggestionSpan(context, null /* locale */, new String[] {} /* suggestions */, OBJ_FLAG_AUTO_CORRECTION, SuggestionSpanPickedNotificationReceiver.class); @@ -65,6 +68,7 @@ public final class SuggestionSpanUtils { return spannable; } + @UsedForTesting public static CharSequence getTextWithSuggestionSpan(final Context context, final String pickedWord, final SuggestedWords suggestedWords) { if (TextUtils.isEmpty(pickedWord) || suggestedWords.isEmpty() @@ -86,6 +90,7 @@ public final class SuggestionSpanUtils { suggestionsList.add(word.toString()); } } + // TODO: Set locale if it is feasible. final SuggestionSpan suggestionSpan = new SuggestionSpan(context, null /* locale */, suggestionsList.toArray(new String[suggestionsList.size()]), 0 /* flags */, SuggestionSpanPickedNotificationReceiver.class); diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 6f3c48c47..10dea749d 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -42,6 +42,8 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import javax.annotation.Nonnull; + /** * Implements a static, compacted, binary dictionary of standard words. */ @@ -199,6 +201,7 @@ public final class BinaryDictionary extends Dictionary { int[] word, int probability, int timestamp); private static native boolean removeNgramEntryNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); + // TODO: Rename to updateEntriesForWordWithNgramContextNative. private static native boolean updateCounterNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word, boolean isValidWord, int count, int timestamp); @@ -493,6 +496,24 @@ public final class BinaryDictionary extends Dictionary { return true; } + // Update entries for the word occurrence with the ngramContext. + public boolean updateEntriesForWordWithNgramContext(@Nonnull final NgramContext ngramContext, + final String word, final boolean isValidWord, final int count, final int timestamp) { + if (TextUtils.isEmpty(word)) { + return false; + } + final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; + final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; + ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); + final int[] wordCodePoints = StringUtils.toCodePointArray(word); + if (!updateCounterNative(mNativeDict, prevWordCodePointArrays, + isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { + return false; + } + mHasUpdated = true; + return true; + } + @UsedForTesting public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { if (!isValidDictionary()) return; diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 1bdadc30b..d24f80a45 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -46,6 +46,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.annotation.Nonnull; + /** * Abstract base class for an expandable dictionary that can be created and updated dynamically * during runtime. When updated it automatically generates a new binary dictionary to handle future @@ -292,13 +294,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - /** - * Adds unigram information of a word to the dictionary. May overwrite an existing entry. - */ - public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency, - final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, - final boolean isBlacklisted, final int timestamp, - final DistracterFilter distracterFilter) { + private void updateDictionaryWithWriteLockIfWordIsNotADistracter( + @Nonnull final Runnable updateTask, + @Nonnull final String word, @Nonnull final DistracterFilter distracterFilter) { reloadDictionaryIfRequired(); asyncPreCheckAndExecuteTaskWithWriteLock( new Callable<Boolean>() { @@ -315,12 +313,27 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return; } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, - isNotAWord, isBlacklisted, timestamp); + updateTask.run(); } }); } + /** + * Adds unigram information of a word to the dictionary. May overwrite an existing entry. + */ + public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency, + final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, + final boolean isBlacklisted, final int timestamp, + @Nonnull final DistracterFilter distracterFilter) { + updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + @Override + public void run() { + addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, + isNotAWord, isBlacklisted, timestamp); + } + }, word, distracterFilter); + } + protected void addUnigramLocked(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { @@ -354,7 +367,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Adds n-gram information of a word to the dictionary. May overwrite an existing entry. */ - public void addNgramEntry(final NgramContext ngramContext, final String word, + public void addNgramEntry(@Nonnull final NgramContext ngramContext, final String word, final int frequency, final int timestamp) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { @@ -369,7 +382,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } - protected void addNgramEntryLocked(final NgramContext ngramContext, final String word, + protected void addNgramEntryLocked(@Nonnull final NgramContext ngramContext, final String word, final int frequency, final int timestamp) { if (!mBinaryDictionary.addNgramEntry(ngramContext, word, frequency, timestamp)) { if (DEBUG) { @@ -383,7 +396,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Dynamically remove the n-gram entry in the dictionary. */ @UsedForTesting - public void removeNgramDynamically(final NgramContext ngramContext, final String word) { + public void removeNgramDynamically(@Nonnull final NgramContext ngramContext, + final String word) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { @Override @@ -402,6 +416,26 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } + /** + * Update dictionary for the word with the ngramContext if the word is not a distracter. + */ + public void updateEntriesForWordWithCheckingDistracter(@Nonnull final NgramContext ngramContext, + final String word, final boolean isValidWord, final int count, final int timestamp, + @Nonnull final DistracterFilter distracterFilter) { + updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + @Override + public void run() { + if (!mBinaryDictionary.updateEntriesForWordWithNgramContext(ngramContext, word, + isValidWord, count, timestamp)) { + if (DEBUG) { + Log.e(TAG, "Cannot update counter. word: " + word + + " context: "+ ngramContext.toString()); + } + } + } + }, word, distracterFilter); + } + public interface AddMultipleDictionaryEntriesCallback { public void onFinished(); } @@ -410,7 +444,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Dynamically add multiple entries to the dictionary. */ public void addMultipleDictionaryEntriesDynamically( - final ArrayList<LanguageModelParam> languageModelParams, + @Nonnull final ArrayList<LanguageModelParam> languageModelParams, final AddMultipleDictionaryEntriesCallback callback) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index a6be75943..8e93761a2 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -755,12 +755,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (prevExtractEditText == nextExtractEditText) { return; } - if (ProductionFlags.ENABLE_CURSOR_ANCHOR_INFO_CALLBACK && prevExtractEditText != null) { + if (prevExtractEditText != null) { prevExtractEditText.getViewTreeObserver().removeOnPreDrawListener( mExtractTextViewPreDrawListener); } mExtractEditText = nextExtractEditText; - if (ProductionFlags.ENABLE_CURSOR_ANCHOR_INFO_CALLBACK && mExtractEditText != null) { + if (mExtractEditText != null) { mExtractEditText.getViewTreeObserver().addOnPreDrawListener( mExtractTextViewPreDrawListener); } @@ -776,8 +776,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen }; private void onExtractTextViewPreDraw() { - if (!ProductionFlags.ENABLE_CURSOR_ANCHOR_INFO_CALLBACK || !isFullscreenMode() - || mExtractEditText == null) { + if (!isFullscreenMode() || mExtractEditText == null) { return; } final CursorAnchorInfo info = CursorAnchorInfoUtils.getCursorAnchorInfo(mExtractEditText); @@ -1060,7 +1059,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // We cannot mark this method as @Override until new SDK becomes publicly available. // @Override public void onUpdateCursorAnchorInfo(final CursorAnchorInfo info) { - if (!ProductionFlags.ENABLE_CURSOR_ANCHOR_INFO_CALLBACK || isFullscreenMode()) { + if (isFullscreenMode()) { return; } mInputLogic.onUpdateCursorAnchorInfo(CursorAnchorInfoCompatWrapper.fromObject(info)); diff --git a/java/src/com/android/inputmethod/latin/NgramContext.java b/java/src/com/android/inputmethod/latin/NgramContext.java index 6d438584f..a02531cc4 100644 --- a/java/src/com/android/inputmethod/latin/NgramContext.java +++ b/java/src/com/android/inputmethod/latin/NgramContext.java @@ -158,11 +158,6 @@ public class NgramContext { } } - public NgramContext getTrimmedNgramContext(final int maxPrevWordCount) { - final int newSize = Math.min(maxPrevWordCount, mPrevWordsCount); - return new NgramContext(this /* prevWordsInfo */, newSize); - } - public int getPrevWordCount() { return mPrevWordsCount; } diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index 1b1d5e7e5..f67a9a6ef 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -169,14 +169,11 @@ public final class InputLogic { mInputLogicHandler.reset(); } - if (ProductionFlags.ENABLE_CURSOR_ANCHOR_INFO_CALLBACK) { - // AcceptTypedWord feature relies on CursorAnchorInfo. - if (settingsValues.mShouldShowUiToAcceptTypedWord) { - mConnection.requestCursorUpdates(true /* enableMonitor */, - true /* requestImmediateCallback */); - } - mTextDecorator.reset(); + if (settingsValues.mShouldShowUiToAcceptTypedWord) { + mConnection.requestCursorUpdates(true /* enableMonitor */, + true /* requestImmediateCallback */); } + mTextDecorator.reset(); } /** diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index d61684698..59761547d 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -65,34 +65,7 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas if (word.length() > Constants.DICTIONARY_MAX_WORD_LENGTH) { return; } - final int frequency = isValid ? - FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; - userHistoryDictionary.addUnigramEntryWithCheckingDistracter(word, frequency, - null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */, - false /* isBlacklisted */, timestamp, distracterFilter); - - final boolean isBeginningOfSentenceContext = ngramContext.isBeginningOfSentenceContext(); - final NgramContext ngramContextToBeSaved = - ngramContext.getTrimmedNgramContext(SUPPORTED_NGRAM - 1); - for (int i = 0; i < ngramContextToBeSaved.getPrevWordCount(); i++) { - final CharSequence prevWord = ngramContextToBeSaved.getNthPrevWord(1 /* n */); - if (prevWord == null || (prevWord.length() > Constants.DICTIONARY_MAX_WORD_LENGTH)) { - return; - } - // Do not insert a word as a bigram of itself - if (i == 0 && TextUtils.equals(word, prevWord)) { - return; - } - if (isBeginningOfSentenceContext) { - // Beginning-of-Sentence n-gram entry is added as an n-gram entry of an OOV word. - userHistoryDictionary.addNgramEntry( - ngramContextToBeSaved.getTrimmedNgramContext(i + 1), word, - FREQUENCY_FOR_WORDS_NOT_IN_DICTS, timestamp); - } else { - userHistoryDictionary.addNgramEntry( - ngramContextToBeSaved.getTrimmedNgramContext(i + 1), word, frequency, - timestamp); - } - } + userHistoryDictionary.updateEntriesForWordWithCheckingDistracter(ngramContext, word, + isValid, 1 /* count */, timestamp, distracterFilter); } } diff --git a/java/src/com/android/inputmethod/latin/settings/CustomInputStyleSettingsFragment.java b/java/src/com/android/inputmethod/latin/settings/CustomInputStyleSettingsFragment.java index 2174f5224..c633fc167 100644 --- a/java/src/com/android/inputmethod/latin/settings/CustomInputStyleSettingsFragment.java +++ b/java/src/com/android/inputmethod/latin/settings/CustomInputStyleSettingsFragment.java @@ -33,7 +33,6 @@ import android.preference.PreferenceGroup; import android.support.v4.view.ViewCompat; import android.text.TextUtils; import android.util.Log; -import android.util.Pair; import android.view.LayoutInflater; import android.view.Menu; import android.view.MenuInflater; @@ -80,25 +79,26 @@ public final class CustomInputStyleSettingsFragment extends PreferenceFragment { "is_subtype_enabler_notification_dialog_open"; private static final String KEY_SUBTYPE_FOR_SUBTYPE_ENABLER = "subtype_for_subtype_enabler"; - static final class SubtypeLocaleItem extends Pair<String, String> - implements Comparable<SubtypeLocaleItem> { - public SubtypeLocaleItem(final String localeString, final String displayName) { - super(localeString, displayName); - } + static final class SubtypeLocaleItem implements Comparable<SubtypeLocaleItem> { + public final String mLocaleString; + private final String mDisplayName; - public SubtypeLocaleItem(final String localeString) { - this(localeString, - SubtypeLocaleUtils.getSubtypeLocaleDisplayNameInSystemLocale(localeString)); + public SubtypeLocaleItem(final InputMethodSubtype subtype) { + mLocaleString = subtype.getLocale(); + mDisplayName = SubtypeLocaleUtils.getSubtypeLocaleDisplayNameInSystemLocale( + mLocaleString); } + // {@link ArrayAdapter<T>} that hosts the instance of this class needs {@link #toString()} + // to get display name. @Override public String toString() { - return second; + return mDisplayName; } @Override public int compareTo(final SubtypeLocaleItem o) { - return first.compareTo(o.first); + return mLocaleString.compareTo(o.mLocaleString); } } @@ -121,32 +121,28 @@ public final class CustomInputStyleSettingsFragment extends PreferenceFragment { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(subtype))); } if (InputMethodSubtypeCompatUtils.isAsciiCapable(subtype)) { - items.add(createItem(context, subtype.getLocale())); + items.add(new SubtypeLocaleItem(subtype)); } } // TODO: Should filter out already existing combinations of locale and layout. addAll(items); } - - public static SubtypeLocaleItem createItem(final Context context, - final String localeString) { - if (localeString.equals(SubtypeLocaleUtils.NO_LANGUAGE)) { - final String displayName = context.getString(R.string.subtype_no_language); - return new SubtypeLocaleItem(localeString, displayName); - } - return new SubtypeLocaleItem(localeString); - } } - static final class KeyboardLayoutSetItem extends Pair<String, String> { + static final class KeyboardLayoutSetItem { + public final String mLayoutName; + private final String mDisplayName; + public KeyboardLayoutSetItem(final InputMethodSubtype subtype) { - super(SubtypeLocaleUtils.getKeyboardLayoutSetName(subtype), - SubtypeLocaleUtils.getKeyboardLayoutSetDisplayName(subtype)); + mLayoutName = SubtypeLocaleUtils.getKeyboardLayoutSetName(subtype); + mDisplayName = SubtypeLocaleUtils.getKeyboardLayoutSetDisplayName(subtype); } + // {@link ArrayAdapter<T>} that hosts the instance of this class needs {@link #toString()} + // to get display name. @Override public String toString() { - return second; + return mDisplayName; } } @@ -255,7 +251,6 @@ public final class CustomInputStyleSettingsFragment extends PreferenceFragment { @Override protected void onPrepareDialogBuilder(final AlertDialog.Builder builder) { - final Context context = builder.getContext(); builder.setCancelable(true).setOnCancelListener(this); if (isIncomplete()) { builder.setPositiveButton(R.string.add, this) @@ -264,8 +259,7 @@ public final class CustomInputStyleSettingsFragment extends PreferenceFragment { builder.setPositiveButton(R.string.save, this) .setNeutralButton(android.R.string.cancel, this) .setNegativeButton(R.string.remove, this); - final SubtypeLocaleItem localeItem = SubtypeLocaleAdapter.createItem( - context, mSubtype.getLocale()); + final SubtypeLocaleItem localeItem = new SubtypeLocaleItem(mSubtype); final KeyboardLayoutSetItem layoutItem = new KeyboardLayoutSetItem(mSubtype); setSpinnerPosition(mSubtypeLocaleSpinner, localeItem); setSpinnerPosition(mKeyboardLayoutSetSpinner, layoutItem); @@ -303,7 +297,7 @@ public final class CustomInputStyleSettingsFragment extends PreferenceFragment { (KeyboardLayoutSetItem) mKeyboardLayoutSetSpinner.getSelectedItem(); final InputMethodSubtype subtype = AdditionalSubtypeUtils.createAsciiEmojiCapableAdditionalSubtype( - locale.first, layout.first); + locale.mLocaleString, layout.mLayoutName); setSubtype(subtype); notifyChanged(); if (isEditing) { @@ -469,8 +463,6 @@ public final class CustomInputStyleSettingsFragment extends PreferenceFragment { KEY_IS_SUBTYPE_ENABLER_NOTIFICATION_DIALOG_OPEN)) { mSubtypePreferenceKeyForSubtypeEnabler = savedInstanceState.getString( KEY_SUBTYPE_FOR_SUBTYPE_ENABLER); - final SubtypePreference subtypePref = (SubtypePreference)findPreference( - mSubtypePreferenceKeyForSubtypeEnabler); mSubtypeEnablerNotificationDialog = createDialog(); mSubtypeEnablerNotificationDialog.show(); } diff --git a/java/src/com/android/inputmethod/latin/setup/SetupActivity.java b/java/src/com/android/inputmethod/latin/setup/SetupActivity.java index b770ea512..7607429f8 100644 --- a/java/src/com/android/inputmethod/latin/setup/SetupActivity.java +++ b/java/src/com/android/inputmethod/latin/setup/SetupActivity.java @@ -17,12 +17,8 @@ package com.android.inputmethod.latin.setup; import android.app.Activity; -import android.content.Context; import android.content.Intent; import android.os.Bundle; -import android.provider.Settings; -import android.view.inputmethod.InputMethodInfo; -import android.view.inputmethod.InputMethodManager; public final class SetupActivity extends Activity { @Override diff --git a/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java b/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java index e455e53d3..54562f39d 100644 --- a/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java +++ b/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java @@ -46,6 +46,8 @@ import java.util.ArrayList; public final class SetupWizardActivity extends Activity implements View.OnClickListener { static final String TAG = SetupWizardActivity.class.getSimpleName(); + // For debugging purpose. + private static final boolean FORCE_TO_SHOW_WELCOME_SCREEN = false; private static final boolean ENABLE_WELCOME_VIDEO = true; private InputMethodManager mImm; @@ -304,6 +306,9 @@ public final class SetupWizardActivity extends Activity implements View.OnClickL private int determineSetupStepNumber() { mHandler.cancelPollingImeSettings(); + if (FORCE_TO_SHOW_WELCOME_SCREEN) { + return STEP_1; + } if (!UncachedInputMethodManagerUtils.isThisImeEnabled(this, mImm)) { return STEP_1; } diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 365217a60..76c7fdd6f 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -252,6 +252,9 @@ static void latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, } else { dictionary->getPredictions(&prevWordsInfo, &suggestionResults); } + if (DEBUG_DICT) { + suggestionResults.dumpSuggestions(); + } suggestionResults.outputSuggestions(env, outSuggestionCount, outCodePointsArray, outScoresArray, outSpaceIndicesArray, outTypesArray, outAutoCommitFirstWordConfidenceArray, inOutWeightOfLangModelVsSpatialModel); diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h index e55c9eb8a..885118524 100644 --- a/native/jni/src/defines.h +++ b/native/jni/src/defines.h @@ -119,7 +119,7 @@ static inline void dumpWordInfo(const int *word, const int length, const int ran const int probability) { static char charBuf[50]; const int N = intArrayToCharArray(word, length, charBuf, NELEMS(charBuf)); - if (N > 1) { + if (N > 0) { AKLOGI("%2d [ %s ] (%d)", rank, charBuf, probability); } } diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index 8d3f8a9f8..7a69d3ceb 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -55,9 +55,6 @@ void Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession suggest->getSuggestions(proximityInfo, traverseSession, xcoordinates, ycoordinates, times, pointerIds, inputCodePoints, inputSize, weightOfLangModelVsSpatialModel, outSuggestionResults); - if (DEBUG_DICT) { - outSuggestionResults->dumpSuggestions(); - } } Dictionary::NgramListenerForPrediction::NgramListenerForPrediction( diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 6ed65d921..4c4dfc578 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -35,23 +35,15 @@ const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE // count. const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO"; const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration -const char *const HeaderPolicy::FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY = - "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP"; const char *const HeaderPolicy::FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY = "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID"; -const char *const HeaderPolicy::FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY = - "FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS"; const char *const HeaderPolicy::MAX_UNIGRAM_COUNT_KEY = "MAX_UNIGRAM_COUNT"; const char *const HeaderPolicy::MAX_BIGRAM_COUNT_KEY = "MAX_BIGRAM_COUNT"; const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100; const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f; -const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP = 2; const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID = 3; -// 30 days -const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS = - 30 * 24 * 60 * 60; const int HeaderPolicy::DEFAULT_MAX_UNIGRAM_COUNT = 10000; const int HeaderPolicy::DEFAULT_MAX_BIGRAM_COUNT = 10000; diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index daf40d4f9..bc8eaded3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -53,15 +53,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)), mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue( &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)), - mForgettingCurveOccurrencesToLevelUp(HeaderReadWriteUtils::readIntAttributeValue( - &mAttributeMap, FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY, - DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)), mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY, DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)), - mForgettingCurveDurationToLevelDown(HeaderReadWriteUtils::readIntAttributeValue( - &mAttributeMap, FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY, - DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS)), mMaxUnigramCount(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, MAX_UNIGRAM_COUNT_KEY, DEFAULT_MAX_UNIGRAM_COUNT)), mMaxBigramCount(HeaderReadWriteUtils::readIntAttributeValue( @@ -86,15 +80,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0), mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue( &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)), - mForgettingCurveOccurrencesToLevelUp(HeaderReadWriteUtils::readIntAttributeValue( - &mAttributeMap, FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY, - DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)), mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY, DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)), - mForgettingCurveDurationToLevelDown(HeaderReadWriteUtils::readIntAttributeValue( - &mAttributeMap, FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY, - DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS)), mMaxUnigramCount(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, MAX_UNIGRAM_COUNT_KEY, DEFAULT_MAX_UNIGRAM_COUNT)), mMaxBigramCount(HeaderReadWriteUtils::readIntAttributeValue( @@ -113,12 +101,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mUnigramCount(headerPolicy->mUnigramCount), mBigramCount(headerPolicy->mBigramCount), mExtendedRegionSize(headerPolicy->mExtendedRegionSize), mHasHistoricalInfoOfWords(headerPolicy->mHasHistoricalInfoOfWords), - mForgettingCurveOccurrencesToLevelUp( - headerPolicy->mForgettingCurveOccurrencesToLevelUp), mForgettingCurveProbabilityValuesTableId( headerPolicy->mForgettingCurveProbabilityValuesTableId), - mForgettingCurveDurationToLevelDown( - headerPolicy->mForgettingCurveDurationToLevelDown), mMaxUnigramCount(headerPolicy->mMaxUnigramCount), mMaxBigramCount(headerPolicy->mMaxBigramCount), mCodePointTable(headerPolicy->mCodePointTable) {} @@ -130,8 +114,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false), mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false), - mForgettingCurveOccurrencesToLevelUp(0), mForgettingCurveProbabilityValuesTableId(0), - mForgettingCurveDurationToLevelDown(0), mMaxUnigramCount(0), mMaxBigramCount(0), + mForgettingCurveProbabilityValuesTableId(0), mMaxUnigramCount(0), mMaxBigramCount(0), mCodePointTable(nullptr) {} ~HeaderPolicy() {} @@ -217,18 +200,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return &mAttributeMap; } - AK_FORCE_INLINE int getForgettingCurveOccurrencesToLevelUp() const { - return mForgettingCurveOccurrencesToLevelUp; - } - AK_FORCE_INLINE int getForgettingCurveProbabilityValuesTableId() const { return mForgettingCurveProbabilityValuesTableId; } - AK_FORCE_INLINE int getForgettingCurveDurationToLevelDown() const { - return mForgettingCurveDurationToLevelDown; - } - AK_FORCE_INLINE int getMaxUnigramCount() const { return mMaxUnigramCount; } @@ -280,9 +255,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { static const char *const MAX_BIGRAM_COUNT_KEY; static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE; static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE; - static const int DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP; static const int DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID; - static const int DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS; static const int DEFAULT_MAX_UNIGRAM_COUNT; static const int DEFAULT_MAX_BIGRAM_COUNT; @@ -300,9 +273,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const int mBigramCount; const int mExtendedRegionSize; const bool mHasHistoricalInfoOfWords; - const int mForgettingCurveOccurrencesToLevelUp; const int mForgettingCurveProbabilityValuesTableId; - const int mForgettingCurveDurationToLevelDown; const int mMaxUnigramCount; const int mMaxBigramCount; const int *const mCodePointTable; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp index 4a740d47b..ef6166ffd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp @@ -74,8 +74,8 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId, return false; } writingPos += getEntrySize(); - mSize++; } + mSize = terminalId + 1; } return writeEntry(probabilityEntry, entryPos); } @@ -100,7 +100,6 @@ bool ProbabilityDictContent::flushToFile(const char *const dictPath) const { bool ProbabilityDictContent::runGC( const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ProbabilityDictContent *const originalProbabilityDictContent) { - mSize = 0; for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin(); it != terminalIdMap->end(); ++it) { const ProbabilityEntry probabilityEntry = @@ -109,7 +108,6 @@ bool ProbabilityDictContent::runGC( AKLOGE("Cannot set probability entry in runGC. terminalId: %d", it->second); return false; } - mSize++; } return true; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp index 11f7b305f..36eafa1e9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.cpp @@ -146,18 +146,15 @@ const WordAttributes Ver4PatriciaTriePolicy::getWordAttributes(const int probabi int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability, const int bigramProbability) const { - if (mHeaderPolicy->isDecayingDict()) { - // Both probabilities are encoded. Decode them and get probability. - return ForgettingCurveUtils::getProbability(unigramProbability, bigramProbability); - } else { - if (unigramProbability == NOT_A_PROBABILITY) { - return NOT_A_PROBABILITY; - } else if (bigramProbability == NOT_A_PROBABILITY) { - return ProbabilityUtils::backoff(unigramProbability); - } else { - return bigramProbability; - } + // In the v4 format, bigramProbability is a conditional probability. + const int bigramConditionalProbability = bigramProbability; + if (unigramProbability == NOT_A_PROBABILITY) { + return NOT_A_PROBABILITY; } + if (bigramConditionalProbability == NOT_A_PROBABILITY) { + return ProbabilityUtils::backoff(unigramProbability); + } + return bigramConditionalProbability; } int Ver4PatriciaTriePolicy::getProbabilityOfWord(const WordIdArrayView prevWordIds, @@ -170,37 +167,66 @@ int Ver4PatriciaTriePolicy::getProbabilityOfWord(const WordIdArrayView prevWordI if (ptNodeParams.isDeleted() || ptNodeParams.isBlacklisted() || ptNodeParams.isNotAWord()) { return NOT_A_PROBABILITY; } - if (!prevWordIds.empty()) { - const int bigramsPosition = getBigramsPositionOfPtNode( - getTerminalPtNodePosFromWordId(prevWordIds[0])); - BinaryDictionaryBigramsIterator bigramsIt(&mBigramPolicy, bigramsPosition); - while (bigramsIt.hasNext()) { - bigramsIt.next(); - if (bigramsIt.getBigramPos() == ptNodePos - && bigramsIt.getProbability() != NOT_A_PROBABILITY) { - return getProbability(ptNodeParams.getProbability(), bigramsIt.getProbability()); - } - } + if (prevWordIds.empty()) { + return getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY); + } + if (prevWordIds[0] == NOT_A_WORD_ID) { return NOT_A_PROBABILITY; } - return getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY); + const PtNodeParams prevWordPtNodeParams = + mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(prevWordIds[0]); + if (prevWordPtNodeParams.isDeleted()) { + return getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY); + } + const int bigramsPosition = mBuffers->getBigramDictContent()->getBigramListHeadPos( + prevWordPtNodeParams.getTerminalId()); + BinaryDictionaryBigramsIterator bigramsIt(&mBigramPolicy, bigramsPosition); + while (bigramsIt.hasNext()) { + bigramsIt.next(); + if (bigramsIt.getBigramPos() == ptNodePos + && bigramsIt.getProbability() != NOT_A_PROBABILITY) { + const int bigramConditionalProbability = getBigramConditionalProbability( + prevWordPtNodeParams.getProbability(), bigramsIt.getProbability()); + return getProbability(ptNodeParams.getProbability(), bigramConditionalProbability); + } + } + return NOT_A_PROBABILITY; } void Ver4PatriciaTriePolicy::iterateNgramEntries(const WordIdArrayView prevWordIds, NgramListener *const listener) const { - if (prevWordIds.empty()) { + if (prevWordIds.firstOrDefault(NOT_A_DICT_POS) == NOT_A_DICT_POS) { + return; + } + const PtNodeParams prevWordPtNodeParams = + mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(prevWordIds[0]); + if (prevWordPtNodeParams.isDeleted()) { return; } - const int bigramsPosition = getBigramsPositionOfPtNode( - getTerminalPtNodePosFromWordId(prevWordIds[0])); + const int bigramsPosition = mBuffers->getBigramDictContent()->getBigramListHeadPos( + prevWordPtNodeParams.getTerminalId()); BinaryDictionaryBigramsIterator bigramsIt(&mBigramPolicy, bigramsPosition); while (bigramsIt.hasNext()) { bigramsIt.next(); - listener->onVisitEntry(bigramsIt.getProbability(), + const int bigramConditionalProbability = getBigramConditionalProbability( + prevWordPtNodeParams.getProbability(), bigramsIt.getProbability()); + listener->onVisitEntry(bigramConditionalProbability, getWordIdFromTerminalPtNodePos(bigramsIt.getBigramPos())); } } +int Ver4PatriciaTriePolicy::getBigramConditionalProbability(const int prevWordUnigramProbability, + const int bigramProbability) const { + if (mHeaderPolicy->hasHistoricalInfoOfWords()) { + // Calculate conditional probability. + return std::min(MAX_PROBABILITY - prevWordUnigramProbability + bigramProbability, + MAX_PROBABILITY); + } else { + // bigramProbability is a conditional probability. + return bigramProbability; + } +} + BinaryDictionaryShortcutIterator Ver4PatriciaTriePolicy::getShortcutIterator( const int wordId) const { const int shortcutPos = getShortcutPositionOfPtNode(getTerminalPtNodePosFromWordId(wordId)); @@ -428,7 +454,10 @@ bool Ver4PatriciaTriePolicy::updateCounter(const PrevWordsInfo *const prevWordsI AKLOGE("Cannot update unigarm entry in updateCounter()."); return false; } - const NgramProperty ngramProperty(wordCodePoints.toVector(), probability, historicalInfo); + const int probabilityForNgram = prevWordsInfo->isNthPrevWordBeginningOfSentence(1 /* n */) + ? NOT_A_PROBABILITY : probability; + const NgramProperty ngramProperty(wordCodePoints.toVector(), probabilityForNgram, + historicalInfo); if (!addNgramEntry(prevWordsInfo, &ngramProperty)) { AKLOGE("Cannot update unigarm entry in updateCounter()."); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h index 995d7764f..b82563e61 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/ver4_patricia_trie_policy.h @@ -174,6 +174,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { int getTerminalPtNodePosFromWordId(const int wordId) const; const WordAttributes getWordAttributes(const int probability, const PtNodeParams &ptNodeParams) const; + int getBigramConditionalProbability(const int prevWordUnigramProbability, + const int bigramProbability) const; }; } // namespace v402 } // namespace backward diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 41b109f95..036197c41 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -378,7 +378,10 @@ bool Ver4PatriciaTriePolicy::updateCounter(const PrevWordsInfo *const prevWordsI AKLOGE("Cannot update unigarm entry in updateCounter()."); return false; } - const NgramProperty ngramProperty(wordCodePoints.toVector(), probability, historicalInfo); + const int probabilityForNgram = prevWordsInfo->isNthPrevWordBeginningOfSentence(1 /* n */) + ? NOT_A_PROBABILITY : probability; + const NgramProperty ngramProperty(wordCodePoints.toVector(), probabilityForNgram, + historicalInfo); for (size_t i = 1; i <= prevWordsInfo->getPrevWordCount(); ++i) { const PrevWordsInfo trimmedPrevWordsInfo(prevWordsInfo->getTrimmedPrevWordsInfo(i)); if (!addNgramEntry(&trimmedPrevWordsInfo, &ngramProperty)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp index af4bc186a..e5ef2abf8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp @@ -29,10 +29,14 @@ namespace latinime { const int ForgettingCurveUtils::MULTIPLIER_TWO_IN_PROBABILITY_SCALE = 8; const int ForgettingCurveUtils::DECAY_INTERVAL_SECONDS = 2 * 60 * 60; -const int ForgettingCurveUtils::MAX_LEVEL = 3; -const int ForgettingCurveUtils::MIN_VISIBLE_LEVEL = 1; -const int ForgettingCurveUtils::MAX_ELAPSED_TIME_STEP_COUNT = 15; -const int ForgettingCurveUtils::DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD = 14; +const int ForgettingCurveUtils::MAX_LEVEL = 15; +const int ForgettingCurveUtils::MIN_VISIBLE_LEVEL = 2; +const int ForgettingCurveUtils::MAX_ELAPSED_TIME_STEP_COUNT = 31; +const int ForgettingCurveUtils::DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD = 30; +const int ForgettingCurveUtils::OCCURRENCES_TO_RAISE_THE_LEVEL = 1; +// TODO: Evaluate whether this should be 7.5 days. +// 15 days +const int ForgettingCurveUtils::DURATION_TO_LOWER_THE_LEVEL_IN_SECONDS = 15 * 24 * 60 * 60; const float ForgettingCurveUtils::UNIGRAM_COUNT_HARD_LIMIT_WEIGHT = 1.2; const float ForgettingCurveUtils::BIGRAM_COUNT_HARD_LIMIT_WEIGHT = 1.2; @@ -54,19 +58,23 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT || (originalHistoricalInfo->getLevel() == newHistoricalInfo->getLevel() && originalHistoricalInfo->getCount() < newHistoricalInfo->getCount())) { // Initial information. + int count = newHistoricalInfo->getCount(); + if (count >= OCCURRENCES_TO_RAISE_THE_LEVEL) { + const int level = clampToValidLevelRange(newHistoricalInfo->getLevel() + 1); + return HistoricalInfo(timestamp, level, 0 /* count */); + } const int level = clampToValidLevelRange(newHistoricalInfo->getLevel()); - const int count = clampToValidCountRange(newHistoricalInfo->getCount(), headerPolicy); - return HistoricalInfo(timestamp, level, count); + return HistoricalInfo(timestamp, level, clampToValidCountRange(count, headerPolicy)); } else { const int updatedCount = originalHistoricalInfo->getCount() + 1; - if (updatedCount >= headerPolicy->getForgettingCurveOccurrencesToLevelUp()) { + if (updatedCount >= OCCURRENCES_TO_RAISE_THE_LEVEL) { // The count exceeds the max value the level can be incremented. if (originalHistoricalInfo->getLevel() >= MAX_LEVEL) { // The level is already max. return HistoricalInfo(timestamp, originalHistoricalInfo->getLevel(), originalHistoricalInfo->getCount()); } else { - // Level up. + // Raise the level. return HistoricalInfo(timestamp, originalHistoricalInfo->getLevel() + 1, 0 /* count */); } @@ -79,31 +87,18 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT /* static */ int ForgettingCurveUtils::decodeProbability( const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy) { const int elapsedTimeStepCount = getElapsedTimeStepCount(historicalInfo->getTimestamp(), - headerPolicy->getForgettingCurveDurationToLevelDown()); + DURATION_TO_LOWER_THE_LEVEL_IN_SECONDS); return sProbabilityTable.getProbability( headerPolicy->getForgettingCurveProbabilityValuesTableId(), clampToValidLevelRange(historicalInfo->getLevel()), clampToValidTimeStepCountRange(elapsedTimeStepCount)); } -/* static */ int ForgettingCurveUtils::getProbability(const int unigramProbability, - const int bigramProbability) { - if (unigramProbability == NOT_A_PROBABILITY) { - return NOT_A_PROBABILITY; - } else if (bigramProbability == NOT_A_PROBABILITY) { - return std::min(backoff(unigramProbability), MAX_PROBABILITY); - } else { - // TODO: Investigate better way to handle bigram probability. - return std::min(std::max(unigramProbability, - bigramProbability + MULTIPLIER_TWO_IN_PROBABILITY_SCALE), MAX_PROBABILITY); - } -} - /* static */ bool ForgettingCurveUtils::needsToKeep(const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy) { return historicalInfo->getLevel() > 0 || getElapsedTimeStepCount(historicalInfo->getTimestamp(), - headerPolicy->getForgettingCurveDurationToLevelDown()) + DURATION_TO_LOWER_THE_LEVEL_IN_SECONDS) < DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD; } @@ -113,14 +108,14 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT if (originalHistoricalInfo->getTimestamp() == NOT_A_TIMESTAMP) { return HistoricalInfo(); } - const int durationToLevelDownInSeconds = headerPolicy->getForgettingCurveDurationToLevelDown(); + const int durationToLevelDownInSeconds = DURATION_TO_LOWER_THE_LEVEL_IN_SECONDS; const int elapsedTimeStep = getElapsedTimeStepCount( originalHistoricalInfo->getTimestamp(), durationToLevelDownInSeconds); if (elapsedTimeStep <= MAX_ELAPSED_TIME_STEP_COUNT) { // No need to update historical info. return *originalHistoricalInfo; } - // Level down. + // Lower the level. const int maxLevelDownAmonut = elapsedTimeStep / (MAX_ELAPSED_TIME_STEP_COUNT + 1); const int levelDownAmount = (maxLevelDownAmonut >= originalHistoricalInfo->getLevel()) ? originalHistoricalInfo->getLevel() : maxLevelDownAmonut; @@ -170,7 +165,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT /* static */ int ForgettingCurveUtils::clampToValidCountRange(const int count, const HeaderPolicy *const headerPolicy) { - return std::min(std::max(count, 0), headerPolicy->getForgettingCurveOccurrencesToLevelUp() - 1); + return std::min(std::max(count, 0), OCCURRENCES_TO_RAISE_THE_LEVEL - 1); } /* static */ int ForgettingCurveUtils::clampToValidLevelRange(const int level) { @@ -187,9 +182,9 @@ const int ForgettingCurveUtils::ProbabilityTable::MODEST_PROBABILITY_TABLE_ID = const int ForgettingCurveUtils::ProbabilityTable::STRONG_PROBABILITY_TABLE_ID = 2; const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_PROBABILITY_TABLE_ID = 3; const int ForgettingCurveUtils::ProbabilityTable::WEAK_MAX_PROBABILITY = 127; -const int ForgettingCurveUtils::ProbabilityTable::MODEST_BASE_PROBABILITY = 32; -const int ForgettingCurveUtils::ProbabilityTable::STRONG_BASE_PROBABILITY = 35; -const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_BASE_PROBABILITY = 40; +const int ForgettingCurveUtils::ProbabilityTable::MODEST_BASE_PROBABILITY = 8; +const int ForgettingCurveUtils::ProbabilityTable::STRONG_BASE_PROBABILITY = 9; +const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_BASE_PROBABILITY = 10; ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTables() { @@ -202,7 +197,7 @@ ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTables() { const float endProbability = getBaseProbabilityForLevel(tableId, level - 1); for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT; ++timeStepCount) { - if (level == 0) { + if (level < MIN_VISIBLE_LEVEL) { mTables[tableId][level][timeStepCount] = NOT_A_PROBABILITY; continue; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h index 10abb405a..ccbc4a98d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h @@ -39,9 +39,6 @@ class ForgettingCurveUtils { static int decodeProbability(const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy); - static int getProbability(const int encodedUnigramProbability, - const int encodedBigramProbability); - static bool needsToKeep(const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy); @@ -101,6 +98,8 @@ class ForgettingCurveUtils { static const int MIN_VISIBLE_LEVEL; static const int MAX_ELAPSED_TIME_STEP_COUNT; static const int DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD; + static const int OCCURRENCES_TO_RAISE_THE_LEVEL; + static const int DURATION_TO_LOWER_THE_LEVEL_IN_SECONDS; static const float UNIGRAM_COUNT_HARD_LIMIT_WEIGHT; static const float BIGRAM_COUNT_HARD_LIMIT_WEIGHT; diff --git a/tests/src/com/android/inputmethod/compat/SuggestionSpanUtilsTest.java b/tests/src/com/android/inputmethod/compat/SuggestionSpanUtilsTest.java new file mode 100644 index 000000000..a6b3af4c2 --- /dev/null +++ b/tests/src/com/android/inputmethod/compat/SuggestionSpanUtilsTest.java @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.compat; + +import android.annotation.TargetApi; +import android.os.Build; +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; +import android.text.Spanned; +import android.text.TextUtils; +import android.text.style.SuggestionSpan; + +import com.android.inputmethod.latin.SuggestedWords; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Locale; + +@SmallTest +public class SuggestionSpanUtilsTest extends AndroidTestCase { + + /** + * Helper method to create a dummy {@link SuggestedWordInfo}. + * + * @param kindAndFlags the kind and flags to be used to create {@link SuggestedWordInfo}. + * @param word the word to be used to create {@link SuggestedWordInfo}. + * @return a new instance of {@link SuggestedWordInfo}. + */ + private static SuggestedWordInfo createWordInfo(final String word, final int kindAndFlags) { + return new SuggestedWordInfo(word, 1 /* score */, kindAndFlags, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); + } + + private static void assertNotSuggestionSpan(final String expectedText, + final CharSequence actualText) { + assertTrue(TextUtils.equals(expectedText, actualText)); + if (!(actualText instanceof Spanned)) { + return; + } + final Spanned spanned = (Spanned)actualText; + final SuggestionSpan[] suggestionSpans = spanned.getSpans(0, spanned.length(), + SuggestionSpan.class); + assertEquals(0, suggestionSpans.length); + } + + private static void assertSuggestionSpan(final String expectedText, + final int reuiredSuggestionSpanFlags, final int requiredSpanFlags, + final String[] expectedSuggestions, + final CharSequence actualText) { + assertTrue(TextUtils.equals(expectedText, actualText)); + assertTrue(actualText instanceof Spanned); + final Spanned spanned = (Spanned)actualText; + final SuggestionSpan[] suggestionSpans = spanned.getSpans(0, spanned.length(), + SuggestionSpan.class); + assertEquals(1, suggestionSpans.length); + final SuggestionSpan suggestionSpan = suggestionSpans[0]; + if (reuiredSuggestionSpanFlags != 0) { + assertTrue((suggestionSpan.getFlags() & reuiredSuggestionSpanFlags) != 0); + } + if (requiredSpanFlags != 0) { + assertTrue((spanned.getSpanFlags(suggestionSpan) & requiredSpanFlags) != 0); + } + if (expectedSuggestions != null) { + final String[] actualSuggestions = suggestionSpan.getSuggestions(); + assertEquals(expectedSuggestions.length, actualSuggestions.length); + for (int i = 0; i < expectedSuggestions.length; ++i) { + assertEquals(expectedSuggestions[i], actualSuggestions[i]); + } + } + } + + @TargetApi(Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) + public void testGetTextWithAutoCorrectionIndicatorUnderline() { + final String ORIGINAL_TEXT = "Hey!"; + final CharSequence text = SuggestionSpanUtils.getTextWithAutoCorrectionIndicatorUnderline( + getContext(), ORIGINAL_TEXT); + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) { + assertNotSuggestionSpan(ORIGINAL_TEXT, text); + return; + } + + assertSuggestionSpan(ORIGINAL_TEXT, + SuggestionSpan.FLAG_AUTO_CORRECTION /* reuiredSuggestionSpanFlags */, + Spanned.SPAN_COMPOSING | Spanned.SPAN_EXCLUSIVE_EXCLUSIVE /* requiredSpanFlags */, + new String[]{}, text); + } + + public void testGetTextWithSuggestionSpan() { + final SuggestedWordInfo predicition1 = + createWordInfo("Quality", SuggestedWordInfo.KIND_PREDICTION); + final SuggestedWordInfo predicition2 = + createWordInfo("Speed", SuggestedWordInfo.KIND_PREDICTION); + final SuggestedWordInfo predicition3 = + createWordInfo("Price", SuggestedWordInfo.KIND_PREDICTION); + + final SuggestedWordInfo typed = + createWordInfo("Hey", SuggestedWordInfo.KIND_TYPED); + + final SuggestedWordInfo[] corrections = + new SuggestedWordInfo[SuggestionSpan.SUGGESTIONS_MAX_SIZE * 2]; + for (int i = 0; i < corrections.length; ++i) { + corrections[i] = createWordInfo("correction" + i, SuggestedWordInfo.KIND_CORRECTION); + } + + // SuggestionSpan will not be attached when {@link SuggestedWords#INPUT_STYLE_PREDICTION} + // is specified. + { + final SuggestedWords predictedWords = new SuggestedWords( + new ArrayList<>(Arrays.asList(predicition1, predicition2, predicition3)), + null /* rawSuggestions */, + false /* typedWordValid */, + false /* willAutoCorrect */, + false /* isObsoleteSuggestions */, + SuggestedWords.INPUT_STYLE_PREDICTION); + final String PICKED_WORD = predicition2.mWord; + assertNotSuggestionSpan( + PICKED_WORD, + SuggestionSpanUtils.getTextWithSuggestionSpan(getContext(), PICKED_WORD, + predictedWords)); + } + + final ArrayList<SuggestedWordInfo> suggestedWordList = new ArrayList<>(); + suggestedWordList.add(typed); + suggestedWordList.add(predicition1); + suggestedWordList.add(predicition2); + suggestedWordList.add(predicition3); + suggestedWordList.addAll(Arrays.asList(corrections)); + final SuggestedWords typedAndCollectedWords = new SuggestedWords( + suggestedWordList, + null /* rawSuggestions */, + false /* typedWordValid */, + false /* willAutoCorrect */, + false /* isObsoleteSuggestions */, + SuggestedWords.INPUT_STYLE_TYPING); + + for (final SuggestedWordInfo pickedWord : suggestedWordList) { + final String PICKED_WORD = pickedWord.mWord; + + final ArrayList<String> expectedSuggestions = new ArrayList<>(); + for (SuggestedWordInfo suggestedWordInfo : suggestedWordList) { + if (expectedSuggestions.size() >= SuggestionSpan.SUGGESTIONS_MAX_SIZE) { + break; + } + if (suggestedWordInfo.isKindOf(SuggestedWordInfo.KIND_PREDICTION)) { + // Currently predictions are not filled into SuggestionSpan. + continue; + } + final String suggestedWord = suggestedWordInfo.mWord; + if (TextUtils.equals(PICKED_WORD, suggestedWord)) { + // Typed word itself is not added to SuggestionSpan. + continue; + } + expectedSuggestions.add(suggestedWord); + } + + assertSuggestionSpan( + PICKED_WORD, + 0 /* reuiredSuggestionSpanFlags */, + Spanned.SPAN_EXCLUSIVE_EXCLUSIVE /* requiredSpanFlags */, + expectedSuggestions.toArray(new String[expectedSuggestions.size()]), + SuggestionSpanUtils.getTextWithSuggestionSpan(getContext(), PICKED_WORD, + typedAndCollectedWords)); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index 6b6ad21b7..0e58b7211 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -75,20 +75,25 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { return formatVersion > FormatSpec.VERSION401; } - private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, - final int probability) { - binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, - BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, - false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, mCurrentTime /* timestamp */); + private void onInputWord(final BinaryDictionary binaryDictionary, final String word, + final boolean isValidWord) { + binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO, + word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } - private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, - final String word1, final int probability) { - binaryDictionary.addNgramEntry(new NgramContext(new WordInfo(word0)), word1, probability, + private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word, + final boolean isValidWord, final String prevWord) { + binaryDictionary.updateEntriesForWordWithNgramContext( + new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } + private void onInputWordWithBeginningOfSentenceContext( + final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) { + binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE, + word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); + } + private static boolean isValidBigram(final BinaryDictionary binaryDictionary, final String word0, final String word1) { return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); @@ -175,10 +180,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a"); binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -229,28 +233,27 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "a", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "a", false /* isValidWord */); + onInputWord(binaryDictionary, "a", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "b", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("b")); - addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertFalse(isValidBigram(binaryDictionary, "a", "b")); - addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); - addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. - addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); - addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); binaryDictionary.close(); @@ -266,36 +269,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); assertFalse(isValidBigram(binaryDictionary, "a", "b")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); assertTrue(isValidBigram(binaryDictionary, "a", "b")); @@ -333,7 +332,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = @@ -380,10 +379,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String strong = "strong"; final String weak = "weak"; for (int j = 0; j < strongUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, strong, true /* isValidWord */); } for (int j = 0; j < weakUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, weak, true /* isValidWord */); } assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(weak)); @@ -391,7 +390,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); for (int j = 0; j < eachUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = @@ -450,12 +449,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final int maxBigramCount = Integer.parseInt( - binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); - addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, bigram.first, true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */, + bigram.first); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = @@ -506,23 +506,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); for (int j = 0; j < unigramTypedCount; j++) { - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); } } final String strong = "strong"; final String weak = "weak"; final String target = "target"; for (int j = 0; j < unigramTypedCount; j++) { - addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, weak, true /* isValidWord */); + onInputWord(binaryDictionary, strong, true /* isValidWord */); } binaryDictionary.flushWithGC(); for (int j = 0; j < strongBigramTypedCount; j++) { - addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong); } for (int j = 0; j < weakBigramTypedCount; j++) { - addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak); } assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); @@ -535,7 +534,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word1 = words.get(word1Index); for (int j = 0; j < eachBigramTypedCount; j++) { - addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = @@ -563,19 +562,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("aaa")); - addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidWord("bbb")); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + + onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); - addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); + assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); @@ -585,11 +583,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); - addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "bbb", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); - addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); binaryDictionary.close(); } @@ -612,28 +610,25 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, mCurrentTime); final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, - mCurrentTime); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, - mCurrentTime); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); - forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); - - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, - mCurrentTime); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); binaryDictionary.close(); @@ -651,10 +646,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "aaa", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("aaa")); for (int i = 0; i < unigramInputCount; i++) { - addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "aaa", false /* isValidWord */); } assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.removeUnigramEntry("aaa")); |
