diff options
33 files changed, 282 insertions, 149 deletions
diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml index 055060642..475e92f2e 100644 --- a/java/res/values/attrs.xml +++ b/java/res/values/attrs.xml @@ -175,7 +175,7 @@ </declare-styleable> <declare-styleable name="SuggestionStripView"> - <attr name="suggestionStripOption" format="integer"> + <attr name="suggestionStripOptions" format="integer"> <!-- This should be aligned with SuggestionStripLayoutHelper.AUTO_CORRECT_* and etc. --> <flag name="autoCorrectBold" value="0x01" /> <flag name="autoCorrectUnderline" value="0x02" /> diff --git a/java/res/values/themes-gb.xml b/java/res/values/themes-gb.xml index a460d4f7f..c189da383 100644 --- a/java/res/values/themes-gb.xml +++ b/java/res/values/themes-gb.xml @@ -133,7 +133,7 @@ parent="SuggestionStripView" > <item name="android:background">@drawable/keyboard_suggest_strip_gb</item> - <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item> + <item name="suggestionStripOptions">autoCorrectBold|validTypedWordBold</item> <item name="colorValidTypedWord">@color/highlight_color_gb</item> <item name="colorTypedWord">@color/typed_word_color_gb</item> <item name="colorAutoCorrect">@color/highlight_color_gb</item> diff --git a/java/res/values/themes-ics.xml b/java/res/values/themes-ics.xml index caea92186..720eda9ce 100644 --- a/java/res/values/themes-ics.xml +++ b/java/res/values/themes-ics.xml @@ -112,7 +112,7 @@ parent="SuggestionStripView" > <item name="android:background">@drawable/keyboard_suggest_strip_holo</item> - <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item> + <item name="suggestionStripOptions">autoCorrectBold|validTypedWordBold</item> <item name="colorValidTypedWord">@color/typed_word_color_ics</item> <item name="colorTypedWord">@color/typed_word_color_ics</item> <item name="colorAutoCorrect">@color/highlight_color_ics</item> diff --git a/java/res/values/themes-klp.xml b/java/res/values/themes-klp.xml index 0599fb65e..830527171 100644 --- a/java/res/values/themes-klp.xml +++ b/java/res/values/themes-klp.xml @@ -112,7 +112,7 @@ parent="SuggestionStripView" > <item name="android:background">@drawable/keyboard_suggest_strip_holo</item> - <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item> + <item name="suggestionStripOptions">autoCorrectBold|validTypedWordBold</item> <item name="colorValidTypedWord">@color/typed_word_color_klp</item> <item name="colorTypedWord">@color/typed_word_color_klp</item> <item name="colorAutoCorrect">@color/highlight_color_klp</item> diff --git a/java/res/xml/rowkeys_symbols_shift1.xml b/java/res/xml/rowkeys_symbols_shift1.xml index 7cb3213f0..4c2722c7e 100644 --- a/java/res/xml/rowkeys_symbols_shift1.xml +++ b/java/res/xml/rowkeys_symbols_shift1.xml @@ -34,11 +34,11 @@ <!-- U+221A: "√" SQUARE ROOT --> <Key latin:keySpec="√" /> - <!-- U+03A0: "Π" GREEK CAPITAL LETTER PI - U+03C0: "π" GREEK SMALL LETTER PI --> + <!-- U+03C0: "π" GREEK SMALL LETTER PI + U+03A0: "Π" GREEK CAPITAL LETTER PI --> <Key - latin:keySpec="Π" - latin:moreKeys="π" /> + latin:keySpec="π" + latin:moreKeys="Π" /> <!-- U+00F7: "÷" DIVISION SIGN --> <Key latin:keySpec="÷" /> diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java index afaf2cc57..816a94300 100644 --- a/java/src/com/android/inputmethod/keyboard/Key.java +++ b/java/src/com/android/inputmethod/keyboard/Key.java @@ -716,10 +716,14 @@ public class Key implements Comparable<Key> { return (attrs != null) ? attrs.mAltCode : CODE_UNSPECIFIED; } + public int getIconId() { + return mIconId; + } + public Drawable getIcon(final KeyboardIconsSet iconSet, final int alpha) { final OptionalAttributes attrs = mOptionalAttributes; final int disabledIconId = (attrs != null) ? attrs.mDisabledIconId : ICON_UNDEFINED; - final int iconId = mEnabled ? mIconId : disabledIconId; + final int iconId = mEnabled ? getIconId() : disabledIconId; final Drawable icon = iconSet.getIconDrawable(iconId); if (icon != null) { icon.setAlpha(alpha); @@ -731,7 +735,7 @@ public class Key implements Comparable<Key> { final OptionalAttributes attrs = mOptionalAttributes; final int previewIconId = (attrs != null) ? attrs.mPreviewIconId : ICON_UNDEFINED; return previewIconId != ICON_UNDEFINED - ? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(mIconId); + ? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(getIconId()); } public int getWidth() { diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index f9758634e..573c60552 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -340,10 +340,6 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { mState.onCodeInput(code, mLatinIME.getCurrentAutoCapsState()); } - private boolean isShowingMainKeyboard() { - return null != mKeyboardView && mKeyboardView.isShown(); - } - public boolean isShowingEmojiPalettes() { return mEmojiPalettesView != null && mEmojiPalettesView.isShown(); } @@ -376,10 +372,6 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } } - public boolean isShowingMainKeyboardOrEmojiPalettes() { - return isShowingMainKeyboard() || isShowingEmojiPalettes(); - } - public View onCreateInputView(final boolean isHardwareAcceleratedDrawingEnabled) { if (mKeyboardView != null) { mKeyboardView.closing(); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java index da8bf7d69..79d088f2e 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardIconsSet.java @@ -102,7 +102,7 @@ public final class KeyboardIconsSet { return isValidIconId(iconId) ? ICON_NAMES[iconId] : "unknown<" + iconId + ">"; } - static int getIconId(final String name) { + public static int getIconId(final String name) { Integer iconId = sNameToIdsMap.get(name); if (iconId != null) { return iconId; diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java index 2dba7131e..cd18a6ba5 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -142,7 +143,8 @@ public class DictionaryFacilitatorForSuggest { @UsedForTesting public DictionaryFacilitatorForSuggest(final Context context, final Locale locale, - final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles) { + final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles, + final Map<String, Map<String, String>> additionalDictAttributes) { mContext = context; mLocale = locale; mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0); @@ -159,6 +161,10 @@ public class DictionaryFacilitatorForSuggest { userHistoryDictionary.reloadDictionaryIfRequired(); userHistoryDictionary.waitAllTasksForTests(); setUserHistoryDictionary(userHistoryDictionary); + if (additionalDictAttributes.containsKey(dictType)) { + userHistoryDictionary.clearAndFlushDictionaryWithAdditionalAttributes( + additionalDictAttributes.get(dictType)); + } } else if (dictType.equals(Dictionary.TYPE_PERSONALIZATION)) { final PersonalizationDictionary personalizationDictionary = PersonalizationHelper.getPersonalizationDictionary(context, locale); @@ -167,6 +173,10 @@ public class DictionaryFacilitatorForSuggest { personalizationDictionary.reloadDictionaryIfRequired(); personalizationDictionary.waitAllTasksForTests(); setPersonalizationDictionary(personalizationDictionary); + if (additionalDictAttributes.containsKey(dictType)) { + personalizationDictionary.clearAndFlushDictionaryWithAdditionalAttributes( + additionalDictAttributes.get(dictType)); + } } else if (dictType.equals(Dictionary.TYPE_USER)) { final File file = dictionaryFiles.get(dictType); final UserBinaryDictionary userDictionary = new UserBinaryDictionary( diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 44282a492..8ab1bb6a1 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -853,7 +853,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } // This will set the punctuation suggestions if next word suggestion is off; // otherwise it will clear the suggestion strip. - setNeutralSuggestionStripInternal(false /* needsInputViewShown */); + setNeutralSuggestionStripInternal(); mHandler.cancelUpdateSuggestionStrip(); mHandler.cancelDoubleSpacePeriodTimer(); @@ -1018,23 +1018,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen null /* rawSuggestions */, false /* typedWordValid */, false /* willAutoCorrect */, false /* isObsoleteSuggestions */, false /* isPrediction */); // When in fullscreen mode, show completions generated by the application forcibly - setSuggestedWords(suggestedWords, true /* isSuggestionStripVisible */, - true /* needsInputViewShown */); + setSuggestedWords(suggestedWords, true /* isSuggestionStripVisible */); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_onDisplayCompletions(applicationSpecifiedCompletions); } } - private void setSuggestionStripShownInternal(final boolean isSuggestionStripVisible, - final boolean needsInputViewShown) { + private void setSuggestionStripShownInternal(final boolean isSuggestionStripVisible) { // TODO: Modify this if we support suggestions with hard keyboard if (!onEvaluateInputViewShown() || !hasSuggestionStripView()) { return; } - final boolean inputViewShown = mKeyboardSwitcher.isShowingMainKeyboardOrEmojiPalettes(); - final boolean shouldShowSuggestions = isSuggestionStripVisible - && (needsInputViewShown ? inputViewShown : true); - if (shouldShowSuggestions) { + if (isSuggestionStripVisible) { mSuggestionStripView.setVisibility(View.VISIBLE); } else { mSuggestionStripView.setVisibility(isFullscreenMode() ? View.GONE : View.INVISIBLE); @@ -1321,8 +1316,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Nothing to do so far. } - // TODO[IL]: Define a clear interface for this - public boolean isSuggestionStripVisible() { + private boolean isSuggestionStripVisible() { if (!hasSuggestionStripView()) { return false; } @@ -1346,7 +1340,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return currentSettings.isSuggestionsRequested(); } - @Override public boolean hasSuggestionStripView() { return null != mSuggestionStripView; } @@ -1366,7 +1359,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // TODO[IL]: Define a clear interface for this public void setSuggestedWords(final SuggestedWords suggestedWords, - final boolean isSuggestionStripVisible, final boolean needsInputViewShown) { + final boolean isSuggestionStripVisible) { mInputLogic.setSuggestedWords(suggestedWords); if (!hasSuggestionStripView()) { return; @@ -1386,7 +1379,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen SubtypeLocaleUtils.isRtlLanguage(mSubtypeSwitcher.getCurrentSubtype())); } mKeyboardSwitcher.onAutoCorrectionStateChanged(suggestedWords.mWillAutoCorrect); - setSuggestionStripShownInternal(isSuggestionStripVisible, needsInputViewShown); + setSuggestionStripShownInternal(isSuggestionStripVisible); } // TODO[IL]: Move this out of LatinIME. @@ -1472,8 +1465,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen setNeutralSuggestionStrip(); } else { mInputLogic.mWordComposer.setAutoCorrection(autoCorrection); - setSuggestedWords( - suggestedWords, isSuggestionStripVisible(), true /* needsInputViewShown */); + setSuggestedWords(suggestedWords, isSuggestionStripVisible()); } // Cache the auto-correction in accessibility code so we can speak it if the user // touches a key that will insert it. @@ -1502,14 +1494,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // punctuation suggestions (if it's disabled). @Override public void setNeutralSuggestionStrip() { - setNeutralSuggestionStripInternal(true /* needsInputViewShown */); + setNeutralSuggestionStripInternal(); } - private void setNeutralSuggestionStripInternal(final boolean needsInputViewShown) { + private void setNeutralSuggestionStripInternal() { final SettingsValues currentSettings = mSettings.getCurrent(); final SuggestedWords neutralSuggestions = currentSettings.mBigramPredictionEnabled ? SuggestedWords.EMPTY : currentSettings.mSpacingAndPunctuations.mSuggestPuncList; - setSuggestedWords(neutralSuggestions, isSuggestionStripVisible(), needsInputViewShown); + setSuggestedWords(neutralSuggestions, isSuggestionStripVisible()); } // TODO: Make this private diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index abf831a28..5e144106f 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -147,6 +147,8 @@ public final class Suggest { } } + final boolean isPrediction = !wordComposer.isComposingWord(); + // We allow auto-correction if we have a whitelisted word, or if the word is not a valid // word of more than 1 char, except if the first suggestion is the same as the typed string // because in this case if it's strong enough to auto-correct that will mistakenly designate @@ -165,7 +167,7 @@ public final class Suggest { // same time, it feels wrong that the SuggestedWord object includes information about // the current settings. It may also be useful to know, when the setting is off, whether // the word *would* have been auto-corrected. - if (!isCorrectionEnabled || !allowsToBeAutoCorrected || !wordComposer.isComposingWord() + if (!isCorrectionEnabled || !allowsToBeAutoCorrected || isPrediction || suggestionsSet.isEmpty() || wordComposer.hasDigits() || wordComposer.isMostlyCaps() || wordComposer.isResumed() || !mDictionaryFacilitator.hasMainDictionary() @@ -225,10 +227,9 @@ public final class Suggest { // TODO: this first argument is lying. If this is a whitelisted word which is an // actual word, it says typedWordValid = false, which looks wrong. We should either // rename the attribute or change the value. - !allowsToBeAutoCorrected /* typedWordValid */, + !isPrediction && !allowsToBeAutoCorrected /* typedWordValid */, hasAutoCorrection, /* willAutoCorrect */ - false /* isObsoleteSuggestions */, - !wordComposer.isComposingWord() /* isPrediction */, sequenceNumber)); + false /* isObsoleteSuggestions */, isPrediction, sequenceNumber)); } // Retrieves suggestions for the batch input diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index 46df3e88c..06bc90c97 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -69,7 +69,7 @@ public class SuggestedWords { final boolean isPrediction, final int sequenceNumber) { this(suggestedWordInfoList, rawSuggestions, - suggestedWordInfoList.isEmpty() ? null + (suggestedWordInfoList.isEmpty() || isPrediction) ? null : suggestedWordInfoList.get(INDEX_OF_TYPED_WORD).mWord, typedWordValid, willAutoCorrect, isObsoleteSuggestions, isPrediction, sequenceNumber); diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index 045d06f0e..f2f9f1e68 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -1360,10 +1360,12 @@ public final class InputLogic { }}); } else { // We found suggestion spans in the word. We'll create the SuggestedWords out of - // them, and make willAutoCorrect false. + // them, and make willAutoCorrect false. We make typedWordValid false, because the + // color of the word in the suggestion strip changes according to this parameter, + // and false gives the correct color. final SuggestedWords suggestedWords = new SuggestedWords(suggestions, null /* rawSuggestions */, typedWord, - true /* typedWordValid */, false /* willAutoCorrect */, + false /* typedWordValid */, false /* willAutoCorrect */, false /* isObsoleteSuggestions */, false /* isPrediction */, SuggestedWords.NOT_A_SEQUENCE_NUMBER); mIsAutoCorrectionIndicatorOn = false; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java b/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java index b99e281da..ed9c39602 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java @@ -38,6 +38,10 @@ public final class DictionaryHeader { public static final String DICTIONARY_DATE_KEY = "date"; public static final String HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO"; public static final String USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE"; + public static final String FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY = + "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP"; + public static final String FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY = + "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID"; public static final String ATTRIBUTE_VALUE_TRUE = "1"; public DictionaryHeader(final int headerSize, final DictionaryOptions dictionaryOptions, diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index d3734d693..8c358cd50 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -50,6 +50,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB public final Locale mLocale; private final String mDictName; + private Map<String, String> mAdditionalAttributeMap = null; protected DecayingExpandableBinaryDictionaryBase(final Context context, final String dictName, final Locale locale, final String dictionaryType, @@ -78,7 +79,10 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB @Override protected Map<String, String> getHeaderAttributeMap() { - HashMap<String, String> attributeMap = new HashMap<String, String>(); + final Map<String, String> attributeMap = new HashMap<String, String>(); + if (mAdditionalAttributeMap != null) { + attributeMap.putAll(mAdditionalAttributeMap); + } attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY, DictionaryHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, @@ -152,6 +156,13 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB asyncFlushBinaryDictionary(); } + @UsedForTesting + public void clearAndFlushDictionaryWithAdditionalAttributes( + final Map<String, String> attributeMap) { + mAdditionalAttributeMap = attributeMap; + clearAndFlushDictionary(); + } + /* package */ void decayIfNeeded() { runGCIfRequired(false /* mindsBlockByGC */); } diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index 8ea712835..afa8fe3a8 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -95,9 +95,9 @@ final class SuggestionStripLayoutHelper { private static final CharacterStyle BOLD_SPAN = new StyleSpan(Typeface.BOLD); private static final CharacterStyle UNDERLINE_SPAN = new UnderlineSpan(); - private final int mSuggestionStripOption; + private final int mSuggestionStripOptions; // These constants are the flag values of - // {@link R.styleable#SuggestionStripView_suggestionStripOption} attribute. + // {@link R.styleable#SuggestionStripView_suggestionStripOptions} attribute. private static final int AUTO_CORRECT_BOLD = 0x01; private static final int AUTO_CORRECT_UNDERLINE = 0x02; private static final int VALID_TYPED_WORD_BOLD = 0x04; @@ -122,8 +122,8 @@ final class SuggestionStripLayoutHelper { final TypedArray a = context.obtainStyledAttributes(attrs, R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripView); - mSuggestionStripOption = a.getInt( - R.styleable.SuggestionStripView_suggestionStripOption, 0); + mSuggestionStripOptions = a.getInt( + R.styleable.SuggestionStripView_suggestionStripOptions, 0); mAlphaObsoleted = ResourceUtils.getFraction(a, R.styleable.SuggestionStripView_alphaObsoleted, 1.0f); mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0); @@ -200,22 +200,24 @@ final class SuggestionStripLayoutHelper { return null; } final String word = suggestedWords.getLabel(indexInSuggestedWords); - final boolean isAutoCorrect = indexInSuggestedWords == 1 - && suggestedWords.mWillAutoCorrect; - final boolean isTypedWordValid = indexInSuggestedWords == 0 - && suggestedWords.mTypedWordValid; - if (!isAutoCorrect && !isTypedWordValid) { + // TODO: don't use the index to decide whether this is the auto-correction/typed word, as + // this is brittle + final boolean isAutoCorrection = suggestedWords.mWillAutoCorrect + && indexInSuggestedWords == SuggestedWords.INDEX_OF_AUTO_CORRECTION; + final boolean isTypedWordValid = suggestedWords.mTypedWordValid + && indexInSuggestedWords == SuggestedWords.INDEX_OF_TYPED_WORD; + if (!isAutoCorrection && !isTypedWordValid) { return word; } final int len = word.length(); final Spannable spannedWord = new SpannableString(word); - final int option = mSuggestionStripOption; - if ((isAutoCorrect && (option & AUTO_CORRECT_BOLD) != 0) - || (isTypedWordValid && (option & VALID_TYPED_WORD_BOLD) != 0)) { + final int options = mSuggestionStripOptions; + if ((isAutoCorrection && (options & AUTO_CORRECT_BOLD) != 0) + || (isTypedWordValid && (options & VALID_TYPED_WORD_BOLD) != 0)) { spannedWord.setSpan(BOLD_SPAN, 0, len, Spanned.SPAN_INCLUSIVE_EXCLUSIVE); } - if (isAutoCorrect && (option & AUTO_CORRECT_UNDERLINE) != 0) { + if (isAutoCorrection && (options & AUTO_CORRECT_UNDERLINE) != 0) { spannedWord.setSpan(UNDERLINE_SPAN, 0, len, Spanned.SPAN_INCLUSIVE_EXCLUSIVE); } return spannedWord; @@ -242,22 +244,23 @@ final class SuggestionStripLayoutHelper { return indexInSuggestedWords; } - private int getSuggestionTextColor(final int indexInSuggestedWords, - final SuggestedWords suggestedWords) { + private int getSuggestionTextColor(final SuggestedWords suggestedWords, + final int indexInSuggestedWords) { final int positionInStrip = getPositionInSuggestionStrip(indexInSuggestedWords, suggestedWords); - // TODO: Need to revisit this logic with bigram suggestions - final boolean isSuggested = (indexInSuggestedWords != SuggestedWords.INDEX_OF_TYPED_WORD); + // Use identity for strings, not #equals : it's the typed word if it's the same object + final boolean isTypedWord = + suggestedWords.getWord(indexInSuggestedWords) == suggestedWords.mTypedWord; final int color; if (positionInStrip == mCenterPositionInStrip && suggestedWords.mWillAutoCorrect) { color = mColorAutoCorrect; - } else if (positionInStrip == mCenterPositionInStrip && suggestedWords.mTypedWordValid) { + } else if (isTypedWord && suggestedWords.mTypedWordValid) { color = mColorValidTypedWord; - } else if (isSuggested) { - color = mColorSuggested; - } else { + } else if (isTypedWord) { color = mColorTypedWord; + } else { + color = mColorSuggested; } if (LatinImeLogger.sDBG && suggestedWords.size() > 1) { // If we auto-correct, then the autocorrection is in slot 0 and the typed word @@ -270,7 +273,7 @@ final class SuggestionStripLayoutHelper { } } - if (suggestedWords.mIsObsoleteSuggestions && isSuggested) { + if (suggestedWords.mIsObsoleteSuggestions && !isTypedWord) { return applyAlpha(color, mAlphaObsoleted); } return color; @@ -438,7 +441,7 @@ final class SuggestionStripLayoutHelper { // {@link SuggestionStripView#onClick(View)}. wordView.setTag(indexInSuggestedWords); wordView.setText(getStyledSuggestedWord(suggestedWords, indexInSuggestedWords)); - wordView.setTextColor(getSuggestionTextColor(positionInStrip, suggestedWords)); + wordView.setTextColor(getSuggestionTextColor(suggestedWords, indexInSuggestedWords)); if (SuggestionStripView.DBG) { mDebugInfoViews.get(positionInStrip).setText( suggestedWords.getDebugString(indexInSuggestedWords)); diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java index 60f1c7a4e..52708455e 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripViewAccessor.java @@ -22,7 +22,6 @@ import com.android.inputmethod.latin.SuggestedWords; * An object that gives basic control of a suggestion strip and some info on it. */ public interface SuggestionStripViewAccessor { - public boolean hasSuggestionStripView(); public void showAddToDictionaryHint(final String word); public boolean isShowingAddToDictionaryHint(); public void dismissAddToDictionaryHint(); diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java index a1d641508..562ff9e8d 100644 --- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java +++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java @@ -37,9 +37,9 @@ public final class LanguageModelParam { // non-0. Thus, it's not meaningful to compare 10, 100, and so on. // TODO: Revise the logic in ForgettingCurveUtils in native code. private static final int UNIGRAM_PROBABILITY_FOR_VALID_WORD = 100; - private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = 10; - private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 0; - private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = 0; + private static final int UNIGRAM_PROBABILITY_FOR_OOV_WORD = Dictionary.NOT_A_PROBABILITY; + private static final int BIGRAM_PROBABILITY_FOR_VALID_WORD = 10; + private static final int BIGRAM_PROBABILITY_FOR_OOV_WORD = Dictionary.NOT_A_PROBABILITY; public final String mTargetWord; public final int[] mWord0; diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h index 59748c80d..a8dab9fcd 100644 --- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h @@ -44,8 +44,6 @@ class DictionaryHeaderStructurePolicy { virtual float getMultiWordCostMultiplier() const = 0; - virtual int getLastDecayedTime() const = 0; - virtual void readHeaderValueOrQuestionMark(const char *const key, int *outValue, int outValueSize) const = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp index cd2243025..7f916677a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp @@ -37,7 +37,8 @@ void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const out if (outProbability) { if (bigramEntry.hasHistoricalInfo()) { *outProbability = - ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()); + ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo(), + mHeaderPolicy); } else { *outProbability = bigramEntry.getProbability(); } @@ -160,7 +161,7 @@ bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const i } } else if (bigramEntry.hasHistoricalInfo()) { const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( - bigramEntry.getHistoricalInfo()); + bigramEntry.getHistoricalInfo(), mHeaderPolicy); if (ForgettingCurveUtils::needsToKeep(&historicalInfo)) { const BigramEntry updatedBigramEntry = bigramEntry.updateHistoricalInfoAndGetEntry(&historicalInfo); @@ -230,7 +231,8 @@ const BigramEntry Ver4BigramListPolicy::createUpdatedBigramEntryFrom( if (mHeaderPolicy->hasHistoricalInfoOfWords()) { const HistoricalInfo updatedHistoricalInfo = ForgettingCurveUtils::createUpdatedHistoricalInfo( - originalBigramEntry->getHistoricalInfo(), newProbability, timestamp); + originalBigramEntry->getHistoricalInfo(), newProbability, timestamp, + mHeaderPolicy); return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo); } else { return originalBigramEntry->updateProbabilityAndGetEntry(newProbability); diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 3ce57d910..2ac417b33 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -18,7 +18,7 @@ namespace latinime { -// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. +// Note that these are corresponding definitions in Java side in DictionaryHeader. const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE"; const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = "REQUIRES_GERMAN_UMLAUT_PROCESSING"; @@ -33,8 +33,15 @@ const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE // count. const char *const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO"; const char *const HeaderPolicy::LOCALE_KEY = "locale"; // match Java declaration +const char *const HeaderPolicy::FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY = + "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP"; +const char *const HeaderPolicy::FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY = + "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID"; + const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100; const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f; +const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP = 4; +const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID = 0; // Used for logging. Question mark is used to indicate that the key is not found. void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *outValue, diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index fc347618c..8fa7e168c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -52,7 +52,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)), mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue( - &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {} + &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)), + mForgettingCurveOccurrencesToLevelUp(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY, + DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)), + mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY, + DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)) {} // Constructs header information using an attribute map. HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion, @@ -71,8 +77,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { DATE_KEY, TimeKeeper::peekCurrentTime() /* defaultValue */)), mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0), mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue( - &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) { - } + &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)), + mForgettingCurveOccurrencesToLevelUp(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY, + DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)), + mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY, + DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)) {} // Temporary dummy header. HeaderPolicy() @@ -80,7 +91,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mAttributeMap(), mLocale(CharUtils::EMPTY_STRING), mMultiWordCostMultiplier(0.0f), mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false), mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), - mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {} + mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false), + mForgettingCurveOccurrencesToLevelUp(0), mForgettingCurveProbabilityValuesTableId(0) {} ~HeaderPolicy() {} @@ -159,6 +171,14 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return &mAttributeMap; } + AK_FORCE_INLINE int getForgettingCurveOccurrencesToLevelUp() const { + return mForgettingCurveOccurrencesToLevelUp; + } + + AK_FORCE_INLINE int getForgettingCurveProbabilityValuesTableId() const { + return mForgettingCurveProbabilityValuesTableId; + } + void readHeaderValueOrQuestionMark(const char *const key, int *outValue, int outValueSize) const; @@ -183,8 +203,12 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { static const char *const EXTENDED_REGION_SIZE_KEY; static const char *const HAS_HISTORICAL_INFO_KEY; static const char *const LOCALE_KEY; + static const char *const FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY; + static const char *const FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY; static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE; static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE; + static const int DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP; + static const int DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID; const FormatUtils::FORMAT_VERSION mDictFormatVersion; const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags; @@ -200,6 +224,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const int mBigramCount; const int mExtendedRegionSize; const bool mHasHistoricalInfoOfWords; + const int mForgettingCurveOccurrencesToLevelUp; + const int mForgettingCurveProbabilityValuesTableId; const std::vector<int> readLocale() const; float readMultipleWordCostMultiplier() const; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp index cb9d450ec..279f5b33a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp @@ -23,6 +23,13 @@ namespace latinime { const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition( int *const bigramEntryPos) const { const BufferWithExtendableBuffer *const bigramListBuffer = getContentBuffer(); + if (*bigramEntryPos < 0 || *bigramEntryPos >= bigramListBuffer->getTailPosition()) { + AKLOGE("Invalid bigram entry position. bigramEntryPos: %d, bufSize: %d", + *bigramEntryPos, bigramListBuffer->getTailPosition()); + ASSERT(false); + return BigramEntry(false /* hasNext */, NOT_A_PROBABILITY, + Ver4DictConstants::NOT_A_TERMINAL_ID); + } const int bigramFlags = bigramListBuffer->readUintAndAdvancePosition( Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, bigramEntryPos); const bool hasNext = (bigramFlags & Ver4DictConstants::BIGRAM_HAS_NEXT_MASK) != 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp index 29972a4e8..64d7bc0a5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp @@ -24,6 +24,19 @@ void ShortcutDictContent::getShortcutEntryAndAdvancePosition(const int maxCodePo int *const outCodePoint, int *const outCodePointCount, int *const outProbability, bool *const outhasNext, int *const shortcutEntryPos) const { const BufferWithExtendableBuffer *const shortcutListBuffer = getContentBuffer(); + if (*shortcutEntryPos < 0 || *shortcutEntryPos >= shortcutListBuffer->getTailPosition()) { + AKLOGE("Invalid shortcut entry position. shortcutEntryPos: %d, bufSize: %d", + *shortcutEntryPos, shortcutListBuffer->getTailPosition()); + ASSERT(false); + if (outhasNext) { + *outhasNext = false; + } + if (outCodePointCount) { + *outCodePointCount = 0; + } + return; + } + const int shortcutFlags = shortcutListBuffer->readUintAndAdvancePosition( Ver4DictConstants::SHORTCUT_FLAGS_FIELD_SIZE, shortcutEntryPos); if (outProbability) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 17fc9483b..f149781f4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -65,7 +65,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce mProbabilityDictContent->getProbabilityEntry(terminalId); if (probabilityEntry.hasHistoricalInfo()) { probability = ForgettingCurveUtils::decodeProbability( - probabilityEntry.getHistoricalInfo()); + probabilityEntry.getHistoricalInfo(), mHeaderPolicy); } else { probability = probabilityEntry.getProbability(); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h index 9d932457c..1db9ea026 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h @@ -26,6 +26,7 @@ namespace latinime { class BufferWithExtendableBuffer; +class HeaderPolicy; class ProbabilityDictContent; /* @@ -35,8 +36,10 @@ class ProbabilityDictContent; class Ver4PatriciaTrieNodeReader : public PtNodeReader { public: Ver4PatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer, - const ProbabilityDictContent *const probabilityDictContent) - : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent) {} + const ProbabilityDictContent *const probabilityDictContent, + const HeaderPolicy *const headerPolicy) + : mBuffer(buffer), mProbabilityDictContent(probabilityDictContent), + mHeaderPolicy(headerPolicy) {} ~Ver4PatriciaTrieNodeReader() {} @@ -50,6 +53,7 @@ class Ver4PatriciaTrieNodeReader : public PtNodeReader { const BufferWithExtendableBuffer *const mBuffer; const ProbabilityDictContent *const mProbabilityDictContent; + const HeaderPolicy *const mHeaderPolicy; const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos, const int siblingNodePos) const; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 32576cf0a..13ae9d923 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -159,7 +159,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA toBeUpdatedPtNodeParams->getTerminalId()); if (originalProbabilityEntry.hasHistoricalInfo()) { const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( - originalProbabilityEntry.getHistoricalInfo()); + originalProbabilityEntry.getHistoricalInfo(), mHeaderPolicy); const ProbabilityEntry probabilityEntry = originalProbabilityEntry.createEntryWithUpdatedHistoricalInfo(&historicalInfo); if (!mBuffers->getMutableProbabilityDictContent()->setProbabilityEntry( @@ -382,10 +382,11 @@ const ProbabilityEntry Ver4PatriciaTrieNodeWriter::createUpdatedEntryFrom( const ProbabilityEntry *const originalProbabilityEntry, const int newProbability, const int timestamp) const { // TODO: Consolidate historical info and probability. - if (mBuffers->getHeaderPolicy()->hasHistoricalInfoOfWords()) { + if (mHeaderPolicy->hasHistoricalInfoOfWords()) { const HistoricalInfo updatedHistoricalInfo = ForgettingCurveUtils::createUpdatedHistoricalInfo( - originalProbabilityEntry->getHistoricalInfo(), newProbability, timestamp); + originalProbabilityEntry->getHistoricalInfo(), newProbability, timestamp, + mHeaderPolicy); return originalProbabilityEntry->createEntryWithUpdatedHistoricalInfo( &updatedHistoricalInfo); } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index 66845bbd6..f01b3af0e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -28,6 +28,7 @@ namespace latinime { class BufferWithExtendableBuffer; +class HeaderPolicy; class Ver4BigramListPolicy; class Ver4DictBuffers; class Ver4PatriciaTrieNodeReader; @@ -40,10 +41,11 @@ class Ver4ShortcutListPolicy; class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { public: Ver4PatriciaTrieNodeWriter(BufferWithExtendableBuffer *const trieBuffer, - Ver4DictBuffers *const buffers, const PtNodeReader *const ptNodeReader, + Ver4DictBuffers *const buffers, const HeaderPolicy *const headerPolicy, + const PtNodeReader *const ptNodeReader, const PtNodeArrayReader *const ptNodeArrayReader, Ver4BigramListPolicy *const bigramPolicy, Ver4ShortcutListPolicy *const shortcutPolicy) - : mTrieBuffer(trieBuffer), mBuffers(buffers), + : mTrieBuffer(trieBuffer), mBuffers(buffers), mHeaderPolicy(headerPolicy), mReadingHelper(ptNodeReader, ptNodeArrayReader), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy) {} @@ -116,6 +118,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { BufferWithExtendableBuffer *const mTrieBuffer; Ver4DictBuffers *const mBuffers; + const HeaderPolicy *const mHeaderPolicy; DynamicPtReadingHelper mReadingHelper; Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index b5d80be1d..197250ff3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -382,7 +382,8 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code bigramWord1CodePoints + codePointCount); const HistoricalInfo *const historicalInfo = bigramEntry.getHistoricalInfo(); const int probability = bigramEntry.hasHistoricalInfo() ? - ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()) : + ForgettingCurveUtils::decodeProbability( + bigramEntry.getHistoricalInfo(), mHeaderPolicy) : bigramEntry.getProbability(); bigrams.push_back(WordProperty::BigramProperty(&word1, probability, historicalInfo->getTimeStamp(), historicalInfo->getLevel(), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 7796e2ddc..639c153a1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -47,10 +47,10 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy), mShortcutPolicy(mBuffers.get()->getMutableShortcutDictContent(), mBuffers.get()->getTerminalPositionLookupTable()), - mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()), + mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent(), mHeaderPolicy), mPtNodeArrayReader(mDictBuffer), - mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mPtNodeArrayReader, - &mBigramPolicy, &mShortcutPolicy), + mNodeWriter(mDictBuffer, mBuffers.get(), mHeaderPolicy, &mNodeReader, + &mPtNodeArrayReader, &mBigramPolicy, &mShortcutPolicy), mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter), mWritingHelper(mBuffers.get()), mUnigramCount(mHeaderPolicy->getUnigramCount()), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 93053c38d..2da295054 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -74,14 +74,15 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount, int *const outBigramCount) { Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(), - mBuffers->getProbabilityDictContent()); + mBuffers->getProbabilityDictContent(), headerPolicy); Ver4PtNodeArrayReader ptNodeArrayReader(mBuffers->getTrieBuffer()); Ver4BigramListPolicy bigramPolicy(mBuffers->getMutableBigramDictContent(), mBuffers->getTerminalPositionLookupTable(), headerPolicy); Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getMutableShortcutDictContent(), mBuffers->getTerminalPositionLookupTable()); Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), - mBuffers, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, &shortcutPolicy); + mBuffers, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, + &shortcutPolicy); DynamicPtReadingHelper readingHelper(&ptNodeReader, &ptNodeArrayReader); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); @@ -126,7 +127,8 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap; readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(), - buffersToWrite, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, &shortcutPolicy); + buffersToWrite, headerPolicy, &ptNodeReader, &ptNodeArrayReader, &bigramPolicy, + &shortcutPolicy); DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers, buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap); @@ -137,14 +139,14 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, // Create policy instances for the GCed dictionary. Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(), - buffersToWrite->getProbabilityDictContent()); + buffersToWrite->getProbabilityDictContent(), headerPolicy); Ver4PtNodeArrayReader newPtNodeArrayreader(buffersToWrite->getTrieBuffer()); Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getMutableBigramDictContent(), buffersToWrite->getTerminalPositionLookupTable(), headerPolicy); Ver4ShortcutListPolicy newShortcutPolicy(buffersToWrite->getMutableShortcutDictContent(), buffersToWrite->getTerminalPositionLookupTable()); Ver4PatriciaTrieNodeWriter newPtNodeWriter(buffersToWrite->getWritableTrieBuffer(), - buffersToWrite, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy, + buffersToWrite, headerPolicy, &newPtNodeReader, &newPtNodeArrayreader, &newBigramPolicy, &newShortcutPolicy); // Re-assign terminal IDs for valid terminal PtNodes. TerminalPositionLookupTable::TerminalIdMap terminalIdMap; @@ -202,8 +204,9 @@ bool Ver4PatriciaTrieWritingHelper::truncateUnigrams( const ProbabilityEntry probabilityEntry = mBuffers->getProbabilityDictContent()->getProbabilityEntry(i); const int probability = probabilityEntry.hasHistoricalInfo() ? - ForgettingCurveUtils::decodeProbability(probabilityEntry.getHistoricalInfo()) : - probabilityEntry.getProbability(); + ForgettingCurveUtils::decodeProbability( + probabilityEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) : + probabilityEntry.getProbability(); priorityQueue.push(DictProbability(terminalPos, probability, probabilityEntry.getHistoricalInfo()->getTimeStamp())); } @@ -245,8 +248,9 @@ bool Ver4PatriciaTrieWritingHelper::truncateBigrams(const int maxBigramCount) { continue; } const int probability = bigramEntry.hasHistoricalInfo() ? - ForgettingCurveUtils::decodeProbability(bigramEntry.getHistoricalInfo()) : - bigramEntry.getProbability(); + ForgettingCurveUtils::decodeProbability( + bigramEntry.getHistoricalInfo(), mBuffers->getHeaderPolicy()) : + bigramEntry.getProbability(); priorityQueue.push(DictProbability(entryPos, probability, bigramEntry.getHistoricalInfo()->getTimeStamp())); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp index d58d25989..51b0eb23f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp @@ -19,7 +19,7 @@ #include <cmath> #include <stdlib.h> -#include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" #include "utils/time_keeper.h" @@ -30,11 +30,10 @@ const int ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC = 10000; const int ForgettingCurveUtils::MAX_BIGRAM_COUNT = 12000; const int ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000; -const int ForgettingCurveUtils::MAX_COMPUTED_PROBABILITY = 127; +const int ForgettingCurveUtils::MULTIPLIER_TWO_IN_PROBABILITY_SCALE = 8; const int ForgettingCurveUtils::DECAY_INTERVAL_SECONDS = 2 * 60 * 60; const int ForgettingCurveUtils::MAX_LEVEL = 3; -const int ForgettingCurveUtils::MAX_COUNT = 3; const int ForgettingCurveUtils::MIN_VALID_LEVEL = 1; const int ForgettingCurveUtils::TIME_STEP_DURATION_IN_SECONDS = 6 * 60 * 60; const int ForgettingCurveUtils::MAX_ELAPSED_TIME_STEP_COUNT = 15; @@ -45,7 +44,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT // TODO: Revise the logic to decide the initial probability depending on the given probability. /* static */ const HistoricalInfo ForgettingCurveUtils::createUpdatedHistoricalInfo( const HistoricalInfo *const originalHistoricalInfo, - const int newProbability, const int timestamp) { + const int newProbability, const int timestamp, const HeaderPolicy *const headerPolicy) { if (newProbability != NOT_A_PROBABILITY && originalHistoricalInfo->getLevel() == 0) { return HistoricalInfo(timestamp, MIN_VALID_LEVEL /* level */, 0 /* count */); } else if (!originalHistoricalInfo->isValid()) { @@ -53,7 +52,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT return HistoricalInfo(timestamp, 0 /* level */, 1 /* count */); } else { const int updatedCount = originalHistoricalInfo->getCount() + 1; - if (updatedCount > MAX_COUNT) { + if (updatedCount >= headerPolicy->getForgettingCurveOccurrencesToLevelUp()) { // The count exceeds the max value the level can be incremented. if (originalHistoricalInfo->getLevel() >= MAX_LEVEL) { // The level is already max. @@ -71,9 +70,10 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } /* static */ int ForgettingCurveUtils::decodeProbability( - const HistoricalInfo *const historicalInfo) { + const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy) { const int elapsedTimeStepCount = getElapsedTimeStepCount(historicalInfo->getTimeStamp()); - return sProbabilityTable.getProbability(historicalInfo->getLevel(), + return sProbabilityTable.getProbability( + headerPolicy->getForgettingCurveProbabilityValuesTableId(), historicalInfo->getLevel(), min(max(elapsedTimeStepCount, 0), MAX_ELAPSED_TIME_STEP_COUNT)); } @@ -82,9 +82,11 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT if (unigramProbability == NOT_A_PROBABILITY) { return NOT_A_PROBABILITY; } else if (bigramProbability == NOT_A_PROBABILITY) { - return min(backoff(unigramProbability), MAX_COMPUTED_PROBABILITY); + return min(backoff(unigramProbability), MAX_PROBABILITY); } else { - return min(max(unigramProbability, bigramProbability), MAX_COMPUTED_PROBABILITY); + // TODO: Investigate better way to handle bigram probability. + return min(max(unigramProbability, bigramProbability + MULTIPLIER_TWO_IN_PROBABILITY_SCALE), + MAX_PROBABILITY); } } @@ -95,7 +97,8 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } /* static */ const HistoricalInfo ForgettingCurveUtils::createHistoricalInfoToSave( - const HistoricalInfo *const originalHistoricalInfo) { + const HistoricalInfo *const originalHistoricalInfo, + const HeaderPolicy *const headerPolicy) { if (originalHistoricalInfo->getTimeStamp() == NOT_A_TIMESTAMP) { return HistoricalInfo(); } @@ -115,8 +118,7 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } /* static */ bool ForgettingCurveUtils::needsToDecay(const bool mindsBlockByDecay, - const int unigramCount, const int bigramCount, - const DictionaryHeaderStructurePolicy *const headerPolicy) { + const int unigramCount, const int bigramCount, const HeaderPolicy *const headerPolicy) { if (unigramCount >= ForgettingCurveUtils::MAX_UNIGRAM_COUNT) { // Unigram count exceeds the limit. return true; @@ -137,37 +139,69 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT // See comments in ProbabilityUtils::backoff(). /* static */ int ForgettingCurveUtils::backoff(const int unigramProbability) { - if (unigramProbability == NOT_A_PROBABILITY) { - return NOT_A_PROBABILITY; - } else { - return max(unigramProbability - 8, 0); - } + // See TODO comments in ForgettingCurveUtils::getProbability(). + return unigramProbability; } /* static */ int ForgettingCurveUtils::getElapsedTimeStepCount(const int timestamp) { return (TimeKeeper::peekCurrentTime() - timestamp) / TIME_STEP_DURATION_IN_SECONDS; } -ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTable() { - mTable.resize(MAX_LEVEL + 1); - for (int level = 0; level <= MAX_LEVEL; ++level) { - mTable[level].resize(MAX_ELAPSED_TIME_STEP_COUNT + 1); - const float initialProbability = - static_cast<float>(MAX_COMPUTED_PROBABILITY / (1 << (MAX_LEVEL - level))); - for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT; ++timeStepCount) { - if (level == 0) { - mTable[level][timeStepCount] = NOT_A_PROBABILITY; - continue; +const int ForgettingCurveUtils::ProbabilityTable::PROBABILITY_TABLE_COUNT = 4; +const int ForgettingCurveUtils::ProbabilityTable::WEAK_PROBABILITY_TABLE_ID = 0; +const int ForgettingCurveUtils::ProbabilityTable::MODEST_PROBABILITY_TABLE_ID = 1; +const int ForgettingCurveUtils::ProbabilityTable::STRONG_PROBABILITY_TABLE_ID = 2; +const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_PROBABILITY_TABLE_ID = 3; +const int ForgettingCurveUtils::ProbabilityTable::WEAK_MAX_PROBABILITY = 127; +const int ForgettingCurveUtils::ProbabilityTable::MODEST_BASE_PROBABILITY = 32; +const int ForgettingCurveUtils::ProbabilityTable::STRONG_BASE_PROBABILITY = 35; +const int ForgettingCurveUtils::ProbabilityTable::AGGRESSIVE_BASE_PROBABILITY = 40; + + +ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTables() { + mTables.resize(PROBABILITY_TABLE_COUNT); + for (int tableId = 0; tableId < PROBABILITY_TABLE_COUNT; ++tableId) { + mTables[tableId].resize(MAX_LEVEL + 1); + for (int level = 0; level <= MAX_LEVEL; ++level) { + mTables[tableId][level].resize(MAX_ELAPSED_TIME_STEP_COUNT + 1); + const float initialProbability = getBaseProbabilityForLevel(tableId, level); + const float endProbability = getBaseProbabilityForLevel(tableId, level - 1); + for (int timeStepCount = 0; timeStepCount <= MAX_ELAPSED_TIME_STEP_COUNT; + ++timeStepCount) { + if (level == 0) { + mTables[tableId][level][timeStepCount] = NOT_A_PROBABILITY; + continue; + } + const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS; + const float probability = initialProbability + * powf(initialProbability / endProbability, + -1.0f * static_cast<float>(elapsedTime) + / static_cast<float>(TIME_STEP_DURATION_IN_SECONDS + * (MAX_ELAPSED_TIME_STEP_COUNT + 1))); + mTables[tableId][level][timeStepCount] = + min(max(static_cast<int>(probability), 1), MAX_PROBABILITY); } - const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS; - const float probability = initialProbability - * powf(2.0f, -1.0f * static_cast<float>(elapsedTime) - / static_cast<float>(TIME_STEP_DURATION_IN_SECONDS - * (MAX_ELAPSED_TIME_STEP_COUNT + 1))); - mTable[level][timeStepCount] = - min(max(static_cast<int>(probability), 1), MAX_COMPUTED_PROBABILITY); } } } +/* static */ int ForgettingCurveUtils::ProbabilityTable::getBaseProbabilityForLevel( + const int tableId, const int level) { + if (tableId == WEAK_PROBABILITY_TABLE_ID) { + // Max probability is 127. + return static_cast<float>(WEAK_MAX_PROBABILITY / (1 << (MAX_LEVEL - level))); + } else if (tableId == MODEST_PROBABILITY_TABLE_ID) { + // Max probability is 128. + return static_cast<float>(MODEST_BASE_PROBABILITY * (level + 1)); + } else if (tableId == STRONG_PROBABILITY_TABLE_ID) { + // Max probability is 140. + return static_cast<float>(STRONG_BASE_PROBABILITY * (level + 1)); + } else if (tableId == AGGRESSIVE_PROBABILITY_TABLE_ID) { + // Max probability is 160. + return static_cast<float>(AGGRESSIVE_BASE_PROBABILITY * (level + 1)); + } else { + return NOT_A_PROBABILITY; + } +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h index b37353455..1a285e573 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h @@ -24,7 +24,7 @@ namespace latinime { -class DictionaryHeaderStructurePolicy; +class HeaderPolicy; class ForgettingCurveUtils { public: @@ -35,12 +35,14 @@ class ForgettingCurveUtils { static const HistoricalInfo createUpdatedHistoricalInfo( const HistoricalInfo *const originalHistoricalInfo, const int newProbability, - const int timestamp); + const int timestamp, const HeaderPolicy *const headerPolicy); static const HistoricalInfo createHistoricalInfoToSave( - const HistoricalInfo *const originalHistoricalInfo); + const HistoricalInfo *const originalHistoricalInfo, + const HeaderPolicy *const headerPolicy); - static int decodeProbability(const HistoricalInfo *const historicalInfo); + static int decodeProbability(const HistoricalInfo *const historicalInfo, + const HeaderPolicy *const headerPolicy); static int getProbability(const int encodedUnigramProbability, const int encodedBigramProbability); @@ -48,7 +50,7 @@ class ForgettingCurveUtils { static bool needsToKeep(const HistoricalInfo *const historicalInfo); static bool needsToDecay(const bool mindsBlockByDecay, const int unigramCount, - const int bigramCount, const DictionaryHeaderStructurePolicy *const headerPolicy); + const int bigramCount, const HeaderPolicy *const headerPolicy); private: DISALLOW_IMPLICIT_CONSTRUCTORS(ForgettingCurveUtils); @@ -57,21 +59,34 @@ class ForgettingCurveUtils { public: ProbabilityTable(); - int getProbability(const int level, const int elapsedTimeStepCount) const { - return mTable[level][elapsedTimeStepCount]; + int getProbability(const int tableId, const int level, + const int elapsedTimeStepCount) const { + return mTables[tableId][level][elapsedTimeStepCount]; } private: DISALLOW_COPY_AND_ASSIGN(ProbabilityTable); - std::vector<std::vector<int> > mTable; + static const int PROBABILITY_TABLE_COUNT; + static const int WEAK_PROBABILITY_TABLE_ID; + static const int MODEST_PROBABILITY_TABLE_ID; + static const int STRONG_PROBABILITY_TABLE_ID; + static const int AGGRESSIVE_PROBABILITY_TABLE_ID; + + static const int WEAK_MAX_PROBABILITY; + static const int MODEST_BASE_PROBABILITY; + static const int STRONG_BASE_PROBABILITY; + static const int AGGRESSIVE_BASE_PROBABILITY; + + std::vector<std::vector<std::vector<int> > > mTables; + + static int getBaseProbabilityForLevel(const int tableId, const int level); }; - static const int MAX_COMPUTED_PROBABILITY; + static const int MULTIPLIER_TWO_IN_PROBABILITY_SCALE; static const int DECAY_INTERVAL_SECONDS; static const int MAX_LEVEL; - static const int MAX_COUNT; static const int MIN_VALID_LEVEL; static const int TIME_STEP_DURATION_IN_SECONDS; static const int MAX_ELAPSED_TIME_STEP_COUNT; |