diff options
Diffstat (limited to 'java/src')
20 files changed, 443 insertions, 244 deletions
diff --git a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java index 668eb925b..743bc8037 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java +++ b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java @@ -47,7 +47,7 @@ public class MetadataDbHelper extends SQLiteOpenHelper { // used to identify the versions for upgrades. This should never change going forward. private static final int METADATA_DATABASE_VERSION_WITH_CLIENTID = 6; // The current database version. - private static final int CURRENT_METADATA_DATABASE_VERSION = 8; + private static final int CURRENT_METADATA_DATABASE_VERSION = 9; private final static long NOT_A_DOWNLOAD_ID = -1; diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index e7ab02ac1..b77540622 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -191,7 +191,8 @@ public final class BinaryDictionary extends Dictionary { private static native void closeNative(long dict); private static native int getFormatVersionNative(long dict); private static native int getProbabilityNative(long dict, int[] word); - private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1); + private static native int getBigramProbabilityNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1); private static native void getWordPropertyNative(long dict, int[] word, int[] outCodePoints, boolean[] outFlags, int[] outProbabilityInfo, ArrayList<int[]> outBigramTargets, ArrayList<int[]> outBigramProbabilityInfo, @@ -200,15 +201,17 @@ public final class BinaryDictionary extends Dictionary { private static native void getSuggestionsNative(long dict, long proximityInfo, long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, - int[] prevWordCodePointArray, int[] outputSuggestionCount, int[] outputCodePoints, - int[] outputScores, int[] outputIndices, int[] outputTypes, - int[] outputAutoCommitFirstWordConfidence, float[] inOutLanguageWeight); + int[] prevWordCodePointArray, boolean isBeginningOfSentence, + int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores, + int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence, + float[] inOutLanguageWeight); private static native void addUnigramWordNative(long dict, int[] word, int probability, - int[] shortcutTarget, int shortcutProbability, boolean isNotAWord, - boolean isBlacklisted, int timestamp); - private static native void addBigramWordsNative(long dict, int[] word0, int[] word1, - int probability, int timestamp); - private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); + int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, + boolean isNotAWord, boolean isBlacklisted, int timestamp); + private static native void addBigramWordsNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1, int probability, int timestamp); + private static native void removeBigramWordsNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1); private static native int addMultipleDictionaryEntriesNative(long dict, LanguageModelParam[] languageModelParams, int startIndex); private static native String getPropertyNative(long dict, String query); @@ -301,7 +304,8 @@ public final class BinaryDictionary extends Dictionary { getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), inputPointers.getYCoordinates(), inputPointers.getTimes(), inputPointers.getPointerIds(), mInputCodePoints, inputSize, - mNativeSuggestOptions.getOptions(), prevWordCodePointArray, mOutputSuggestionCount, + mNativeSuggestOptions.getOptions(), prevWordCodePointArray, + prevWordsInfo.mIsBeginningOfSentence, mOutputSuggestionCount, mOutputCodePoints, mOutputScores, mSpaceIndices, mOutputTypes, mOutputAutoCommitFirstWordConfidence, mInputOutputLanguageWeight); if (inOutLanguageWeight != null) { @@ -364,12 +368,13 @@ public final class BinaryDictionary extends Dictionary { } public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) { - if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return NOT_A_PROBABILITY; } final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); final int[] codePoints1 = StringUtils.toCodePointArray(word); - return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); + return getBigramProbabilityNative(mNativeDict, codePoints0, + prevWordsInfo.mIsBeginningOfSentence, codePoints1); } public WordProperty getWordProperty(final String word) { @@ -420,16 +425,17 @@ public final class BinaryDictionary extends Dictionary { // Add a unigram entry to binary dictionary with unigram attributes in native code. public void addUnigramEntry(final String word, final int probability, - final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord, + final String shortcutTarget, final int shortcutProbability, + final boolean isBeginningOfSentence, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { - if (TextUtils.isEmpty(word)) { + if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { return; } final int[] codePoints = StringUtils.toCodePointArray(word); final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? StringUtils.toCodePointArray(shortcutTarget) : null; addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, - shortcutProbability, isNotAWord, isBlacklisted, timestamp); + shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp); mHasUpdated = true; } @@ -437,23 +443,25 @@ public final class BinaryDictionary extends Dictionary { public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word, final int probability, final int timestamp) { - if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return; } final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); final int[] codePoints1 = StringUtils.toCodePointArray(word); - addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp); + addBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, + codePoints1, probability, timestamp); mHasUpdated = true; } // Remove an n-gram entry from the binary dictionary in native code. public void removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) { - if (TextUtils.isEmpty(prevWordsInfo.mPrevWord) || TextUtils.isEmpty(word)) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return; } final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); final int[] codePoints1 = StringUtils.toCodePointArray(word); - removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); + removeBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, + codePoints1); mHasUpdated = true; } diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index 67ca59540..efc5a618b 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -192,7 +192,6 @@ public final class Constants { public static final int CODE_SPACE = ' '; public static final int CODE_PERIOD = '.'; public static final int CODE_COMMA = ','; - public static final int CODE_ARMENIAN_PERIOD = 0x0589; public static final int CODE_DASH = '-'; public static final int CODE_SINGLE_QUOTE = '\''; public static final int CODE_DOUBLE_QUOTE = '"'; @@ -208,6 +207,8 @@ public final class Constants { public static final int CODE_CLOSING_SQUARE_BRACKET = ']'; public static final int CODE_CLOSING_CURLY_BRACKET = '}'; public static final int CODE_CLOSING_ANGLE_BRACKET = '>'; + public static final int CODE_INVERTED_QUESTION_MARK = 0xBF; // ¿ + public static final int CODE_INVERTED_EXCLAMATION_MARK = 0xA1; // ¡ /** * Special keys code. Must be negative. diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index 3fb76b142..538bfc06f 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -224,7 +224,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { */ private void addNameLocked(final String name) { int len = StringUtils.codePointCount(name); - PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null); + PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; // TODO: Better tokenization for non-Latin writing systems for (int i = 0; i < len; i++) { if (Character.isLetter(name.codePointAt(i))) { diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index a97916733..212363895 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -19,14 +19,18 @@ package com.android.inputmethod.latin; import android.content.Context; import android.text.TextUtils; import android.util.Log; +import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.ContextualDictionary; +import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.UserHistoryDictionary; +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.DistracterFilter; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.SuggestionResults; @@ -37,6 +41,7 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -57,6 +62,7 @@ public class DictionaryFacilitator { private volatile CountDownLatch mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0); // To synchronize assigning mDictionaries to ensure closing dictionaries. private final Object mLock = new Object(); + private final DistracterFilter mDistracterFilter; private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTION = new String[] { @@ -162,7 +168,17 @@ public class DictionaryFacilitator { public void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable); } - public DictionaryFacilitator() {} + public DictionaryFacilitator() { + mDistracterFilter = new DistracterFilter.EmptyDistracterFilter(); + } + + public DictionaryFacilitator(final DistracterFilter distracterFilter) { + mDistracterFilter = distracterFilter; + } + + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { + mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); + } public Locale getLocale() { return mDictionaries.mLocale; @@ -321,6 +337,7 @@ public class DictionaryFacilitator { for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTION) { dictionaries.closeDict(dictType); } + mDistracterFilter.close(); } // The main dictionary could have been loaded asynchronously. Don't cache the return value @@ -537,9 +554,16 @@ public class DictionaryFacilitator { personalizationDict.clear(); } - public void addMultipleDictionaryEntriesToPersonalizationDictionary( - final ArrayList<LanguageModelParam> languageModelParams, + public void addEntriesToPersonalizationDictionary( + final PersonalizationDataChunk personalizationDataChunk, + final SpacingAndPunctuations spacingAndPunctuations, final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { + final ArrayList<LanguageModelParam> languageModelParams = + LanguageModelParam.createLanguageModelParamsFrom( + personalizationDataChunk.mTokens, + personalizationDataChunk.mTimestampInSeconds, + this /* dictionaryFacilitator */, spacingAndPunctuations, + mDistracterFilter); final ExpandableBinaryDictionary personalizationDict = mDictionaries.getSubDict(Dictionary.TYPE_PERSONALIZATION); if (personalizationDict == null || languageModelParams == null diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index d67253c3b..2cbce045d 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -114,7 +114,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private boolean needsToMigrateDictionary(final int formatVersion) { // When we bump up the dictionary format version, the old version should be added to here // for supporting migration. Note that native code has to support reading such formats. - return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING; + return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING + || formatVersion == FormatSpec.VERSION401; } public boolean isValidDictionaryLocked() { @@ -292,7 +293,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq, - isNotAWord, isBlacklisted, timestamp); + false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, timestamp); } /** diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 4863326ab..34d5f714c 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -81,9 +81,10 @@ import com.android.inputmethod.latin.suggestions.SuggestionStripView; import com.android.inputmethod.latin.suggestions.SuggestionStripViewAccessor; import com.android.inputmethod.latin.utils.ApplicationUtils; import com.android.inputmethod.latin.utils.CapsModeUtils; +import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CoordinateUtils; import com.android.inputmethod.latin.utils.DialogUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; +import com.android.inputmethod.latin.utils.DistracterFilterUsingSuggestion; import com.android.inputmethod.latin.utils.ImportantNoticeUtils; import com.android.inputmethod.latin.utils.IntentUtils; import com.android.inputmethod.latin.utils.JniUtils; @@ -95,6 +96,7 @@ import com.android.inputmethod.research.ResearchLogger; import java.io.FileDescriptor; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -122,7 +124,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private static final String SCHEME_PACKAGE = "package"; private final Settings mSettings; - private final DictionaryFacilitator mDictionaryFacilitator = new DictionaryFacilitator(); + private final DictionaryFacilitator mDictionaryFacilitator = + new DictionaryFacilitator(new DistracterFilterUsingSuggestion(this /* context */)); private final InputLogic mInputLogic = new InputLogic(this /* LatinIME */, this /* SuggestionStripViewAccessor */, mDictionaryFacilitator); // We expect to have only one decoder in almost all cases, hence the default capacity of 1. @@ -538,6 +541,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!mHandler.hasPendingReopenDictionaries()) { resetSuggestForLocale(locale); } + mDictionaryFacilitator.updateEnabledSubtypes(mRichImm.getMyEnabledInputMethodSubtypeList( + true /* allowsImplicitlySelectedSubtypes */)); refreshPersonalizationDictionarySession(); StatsUtils.onLoadSettings(currentSettingsValues); } @@ -564,9 +569,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen PersonalizationHelper.removeAllPersonalizationDictionaries(this); PersonalizationDictionarySessionRegistrar.resetAll(this); } else { - final DistracterFilter distracterFilter = createDistracterFilter(); - PersonalizationDictionarySessionRegistrar.init( - this, mDictionaryFacilitator, distracterFilter); + PersonalizationDictionarySessionRegistrar.init(this, mDictionaryFacilitator); } } @@ -660,9 +663,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mInputLogic.mConnection.finishComposingText(); mInputLogic.mConnection.endBatchEdit(); } - final DistracterFilter distracterFilter = createDistracterFilter(); PersonalizationDictionarySessionRegistrar.onConfigurationChanged(this, conf, - mDictionaryFacilitator, distracterFilter); + mDictionaryFacilitator); super.onConfigurationChanged(conf); } @@ -1739,11 +1741,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } @UsedForTesting - /* package for test */ DistracterFilter createDistracterFilter() { - // Return an empty distracter filter when this method is called before onCreate(). - return (mRichImm != null) ? new DistracterFilter(this /* Context */, - mRichImm.getMyEnabledInputMethodSubtypeList( - true /* allowsImplicitlySelectedSubtypes */)) : new DistracterFilter(); + /* package for test */ List<InputMethodSubtype> getEnabledSubtypesForTest() { + return (mRichImm != null) ? mRichImm.getMyEnabledInputMethodSubtypeList( + true /* allowsImplicitlySelectedSubtypes */) : new ArrayList<InputMethodSubtype>(); } public void dumpDictionaryForDebug(final String dictName) { diff --git a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java index ecc8947db..e44239f1d 100644 --- a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java +++ b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java @@ -16,23 +16,32 @@ package com.android.inputmethod.latin; -import android.util.Log; - +/** + * Class to represent information of previous words. This class is used to add n-gram entries + * into binary dictionaries, to get predictions, and to get suggestions. + */ // TODO: Support multiple previous words for n-gram. public class PrevWordsInfo { - // The previous word. May be null after resetting and before starting a new composing word, or - // when there is no context like at the start of text for example. It can also be set to null - // externally when the user enters a separator that does not let bigrams across, like a period - // or a comma. + public static final PrevWordsInfo EMPTY_PREV_WORDS_INFO = new PrevWordsInfo(null); + public static final PrevWordsInfo BEGINNING_OF_SENTENCE = new PrevWordsInfo(); + + // The word immediately before the considered word. null means we don't have any context + // including the "beginning of sentence context" - we just don't know what to predict. + // An example of that is after a comma. + // For simplicity of implementation, this may also be null transiently after the WordComposer + // was reset and before starting a new composing word, but we should never be calling + // getSuggetions* in this situation. + // This is an empty string when mIsBeginningOfSentence is true. public final String mPrevWord; // TODO: Have sentence separator. - // Whether the current context is beginning of sentence or not. + // Whether the current context is beginning of sentence or not. This is true when composing at + // the beginning of an input field or composing a word after a sentence separator. public final boolean mIsBeginningOfSentence; // Beginning of sentence. public PrevWordsInfo() { - mPrevWord = null; + mPrevWord = ""; mIsBeginningOfSentence = true; } @@ -40,4 +49,8 @@ public class PrevWordsInfo { mPrevWord = prevWord; mIsBeginningOfSentence = false; } + + public boolean isValid() { + return mPrevWord != null; + } } diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 2c54e10aa..e7c163606 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -542,7 +542,7 @@ public final class RichInputConnection { final SpacingAndPunctuations spacingAndPunctuations, final int n) { mIC = mParent.getCurrentInputConnection(); if (null == mIC) { - return new PrevWordsInfo(null); + return PrevWordsInfo.EMPTY_PREV_WORDS_INFO; } final CharSequence prev = getTextBeforeCursor(LOOKBACK_CHARACTER_NUM, 0); if (DEBUG_PREVIOUS_TEXT && null != prev) { @@ -588,30 +588,30 @@ public final class RichInputConnection { // (n = 2) "abc. def|" -> beginning-of-sentence public static PrevWordsInfo getPrevWordsInfoFromNthPreviousWord(final CharSequence prev, final SpacingAndPunctuations spacingAndPunctuations, final int n) { - if (prev == null) return new PrevWordsInfo(null); + if (prev == null) return PrevWordsInfo.EMPTY_PREV_WORDS_INFO; final String[] w = spaceRegex.split(prev); // If we can't find n words, or we found an empty word, the context is // beginning-of-sentence. if (w.length < n) { - return new PrevWordsInfo(); + return PrevWordsInfo.BEGINNING_OF_SENTENCE; } final String nthPrevWord = w[w.length - n]; final int length = nthPrevWord.length(); if (length <= 0) { - return new PrevWordsInfo(); + return PrevWordsInfo.BEGINNING_OF_SENTENCE; } // If ends in a sentence separator, the context is beginning-of-sentence. final char lastChar = nthPrevWord.charAt(length - 1); if (spacingAndPunctuations.isSentenceSeparator(lastChar)) { - new PrevWordsInfo(); + return PrevWordsInfo.BEGINNING_OF_SENTENCE; } // If ends in a word separator or connector, the context is unclear. // TODO: Return meaningful context for this case. if (spacingAndPunctuations.isWordSeparator(lastChar) || spacingAndPunctuations.isWordConnector(lastChar)) { - return new PrevWordsInfo(null); + return PrevWordsInfo.EMPTY_PREV_WORDS_INFO; } return new PrevWordsInfo(nthPrevWord); } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 1eccf2ca2..daa7f4b47 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.text.TextUtils; +import com.android.inputmethod.event.Event; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.define.ProductionFlag; diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index 6ecb37346..c53a8fda4 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -85,7 +85,7 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPrevWordsInfo = new PrevWordsInfo(null); + mPrevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; refreshTypedWordCache(); } @@ -117,7 +117,7 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPrevWordsInfo = new PrevWordsInfo(null); + mPrevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; refreshTypedWordCache(); } @@ -445,7 +445,7 @@ public final class WordComposer { // when the user inputs a separator that's not whitespace (including the case of the // double-space-to-period feature). public void discardPreviousWordForSuggestion() { - mPrevWordsInfo = new PrevWordsInfo(null); + mPrevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; } public void resumeSuggestionOnLastComposedWord(final LastComposedWord lastComposedWord, diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index 78d4bc807..237b43bfb 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -809,11 +809,10 @@ public final class InputLogic { final int codePoint = inputTransaction.mEvent.mCodePoint; final SettingsValues settingsValues = inputTransaction.mSettingsValues; boolean didAutoCorrect = false; - final boolean wasComposingWord = mWordComposer.isComposingWord(); // We avoid sending spaces in languages without spaces if we were composing. final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == codePoint && !settingsValues.mSpacingAndPunctuations.mCurrentLanguageHasSpaces - && wasComposingWord; + && mWordComposer.isComposingWord(); if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the separator at the current cursor position. @@ -857,7 +856,7 @@ public final class InputLogic { promotePhantomSpace(settingsValues); } if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleSeparator(codePoint, wasComposingWord); + ResearchLogger.latinIME_handleSeparator(codePoint, mWordComposer.isComposingWord()); } if (!shouldAvoidSendingCode) { @@ -873,9 +872,7 @@ public final class InputLogic { } startDoubleSpacePeriodCountdown(inputTransaction); - if (wasComposingWord) { - inputTransaction.setRequiresUpdateSuggestions(); - } + inputTransaction.setRequiresUpdateSuggestions(); } else { if (swapWeakSpace) { swapSwapperAndSpace(inputTransaction); @@ -1612,8 +1609,9 @@ public final class InputLogic { return mConnection.getPrevWordsInfoFromNthPreviousWord( spacingAndPunctuations, nthPreviousWord); } else { - return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? new PrevWordsInfo() - : new PrevWordsInfo(mLastComposedWord.mCommittedWord.toString()); + return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? + PrevWordsInfo.BEGINNING_OF_SENTENCE : + new PrevWordsInfo(mLastComposedWord.mCommittedWord.toString()); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index f5f072b7a..a2ae74b20 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -192,8 +192,9 @@ public final class FormatSpec { public static final int VERSION2 = 2; // Dictionary version used for testing. public static final int VERSION4_ONLY_FOR_TESTING = 399; - public static final int VERSION4 = 401; - public static final int VERSION4_DEV = 402; + public static final int VERSION401 = 401; + public static final int VERSION4 = 402; + public static final int VERSION4_DEV = 403; static final int MINIMUM_SUPPORTED_VERSION = VERSION2; static final int MAXIMUM_SUPPORTED_VERSION = VERSION4_DEV; diff --git a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java index 853392200..ed832510c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java +++ b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java @@ -35,6 +35,8 @@ public final class WordProperty implements Comparable<WordProperty> { public final ProbabilityInfo mProbabilityInfo; public final ArrayList<WeightedString> mShortcutTargets; public final ArrayList<WeightedString> mBigrams; + // TODO: Support mIsBeginningOfSentence. + public final boolean mIsBeginningOfSentence; public final boolean mIsNotAWord; public final boolean mIsBlacklistEntry; public final boolean mHasShortcuts; @@ -51,6 +53,7 @@ public final class WordProperty implements Comparable<WordProperty> { mProbabilityInfo = probabilityInfo; mShortcutTargets = shortcutTargets; mBigrams = bigrams; + mIsBeginningOfSentence = false; mIsNotAWord = isNotAWord; mIsBlacklistEntry = isBlacklistEntry; mHasBigrams = bigrams != null && !bigrams.isEmpty(); @@ -77,6 +80,7 @@ public final class WordProperty implements Comparable<WordProperty> { mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo); mShortcutTargets = CollectionUtils.newArrayList(); mBigrams = CollectionUtils.newArrayList(); + mIsBeginningOfSentence = false; mIsNotAWord = isNotAWord; mIsBlacklistEntry = isBlacklisted; mHasShortcuts = hasShortcuts; diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDataChunk.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDataChunk.java new file mode 100644 index 000000000..9d72de8c5 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDataChunk.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public class PersonalizationDataChunk { + public final boolean mInputByUser; + public final List<String> mTokens; + public final int mTimestampInSeconds; + public final String mPackageName; + public final Locale mlocale = null; + + public PersonalizationDataChunk(boolean inputByUser, final List<String> tokens, + final int timestampInSeconds, final String packageName) { + mInputByUser = inputByUser; + mTokens = Collections.unmodifiableList(tokens); + mTimestampInSeconds = timestampInSeconds; + mPackageName = packageName; + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java index 805f4220f..450644032 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java @@ -20,17 +20,14 @@ import android.content.Context; import android.content.res.Configuration; import com.android.inputmethod.latin.DictionaryFacilitator; -import com.android.inputmethod.latin.utils.DistracterFilter; public class PersonalizationDictionarySessionRegistrar { public static void init(final Context context, - final DictionaryFacilitator dictionaryFacilitator, - final DistracterFilter distracterFilter) { + final DictionaryFacilitator dictionaryFacilitator) { } public static void onConfigurationChanged(final Context context, final Configuration conf, - final DictionaryFacilitator dictionaryFacilitator, - final DistracterFilter distracterFilter) { + final DictionaryFacilitator dictionaryFacilitator) { } public static void onUpdateData(final Context context, final String type) { diff --git a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java index 702688f93..936219332 100644 --- a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java @@ -62,6 +62,22 @@ public final class CapsModeUtils { } /** + * Helper method to find out if a code point is starting punctuation. + * + * This include the Unicode START_PUNCTUATION category, but also some other symbols that are + * starting, like the inverted question mark or the double quote. + * + * @param codePoint the code point + * @return true if it's starting punctuation, false otherwise. + */ + private static boolean isStartPunctuation(final int codePoint) { + return (codePoint == Constants.CODE_DOUBLE_QUOTE || codePoint == Constants.CODE_SINGLE_QUOTE + || codePoint == Constants.CODE_INVERTED_QUESTION_MARK + || codePoint == Constants.CODE_INVERTED_EXCLAMATION_MARK + || Character.getType(codePoint) == Character.START_PUNCTUATION); + } + + /** * Determine what caps mode should be in effect at the current offset in * the text. Only the mode bits set in <var>reqModes</var> will be * checked. Note that the caps mode flags here are explicitly defined @@ -115,8 +131,7 @@ public final class CapsModeUtils { } else { for (i = cs.length(); i > 0; i--) { final char c = cs.charAt(i - 1); - if (c != Constants.CODE_DOUBLE_QUOTE && c != Constants.CODE_SINGLE_QUOTE - && Character.getType(c) != Character.START_PUNCTUATION) { + if (!isStartPunctuation(c)) { break; } } @@ -210,11 +225,14 @@ public final class CapsModeUtils { // We found out that we have a period. We need to determine if this is a full stop or // otherwise sentence-ending period, or an abbreviation like "e.g.". An abbreviation - // looks like (\w\.){2,} + // looks like (\w\.){2,}. Moreover, in German, you put periods after digits for dates + // and some other things, and in German specifically we need to not go into autocaps after + // a whitespace-digits-period sequence. // To find out, we will have a simple state machine with the following states : - // START, WORD, PERIOD, ABBREVIATION + // START, WORD, PERIOD, ABBREVIATION, NUMBER // On START : (just before the first period) // letter => WORD + // digit => NUMBER if German; end with caps otherwise // whitespace => end with no caps (it was a stand-alone period) // otherwise => end with caps (several periods/symbols in a row) // On WORD : (within the word just before the first period) @@ -228,6 +246,11 @@ public final class CapsModeUtils { // letter => LETTER // period => PERIOD // otherwise => end with no caps (it was an abbreviation) + // On NUMBER : (period immediately preceded by one or more digits) + // digit => NUMBER + // letter => LETTER (promote to word) + // otherwise => end with no caps (it was a whitespace-digits-period sequence, + // or a punctuation-digits-period sequence like "11.11.") // "Not an abbreviation" in the above chart essentially covers cases like "...yes.". This // should capitalize. @@ -235,6 +258,7 @@ public final class CapsModeUtils { final int WORD = 1; final int PERIOD = 2; final int LETTER = 3; + final int NUMBER = 4; final int caps = (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_WORDS | TextUtils.CAP_MODE_SENTENCES) & reqModes; final int noCaps = (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_WORDS) & reqModes; @@ -247,6 +271,8 @@ public final class CapsModeUtils { state = WORD; } else if (Character.isWhitespace(c)) { return noCaps; + } else if (Character.isDigit(c) && spacingAndPunctuations.mUsesGermanRules) { + state = NUMBER; } else { return caps; } @@ -275,6 +301,15 @@ public final class CapsModeUtils { } else { return noCaps; } + break; + case NUMBER: + if (Character.isLetter(c)) { + state = WORD; + } else if (Character.isDigit(c)) { + state = NUMBER; + } else { + return noCaps; + } } } // Here we arrived at the start of the line. This should behave exactly like whitespace. diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java index 19a01eea8..6e0fab32a 100644 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java @@ -16,132 +16,14 @@ package com.android.inputmethod.latin.utils; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Locale; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import android.content.Context; -import android.content.res.Resources; -import android.text.InputType; -import android.util.Log; -import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputMethodSubtype; -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.keyboard.KeyboardId; -import com.android.inputmethod.keyboard.KeyboardLayoutSet; -import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.PrevWordsInfo; -import com.android.inputmethod.latin.Suggest; -import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; -import com.android.inputmethod.latin.SuggestedWords; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; - -/** - * This class is used to prevent distracters being added to personalization - * or user history dictionaries - */ -public class DistracterFilter { - private static final String TAG = DistracterFilter.class.getSimpleName(); - - private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; - - private final Context mContext; - private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap; - private final Map<Locale, Keyboard> mLocaleToKeyboardMap; - private final DictionaryFacilitator mDictionaryFacilitator; - private final Suggest mSuggest; - private Keyboard mKeyboard; - - // If the score of the top suggestion exceeds this value, the tested word (e.g., - // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to - // words in dictionary. The greater the threshold is, the less likely the tested word would - // become a distracter, which means the tested word will be more likely to be added to - // the dictionary. - private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 2.0f; - - // Create empty distracter filter. - public DistracterFilter() { - this(null, new ArrayList<InputMethodSubtype>()); - } - - /** - * Create a DistracterFilter instance. - * - * @param context the context. - * @param enabledSubtypes the enabled subtypes. - */ - public DistracterFilter(final Context context, final List<InputMethodSubtype> enabledSubtypes) { - mContext = context; - mLocaleToSubtypeMap = new HashMap<>(); - if (enabledSubtypes != null) { - for (final InputMethodSubtype subtype : enabledSubtypes) { - final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); - if (mLocaleToSubtypeMap.containsKey(locale)) { - // Multiple subtypes are enabled for one locale. - // TODO: Investigate what we should do for this case. - continue; - } - mLocaleToSubtypeMap.put(locale, subtype); - } - } - mLocaleToKeyboardMap = new HashMap<>(); - mDictionaryFacilitator = new DictionaryFacilitator(); - mSuggest = new Suggest(mDictionaryFacilitator); - mKeyboard = null; - } - - private static boolean suggestionExceedsDistracterThreshold( - final SuggestedWordInfo suggestion, final String consideredWord, - final float distracterThreshold) { - if (null != suggestion) { - final int suggestionScore = suggestion.mScore; - final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( - consideredWord, suggestion.mWord, suggestionScore); - if (normalizedScore > distracterThreshold) { - return true; - } - } - return false; - } - - private void loadKeyboardForLocale(final Locale newLocale) { - final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale); - if (cachedKeyboard != null) { - mKeyboard = cachedKeyboard; - return; - } - final InputMethodSubtype subtype = mLocaleToSubtypeMap.get(newLocale); - if (subtype == null) { - return; - } - final EditorInfo editorInfo = new EditorInfo(); - editorInfo.inputType = InputType.TYPE_CLASS_TEXT; - final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( - mContext, editorInfo); - final Resources res = mContext.getResources(); - final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); - final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); - builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); - builder.setSubtype(subtype); - builder.setIsSpellChecker(false /* isSpellChecker */); - final KeyboardLayoutSet layoutSet = builder.build(); - mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); - } - - private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { - mDictionaryFacilitator.resetDictionaries(mContext, newlocale, - false /* useContactsDict */, false /* usePersonalizedDicts */, - false /* forceReloadMainDictionary */, null /* listener */); - mDictionaryFacilitator.waitForLoadingMainDictionary( - TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS); - } +public interface DistracterFilter { /** * Determine whether a word is a distracter to words in dictionaries. * @@ -152,56 +34,25 @@ public class DistracterFilter { * @return true if testedWord is a distracter, otherwise false. */ public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, - final String testedWord, final Locale locale) { - if (locale == null) { - return false; - } - if (!locale.equals(mDictionaryFacilitator.getLocale())) { - if (!mLocaleToSubtypeMap.containsKey(locale)) { - Log.e(TAG, "Locale " + locale + " is not enabled."); - // TODO: Investigate what we should do for disabled locales. - return false; - } - loadKeyboardForLocale(locale); - // Reset dictionaries for the locale. - try { - loadDictionariesForLocale(locale); - } catch (final InterruptedException e) { - Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter", e); - return false; - } - } - if (mKeyboard == null) { + final String testedWord, final Locale locale); + + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes); + + public void close(); + + public static final class EmptyDistracterFilter implements DistracterFilter { + @Override + public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo, + String testedWord, Locale locale) { return false; } - final WordComposer composer = new WordComposer(); - final int[] codePoints = StringUtils.toCodePointArray(testedWord); - final int[] coordinates = mKeyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates, prevWordsInfo); - final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); - final String consideredWord = trailingSingleQuotesCount > 0 ? - testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : - testedWord; - final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); - final OnGetSuggestedWordsCallback callback = new OnGetSuggestedWordsCallback() { - @Override - public void onGetSuggestedWords(final SuggestedWords suggestedWords) { - if (suggestedWords != null && suggestedWords.size() > 1) { - // The suggestedWordInfo at 0 is the typed word. The 1st suggestion from - // the decoder is at index 1. - final SuggestedWordInfo firstSuggestion = suggestedWords.getInfo(1); - final boolean hasStrongDistractor = suggestionExceedsDistracterThreshold( - firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); - holder.set(hasStrongDistractor); - } - } - }; - mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), - true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, - null /* additionalFeaturesOptions */, 0 /* sessionId */, - SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); + @Override + public void close() { + } - return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + @Override + public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) { + } } } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java new file mode 100644 index 000000000..92033b76f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import android.content.Context; +import android.content.res.Resources; +import android.text.InputType; +import android.util.Log; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputMethodSubtype; + +import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.keyboard.KeyboardId; +import com.android.inputmethod.keyboard.KeyboardLayoutSet; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.DictionaryFacilitator; +import com.android.inputmethod.latin.PrevWordsInfo; +import com.android.inputmethod.latin.Suggest; +import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; +import com.android.inputmethod.latin.SuggestedWords; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.WordComposer; + +/** + * This class is used to prevent distracters being added to personalization + * or user history dictionaries + */ +public class DistracterFilterUsingSuggestion implements DistracterFilter { + private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName(); + + private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; + + private final Context mContext; + private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap; + private final Map<Locale, Keyboard> mLocaleToKeyboardMap; + private final DictionaryFacilitator mDictionaryFacilitator; + private final Suggest mSuggest; + private Keyboard mKeyboard; + private final Object mLock = new Object(); + + // If the score of the top suggestion exceeds this value, the tested word (e.g., + // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to + // words in dictionary. The greater the threshold is, the less likely the tested word would + // become a distracter, which means the tested word will be more likely to be added to + // the dictionary. + private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 2.0f; + + /** + * Create a DistracterFilter instance. + * + * @param context the context. + */ + public DistracterFilterUsingSuggestion(final Context context) { + mContext = context; + mLocaleToSubtypeMap = new HashMap<>(); + mLocaleToKeyboardMap = new HashMap<>(); + mDictionaryFacilitator = new DictionaryFacilitator(); + mSuggest = new Suggest(mDictionaryFacilitator); + mKeyboard = null; + } + + @Override + public void close() { + mDictionaryFacilitator.closeDictionaries(); + } + + @Override + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { + final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>(); + if (enabledSubtypes != null) { + for (final InputMethodSubtype subtype : enabledSubtypes) { + final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); + if (newLocaleToSubtypeMap.containsKey(locale)) { + // Multiple subtypes are enabled for one locale. + // TODO: Investigate what we should do for this case. + continue; + } + newLocaleToSubtypeMap.put(locale, subtype); + } + } + if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) { + // Enabled subtypes have not been changed. + return; + } + synchronized (mLock) { + mLocaleToSubtypeMap.clear(); + mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap); + mLocaleToKeyboardMap.clear(); + } + } + + private static boolean suggestionExceedsDistracterThreshold( + final SuggestedWordInfo suggestion, final String consideredWord, + final float distracterThreshold) { + if (null != suggestion) { + final int suggestionScore = suggestion.mScore; + final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( + consideredWord, suggestion.mWord, suggestionScore); + if (normalizedScore > distracterThreshold) { + return true; + } + } + return false; + } + + private void loadKeyboardForLocale(final Locale newLocale) { + final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale); + if (cachedKeyboard != null) { + mKeyboard = cachedKeyboard; + return; + } + final InputMethodSubtype subtype; + synchronized (mLock) { + subtype = mLocaleToSubtypeMap.get(newLocale); + } + if (subtype == null) { + return; + } + final EditorInfo editorInfo = new EditorInfo(); + editorInfo.inputType = InputType.TYPE_CLASS_TEXT; + final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( + mContext, editorInfo); + final Resources res = mContext.getResources(); + final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); + final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); + builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); + builder.setSubtype(subtype); + builder.setIsSpellChecker(false /* isSpellChecker */); + final KeyboardLayoutSet layoutSet = builder.build(); + mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); + } + + private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { + mDictionaryFacilitator.resetDictionaries(mContext, newlocale, + false /* useContactsDict */, false /* usePersonalizedDicts */, + false /* forceReloadMainDictionary */, null /* listener */); + mDictionaryFacilitator.waitForLoadingMainDictionary( + TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS); + } + + /** + * Determine whether a word is a distracter to words in dictionaries. + * + * @param prevWordsInfo the information of previous words. + * @param testedWord the word that will be tested to see whether it is a distracter to words + * in dictionaries. + * @param locale the locale of word. + * @return true if testedWord is a distracter, otherwise false. + */ + @Override + public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, + final String testedWord, final Locale locale) { + if (locale == null) { + return false; + } + if (!locale.equals(mDictionaryFacilitator.getLocale())) { + synchronized (mLock) { + if (!mLocaleToSubtypeMap.containsKey(locale)) { + Log.e(TAG, "Locale " + locale + " is not enabled."); + // TODO: Investigate what we should do for disabled locales. + return false; + } + loadKeyboardForLocale(locale); + // Reset dictionaries for the locale. + try { + loadDictionariesForLocale(locale); + } catch (final InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter", + e); + return false; + } + } + } + if (mKeyboard == null) { + return false; + } + final WordComposer composer = new WordComposer(); + final int[] codePoints = StringUtils.toCodePointArray(testedWord); + final int[] coordinates = mKeyboard.getCoordinates(codePoints); + composer.setComposingWord(codePoints, coordinates, prevWordsInfo); + + final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); + final String consideredWord = trailingSingleQuotesCount > 0 ? + testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : + testedWord; + final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); + final OnGetSuggestedWordsCallback callback = new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + if (suggestedWords != null && suggestedWords.size() > 1) { + // The suggestedWordInfo at 0 is the typed word. The 1st suggestion from + // the decoder is at index 1. + final SuggestedWordInfo firstSuggestion = suggestedWords.getInfo(1); + final boolean hasStrongDistractor = suggestionExceedsDistracterThreshold( + firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); + holder.set(hasStrongDistractor); + } + } + }; + mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), + true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, + null /* additionalFeaturesOptions */, 0 /* sessionId */, + SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); + + return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + } +} diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java index 36543cca6..9c759edb6 100644 --- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java +++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java @@ -24,6 +24,7 @@ import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import java.util.ArrayList; +import java.util.List; import java.util.Locale; // Note: this class is used as a parameter type of a native method. You should be careful when you @@ -79,14 +80,14 @@ public final class LanguageModelParam { // Process a list of words and return a list of {@link LanguageModelParam} objects. public static ArrayList<LanguageModelParam> createLanguageModelParamsFrom( - final ArrayList<String> tokens, final int timestamp, + final List<String> tokens, final int timestamp, final DictionaryFacilitator dictionaryFacilitator, final SpacingAndPunctuations spacingAndPunctuations, final DistracterFilter distracterFilter) { final ArrayList<LanguageModelParam> languageModelParams = CollectionUtils.newArrayList(); final int N = tokens.size(); - PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null); + PrevWordsInfo prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; for (int i = 0; i < N; ++i) { final String tempWord = tokens.get(i); if (StringUtils.isEmptyStringOrWhiteSpaces(tempWord)) { @@ -103,7 +104,7 @@ public final class LanguageModelParam { + tempWord + "\""); } // Sentence terminator found. Split. - prevWordsInfo = new PrevWordsInfo(null); + prevWordsInfo = PrevWordsInfo.EMPTY_PREV_WORDS_INFO; continue; } if (DEBUG_TOKEN) { |