diff options
Diffstat (limited to 'java/src')
16 files changed, 305 insertions, 180 deletions
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 335e52fef..95e1340ed 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -174,7 +174,7 @@ public final class BinaryDictionary extends Dictionary { private static native int getFormatVersionNative(long dict); private static native int getProbabilityNative(long dict, int[] word); private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); - private static native int getBigramProbabilityNative(long dict, int[] word0, + private static native int getNgramProbabilityNative(long dict, int[] word0, boolean isBeginningOfSentence, int[] word1); private static native void getWordPropertyNative(long dict, int[] word, boolean isBeginningOfSentence, int[] outCodePoints, boolean[] outFlags, @@ -190,13 +190,13 @@ public final class BinaryDictionary extends Dictionary { int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence, float[] inOutLanguageWeight); - private static native boolean addUnigramWordNative(long dict, int[] word, int probability, + private static native boolean addUnigramEntryNative(long dict, int[] word, int probability, int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, boolean isNotAWord, boolean isBlacklisted, int timestamp); - private static native boolean removeUnigramWordNative(long dict, int[] word); - private static native boolean addBigramWordsNative(long dict, int[] word0, + private static native boolean removeUnigramEntryNative(long dict, int[] word); + private static native boolean addNgramEntryNative(long dict, int[] word0, boolean isBeginningOfSentence, int[] word1, int probability, int timestamp); - private static native boolean removeBigramWordsNative(long dict, int[] word0, + private static native boolean removeNgramEntryNative(long dict, int[] word0, boolean isBeginningOfSentence, int[] word1); private static native int addMultipleDictionaryEntriesNative(long dict, LanguageModelParam[] languageModelParams, int startIndex); @@ -262,9 +262,8 @@ public final class BinaryDictionary extends Dictionary { } final DicTraverseSession session = getTraverseSession(sessionId); Arrays.fill(session.mInputCodePoints, Constants.NOT_A_CODE); - // TODO: toLowerCase in the native code - final int[] prevWordCodePointArray = (null == prevWordsInfo.mPrevWord) - ? null : StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + prevWordsInfo.outputToArray(session.mPrevWordCodePointArrays, + session.mIsBeginningOfSentenceArray); final InputPointers inputPointers = composer.getInputPointers(); final boolean isGesture = composer.isBatchMode(); final int inputSize; @@ -286,13 +285,13 @@ public final class BinaryDictionary extends Dictionary { } else { session.mInputOutputLanguageWeight[0] = Dictionary.NOT_A_LANGUAGE_WEIGHT; } - // proximityInfo and/or prevWordForBigrams may not be null. + // TOOD: Pass multiple previous words information for n-gram. getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(), getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), inputPointers.getYCoordinates(), inputPointers.getTimes(), inputPointers.getPointerIds(), session.mInputCodePoints, inputSize, - session.mNativeSuggestOptions.getOptions(), prevWordCodePointArray, - prevWordsInfo.mIsBeginningOfSentence, session.mOutputSuggestionCount, + session.mNativeSuggestOptions.getOptions(), session.mPrevWordCodePointArrays[0], + session.mIsBeginningOfSentenceArray[0], session.mOutputSuggestionCount, session.mOutputCodePoints, session.mOutputScores, session.mSpaceIndices, session.mOutputTypes, session.mOutputAutoCommitFirstWordConfidence, session.mInputOutputLanguageWeight); @@ -355,10 +354,13 @@ public final class BinaryDictionary extends Dictionary { if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return NOT_A_PROBABILITY; } - final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); - final int[] codePoints1 = StringUtils.toCodePointArray(word); - return getBigramProbabilityNative(mNativeDict, codePoints0, - prevWordsInfo.mIsBeginningOfSentence, codePoints1); + final int[][] prevWordCodePointArrays = new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; + final boolean[] isBeginningOfSentenceArray = + new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); + final int[] wordCodePoints = StringUtils.toCodePointArray(word); + return getNgramProbabilityNative(mNativeDict, prevWordCodePointArrays[0], + isBeginningOfSentenceArray[0], wordCodePoints); } public WordProperty getWordProperty(final String word, final boolean isBeginningOfSentence) { @@ -422,7 +424,7 @@ public final class BinaryDictionary extends Dictionary { final int[] codePoints = StringUtils.toCodePointArray(word); final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? StringUtils.toCodePointArray(shortcutTarget) : null; - if (!addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, + if (!addUnigramEntryNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp)) { return false; } @@ -436,7 +438,7 @@ public final class BinaryDictionary extends Dictionary { return false; } final int[] codePoints = StringUtils.toCodePointArray(word); - if (!removeUnigramWordNative(mNativeDict, codePoints)) { + if (!removeUnigramEntryNative(mNativeDict, codePoints)) { return false; } mHasUpdated = true; @@ -449,10 +451,13 @@ public final class BinaryDictionary extends Dictionary { if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return false; } - final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); - final int[] codePoints1 = StringUtils.toCodePointArray(word); - if (!addBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, - codePoints1, probability, timestamp)) { + final int[][] prevWordCodePointArrays = new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; + final boolean[] isBeginningOfSentenceArray = + new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); + final int[] wordCodePoints = StringUtils.toCodePointArray(word); + if (!addNgramEntryNative(mNativeDict, prevWordCodePointArrays[0], + isBeginningOfSentenceArray[0], wordCodePoints, probability, timestamp)) { return false; } mHasUpdated = true; @@ -464,10 +469,13 @@ public final class BinaryDictionary extends Dictionary { if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return false; } - final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); - final int[] codePoints1 = StringUtils.toCodePointArray(word); - if (!removeBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, - codePoints1)) { + final int[][] prevWordCodePointArrays = new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; + final boolean[] isBeginningOfSentenceArray = + new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + prevWordsInfo.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); + final int[] wordCodePoints = StringUtils.toCodePointArray(word); + if (!removeNgramEntryNative(mNativeDict, prevWordCodePointArrays[0], + isBeginningOfSentenceArray[0], wordCodePoints)) { return false; } mHasUpdated = true; diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index f9339361a..b4e115c7d 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -166,6 +166,10 @@ public final class Constants { // Must be equal to MAX_WORD_LENGTH in native/jni/src/defines.h public static final int DICTIONARY_MAX_WORD_LENGTH = 48; + // (MAX_PREV_WORD_COUNT_FOR_N_GRAM + 1)-gram is supported in Java side. Needs to modify + // MAX_PREV_WORD_COUNT_FOR_N_GRAM in native/jni/src/defines.h for suggestions. + public static final int MAX_PREV_WORD_COUNT_FOR_N_GRAM = 2; + // Key events coming any faster than this are long-presses. public static final int LONG_PRESS_MILLISECONDS = 200; // TODO: Set this value appropriately. diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index 96160fa4e..ad14c06ef 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -233,19 +233,19 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { final int wordLen = StringUtils.codePointCount(word); if (wordLen < MAX_WORD_LENGTH && wordLen > 1) { if (DEBUG) { - Log.d(TAG, "addName " + name + ", " + word + ", " - + prevWordsInfo.mPrevWord); + Log.d(TAG, "addName " + name + ", " + word + ", " + prevWordsInfo); } runGCIfRequiredLocked(true /* mindsBlockByGC */); addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); - if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && mUseFirstLastBigrams) { + if (!prevWordsInfo.isValid() && mUseFirstLastBigrams) { runGCIfRequiredLocked(true /* mindsBlockByGC */); addNgramEntryLocked(prevWordsInfo, word, FREQUENCY_FOR_CONTACTS_BIGRAM, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } - prevWordsInfo = new PrevWordsInfo(word); + prevWordsInfo = prevWordsInfo.getNextPrevWordsInfo( + new PrevWordsInfo.WordInfo(word)); } } } diff --git a/java/src/com/android/inputmethod/latin/DicTraverseSession.java b/java/src/com/android/inputmethod/latin/DicTraverseSession.java index 8bbf426e5..b341f623e 100644 --- a/java/src/com/android/inputmethod/latin/DicTraverseSession.java +++ b/java/src/com/android/inputmethod/latin/DicTraverseSession.java @@ -28,6 +28,10 @@ public final class DicTraverseSession { // Must be equal to MAX_RESULTS in native/jni/src/defines.h private static final int MAX_RESULTS = 18; public final int[] mInputCodePoints = new int[Constants.DICTIONARY_MAX_WORD_LENGTH]; + public final int[][] mPrevWordCodePointArrays = + new int[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM][]; + public final boolean[] mIsBeginningOfSentenceArray = + new boolean[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; public final int[] mOutputSuggestionCount = new int[1]; public final int[] mOutputCodePoints = new int[Constants.DICTIONARY_MAX_WORD_LENGTH * MAX_RESULTS]; diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index 304c450ab..b8feb2278 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -23,6 +23,7 @@ import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.ContextualDictionary; import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; @@ -407,13 +408,14 @@ public class DictionaryFacilitator { final boolean blockPotentiallyOffensive) { final Dictionaries dictionaries = mDictionaries; final String[] words = suggestion.split(Constants.WORD_SEPARATOR); + PrevWordsInfo prevWordsInfoForCurrentWord = prevWordsInfo; for (int i = 0; i < words.length; i++) { final String currentWord = words[i]; - final PrevWordsInfo prevWordsInfoForCurrentWord = - (i == 0) ? prevWordsInfo : new PrevWordsInfo(words[i - 1]); final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false; addWordToUserHistory(dictionaries, prevWordsInfoForCurrentWord, currentWord, wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive); + prevWordsInfoForCurrentWord = + prevWordsInfoForCurrentWord.getNextPrevWordsInfo(new WordInfo(currentWord)); } } @@ -639,7 +641,8 @@ public class DictionaryFacilitator { contextualDict.addNgramEntry(prevWordsInfo, phrase[i], bigramProbabilityForWords, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } - prevWordsInfo = new PrevWordsInfo(phrase[i]); + prevWordsInfo = + prevWordsInfo.getNextPrevWordsInfo(new PrevWordsInfo.WordInfo(phrase[i])); } } diff --git a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java index 42b311c69..5dda44445 100644 --- a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java +++ b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java @@ -16,47 +16,122 @@ package com.android.inputmethod.latin; +import java.util.Arrays; + +import com.android.inputmethod.latin.utils.StringUtils; + /** * Class to represent information of previous words. This class is used to add n-gram entries * into binary dictionaries, to get predictions, and to get suggestions. */ -// TODO: Support multiple previous words for n-gram. public class PrevWordsInfo { - public static final PrevWordsInfo EMPTY_PREV_WORDS_INFO = new PrevWordsInfo(null); + public static final PrevWordsInfo EMPTY_PREV_WORDS_INFO = + new PrevWordsInfo(WordInfo.EMPTY_WORD_INFO); public static final PrevWordsInfo BEGINNING_OF_SENTENCE = new PrevWordsInfo(); - // The word immediately before the considered word. null means we don't have any context - // including the "beginning of sentence context" - we just don't know what to predict. - // An example of that is after a comma. - // For simplicity of implementation, this may also be null transiently after the WordComposer - // was reset and before starting a new composing word, but we should never be calling - // getSuggetions* in this situation. - // This is an empty string when mIsBeginningOfSentence is true. - public final String mPrevWord; + /** + * Word information used to represent previous words information. + */ + public static class WordInfo { + public static final WordInfo EMPTY_WORD_INFO = new WordInfo(null); + public static final WordInfo BEGINNING_OF_SENTENCE = new WordInfo(); + + // This is an empty string when mIsBeginningOfSentence is true. + public final String mWord; + // TODO: Have sentence separator. + // Whether the current context is beginning of sentence or not. This is true when composing + // at the beginning of an input field or composing a word after a sentence separator. + public final boolean mIsBeginningOfSentence; + + // Beginning of sentence. + public WordInfo() { + mWord = ""; + mIsBeginningOfSentence = true; + } + + public WordInfo(final String word) { + mWord = word; + mIsBeginningOfSentence = false; + } + + public boolean isValid() { + return mWord != null; + } + } - // TODO: Have sentence separator. - // Whether the current context is beginning of sentence or not. This is true when composing at - // the beginning of an input field or composing a word after a sentence separator. - public final boolean mIsBeginningOfSentence; + // The words immediately before the considered word. EMPTY_WORD_INFO element means we don't + // have any context for that previous word including the "beginning of sentence context" - we + // just don't know what to predict using the information. An example of that is after a comma. + // For simplicity of implementation, elements may also be EMPTY_WORD_INFO transiently after the + // WordComposer was reset and before starting a new composing word, but we should never be + // calling getSuggetions* in this situation. + public WordInfo[] mPrevWordsInfo = new WordInfo[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; // Beginning of sentence. public PrevWordsInfo() { - mPrevWord = ""; - mIsBeginningOfSentence = true; + mPrevWordsInfo[0] = WordInfo.BEGINNING_OF_SENTENCE; + Arrays.fill(mPrevWordsInfo, 1 /* start */, mPrevWordsInfo.length, WordInfo.EMPTY_WORD_INFO); } - public PrevWordsInfo(final String prevWord) { - mPrevWord = prevWord; - mIsBeginningOfSentence = false; + // Construct from the previous word information. + public PrevWordsInfo(final WordInfo prevWordInfo) { + mPrevWordsInfo[0] = prevWordInfo; + Arrays.fill(mPrevWordsInfo, 1 /* start */, mPrevWordsInfo.length, WordInfo.EMPTY_WORD_INFO); + } + + // Construct from WordInfo array. n-th element represents (n+1)-th previous word's information. + public PrevWordsInfo(final WordInfo[] prevWordsInfo) { + for (int i = 0; i < Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM; i++) { + mPrevWordsInfo[i] = + (prevWordsInfo.length > i) ? prevWordsInfo[i] : WordInfo.EMPTY_WORD_INFO; + } + } + + // Create next prevWordsInfo using current prevWordsInfo. + public PrevWordsInfo getNextPrevWordsInfo(final WordInfo wordInfo) { + final WordInfo[] prevWordsInfo = new WordInfo[Constants.MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + prevWordsInfo[0] = wordInfo; + for (int i = 1; i < prevWordsInfo.length; i++) { + prevWordsInfo[i] = mPrevWordsInfo[i - 1]; + } + return new PrevWordsInfo(prevWordsInfo); } public boolean isValid() { - return mPrevWord != null; + return mPrevWordsInfo[0].isValid(); + } + + public void outputToArray(final int[][] codePointArrays, + final boolean[] isBeginningOfSentenceArray) { + for (int i = 0; i < mPrevWordsInfo.length; i++) { + final WordInfo wordInfo = mPrevWordsInfo[i]; + if (wordInfo == null || !wordInfo.isValid()) { + codePointArrays[i] = new int[0]; + isBeginningOfSentenceArray[i] = false; + continue; + } + codePointArrays[i] = StringUtils.toCodePointArray(wordInfo.mWord); + isBeginningOfSentenceArray[i] = wordInfo.mIsBeginningOfSentence; + } } @Override public String toString() { - return "PrevWord: " + mPrevWord + ", isBeginningOfSentence: " - + mIsBeginningOfSentence + "."; + final StringBuffer builder = new StringBuffer(); + for (int i = 0; i < mPrevWordsInfo.length; i++) { + final WordInfo wordInfo = mPrevWordsInfo[i]; + builder.append("PrevWord["); + builder.append(i); + builder.append("]: "); + if (!wordInfo.isValid()) { + builder.append("Empty. "); + continue; + } + builder.append(wordInfo.mWord); + builder.append(", isBeginningOfSentence: "); + builder.append(wordInfo.mIsBeginningOfSentence); + builder.append(". "); + } + return builder.toString(); } } diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 96476b2ee..3be6bccc6 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -603,7 +603,7 @@ public final class RichInputConnection { || spacingAndPunctuations.isWordConnector(lastChar)) { return PrevWordsInfo.EMPTY_PREV_WORDS_INFO; } - return new PrevWordsInfo(nthPrevWord); + return new PrevWordsInfo(new PrevWordsInfo.WordInfo(nthPrevWord)); } /** diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 9d03e8a43..2a56861b4 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -123,21 +123,15 @@ public final class Suggest { suggestionsContainer.set(i, transformedWordInfo); } } - SuggestedWordInfo.removeDups(typedWord, suggestionsContainer); + final boolean didRemoveTypedWord = + SuggestedWordInfo.removeDups(typedWord, suggestionsContainer); - // If resumed, then we don't want to upcase everything: resuming on a fully-capitalized - // words is rarely done to switch to another fully-capitalized word, but usually to a - // normal, non-capitalized suggestion. - final String firstSuggestion; final String whitelistedWord; - if (suggestionResults.isEmpty()) { + if (suggestionsContainer.isEmpty()) { whitelistedWord = firstSuggestion = null; } else { - final SuggestedWordInfo firstSuggestedWordInfo = getTransformedSuggestedWordInfo( - suggestionResults.first(), suggestionResults.mLocale, - shouldMakeSuggestionsAllUpperCase, isOnlyFirstCharCapitalized, - trailingSingleQuotesCount); - firstSuggestion = firstSuggestedWordInfo.mWord; + final SuggestedWordInfo firstSuggestedWordInfo = suggestionsContainer.get(0); + final String firstSuggestion = firstSuggestedWordInfo.mWord; if (!firstSuggestedWordInfo.isKindOf(SuggestedWordInfo.KIND_WHITELIST)) { whitelistedWord = null; } else { @@ -145,17 +139,10 @@ public final class Suggest { } } - // We allow auto-correction if we have a whitelisted word, or if the word is not a valid - // word of more than 1 char, except if the first suggestion is the same as the typed string - // because in this case if it's strong enough to auto-correct that will mistakenly designate - // the second candidate for auto-correction. - // TODO: stop relying on indices to find where is the auto-correction in the suggested - // words, and correct this test. - final boolean allowsToBeAutoCorrected = (null != whitelistedWord - && !whitelistedWord.equals(typedWord)) - || (consideredWord.length() > 1 && !mDictionaryFacilitator.isValidWord( - consideredWord, isOnlyFirstCharCapitalized) - && !typedWord.equals(firstSuggestion)); + // We allow auto-correction if we have a whitelisted word, or if the word had more than + // one char and was not suggested. + final boolean allowsToBeAutoCorrected = (null != whitelistedWord) + || (consideredWord.length() > 1 && !didRemoveTypedWord); final boolean hasAutoCorrection; // TODO: using isCorrectionEnabled here is not very good. It's probably useless, because diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index f22af7991..e587b18c9 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -330,29 +330,37 @@ public class SuggestedWords { } // This will always remove the higher index if a duplicate is found. - public static void removeDups(final String typedWord, + public static boolean removeDups(final String typedWord, ArrayList<SuggestedWordInfo> candidates) { if (candidates.isEmpty()) { - return; + return false; } + final boolean didRemoveTypedWord; if (!TextUtils.isEmpty(typedWord)) { - removeSuggestedWordInfoFrom(typedWord, candidates, -1 /* startIndexExclusive */); + didRemoveTypedWord = removeSuggestedWordInfoFrom(typedWord, candidates, + -1 /* startIndexExclusive */); + } else { + didRemoveTypedWord = false; } for (int i = 0; i < candidates.size(); ++i) { removeSuggestedWordInfoFrom(candidates.get(i).mWord, candidates, i /* startIndexExclusive */); } + return didRemoveTypedWord; } - private static void removeSuggestedWordInfoFrom(final String word, + private static boolean removeSuggestedWordInfoFrom(final String word, final ArrayList<SuggestedWordInfo> candidates, final int startIndexExclusive) { + boolean didRemove = false; for (int i = startIndexExclusive + 1; i < candidates.size(); ++i) { final SuggestedWordInfo previous = candidates.get(i); if (word.equals(previous.mWord)) { + didRemove = true; candidates.remove(i); --i; } } + return didRemove; } } diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index dd2a95ee0..ec57cd71f 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -1536,7 +1536,8 @@ public final class InputLogic { } else { return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? PrevWordsInfo.BEGINNING_OF_SENTENCE : - new PrevWordsInfo(mLastComposedWord.mCommittedWord.toString()); + new PrevWordsInfo(new PrevWordsInfo.WordInfo( + mLastComposedWord.mCommittedWord.toString())); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 3916fc24c..a98b0f156 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -60,7 +60,7 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary, final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid, final int timestamp, final DistracterFilter distracterFilter) { - final String prevWord = prevWordsInfo.mPrevWord; + final String prevWord = prevWordsInfo.mPrevWordsInfo[0].mWord; if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || (prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { return; @@ -75,7 +75,13 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas return; } if (null != prevWord) { - userHistoryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp); + if (prevWordsInfo.mPrevWordsInfo[0].mIsBeginningOfSentence) { + // Beginning-of-Sentence n-gram entry is treated as a n-gram entry of invalid word. + userHistoryDictionary.addNgramEntry(prevWordsInfo, word, + FREQUENCY_FOR_WORDS_NOT_IN_DICTS, timestamp); + } else { + userHistoryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp); + } } } } diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java index 8d495646d..d0316242b 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java @@ -37,6 +37,7 @@ import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.LocaleUtils; +import com.android.inputmethod.latin.utils.ScriptUtils; import com.android.inputmethod.latin.utils.StringUtils; import java.lang.ref.WeakReference; @@ -78,40 +79,8 @@ public final class AndroidSpellCheckerService extends SpellCheckerService private final HashSet<WeakReference<DictionaryCollection>> mDictionaryCollectionsList = new HashSet<>(); - public static final int SCRIPT_LATIN = 0; - public static final int SCRIPT_CYRILLIC = 1; - public static final int SCRIPT_GREEK = 2; public static final String SINGLE_QUOTE = "\u0027"; public static final String APOSTROPHE = "\u2019"; - private static final TreeMap<String, Integer> mLanguageToScript; - static { - // List of the supported languages and their associated script. We won't check - // words written in another script than the selected script, because we know we - // don't have those in our dictionary so we will underline everything and we - // will never have any suggestions, so it makes no sense checking them, and this - // is done in {@link #shouldFilterOut}. Also, the script is used to choose which - // proximity to pass to the dictionary descent algorithm. - // IMPORTANT: this only contains languages - do not write countries in there. - // Only the language is searched from the map. - mLanguageToScript = new TreeMap<>(); - mLanguageToScript.put("cs", SCRIPT_LATIN); - mLanguageToScript.put("da", SCRIPT_LATIN); - mLanguageToScript.put("de", SCRIPT_LATIN); - mLanguageToScript.put("el", SCRIPT_GREEK); - mLanguageToScript.put("en", SCRIPT_LATIN); - mLanguageToScript.put("es", SCRIPT_LATIN); - mLanguageToScript.put("fi", SCRIPT_LATIN); - mLanguageToScript.put("fr", SCRIPT_LATIN); - mLanguageToScript.put("hr", SCRIPT_LATIN); - mLanguageToScript.put("it", SCRIPT_LATIN); - mLanguageToScript.put("lt", SCRIPT_LATIN); - mLanguageToScript.put("lv", SCRIPT_LATIN); - mLanguageToScript.put("nb", SCRIPT_LATIN); - mLanguageToScript.put("nl", SCRIPT_LATIN); - mLanguageToScript.put("pt", SCRIPT_LATIN); - mLanguageToScript.put("sl", SCRIPT_LATIN); - mLanguageToScript.put("ru", SCRIPT_CYRILLIC); - } @Override public void onCreate() { super.onCreate(); @@ -122,22 +91,13 @@ public final class AndroidSpellCheckerService extends SpellCheckerService onSharedPreferenceChanged(prefs, PREF_USE_CONTACTS_KEY); } - public static int getScriptFromLocale(final Locale locale) { - final Integer script = mLanguageToScript.get(locale.getLanguage()); - if (null == script) { - throw new RuntimeException("We have been called with an unsupported language: \"" - + locale.getLanguage() + "\". Framework bug?"); - } - return script; - } - private static String getKeyboardLayoutNameForScript(final int script) { switch (script) { - case AndroidSpellCheckerService.SCRIPT_LATIN: + case ScriptUtils.SCRIPT_LATIN: return "qwerty"; - case AndroidSpellCheckerService.SCRIPT_CYRILLIC: + case ScriptUtils.SCRIPT_CYRILLIC: return "east_slavic"; - case AndroidSpellCheckerService.SCRIPT_GREEK: + case ScriptUtils.SCRIPT_GREEK: return "greek"; default: throw new RuntimeException("Wrong script supplied: " + script); @@ -413,7 +373,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService } public DictAndKeyboard createDictAndKeyboard(final Locale locale) { - final int script = getScriptFromLocale(locale); + final int script = ScriptUtils.getScriptFromLocale(locale); final String keyboardLayoutName = getKeyboardLayoutNameForScript(script); final InputMethodSubtype subtype = AdditionalSubtypeUtils.createAdditionalSubtype( locale.toString(), keyboardLayoutName, null); diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java index 55274cfe2..6bfd354ea 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerSession.java @@ -61,7 +61,8 @@ public final class AndroidSpellCheckerSession extends AndroidWordLevelSpellCheck final int offset = ssi.getOffsetAt(i); final int length = ssi.getLengthAt(i); final String subText = typedText.substring(offset, offset + length); - final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(currentWord); + final PrevWordsInfo prevWordsInfo = + new PrevWordsInfo(new PrevWordsInfo.WordInfo(currentWord)); currentWord = subText; if (!subText.contains(AndroidSpellCheckerService.SINGLE_QUOTE)) { continue; @@ -203,7 +204,8 @@ public final class AndroidSpellCheckerSession extends AndroidWordLevelSpellCheck } else { prevWord = null; } - final PrevWordsInfo prevWordsInfo = new PrevWordsInfo(prevWord); + final PrevWordsInfo prevWordsInfo = + new PrevWordsInfo(new PrevWordsInfo.WordInfo(prevWord)); retval[i] = onGetSuggestionsInternal(textInfos[i], prevWordsInfo, suggestionsLimit); retval[i].setCookieAndSequence(textInfos[i].getCookie(), textInfos[i].getSequence()); diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java index 54eebe399..db15c1cc6 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java @@ -36,6 +36,7 @@ import com.android.inputmethod.latin.WordComposer; import com.android.inputmethod.latin.spellcheck.AndroidSpellCheckerService.SuggestionsGatherer; import com.android.inputmethod.latin.utils.CoordinateUtils; import com.android.inputmethod.latin.utils.LocaleUtils; +import com.android.inputmethod.latin.utils.ScriptUtils; import com.android.inputmethod.latin.utils.StringUtils; import java.util.ArrayList; @@ -72,10 +73,10 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { // TODO: Support n-gram input private static String generateKey(final String query, final PrevWordsInfo prevWordsInfo) { - if (TextUtils.isEmpty(query) || TextUtils.isEmpty(prevWordsInfo.mPrevWord)) { + if (TextUtils.isEmpty(query) || !prevWordsInfo.isValid()) { return query; } - return query + CHAR_DELIMITER + prevWordsInfo.mPrevWord; + return query + CHAR_DELIMITER + prevWordsInfo; } public SuggestionsParams getSuggestionsFromCache(String query, @@ -116,7 +117,7 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { final String localeString = getLocale(); mDictionaryPool = mService.getDictionaryPool(localeString); mLocale = LocaleUtils.constructLocaleFromString(localeString); - mScript = AndroidSpellCheckerService.getScriptFromLocale(mLocale); + mScript = ScriptUtils.getScriptFromLocale(mLocale); } @Override @@ -125,44 +126,6 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { cres.unregisterContentObserver(mObserver); } - /* - * Returns whether the code point is a letter that makes sense for the specified - * locale for this spell checker. - * The dictionaries supported by Latin IME are described in res/xml/spellchecker.xml - * and is limited to EFIGS languages and Russian. - * Hence at the moment this explicitly tests for Cyrillic characters or Latin characters - * as appropriate, and explicitly excludes CJK, Arabic and Hebrew characters. - */ - private static boolean isLetterCheckableByLanguage(final int codePoint, - final int script) { - switch (script) { - case AndroidSpellCheckerService.SCRIPT_LATIN: - // Our supported latin script dictionaries (EFIGS) at the moment only include - // characters in the C0, C1, Latin Extended A and B, IPA extensions unicode - // blocks. As it happens, those are back-to-back in the code range 0x40 to 0x2AF, - // so the below is a very efficient way to test for it. As for the 0-0x3F, it's - // excluded from isLetter anyway. - return codePoint <= 0x2AF && Character.isLetter(codePoint); - case AndroidSpellCheckerService.SCRIPT_CYRILLIC: - // All Cyrillic characters are in the 400~52F block. There are some in the upper - // Unicode range, but they are archaic characters that are not used in modern - // Russian and are not used by our dictionary. - return codePoint >= 0x400 && codePoint <= 0x52F && Character.isLetter(codePoint); - case AndroidSpellCheckerService.SCRIPT_GREEK: - // Greek letters are either in the 370~3FF range (Greek & Coptic), or in the - // 1F00~1FFF range (Greek extended). Our dictionary contains both sort of characters. - // Our dictionary also contains a few words with 0xF2; it would be best to check - // if that's correct, but a web search does return results for these words so - // they are probably okay. - return (codePoint >= 0x370 && codePoint <= 0x3FF) - || (codePoint >= 0x1F00 && codePoint <= 0x1FFF) - || codePoint == 0xF2; - default: - // Should never come here - throw new RuntimeException("Impossible value of script: " + script); - } - } - private static final int CHECKABILITY_CHECKABLE = 0; private static final int CHECKABILITY_TOO_MANY_NON_LETTERS = 1; private static final int CHECKABILITY_CONTAINS_PERIOD = 2; @@ -189,7 +152,7 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { // Filter by first letter final int firstCodePoint = text.codePointAt(0); // Filter out words that don't start with a letter or an apostrophe - if (!isLetterCheckableByLanguage(firstCodePoint, script) + if (!ScriptUtils.isLetterCheckableByScript(firstCodePoint, script) && '\'' != firstCodePoint) return CHECKABILITY_FIRST_LETTER_UNCHECKABLE; // Filter contents @@ -210,7 +173,7 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { if (Constants.CODE_PERIOD == codePoint) { return CHECKABILITY_CONTAINS_PERIOD; } - if (isLetterCheckableByLanguage(codePoint, script)) ++letterCount; + if (ScriptUtils.isLetterCheckableByScript(codePoint, script)) ++letterCount; } // Guestimate heuristic: perform spell checking if at least 3/4 of the characters // in this word are letters diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java index 4248bebf6..9ec19efa8 100644 --- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java +++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java @@ -117,7 +117,8 @@ public final class LanguageModelParam { continue; } languageModelParams.add(languageModelParam); - prevWordsInfo = new PrevWordsInfo(languageModelParam.mTargetWord); + prevWordsInfo = prevWordsInfo.getNextPrevWordsInfo( + new PrevWordsInfo.WordInfo(tempWord)); } return languageModelParams; } @@ -153,7 +154,7 @@ public final class LanguageModelParam { final DistracterFilter distracterFilter) { final String word; if (StringUtils.getCapitalizationType(targetWord) == StringUtils.CAPITALIZE_FIRST - && prevWordsInfo.mPrevWord == null && !isValidWord) { + && !prevWordsInfo.isValid() && !isValidWord) { word = targetWord.toLowerCase(locale); } else { word = targetWord; @@ -167,7 +168,7 @@ public final class LanguageModelParam { } final int unigramProbability = isValidWord ? UNIGRAM_PROBABILITY_FOR_VALID_WORD : UNIGRAM_PROBABILITY_FOR_OOV_WORD; - if (prevWordsInfo.mPrevWord == null) { + if (!prevWordsInfo.isValid()) { if (DEBUG) { Log.d(TAG, "--- add unigram: current(" + (isValidWord ? "Valid" : "OOV") + ") = " + word); @@ -175,12 +176,12 @@ public final class LanguageModelParam { return new LanguageModelParam(word, unigramProbability, timestamp); } if (DEBUG) { - Log.d(TAG, "--- add bigram: prev = " + prevWordsInfo.mPrevWord + ", current(" + Log.d(TAG, "--- add bigram: prev = " + prevWordsInfo + ", current(" + (isValidWord ? "Valid" : "OOV") + ") = " + word); } final int bigramProbability = isValidWord ? BIGRAM_PROBABILITY_FOR_VALID_WORD : BIGRAM_PROBABILITY_FOR_OOV_WORD; - return new LanguageModelParam(prevWordsInfo.mPrevWord, word, unigramProbability, - bigramProbability, timestamp); + return new LanguageModelParam(prevWordsInfo.mPrevWordsInfo[0].mWord, word, + unigramProbability, bigramProbability, timestamp); } } diff --git a/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java b/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java new file mode 100644 index 000000000..4dfb38d80 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/ScriptUtils.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import java.util.Locale; +import java.util.TreeMap; + +/** + * A class to help with handling different writing scripts. + */ +public class ScriptUtils { + public static final int SCRIPT_LATIN = 0; + public static final int SCRIPT_CYRILLIC = 1; + public static final int SCRIPT_GREEK = 2; + public static final TreeMap<String, Integer> mLanguageToScript; + static { + // List of the supported languages and their associated script. We won't check + // words written in another script than the selected script, because we know we + // don't have those in our dictionary so we will underline everything and we + // will never have any suggestions, so it makes no sense checking them, and this + // is done in {@link #shouldFilterOut}. Also, the script is used to choose which + // proximity to pass to the dictionary descent algorithm. + // IMPORTANT: this only contains languages - do not write countries in there. + // Only the language is searched from the map. + mLanguageToScript = new TreeMap<>(); + mLanguageToScript.put("cs", SCRIPT_LATIN); + mLanguageToScript.put("da", SCRIPT_LATIN); + mLanguageToScript.put("de", SCRIPT_LATIN); + mLanguageToScript.put("el", SCRIPT_GREEK); + mLanguageToScript.put("en", SCRIPT_LATIN); + mLanguageToScript.put("es", SCRIPT_LATIN); + mLanguageToScript.put("fi", SCRIPT_LATIN); + mLanguageToScript.put("fr", SCRIPT_LATIN); + mLanguageToScript.put("hr", SCRIPT_LATIN); + mLanguageToScript.put("it", SCRIPT_LATIN); + mLanguageToScript.put("lt", SCRIPT_LATIN); + mLanguageToScript.put("lv", SCRIPT_LATIN); + mLanguageToScript.put("nb", SCRIPT_LATIN); + mLanguageToScript.put("nl", SCRIPT_LATIN); + mLanguageToScript.put("pt", SCRIPT_LATIN); + mLanguageToScript.put("sl", SCRIPT_LATIN); + mLanguageToScript.put("ru", SCRIPT_CYRILLIC); + } + /* + * Returns whether the code point is a letter that makes sense for the specified + * locale for this spell checker. + * The dictionaries supported by Latin IME are described in res/xml/spellchecker.xml + * and is limited to EFIGS languages and Russian. + * Hence at the moment this explicitly tests for Cyrillic characters or Latin characters + * as appropriate, and explicitly excludes CJK, Arabic and Hebrew characters. + */ + public static boolean isLetterCheckableByScript(final int codePoint, final int script) { + switch (script) { + case SCRIPT_LATIN: + // Our supported latin script dictionaries (EFIGS) at the moment only include + // characters in the C0, C1, Latin Extended A and B, IPA extensions unicode + // blocks. As it happens, those are back-to-back in the code range 0x40 to 0x2AF, + // so the below is a very efficient way to test for it. As for the 0-0x3F, it's + // excluded from isLetter anyway. + return codePoint <= 0x2AF && Character.isLetter(codePoint); + case SCRIPT_CYRILLIC: + // All Cyrillic characters are in the 400~52F block. There are some in the upper + // Unicode range, but they are archaic characters that are not used in modern + // Russian and are not used by our dictionary. + return codePoint >= 0x400 && codePoint <= 0x52F && Character.isLetter(codePoint); + case SCRIPT_GREEK: + // Greek letters are either in the 370~3FF range (Greek & Coptic), or in the + // 1F00~1FFF range (Greek extended). Our dictionary contains both sort of characters. + // Our dictionary also contains a few words with 0xF2; it would be best to check + // if that's correct, but a web search does return results for these words so + // they are probably okay. + return (codePoint >= 0x370 && codePoint <= 0x3FF) + || (codePoint >= 0x1F00 && codePoint <= 0x1FFF) + || codePoint == 0xF2; + default: + // Should never come here + throw new RuntimeException("Impossible value of script: " + script); + } + } + + public static int getScriptFromLocale(final Locale locale) { + final Integer script = mLanguageToScript.get(locale.getLanguage()); + if (null == script) { + throw new RuntimeException("We have been called with an unsupported language: \"" + + locale.getLanguage() + "\". Framework bug?"); + } + return script; + } +} |