diff options
17 files changed, 144 insertions, 134 deletions
diff --git a/dictionaries/de_wordlist.combined.gz b/dictionaries/de_wordlist.combined.gz Binary files differindex 400718df1..803211c01 100644 --- a/dictionaries/de_wordlist.combined.gz +++ b/dictionaries/de_wordlist.combined.gz diff --git a/dictionaries/en_GB_wordlist.combined.gz b/dictionaries/en_GB_wordlist.combined.gz Binary files differindex bf637e93d..1fa9b85ea 100644 --- a/dictionaries/en_GB_wordlist.combined.gz +++ b/dictionaries/en_GB_wordlist.combined.gz diff --git a/dictionaries/en_US_wordlist.combined.gz b/dictionaries/en_US_wordlist.combined.gz Binary files differindex 9ea04b146..2e039ff05 100644 --- a/dictionaries/en_US_wordlist.combined.gz +++ b/dictionaries/en_US_wordlist.combined.gz diff --git a/dictionaries/en_wordlist.combined.gz b/dictionaries/en_wordlist.combined.gz Binary files differindex 87a863330..e845346d6 100644 --- a/dictionaries/en_wordlist.combined.gz +++ b/dictionaries/en_wordlist.combined.gz diff --git a/dictionaries/es_wordlist.combined.gz b/dictionaries/es_wordlist.combined.gz Binary files differindex 8bb21027f..3391e64b4 100644 --- a/dictionaries/es_wordlist.combined.gz +++ b/dictionaries/es_wordlist.combined.gz diff --git a/dictionaries/ro_wordlist.combined.gz b/dictionaries/ro_wordlist.combined.gz Binary files differindex 829abd9b8..53078553c 100644 --- a/dictionaries/ro_wordlist.combined.gz +++ b/dictionaries/ro_wordlist.combined.gz diff --git a/dictionaries/ru_wordlist.combined.gz b/dictionaries/ru_wordlist.combined.gz Binary files differindex 7794f7bed..401ad08b0 100644 --- a/dictionaries/ru_wordlist.combined.gz +++ b/dictionaries/ru_wordlist.combined.gz diff --git a/java/res/raw/main_de.dict b/java/res/raw/main_de.dict Binary files differindex 3cbf7105d..45b288375 100644 --- a/java/res/raw/main_de.dict +++ b/java/res/raw/main_de.dict diff --git a/java/res/raw/main_en.dict b/java/res/raw/main_en.dict Binary files differindex d631d6fbc..5bbb85761 100644 --- a/java/res/raw/main_en.dict +++ b/java/res/raw/main_en.dict diff --git a/java/res/raw/main_es.dict b/java/res/raw/main_es.dict Binary files differindex 8c42fda8e..fae131850 100644 --- a/java/res/raw/main_es.dict +++ b/java/res/raw/main_es.dict diff --git a/java/res/raw/main_ru.dict b/java/res/raw/main_ru.dict Binary files differindex 9fdaf3148..8d1576987 100644 --- a/java/res/raw/main_ru.dict +++ b/java/res/raw/main_ru.dict diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 6f3c48c47..10dea749d 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -42,6 +42,8 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import javax.annotation.Nonnull; + /** * Implements a static, compacted, binary dictionary of standard words. */ @@ -199,6 +201,7 @@ public final class BinaryDictionary extends Dictionary { int[] word, int probability, int timestamp); private static native boolean removeNgramEntryNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); + // TODO: Rename to updateEntriesForWordWithNgramContextNative. private static native boolean updateCounterNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word, boolean isValidWord, int count, int timestamp); @@ -493,6 +496,24 @@ public final class BinaryDictionary extends Dictionary { return true; } + // Update entries for the word occurrence with the ngramContext. + public boolean updateEntriesForWordWithNgramContext(@Nonnull final NgramContext ngramContext, + final String word, final boolean isValidWord, final int count, final int timestamp) { + if (TextUtils.isEmpty(word)) { + return false; + } + final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; + final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; + ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); + final int[] wordCodePoints = StringUtils.toCodePointArray(word); + if (!updateCounterNative(mNativeDict, prevWordCodePointArrays, + isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { + return false; + } + mHasUpdated = true; + return true; + } + @UsedForTesting public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { if (!isValidDictionary()) return; diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 1bdadc30b..d24f80a45 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -46,6 +46,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.annotation.Nonnull; + /** * Abstract base class for an expandable dictionary that can be created and updated dynamically * during runtime. When updated it automatically generates a new binary dictionary to handle future @@ -292,13 +294,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - /** - * Adds unigram information of a word to the dictionary. May overwrite an existing entry. - */ - public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency, - final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, - final boolean isBlacklisted, final int timestamp, - final DistracterFilter distracterFilter) { + private void updateDictionaryWithWriteLockIfWordIsNotADistracter( + @Nonnull final Runnable updateTask, + @Nonnull final String word, @Nonnull final DistracterFilter distracterFilter) { reloadDictionaryIfRequired(); asyncPreCheckAndExecuteTaskWithWriteLock( new Callable<Boolean>() { @@ -315,12 +313,27 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return; } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, - isNotAWord, isBlacklisted, timestamp); + updateTask.run(); } }); } + /** + * Adds unigram information of a word to the dictionary. May overwrite an existing entry. + */ + public void addUnigramEntryWithCheckingDistracter(final String word, final int frequency, + final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, + final boolean isBlacklisted, final int timestamp, + @Nonnull final DistracterFilter distracterFilter) { + updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + @Override + public void run() { + addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, + isNotAWord, isBlacklisted, timestamp); + } + }, word, distracterFilter); + } + protected void addUnigramLocked(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { @@ -354,7 +367,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Adds n-gram information of a word to the dictionary. May overwrite an existing entry. */ - public void addNgramEntry(final NgramContext ngramContext, final String word, + public void addNgramEntry(@Nonnull final NgramContext ngramContext, final String word, final int frequency, final int timestamp) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { @@ -369,7 +382,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } - protected void addNgramEntryLocked(final NgramContext ngramContext, final String word, + protected void addNgramEntryLocked(@Nonnull final NgramContext ngramContext, final String word, final int frequency, final int timestamp) { if (!mBinaryDictionary.addNgramEntry(ngramContext, word, frequency, timestamp)) { if (DEBUG) { @@ -383,7 +396,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Dynamically remove the n-gram entry in the dictionary. */ @UsedForTesting - public void removeNgramDynamically(final NgramContext ngramContext, final String word) { + public void removeNgramDynamically(@Nonnull final NgramContext ngramContext, + final String word) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { @Override @@ -402,6 +416,26 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } + /** + * Update dictionary for the word with the ngramContext if the word is not a distracter. + */ + public void updateEntriesForWordWithCheckingDistracter(@Nonnull final NgramContext ngramContext, + final String word, final boolean isValidWord, final int count, final int timestamp, + @Nonnull final DistracterFilter distracterFilter) { + updateDictionaryWithWriteLockIfWordIsNotADistracter(new Runnable() { + @Override + public void run() { + if (!mBinaryDictionary.updateEntriesForWordWithNgramContext(ngramContext, word, + isValidWord, count, timestamp)) { + if (DEBUG) { + Log.e(TAG, "Cannot update counter. word: " + word + + " context: "+ ngramContext.toString()); + } + } + } + }, word, distracterFilter); + } + public interface AddMultipleDictionaryEntriesCallback { public void onFinished(); } @@ -410,7 +444,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Dynamically add multiple entries to the dictionary. */ public void addMultipleDictionaryEntriesDynamically( - final ArrayList<LanguageModelParam> languageModelParams, + @Nonnull final ArrayList<LanguageModelParam> languageModelParams, final AddMultipleDictionaryEntriesCallback callback) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { diff --git a/java/src/com/android/inputmethod/latin/NgramContext.java b/java/src/com/android/inputmethod/latin/NgramContext.java index 6d438584f..a02531cc4 100644 --- a/java/src/com/android/inputmethod/latin/NgramContext.java +++ b/java/src/com/android/inputmethod/latin/NgramContext.java @@ -158,11 +158,6 @@ public class NgramContext { } } - public NgramContext getTrimmedNgramContext(final int maxPrevWordCount) { - final int newSize = Math.min(maxPrevWordCount, mPrevWordsCount); - return new NgramContext(this /* prevWordsInfo */, newSize); - } - public int getPrevWordCount() { return mPrevWordsCount; } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index d61684698..59761547d 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -65,34 +65,7 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas if (word.length() > Constants.DICTIONARY_MAX_WORD_LENGTH) { return; } - final int frequency = isValid ? - FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; - userHistoryDictionary.addUnigramEntryWithCheckingDistracter(word, frequency, - null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */, - false /* isBlacklisted */, timestamp, distracterFilter); - - final boolean isBeginningOfSentenceContext = ngramContext.isBeginningOfSentenceContext(); - final NgramContext ngramContextToBeSaved = - ngramContext.getTrimmedNgramContext(SUPPORTED_NGRAM - 1); - for (int i = 0; i < ngramContextToBeSaved.getPrevWordCount(); i++) { - final CharSequence prevWord = ngramContextToBeSaved.getNthPrevWord(1 /* n */); - if (prevWord == null || (prevWord.length() > Constants.DICTIONARY_MAX_WORD_LENGTH)) { - return; - } - // Do not insert a word as a bigram of itself - if (i == 0 && TextUtils.equals(word, prevWord)) { - return; - } - if (isBeginningOfSentenceContext) { - // Beginning-of-Sentence n-gram entry is added as an n-gram entry of an OOV word. - userHistoryDictionary.addNgramEntry( - ngramContextToBeSaved.getTrimmedNgramContext(i + 1), word, - FREQUENCY_FOR_WORDS_NOT_IN_DICTS, timestamp); - } else { - userHistoryDictionary.addNgramEntry( - ngramContextToBeSaved.getTrimmedNgramContext(i + 1), word, frequency, - timestamp); - } - } + userHistoryDictionary.updateEntriesForWordWithCheckingDistracter(ngramContext, word, + isValid, 1 /* count */, timestamp, distracterFilter); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp index 4a740d47b..ef6166ffd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/backward/v402/content/probability_dict_content.cpp @@ -74,8 +74,8 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId, return false; } writingPos += getEntrySize(); - mSize++; } + mSize = terminalId + 1; } return writeEntry(probabilityEntry, entryPos); } @@ -100,7 +100,6 @@ bool ProbabilityDictContent::flushToFile(const char *const dictPath) const { bool ProbabilityDictContent::runGC( const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ProbabilityDictContent *const originalProbabilityDictContent) { - mSize = 0; for (TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->begin(); it != terminalIdMap->end(); ++it) { const ProbabilityEntry probabilityEntry = @@ -109,7 +108,6 @@ bool ProbabilityDictContent::runGC( AKLOGE("Cannot set probability entry in runGC. terminalId: %d", it->second); return false; } - mSize++; } return true; } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index 6b6ad21b7..c3c2bb2fd 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -75,20 +75,25 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { return formatVersion > FormatSpec.VERSION401; } - private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, - final int probability) { - binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, - BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, - false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, mCurrentTime /* timestamp */); + private void onInputWord(final BinaryDictionary binaryDictionary, final String word, + final boolean isValidWord) { + binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO, + word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } - private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, - final String word1, final int probability) { - binaryDictionary.addNgramEntry(new NgramContext(new WordInfo(word0)), word1, probability, + private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word, + final boolean isValidWord, final String prevWord) { + binaryDictionary.updateEntriesForWordWithNgramContext( + new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } + private void onInputWordWithBeginningOfSentenceContext( + final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) { + binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE, + word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); + } + private static boolean isValidBigram(final BinaryDictionary binaryDictionary, final String word0, final String word1) { return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); @@ -175,10 +180,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a"); binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -229,28 +233,27 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "a", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "a", false /* isValidWord */); + onInputWord(binaryDictionary, "a", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "b", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("b")); - addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertFalse(isValidBigram(binaryDictionary, "a", "b")); - addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); - addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. - addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); - addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); binaryDictionary.close(); @@ -266,36 +269,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); assertFalse(isValidBigram(binaryDictionary, "a", "b")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); assertTrue(isValidBigram(binaryDictionary, "a", "b")); @@ -333,7 +332,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = @@ -380,10 +379,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String strong = "strong"; final String weak = "weak"; for (int j = 0; j < strongUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, strong, true /* isValidWord */); } for (int j = 0; j < weakUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, weak, true /* isValidWord */); } assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(weak)); @@ -391,7 +390,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); for (int j = 0; j < eachUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = @@ -450,12 +449,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final int maxBigramCount = Integer.parseInt( - binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); - addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, bigram.first, true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */, + bigram.first); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = @@ -506,23 +506,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); for (int j = 0; j < unigramTypedCount; j++) { - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); } } final String strong = "strong"; final String weak = "weak"; final String target = "target"; for (int j = 0; j < unigramTypedCount; j++) { - addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, weak, true /* isValidWord */); + onInputWord(binaryDictionary, strong, true /* isValidWord */); } binaryDictionary.flushWithGC(); for (int j = 0; j < strongBigramTypedCount; j++) { - addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong); } for (int j = 0; j < weakBigramTypedCount; j++) { - addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak); } assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); @@ -535,7 +534,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word1 = words.get(word1Index); for (int j = 0; j < eachBigramTypedCount; j++) { - addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = @@ -563,19 +562,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("aaa")); - addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidWord("bbb")); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + + onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); - addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); + assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); @@ -585,11 +583,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); - addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "bbb", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); - addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); binaryDictionary.close(); } @@ -612,28 +610,19 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, mCurrentTime); final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, - mCurrentTime); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, - mCurrentTime); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); - forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, - mCurrentTime); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); binaryDictionary.close(); @@ -651,10 +640,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "aaa", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("aaa")); for (int i = 0; i < unigramInputCount; i++) { - addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "aaa", false /* isValidWord */); } assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.removeUnigramEntry("aaa")); |