diff options
12 files changed, 187 insertions, 47 deletions
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeySpecParser.java b/java/src/com/android/inputmethod/keyboard/internal/KeySpecParser.java index 22f5b3dd1..cb4663777 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeySpecParser.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeySpecParser.java @@ -483,7 +483,7 @@ public final class KeySpecParser { public static int toUpperCaseOfCodeForLocale(final int code, final boolean needsToUpperCase, final Locale locale) { if (!Constants.isLetterCode(code) || !needsToUpperCase) return code; - final String text = new String(new int[] { code } , 0, 1); + final String text = StringUtils.newSingleCodePointString(code); final String casedText = KeySpecParser.toUpperCaseOfStringForLocale( text, needsToUpperCase, locale); return StringUtils.codePointCount(casedText) == 1 diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 1abbf3084..51ae63631 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -57,6 +57,8 @@ public final class BinaryDictionary extends Dictionary { @UsedForTesting public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; + private static final int NOT_A_VALID_TIME_STAMP = -1; + private long mNativeDict; private final Locale mLocale; private final long mDictSize; @@ -133,9 +135,11 @@ public final class BinaryDictionary extends Dictionary { int[] outputAutoCommitFirstWordConfidence); private static native float calcNormalizedScoreNative(int[] before, int[] after, int score); private static native int editDistanceNative(int[] before, int[] after); - private static native void addUnigramWordNative(long dict, int[] word, int probability); + private static native void addUnigramWordNative(long dict, int[] word, int probability, + int[] shortcutTarget, int shortcutProbability, boolean isNotAWord, + boolean isBlacklisted, int timeStamp); private static native void addBigramWordsNative(long dict, int[] word0, int[] word1, - int probability); + int probability, int timeStamp); private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); private static native int addMultipleDictionaryEntriesNative(long dict, LanguageModelParam[] languageModelParams, int startIndex); @@ -282,7 +286,10 @@ public final class BinaryDictionary extends Dictionary { return; } final int[] codePoints = StringUtils.toCodePointArray(word); - addUnigramWordNative(mNativeDict, codePoints, probability); + final int[] shortcutTarget = new int[0]; + addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTarget, + NOT_A_PROBABILITY, false /* isNotAWord */, false /* isBlacklisted */, + NOT_A_VALID_TIME_STAMP); } // Add a bigram entry to binary dictionary in native code. @@ -292,7 +299,8 @@ public final class BinaryDictionary extends Dictionary { } final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); - addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability); + addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, + NOT_A_VALID_TIME_STAMP); } // Remove a bigram entry form binary dictionary in native code. @@ -308,15 +316,25 @@ public final class BinaryDictionary extends Dictionary { public static class LanguageModelParam { public final int[] mWord0; public final int[] mWord1; + public final int[] mShortcutTarget; public final int mUnigramProbability; public final int mBigramProbability; + public final int mShortcutProbability; + public final boolean mIsNotAWord; + public final boolean mIsBlacklisted; + public final int mTimeStamp; // Constructor for unigram. public LanguageModelParam(final String word, final int unigramProbability) { mWord0 = null; mWord1 = StringUtils.toCodePointArray(word); + mShortcutTarget = null; mUnigramProbability = unigramProbability; mBigramProbability = NOT_A_PROBABILITY; + mShortcutProbability = NOT_A_PROBABILITY; + mIsNotAWord = false; + mIsBlacklisted = false; + mTimeStamp = NOT_A_VALID_TIME_STAMP; } // Constructor for unigram and bigram. @@ -324,8 +342,13 @@ public final class BinaryDictionary extends Dictionary { final int unigramProbability, final int bigramProbability) { mWord0 = StringUtils.toCodePointArray(word0); mWord1 = StringUtils.toCodePointArray(word1); + mShortcutTarget = null; mUnigramProbability = unigramProbability; mBigramProbability = bigramProbability; + mShortcutProbability = NOT_A_PROBABILITY; + mIsNotAWord = false; + mIsBlacklisted = false; + mTimeStamp = NOT_A_VALID_TIME_STAMP; } } diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 5aa824ed4..b5ea0de01 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -428,7 +428,7 @@ public final class RichInputConnection { } break; default: - final String text = new String(new int[] { keyEvent.getUnicodeChar() }, 0, 1); + final String text = StringUtils.newSingleCodePointString(keyEvent.getUnicodeChar()); mCommittedTextBeforeComposingText.append(text); mExpectedSelStart += text.length(); mExpectedSelEnd = mExpectedSelStart; diff --git a/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java b/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java index e958a7e71..d14ba508b 100644 --- a/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java @@ -35,7 +35,7 @@ public final class LatinImeLoggerUtils { public static void onSeparator(final int code, final int x, final int y) { // Helper method to log a single code point separator // TODO: cache this mapping of a code point to a string in a sparse array in StringUtils - onSeparator(new String(new int[]{code}, 0, 1), x, y); + onSeparator(StringUtils.newSingleCodePointString(code), x, y); } public static void onSeparator(final String separator, final int x, final int y) { diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index b1aa034ea..5fc20eaa0 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -277,7 +277,8 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji } static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jlong dict, - jintArray word, jint probability) { + jintArray word, jint probability, jintArray shortcutTarget, jint shortuctProbability, + jboolean isNotAWord, jboolean isBlacklisted, jint timeStamp) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) { return; @@ -285,11 +286,16 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jsize wordLength = env->GetArrayLength(word); int codePoints[wordLength]; env->GetIntArrayRegion(word, 0, wordLength, codePoints); + jsize shortcutLength = shortcutTarget ? env->GetArrayLength(shortcutTarget) : 0; + int shortcutTargetCodePoints[wordLength]; + if (shortcutTarget) { + env->GetIntArrayRegion(shortcutTarget, 0, shortcutLength, shortcutTargetCodePoints); + } dictionary->addUnigramWord(codePoints, wordLength, probability); } static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jlong dict, - jintArray word0, jintArray word1, jint probability) { + jintArray word0, jintArray word1, jint probability, jint timeStamp) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) { return; @@ -335,6 +341,8 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j jobject languageModelParam = env->GetObjectArrayElement(languageModelParams, 0); jclass languageModelParamClass = env->GetObjectClass(languageModelParam); env->DeleteLocalRef(languageModelParam); + + // TODO: Support shortcut, timestamp and flags. jfieldID word0FieldId = env->GetFieldID(languageModelParamClass, "mWord0", "[I"); jfieldID word1FieldId = env->GetFieldID(languageModelParamClass, "mWord1", "[I"); jfieldID unigramProbabilityFieldId = @@ -345,6 +353,9 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j for (int i = startIndex; i < languageModelParamCount; ++i) { jobject languageModelParam = env->GetObjectArrayElement(languageModelParams, i); + // languageModelParam is a set of params for word1; thus, word1 cannot be null. On the + // other hand, word0 can be null and then it means the set of params doesn't contain bigram + // information. jintArray word0 = static_cast<jintArray>( env->GetObjectField(languageModelParam, word0FieldId)); jsize word0Length = word0 ? env->GetArrayLength(word0) : 0; @@ -459,12 +470,12 @@ static const JNINativeMethod sMethods[] = { }, { const_cast<char *>("addUnigramWordNative"), - const_cast<char *>("(J[II)V"), + const_cast<char *>("(J[II[IIZZI)V"), reinterpret_cast<void *>(latinime_BinaryDictionary_addUnigramWord) }, { const_cast<char *>("addBigramWordsNative"), - const_cast<char *>("(J[I[II)V"), + const_cast<char *>("(J[I[III)V"), reinterpret_cast<void *>(latinime_BinaryDictionary_addBigramWords) }, { diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp index 628d3ab38..dc2adb44e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp @@ -20,6 +20,7 @@ #include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" namespace latinime { @@ -46,10 +47,12 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget if (!mBigramDictContent->createNewBigramList(terminalId)) { return false; } + const int probabilityToWrite = getUpdatedProbability( + NOT_A_PROBABILITY /* originalProbability */, newProbability); // Write an entry. - int writingPos = mBigramDictContent->getBigramListHeadPos(terminalId); - if (!mBigramDictContent->writeBigramEntryAndAdvancePosition(newProbability, - false /* hasNext */, newTargetTerminalId, &writingPos)) { + const int writingPos = mBigramDictContent->getBigramListHeadPos(terminalId); + if (!mBigramDictContent->writeBigramEntry(probabilityToWrite, false /* hasNext */, + newTargetTerminalId, writingPos)) { return false; } if (outAddedNewEntry) { @@ -61,19 +64,18 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget const int entryPosToUpdate = getEntryPosToUpdate(newTargetTerminalId, bigramListPos); if (entryPosToUpdate != NOT_A_DICT_POS) { // Overwrite existing entry. - int readingPos = entryPosToUpdate; bool hasNext = false; int probability = NOT_A_PROBABILITY; int targetTerminalId = Ver4DictConstants::NOT_A_TERMINAL_ID; - mBigramDictContent->getBigramEntryAndAdvancePosition(&probability, &hasNext, - &targetTerminalId, &readingPos); + mBigramDictContent->getBigramEntry(&probability, &hasNext, &targetTerminalId, + entryPosToUpdate); + const int probabilityToWrite = getUpdatedProbability(probability, newProbability); if (targetTerminalId == Ver4DictConstants::NOT_A_TERMINAL_ID && outAddedNewEntry) { // Reuse invalid entry. *outAddedNewEntry = true; } - int writingPos = entryPosToUpdate; - return mBigramDictContent->writeBigramEntryAndAdvancePosition(newProbability, hasNext, - newTargetTerminalId, &writingPos); + return mBigramDictContent->writeBigramEntry(probabilityToWrite, hasNext, + newTargetTerminalId, entryPosToUpdate); } // Add new entry to the bigram list. @@ -83,7 +85,9 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget } // Write new entry at a head position of the bigram list. int writingPos = mBigramDictContent->getBigramListHeadPos(terminalId); - if (!mBigramDictContent->writeBigramEntryAndAdvancePosition(newProbability, + const int probabilityToWrite = getUpdatedProbability( + NOT_A_PROBABILITY /* originalProbability */, newProbability); + if (!mBigramDictContent->writeBigramEntryAndAdvancePosition(probabilityToWrite, true /* hasNext */, newTargetTerminalId, &writingPos)) { return false; } @@ -105,20 +109,18 @@ bool Ver4BigramListPolicy::removeEntry(const int terminalId, const int targetTer // Bigram entry doesn't exist. return false; } - int readingPos = entryPosToUpdate; bool hasNext = false; int probability = NOT_A_PROBABILITY; int originalTargetTerminalId = Ver4DictConstants::NOT_A_TERMINAL_ID; - mBigramDictContent->getBigramEntryAndAdvancePosition(&probability, &hasNext, - &originalTargetTerminalId, &readingPos); + mBigramDictContent->getBigramEntry(&probability, &hasNext, &originalTargetTerminalId, + entryPosToUpdate); if (targetTerminalId != originalTargetTerminalId) { // Bigram entry doesn't exist. return false; } - int writingPos = entryPosToUpdate; // Remove bigram entry by overwriting target terminal Id. - return mBigramDictContent->writeBigramEntryAndAdvancePosition(probability, hasNext, - Ver4DictConstants::NOT_A_TERMINAL_ID /* targetTerminalId */, &writingPos); + return mBigramDictContent->writeBigramEntry(probability, hasNext, + Ver4DictConstants::NOT_A_TERMINAL_ID /* targetTerminalId */, entryPosToUpdate); } bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const int terminalId, @@ -143,9 +145,28 @@ bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const i targetTerminalId); if (targetPtNodePos == NOT_A_DICT_POS) { // Invalidate bigram entry. - int writingPos = entryPos; - return mBigramDictContent->writeBigramEntryAndAdvancePosition(probability, hasNext, - Ver4DictConstants::NOT_A_TERMINAL_ID /* targetTerminalId */, &writingPos); + if (!mBigramDictContent->writeBigramEntry(probability, hasNext, + Ver4DictConstants::NOT_A_TERMINAL_ID /* targetTerminalId */, entryPos)) { + return false; + } + } else if (mNeedsToDecayWhenUpdating) { + probability = ForgettingCurveUtils::getEncodedProbabilityToSave( + probability, mHeaderPolicy); + if (ForgettingCurveUtils::isValidEncodedProbability(probability)) { + if (!mBigramDictContent->writeBigramEntry(probability, hasNext, targetTerminalId, + entryPos)) { + return false; + } + *outBigramCount += 1; + } else { + // Remove entry. + if (!mBigramDictContent->writeBigramEntry(probability, hasNext, + Ver4DictConstants::NOT_A_TERMINAL_ID /* targetTerminalId */, entryPos)) { + return false; + } + } + } else { + *outBigramCount += 1; } } return true; @@ -192,4 +213,14 @@ int Ver4BigramListPolicy::getEntryPosToUpdate(const int targetTerminalIdToFind, return invalidEntryPos; } +int Ver4BigramListPolicy::getUpdatedProbability(const int originalProbability, + const int newProbability) const { + if (mNeedsToDecayWhenUpdating) { + return ForgettingCurveUtils::getUpdatedEncodedProbability(originalProbability, + newProbability); + } else { + return newProbability; + } +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h index 5b7d5b527..ed8bdb84d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h @@ -23,14 +23,18 @@ namespace latinime { class BigramDictContent; +class DictionaryHeaderStructurePolicy; class TerminalPositionLookupTable; class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy { public: Ver4BigramListPolicy(BigramDictContent *const bigramDictContent, - const TerminalPositionLookupTable *const terminalPositionLookupTable) + const TerminalPositionLookupTable *const terminalPositionLookupTable, + const DictionaryHeaderStructurePolicy *const headerPolicy, + const bool needsToDecayWhenUpdating) : mBigramDictContent(bigramDictContent), - mTerminalPositionLookupTable(terminalPositionLookupTable) {} + mTerminalPositionLookupTable(terminalPositionLookupTable), + mHeaderPolicy(headerPolicy), mNeedsToDecayWhenUpdating(needsToDecayWhenUpdating) {} void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, int *const bigramEntryPos) const; @@ -54,8 +58,12 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy { int getEntryPosToUpdate(const int targetTerminalIdToFind, const int bigramListPos) const; + int getUpdatedProbability(const int originalProbability, const int newProbability) const; + BigramDictContent *const mBigramDictContent; const TerminalPositionLookupTable *const mTerminalPositionLookupTable; + const DictionaryHeaderStructurePolicy *const mHeaderPolicy; + const bool mNeedsToDecayWhenUpdating; }; } // namespace latinime #endif /* LATINIME_VER4_BIGRAM_LIST_POLICY_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp index 906687647..4cd96722e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp @@ -103,6 +103,7 @@ bool BigramDictContent::runGC(const TerminalPositionLookupTable::TerminalIdMap * return true; } +// Returns whether GC for the bigram list was succeeded or not. bool BigramDictContent::runGCBigramList(const int bigramListPos, const BigramDictContent *const sourceBigramDictContent, const int toPos, const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, @@ -121,9 +122,8 @@ bool BigramDictContent::runGCBigramList(const int bigramListPos, TerminalPositionLookupTable::TerminalIdMap::const_iterator it = terminalIdMap->find(targetTerminalId); if (it == terminalIdMap->end()) { - AKLOGE("terminal Id %d is not in the terminal position map. map size: %zd", - targetTerminalId, terminalIdMap->size()); - return false; + // Target word has been removed. + continue; } if (!writeBigramEntryAndAdvancePosition(probability, hasNext, it->second, &writingPos)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h index ec0d756d8..cf380f403 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h @@ -38,6 +38,13 @@ class BigramDictContent : public SparseTableDictContent { : SparseTableDictContent(Ver4DictConstants::BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, Ver4DictConstants::BIGRAM_ADDRESS_TABLE_DATA_SIZE) {} + void getBigramEntry(int *const outProbability, bool *const outHasNext, + int *const outTargetTerminalId, const int bigramEntryPos) const { + int readingPos = bigramEntryPos; + getBigramEntryAndAdvancePosition(outProbability, outHasNext, outTargetTerminalId, + &readingPos); + } + void getBigramEntryAndAdvancePosition(int *const outProbability, bool *const outHasNext, int *const outTargetTerminalId, int *const bigramEntryPos) const; @@ -50,6 +57,13 @@ class BigramDictContent : public SparseTableDictContent { return addressLookupTable->get(terminalId); } + bool writeBigramEntry(const int probability, const int hasNext, const int targetTerminalId, + const int entryWritingPos) { + int writingPos = entryWritingPos; + return writeBigramEntryAndAdvancePosition(probability, hasNext, targetTerminalId, + &writingPos); + } + bool writeBigramEntryAndAdvancePosition(const int probability, const int hasNext, const int targetTerminalId, int *const entryWritingPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 6fe978d0f..3606a2ae9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -43,7 +43,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { false /* usesAdditionalBuffer*/), FormatUtils::VERSION_4), mDictBuffer(mBuffers.get()->getWritableTrieBuffer()), mBigramPolicy(mBuffers.get()->getUpdatableBigramDictContent(), - mBuffers.get()->getTerminalPositionLookupTable()), + mBuffers.get()->getTerminalPositionLookupTable(), &mHeaderPolicy, + mHeaderPolicy.isDecayingDict()), mShortcutPolicy(mBuffers.get()->getShortcutDictContent(), mBuffers.get()->getTerminalPositionLookupTable()), mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index f141d52f5..ea03c72fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -83,7 +83,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(), mBuffers->getProbabilityDictContent()); Ver4BigramListPolicy bigramPolicy(mBuffers->getUpdatableBigramDictContent(), - mBuffers->getTerminalPositionLookupTable()); + mBuffers->getTerminalPositionLookupTable(), headerPolicy, needsToDecay); Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getShortcutDictContent(), mBuffers->getTerminalPositionLookupTable()); Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), @@ -134,7 +134,8 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, Ver4PatriciaTrieNodeReader newPtNodeReader(buffersToWrite->getTrieBuffer(), buffersToWrite->getProbabilityDictContent()); Ver4BigramListPolicy newBigramPolicy(buffersToWrite->getUpdatableBigramDictContent(), - buffersToWrite->getTerminalPositionLookupTable()); + buffersToWrite->getTerminalPositionLookupTable(), headerPolicy, + false /* needsToDecay */); Ver4ShortcutListPolicy newShortcutPolicy(buffersToWrite->getShortcutDictContent(), buffersToWrite->getTerminalPositionLookupTable()); Ver4PatriciaTrieNodeWriter newPtNodeWriter(buffersToWrite->getWritableTrieBuffer(), diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index 12b1caab1..140ab63b0 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -72,26 +72,63 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } } - private File createEmptyDictionaryAndGetFile(final String filename) throws IOException { - final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, + private File createEmptyDictionaryAndGetFile(final String dictId, + final int formatVersion) throws IOException { + if (formatVersion == 3) { + return createEmptyVer3DictionaryAndGetFile(dictId); + } else if (formatVersion == 4) { + return createEmptyVer4DictionaryAndGetFile(dictId); + } else { + throw new IOException("Dictionary format version " + formatVersion + + " is not supported."); + } + } + private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException { + final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); + file.delete(); + file.mkdir(); Map<String, String> attributeMap = new HashMap<String, String>(); attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - 3 /* dictVersion */, attributeMap)) { + 4 /* dictVersion */, attributeMap)) { + return new File(file, FormatSpec.TRIE_FILE_EXTENSION); + } else { + throw new IOException("Empty dictionary " + file.getAbsolutePath() + " " + + FormatSpec.TRIE_FILE_EXTENSION + " cannot be created."); + } + } + + private File createEmptyVer3DictionaryAndGetFile(final String dictId) throws IOException { + final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); + file.delete(); + Map<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), 3 /* dictVersion */, + attributeMap)) { return file; } else { - throw new IOException("Empty dictionary cannot be created."); + throw new IOException( + "Empty dictionary " + file.getAbsolutePath() + " cannot be created."); } } public void testAddValidAndInvalidWords() { + testAddValidAndInvalidWords(3 /* formatVersion */); + testAddValidAndInvalidWords(4 /* formatVersion */); + } + + private void testAddValidAndInvalidWords(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -111,7 +148,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidWord("b")); - final int unigramProbability = binaryDictionary.getFrequency("a"); binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidBigram("a", "b")); binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); @@ -136,9 +172,14 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testDecayingProbability() { + testDecayingProbability(3 /* formatVersion */); + testDecayingProbability(4 /* formatVersion */); + } + + private void testDecayingProbability(final int formatVersion) { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -190,6 +231,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testAddManyUnigramsToDecayingDict() { + testAddManyUnigramsToDecayingDict(3 /* formatVersion */); + testAddManyUnigramsToDecayingDict(4 /* formatVersion */); + } + + private void testAddManyUnigramsToDecayingDict(final int formatVersion) { final int unigramCount = 30000; final int unigramTypedCount = 100000; final int codePointSetSize = 50; @@ -198,7 +244,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } @@ -242,6 +288,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testAddManyBigramsToDecayingDict() { + testAddManyBigramsToDecayingDict(3 /* formatVersion */); + testAddManyBigramsToDecayingDict(4 /* formatVersion */); + } + + private void testAddManyBigramsToDecayingDict(final int formatVersion) { final int unigramCount = 5000; final int bigramCount = 30000; final int bigramTypedCount = 100000; @@ -251,7 +302,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { File dictFile = null; try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } |