diff options
17 files changed, 285 insertions, 170 deletions
diff --git a/java/src/com/android/inputmethod/latin/AssetFileAddress.java b/java/src/com/android/inputmethod/latin/AssetFileAddress.java index 875192554..855a8d8c7 100644 --- a/java/src/com/android/inputmethod/latin/AssetFileAddress.java +++ b/java/src/com/android/inputmethod/latin/AssetFileAddress.java @@ -52,4 +52,12 @@ public final class AssetFileAddress { if (!f.isFile()) return null; return new AssetFileAddress(filename, offset, length); } + + public boolean pointsToPhysicalFile() { + return 0 == mOffset; + } + + public void deleteUnderlyingFile() { + new File(mFilename).delete(); + } } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 0e03b5280..aa530ffb9 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -57,7 +57,7 @@ public final class BinaryDictionary extends Dictionary { @UsedForTesting public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; - public static final int NOT_A_VALID_TIME_STAMP = -1; + public static final int NOT_A_VALID_TIMESTAMP = -1; private long mNativeDict; private final Locale mLocale; @@ -121,6 +121,7 @@ public final class BinaryDictionary extends Dictionary { String[] attributeKeyStringArray, String[] attributeValueStringArray); private static native long openNative(String sourceDir, long dictOffset, long dictSize, boolean isUpdatable); + private static native boolean hasValidContentsNative(long dict); private static native void flushNative(long dict, String filePath); private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); private static native void flushWithGCNative(long dict, String filePath); @@ -242,6 +243,10 @@ public final class BinaryDictionary extends Dictionary { return mNativeDict != 0; } + public boolean hasValidContents() { + return hasValidContentsNative(mNativeDict); + } + public int getFormatVersion() { return getFormatVersionNative(mNativeDict); } @@ -285,18 +290,6 @@ public final class BinaryDictionary extends Dictionary { return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); } - // Add a unigram entry to binary dictionary in native code. - public void addUnigramWord(final String word, final int probability) { - if (TextUtils.isEmpty(word)) { - return; - } - final int[] codePoints = StringUtils.toCodePointArray(word); - final int[] shortcutTargetCodePoints = new int[0]; - addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, - NOT_A_PROBABILITY, false /* isNotAWord */, false /* isBlacklisted */, - NOT_A_VALID_TIME_STAMP); - } - // Add a unigram entry to binary dictionary with unigram attributes in native code. public void addUnigramWord(final String word, final int probability, final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord, @@ -311,17 +304,6 @@ public final class BinaryDictionary extends Dictionary { shortcutProbability, isNotAWord, isBlacklisted, timestamp); } - // Add a bigram entry to binary dictionary in native code. - public void addBigramWords(final String word0, final String word1, final int probability) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { - return; - } - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, - NOT_A_VALID_TIME_STAMP); - } - // Add a bigram entry to binary dictionary with timestamp in native code. public void addBigramWords(final String word0, final String word1, final int probability, final int timestamp) { @@ -403,7 +385,7 @@ public final class BinaryDictionary extends Dictionary { private void reopen() { close(); final File dictFile = new File(mDictFilePath); - // WARNING: Because we pass 0 as the offstet and file.length() as the length, this can + // WARNING: Because we pass 0 as the offset and file.length() as the length, this can // only be called for actual files. Right now it's only called by the flush() family of // functions, which require an updatable dictionary, so it's okay. But beware. loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java index ad94a0493..b4382bc2c 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java @@ -98,7 +98,7 @@ public final class BinaryDictionaryFileDumper { * This creates a URI builder able to build a URI pointing to the dictionary * pack content provider for a specific dictionary id. */ - private static Uri.Builder getProviderUriBuilder(final String path) { + public static Uri.Builder getProviderUriBuilder(final String path) { return new Uri.Builder().scheme(ContentResolver.SCHEME_CONTENT) .authority(DictionaryPackConstants.AUTHORITY).appendPath(path); } @@ -339,15 +339,25 @@ public final class BinaryDictionaryFileDumper { Log.e(TAG, "Could not copy a word list. Will not be able to use it."); // If we can't copy it we should warn the dictionary provider so that it can mark it // as invalid. - wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT, - QUERY_PARAMETER_FAILURE); + reportBrokenFileToDictionaryProvider(providerClient, clientId, wordlistId); + } + + public static boolean reportBrokenFileToDictionaryProvider( + final ContentProviderClient providerClient, final String clientId, + final String wordlistId) { try { + final Uri.Builder wordListUriBuilder = getContentUriBuilderForType(clientId, + providerClient, QUERY_PATH_DATAFILE, wordlistId /* extraPath */); + wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT, + QUERY_PARAMETER_FAILURE); if (0 >= providerClient.delete(wordListUriBuilder.build(), null, null)) { - Log.e(TAG, "In addition, we were unable to delete it."); + Log.e(TAG, "Unable to delete a word list."); } } catch (RemoteException e) { - Log.e(TAG, "In addition, communication with the dictionary provider was cut", e); + Log.e(TAG, "Communication with the dictionary provider was cut", e); + return false; } + return true; } // Ideally the two following methods should be merged, but AssetFileDescriptor does not diff --git a/java/src/com/android/inputmethod/latin/DictionaryFactory.java b/java/src/com/android/inputmethod/latin/DictionaryFactory.java index 828e54f14..bcb38da38 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFactory.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFactory.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin; +import android.content.ContentProviderClient; import android.content.Context; import android.content.res.AssetFileDescriptor; import android.content.res.Resources; @@ -62,8 +63,12 @@ public final class DictionaryFactory { final ReadOnlyBinaryDictionary readOnlyBinaryDictionary = new ReadOnlyBinaryDictionary(f.mFilename, f.mOffset, f.mLength, useFullEditDistance, locale, Dictionary.TYPE_MAIN); - if (readOnlyBinaryDictionary.isValidDictionary()) { + if (readOnlyBinaryDictionary.hasValidContents()) { dictList.add(readOnlyBinaryDictionary); + } else { + readOnlyBinaryDictionary.close(); + // Prevent this dictionary to do any further harm. + killDictionary(context, f); } } } @@ -75,6 +80,51 @@ public final class DictionaryFactory { } /** + * Kills a dictionary so that it is never used again, if possible. + * @param context The context to contact the dictionary provider, if possible. + * @param f A file address to the dictionary to kill. + */ + private static void killDictionary(final Context context, final AssetFileAddress f) { + if (f.pointsToPhysicalFile()) { + f.deleteUnderlyingFile(); + // Warn the dictionary provider if the dictionary came from there. + final ContentProviderClient providerClient; + try { + providerClient = context.getContentResolver().acquireContentProviderClient( + BinaryDictionaryFileDumper.getProviderUriBuilder("").build()); + } catch (final SecurityException e) { + Log.e(TAG, "No permission to communicate with the dictionary provider", e); + return; + } + if (null == providerClient) { + Log.e(TAG, "Can't establish communication with the dictionary provider"); + return; + } + final String wordlistId = + DictionaryInfoUtils.getWordListIdFromFileName(new File(f.mFilename).getName()); + if (null != wordlistId) { + // TODO: this is a reasonable last resort, but it is suboptimal. + // The following will remove the entry for this dictionary with the dictionary + // provider. When the metadata is downloaded again, we will try downloading it + // again. + // However, in the practice that will mean the user will find themselves without + // the new dictionary. That's fine for languages where it's included in the APK, + // but for other languages it will leave the user without a dictionary at all until + // the next update, which may be a few days away. + // Ideally, we would trigger a new download right away, and use increasing retry + // delays for this particular id/version combination. + // Then again, this is expected to only ever happen in case of human mistake. If + // the wrong file is on the server, the following is still doing the right thing. + // If it's a file left over from the last version however, it's not great. + BinaryDictionaryFileDumper.reportBrokenFileToDictionaryProvider( + providerClient, + context.getString(R.string.dictionary_pack_client_id), + wordlistId); + } + } + } + + /** * Initializes a main dictionary collection from a dictionary pack, with default flags. * * This searches for a content provider providing a dictionary pack for the specified diff --git a/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java index 68505ce38..c8e4014bb 100644 --- a/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ReadOnlyBinaryDictionary.java @@ -44,6 +44,15 @@ public final class ReadOnlyBinaryDictionary extends Dictionary { locale, dictType, false /* isUpdatable */); } + public boolean hasValidContents() { + mLock.readLock().lock(); + try { + return mBinaryDictionary.hasValidContents(); + } finally { + mLock.readLock().unlock(); + } + } + public boolean isValidDictionary() { return mBinaryDictionary.isValidDictionary(); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 555c71b4c..93ccc62b4 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -368,6 +368,7 @@ public final class FormatSpec { // and latinime::HeaderReadWriteUtils. public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; + public static final String HAS_HISTORICAL_INFO_ATTRIBUTE = "HAS_HISTORICAL_INFO"; public static final String ATTRIBUTE_VALUE_TRUE = "1"; public static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index f4b7d6e5c..529ddc257 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -94,6 +94,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName); attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString()); attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE, diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 57c170f59..71567e224 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -135,6 +135,14 @@ static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dic delete dictionary; } +static bool latinime_BinaryDictionary_hasValidContents(JNIEnv *env, jclass clazz, + jlong dict) { + Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); + if (!dictionary) return false; + // TODO: check format version + return true; +} + static int latinime_BinaryDictionary_getFormatVersion(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return 0; @@ -438,6 +446,11 @@ static const JNINativeMethod sMethods[] = { reinterpret_cast<void *>(latinime_BinaryDictionary_close) }, { + const_cast<char *>("hasValidContentsNative"), + const_cast<char *>("(J)Z"), + reinterpret_cast<void *>(latinime_BinaryDictionary_hasValidContents) + }, + { const_cast<char *>("getFormatVersionNative"), const_cast<char *>("(J)I"), reinterpret_cast<void *>(latinime_BinaryDictionary_getFormatVersion) diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 9726fb856..4c4e43b95 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -68,6 +68,13 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mHasHistoricalInfoOfWords(HeaderReadWriteUtils::readBoolAttributeValue( &mAttributeMap, HAS_HISTORICAL_INFO_KEY, false /* defaultValue */)) {} + // Temporary dummy header. + HeaderPolicy() + : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0), + mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false), + mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), + mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {} + ~HeaderPolicy() {} virtual int getFormatVersionNumber() const { @@ -139,7 +146,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const int unigramCount, const int bigramCount, const int extendedRegionSize) const; private: - DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy); + DISALLOW_COPY_AND_ASSIGN(HeaderPolicy); static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; static const char *const IS_DECAYING_DICT_KEY; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index 07b8f181d..40f6469fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -18,7 +18,7 @@ #define LATINIME_VER4_DICT_BUFFER_H #include "defines.h" -#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h" +#include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h" @@ -39,8 +39,9 @@ class Ver4DictBuffers { return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable)); } - static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers() { - return Ver4DictBuffersPtr(new Ver4DictBuffers()); + static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers( + const HeaderPolicy *const headerPolicy) { + return Ver4DictBuffersPtr(new Ver4DictBuffers(headerPolicy)); } AK_FORCE_INLINE bool isValid() const { @@ -57,6 +58,10 @@ class Ver4DictBuffers { || mShortcutDictContent.isNearSizeLimit(); } + AK_FORCE_INLINE const HeaderPolicy *getHeaderPolicy() const { + return &mHeaderPolicy; + } + AK_FORCE_INLINE BufferWithExtendableBuffer *getWritableHeaderBuffer() { return &mExpandableHeaderBuffer; } @@ -118,30 +123,31 @@ class Ver4DictBuffers { AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable) : mDictBuffer(dictBuffer), - mHeaderSize(HeaderReadWriteUtils::getHeaderSize(mDictBuffer.get()->getBuffer())), - mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderSize, + mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4), + mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderSize, - dictBuffer.get()->getBufferSize() - mHeaderSize, + mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(), + dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), // TODO: Quit using header size. - mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderSize), - mProbabilityDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable), - mBigramDictContent(dictDirPath, false /* hasHistoricalInfo */, isUpdatable), + mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()), + mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(), + isUpdatable), + mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistricalInfoOfWords(), isUpdatable), mShortcutDictContent(dictDirPath, isUpdatable), mIsUpdatable(isUpdatable) {} - AK_FORCE_INLINE Ver4DictBuffers() - : mDictBuffer(0), mHeaderSize(0), + AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy) + : mDictBuffer(0), mHeaderPolicy(), mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mTerminalPositionLookupTable(), - mProbabilityDictContent(false /* hasHistoricalInfo */), - mBigramDictContent(false /* hasHistoricalInfo */), mShortcutDictContent(), + mProbabilityDictContent(headerPolicy->hasHistricalInfoOfWords()), + mBigramDictContent(headerPolicy->hasHistricalInfoOfWords()), mShortcutDictContent(), mIsUpdatable(true) {} const MmappedBuffer::MmappedBufferPtr mDictBuffer; - const int mHeaderSize; + const HeaderPolicy mHeaderPolicy; BufferWithExtendableBuffer mExpandableHeaderBuffer; BufferWithExtendableBuffer mExpandableTrieBuffer; TerminalPositionLookupTable mTerminalPositionLookupTable; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 719f35807..f9baeee31 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -50,7 +50,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d break; } bool isTerminal = ptNodeParams.isTerminal() && !ptNodeParams.isDeleted(); - if (isTerminal && mHeaderPolicy.isDecayingDict()) { + if (isTerminal && mHeaderPolicy->isDecayingDict()) { // A DecayingDict may have a terminal PtNode that has a terminal DicNode whose // probability is NOT_A_PROBABILITY. In such case, we don't want to treat it as a // valid terminal DicNode. @@ -85,7 +85,7 @@ int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inW int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability, const int bigramProbability) const { - if (mHeaderPolicy.isDecayingDict()) { + if (mHeaderPolicy->isDecayingDict()) { // Both probabilities are encoded. Decode them and get probability. return ForgettingCurveUtils::getProbability(unigramProbability, bigramProbability); } else { @@ -229,7 +229,7 @@ void Ver4PatriciaTriePolicy::flush(const char *const filePath) { AKLOGI("Warning: flush() is called for non-updatable dictionary. filePath: %s", filePath); return; } - mWritingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount); + mWritingHelper.writeToDictFile(filePath, mUnigramCount, mBigramCount); } void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) { @@ -237,10 +237,10 @@ void Ver4PatriciaTriePolicy::flushWithGC(const char *const filePath) { AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); return; } - const bool needsToDecay = mHeaderPolicy.isDecayingDict() + const bool needsToDecay = mHeaderPolicy->isDecayingDict() && (mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay( - false /* mindsBlockByDecay */, mUnigramCount, mBigramCount, &mHeaderPolicy)); - mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy, needsToDecay); + false /* mindsBlockByDecay */, mUnigramCount, mBigramCount, mHeaderPolicy)); + mWritingHelper.writeToDictFileWithGC(getRootPosition(), filePath, needsToDecay); mNeedsToDecayForTesting = false; } @@ -252,7 +252,7 @@ bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { if (mBuffers.get()->isNearSizeLimit()) { // Additional buffer size is near the limit. return true; - } else if (mHeaderPolicy.getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize() + } else if (mHeaderPolicy->getExtendedRegionSize() + mDictBuffer->getUsedAdditionalBufferSize() > Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE) { // Total extended region size of the trie exceeds the limit. return true; @@ -260,9 +260,9 @@ bool Ver4PatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { && mDictBuffer->getUsedAdditionalBufferSize() > 0) { // Needs to reduce dictionary size. return true; - } else if (mHeaderPolicy.isDecayingDict()) { + } else if (mHeaderPolicy->isDecayingDict()) { return mNeedsToDecayForTesting || ForgettingCurveUtils::needsToDecay( - mindsBlockByGC, mUnigramCount, mBigramCount, &mHeaderPolicy); + mindsBlockByGC, mUnigramCount, mBigramCount, mHeaderPolicy); } return false; } @@ -276,11 +276,11 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer snprintf(outResult, maxResultLength, "%d", mBigramCount); } else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, compareLength) == 0) { snprintf(outResult, maxResultLength, "%d", - mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT : + mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT : static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE)); } else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, compareLength) == 0) { snprintf(outResult, maxResultLength, "%d", - mHeaderPolicy.isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT : + mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT : static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE)); } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, compareLength) == 0) { mNeedsToDecayForTesting = true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 360b6b82e..e38510708 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -38,22 +38,20 @@ class DicNodeVector; class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: Ver4PatriciaTriePolicy(const Ver4DictBuffers::Ver4DictBuffersPtr &buffers) - : mBuffers(buffers), - mHeaderPolicy(mBuffers.get()->getWritableHeaderBuffer()->getBuffer( - false /* usesAdditionalBuffer*/), FormatUtils::VERSION_4), + : mBuffers(buffers), mHeaderPolicy(mBuffers.get()->getHeaderPolicy()), mDictBuffer(mBuffers.get()->getWritableTrieBuffer()), mBigramPolicy(mBuffers.get()->getUpdatableBigramDictContent(), - mBuffers.get()->getTerminalPositionLookupTable(), &mHeaderPolicy, - mHeaderPolicy.isDecayingDict()), + mBuffers.get()->getTerminalPositionLookupTable(), mHeaderPolicy, + mHeaderPolicy->isDecayingDict()), mShortcutPolicy(mBuffers.get()->getShortcutDictContent(), mBuffers.get()->getTerminalPositionLookupTable()), mNodeReader(mDictBuffer, mBuffers.get()->getProbabilityDictContent()), mNodeWriter(mDictBuffer, mBuffers.get(), &mNodeReader, &mBigramPolicy, - &mShortcutPolicy, mHeaderPolicy.isDecayingDict()), + &mShortcutPolicy, mHeaderPolicy->isDecayingDict()), mUpdatingHelper(mDictBuffer, &mNodeReader, &mNodeWriter), mWritingHelper(mBuffers.get()), - mUnigramCount(mHeaderPolicy.getUnigramCount()), - mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {}; + mUnigramCount(mHeaderPolicy->getUnigramCount()), + mBigramCount(mHeaderPolicy->getBigramCount()), mNeedsToDecayForTesting(false) {}; AK_FORCE_INLINE int getRootPosition() const { return 0; @@ -78,7 +76,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { int getBigramsPositionOfPtNode(const int ptNodePos) const; const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const { - return &mHeaderPolicy; + return mHeaderPolicy; } const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const { @@ -121,7 +119,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS; Ver4DictBuffers::Ver4DictBuffersPtr mBuffers; - const HeaderPolicy mHeaderPolicy; + const HeaderPolicy *const mHeaderPolicy; BufferWithExtendableBuffer *const mDictBuffer; Ver4BigramListPolicy mBigramPolicy; Ver4ShortcutListPolicy mShortcutPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index e8368af27..dd62b9c32 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -32,8 +32,8 @@ namespace latinime { void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePath, - const HeaderPolicy *const headerPolicy, const int unigramCount, - const int bigramCount) const { + const int unigramCount, const int bigramCount) const { + const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy(); const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */; char dirPath[dirPathBufSize]; FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath); @@ -53,9 +53,10 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa } void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, - const char *const trieFilePath, const HeaderPolicy *const headerPolicy, - const bool needsToDecay) { - Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers()); + const char *const trieFilePath, const bool needsToDecay) { + const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy(); + Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( + Ver4DictBuffers::createVer4DictBuffers(headerPolicy)); int unigramCount = 0; int bigramCount = 0; if (needsToDecay) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h index ea2fd532b..9344bde39 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h @@ -32,12 +32,11 @@ class Ver4PatriciaTrieWritingHelper { Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers) : mBuffers(buffers) {} - void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy, - const int unigramCount, const int bigramCount) const; + void writeToDictFile(const char *const trieFilePath, const int unigramCount, + const int bigramCount) const; void writeToDictFileWithGC(const int rootPtNodeArrayPos, - const char *const trieFilePath, const HeaderPolicy *const headerPolicy, - const bool needsToDecay); + const char *const trieFilePath, const bool needsToDecay); private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index ff762d75d..d959b26a8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -43,8 +43,9 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = /* static */ bool DictFileWritingUtils::createEmptyV4DictFile(const char *const dirPath, const HeaderReadWriteUtils::AttributeMap *const attributeMap) { - Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = Ver4DictBuffers::createVer4DictBuffers(); HeaderPolicy headerPolicy(FormatUtils::VERSION_4, attributeMap); + Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = + Ver4DictBuffers::createVer4DictBuffers(&headerPolicy); headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index 5b888ab2c..95504ef14 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -53,6 +53,20 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { super.tearDown(); } + private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, + final int probability) { + binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */, + BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + false /* isNotAWord */, false /* isBlacklisted */, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + + private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, + final String word1, final int probability) { + binaryDictionary.addBigramWords(word0, word1, probability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { // Entries having low probability would be suppressed once in 3 GCs. final int count = 3; @@ -115,35 +129,35 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); assertTrue(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidWord("b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); assertTrue(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addUnigramWord("c", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "c", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidBigram("a", "c")); // Add bigrams of not valid unigrams. - binaryDictionary.addBigramWords("x", "y", Dictionary.NOT_A_PROBABILITY); + addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); assertFalse(binaryDictionary.isValidBigram("x", "y")); - binaryDictionary.addBigramWords("x", "y", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); assertFalse(binaryDictionary.isValidBigram("x", "y")); binaryDictionary.close(); @@ -165,39 +179,39 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidBigram("a", "b")); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidBigram("a", "b")); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); - binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidBigram("a", "b")); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidBigram("a", "b")); @@ -241,7 +255,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); - binaryDictionary.addUnigramWord(word, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = @@ -310,9 +324,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.getPropertyForTests(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); - binaryDictionary.addUnigramWord(bigram.first, DUMMY_PROBABILITY); - binaryDictionary.addUnigramWord(bigram.second, DUMMY_PROBABILITY); - binaryDictionary.addBigramWords(bigram.first, bigram.second, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 8565db919..9cccdedda 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -105,6 +105,20 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.close(); } + private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, + final int probability) { + binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */, + BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + false /* isNotAWord */, false /* isBlacklisted */, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + + private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, + final String word1, final int probability) { + binaryDictionary.addBigramWords(word0, word1, probability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + public void testAddUnigramWord() { testAddUnigramWord(4 /* formatVersion */); } @@ -121,21 +135,21 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = 100; - binaryDictionary.addUnigramWord("aaa", probability); + addUnigramWord(binaryDictionary, "aaa", probability); // Reallocate and create. - binaryDictionary.addUnigramWord("aab", probability); + addUnigramWord(binaryDictionary, "aab", probability); // Insert into children. - binaryDictionary.addUnigramWord("aac", probability); + addUnigramWord(binaryDictionary, "aac", probability); // Make terminal. - binaryDictionary.addUnigramWord("aa", probability); + addUnigramWord(binaryDictionary, "aa", probability); // Create children. - binaryDictionary.addUnigramWord("aaaa", probability); + addUnigramWord(binaryDictionary, "aaaa", probability); // Reallocate and make termianl. - binaryDictionary.addUnigramWord("a", probability); + addUnigramWord(binaryDictionary, "a", probability); final int updatedProbability = 200; // Update. - binaryDictionary.addUnigramWord("aaa", updatedProbability); + addUnigramWord(binaryDictionary, "aaa", updatedProbability); assertEquals(probability, binaryDictionary.getFrequency("aab")); assertEquals(probability, binaryDictionary.getFrequency("aac")); @@ -175,7 +189,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { probabilityMap.put(word, random.nextInt(0xFF)); } for (String word : probabilityMap.keySet()) { - binaryDictionary.addUnigramWord(word, probabilityMap.get(word)); + addUnigramWord(binaryDictionary, word, probabilityMap.get(word)); } for (String word : probabilityMap.keySet()) { assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word)); @@ -201,13 +215,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int unigramProbability = 100; final int bigramProbability = 10; final int updatedBigramProbability = 15; - binaryDictionary.addUnigramWord("aaa", unigramProbability); - binaryDictionary.addUnigramWord("abb", unigramProbability); - binaryDictionary.addUnigramWord("bcc", unigramProbability); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); - binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); - binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); + addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); + addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); final int probability = binaryDictionary.calculateProbability(unigramProbability, bigramProbability); @@ -220,7 +234,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); - binaryDictionary.addBigramWords("aaa", "abb", updatedBigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability); final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability, updatedBigramProbability); assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb")); @@ -236,16 +250,16 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.getBigramProbability("aaa", "aaa")); // Testing bigram link. - binaryDictionary.addUnigramWord("abcde", unigramProbability); - binaryDictionary.addUnigramWord("fghij", unigramProbability); - binaryDictionary.addBigramWords("abcde", "fghij", bigramProbability); - binaryDictionary.addUnigramWord("fgh", unigramProbability); - binaryDictionary.addUnigramWord("abc", unigramProbability); - binaryDictionary.addUnigramWord("f", unigramProbability); + addUnigramWord(binaryDictionary, "abcde", unigramProbability); + addUnigramWord(binaryDictionary, "fghij", unigramProbability); + addBigramWords(binaryDictionary, "abcde", "fghij", bigramProbability); + addUnigramWord(binaryDictionary, "fgh", unigramProbability); + addUnigramWord(binaryDictionary, "abc", unigramProbability); + addUnigramWord(binaryDictionary, "f", unigramProbability); assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getBigramProbability("abcde", "fgh")); - binaryDictionary.addBigramWords("abcde", "fghij", updatedBigramProbability); + addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability); assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij")); dictFile.delete(); @@ -284,7 +298,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int i = 0; i < bigramCount; i++) { @@ -297,7 +311,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { bigramWords.add(bigram); final int bigramProbability = random.nextInt(0xF); bigramProbabilities.put(bigram, bigramProbability); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } for (final Pair<String, String> bigram : bigramWords) { @@ -328,13 +342,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int unigramProbability = 100; final int bigramProbability = 10; - binaryDictionary.addUnigramWord("aaa", unigramProbability); - binaryDictionary.addUnigramWord("abb", unigramProbability); - binaryDictionary.addUnigramWord("bcc", unigramProbability); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); - binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); - binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); + addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); + addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); @@ -343,7 +357,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.removeBigramWords("aaa", "abb"); assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb")); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); @@ -378,8 +392,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final int probability = 100; - binaryDictionary.addUnigramWord("aaa", probability); - binaryDictionary.addUnigramWord("abcd", probability); + addUnigramWord(binaryDictionary, "aaa", probability); + addUnigramWord(binaryDictionary, "abcd", probability); // Close without flushing. binaryDictionary.close(); @@ -390,8 +404,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa")); assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd")); - binaryDictionary.addUnigramWord("aaa", probability); - binaryDictionary.addUnigramWord("abcd", probability); + addUnigramWord(binaryDictionary, "aaa", probability); + addUnigramWord(binaryDictionary, "abcd", probability); binaryDictionary.flush(); binaryDictionary.close(); @@ -401,7 +415,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(probability, binaryDictionary.getFrequency("aaa")); assertEquals(probability, binaryDictionary.getFrequency("abcd")); - binaryDictionary.addUnigramWord("bcde", probability); + addUnigramWord(binaryDictionary, "bcde", probability); binaryDictionary.flush(); binaryDictionary.close(); @@ -431,13 +445,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int unigramProbability = 100; final int bigramProbability = 10; - binaryDictionary.addUnigramWord("aaa", unigramProbability); - binaryDictionary.addUnigramWord("abb", unigramProbability); - binaryDictionary.addUnigramWord("bcc", unigramProbability); - binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); - binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); - binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); - binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + addBigramWords(binaryDictionary, "aaa", "abb", bigramProbability); + addBigramWords(binaryDictionary, "aaa", "bcc", bigramProbability); + addBigramWords(binaryDictionary, "abb", "aaa", bigramProbability); + addBigramWords(binaryDictionary, "abb", "bcc", bigramProbability); binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -497,7 +511,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int i = 0; i < bigramCount; i++) { @@ -510,7 +524,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { bigramWords.add(bigram); final int bigramProbability = random.nextInt(0xF); bigramProbabilities.put(bigram, bigramProbability); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } binaryDictionary.flushWithGC(); @@ -568,7 +582,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -584,7 +598,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } // Add bigram. if (random.nextFloat() < addBigramProb && words.size() > 2) { @@ -602,7 +616,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final Pair<String, String> bigram = new Pair<String, String>(word0, word1); bigramWords.add(bigram); bigramProbabilities.put(bigram, bigramProbability); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } // Remove bigram. if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) { @@ -674,7 +688,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { words.add(word); final int unigramProbability = random.nextInt(0xFF); unigramProbabilities.put(word, unigramProbability); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int j = 0; j < words.size(); j++) { @@ -722,7 +736,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); - binaryDictionary.addUnigramWord(word, unigramProbability); + addUnigramWord(binaryDictionary, word, unigramProbability); } for (int j = 0; j < bigramCountPerIteration; j++) { final String word0 = words.get(random.nextInt(words.size())); @@ -732,7 +746,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } bigrams.add(new Pair<String, String>(word0, word1)); final int bigramProbability = random.nextInt(0xF); - binaryDictionary.addBigramWords(word0, word1, bigramProbability); + addBigramWords(binaryDictionary, word0, word1, bigramProbability); } assertEquals(new HashSet<String>(words).size(), Integer.parseInt( binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY))); @@ -781,10 +795,10 @@ public class BinaryDictionaryTests extends AndroidTestCase { unigramProbabilities.put(word, probability); if (prevWord == null) { languageModelParams[i] = new LanguageModelParam(word, probability, - BinaryDictionary.NOT_A_VALID_TIME_STAMP); + BinaryDictionary.NOT_A_VALID_TIMESTAMP); } else { languageModelParams[i] = new LanguageModelParam(prevWord, word, probability, - bigramProbability, BinaryDictionary.NOT_A_VALID_TIME_STAMP); + bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); bigramProbabilities.put(new Pair<String, String>(prevWord, word), bigramProbability); } |