diff options
4 files changed, 88 insertions, 2 deletions
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 8648d8dde..3ac424fea 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -32,6 +32,7 @@ #include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h" #include "utils/char_utils.h" #include "utils/jni_data_utils.h" +#include "utils/log_utils.h" #include "utils/time_keeper.h" namespace latinime { @@ -489,14 +490,85 @@ static bool latinime_BinaryDictionary_isCorruptedNative(JNIEnv *env, jclass claz return dictionary->getDictionaryStructurePolicy()->isCorrupted(); } +static DictionaryStructureWithBufferPolicy::StructurePolicyPtr runGCAndGetNewStructurePolicy( + DictionaryStructureWithBufferPolicy::StructurePolicyPtr structurePolicy, + const char *const dictFilePath) { + structurePolicy->flushWithGC(dictFilePath); + structurePolicy.release(); + return DictionaryStructureWithBufferPolicyFactory::newPolicyForExistingDictFile( + dictFilePath, 0 /* offset */, 0 /* size */, true /* isUpdatable */); +} + static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, jlong dict, jstring dictFilePath, jlong newFormatVersion) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) { return false; } - // TODO: Implement. - return false; + const jsize filePathUtf8Length = env->GetStringUTFLength(dictFilePath); + char dictFilePathChars[filePathUtf8Length + 1]; + env->GetStringUTFRegion(dictFilePath, 0, env->GetStringLength(dictFilePath), dictFilePathChars); + dictFilePathChars[filePathUtf8Length] = '\0'; + + const DictionaryHeaderStructurePolicy *const headerPolicy = + dictionary->getDictionaryStructurePolicy()->getHeaderStructurePolicy(); + DictionaryStructureWithBufferPolicy::StructurePolicyPtr dictionaryStructureWithBufferPolicy = + DictionaryStructureWithBufferPolicyFactory::newPolicyForOnMemoryDict( + newFormatVersion, *headerPolicy->getLocale(), headerPolicy->getAttributeMap()); + if (!dictionaryStructureWithBufferPolicy) { + LogUtils::logToJava(env, "Cannot migrate header."); + return false; + } + + // TODO: Migrate historical information. + int wordCodePoints[MAX_WORD_LENGTH]; + int token = 0; + // Add unigrams. + do { + token = dictionary->getNextWordAndNextToken(token, wordCodePoints); + const int wordLength = CharUtils::getCodePointCount(MAX_WORD_LENGTH, wordCodePoints); + const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, wordLength); + if (dictionaryStructureWithBufferPolicy->needsToRunGC(true /* mindsBlockByGC */)) { + dictionaryStructureWithBufferPolicy = runGCAndGetNewStructurePolicy( + std::move(dictionaryStructureWithBufferPolicy), dictFilePathChars); + if (!dictionaryStructureWithBufferPolicy) { + LogUtils::logToJava(env, "Cannot open dict after GC."); + return false; + } + } + if (!dictionaryStructureWithBufferPolicy->addUnigramWord(wordCodePoints, wordLength, + wordProperty.getUnigramProperty())) { + LogUtils::logToJava(env, "Cannot add unigram to the new dict."); + return false; + } + } while (token != 0); + + // Add bigrams. + do { + token = dictionary->getNextWordAndNextToken(token, wordCodePoints); + const int wordLength = CharUtils::getCodePointCount(MAX_WORD_LENGTH, wordCodePoints); + const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, wordLength); + if (dictionaryStructureWithBufferPolicy->needsToRunGC(true /* mindsBlockByGC */)) { + dictionaryStructureWithBufferPolicy = runGCAndGetNewStructurePolicy( + std::move(dictionaryStructureWithBufferPolicy), dictFilePathChars); + if (!dictionaryStructureWithBufferPolicy) { + LogUtils::logToJava(env, "Cannot open dict after GC."); + return false; + } + } + for (const BigramProperty &bigarmProperty : *wordProperty.getBigramProperties()) { + const std::vector<int> *targetCodePoints = bigarmProperty.getTargetCodePoints(); + if (!dictionaryStructureWithBufferPolicy->addBigramWords(wordCodePoints, wordLength, + targetCodePoints->data(), targetCodePoints->size(), + bigarmProperty.getProbability(), bigarmProperty.getTimestamp())) { + LogUtils::logToJava(env, "Cannot add bigram to the new dict."); + return false; + } + } + } while (token != 0); + // Save to File. + dictionaryStructureWithBufferPolicy->flushWithGC(dictFilePathChars); + return true; } static const JNINativeMethod sMethods[] = { diff --git a/native/jni/src/suggest/core/dictionary/property/word_property.h b/native/jni/src/suggest/core/dictionary/property/word_property.h index 5519a917c..aa3e0b68a 100644 --- a/native/jni/src/suggest/core/dictionary/property/word_property.h +++ b/native/jni/src/suggest/core/dictionary/property/word_property.h @@ -42,6 +42,14 @@ class WordProperty { jintArray outProbabilityInfo, jobject outBigramTargets, jobject outBigramProbabilities, jobject outShortcutTargets, jobject outShortcutProbabilities) const; + const UnigramProperty *getUnigramProperty() const { + return &mUnigramProperty; + } + + const std::vector<BigramProperty> *getBigramProperties() const { + return &mBigrams; + } + private: // Default copy constructor is used for using as a return value. DISALLOW_ASSIGNMENT_OPERATOR(WordProperty); diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h index a8dab9fcd..845e629e6 100644 --- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h @@ -49,6 +49,8 @@ class DictionaryHeaderStructurePolicy { virtual bool shouldBoostExactMatches() const = 0; + virtual const std::vector<int> *getLocale() const = 0; + protected: DictionaryHeaderStructurePolicy() {} diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 251a71941..e4a6dc594 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -238,6 +238,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const int unigramCount, const int bigramCount, const int extendedRegionSize, DictionaryHeaderStructurePolicy::AttributeMap *outAttributeMap) const; + AK_FORCE_INLINE const std::vector<int> *getLocale() const { + return &mLocale; + } + private: DISALLOW_COPY_AND_ASSIGN(HeaderPolicy); |