aboutsummaryrefslogtreecommitdiffstats
path: root/native
diff options
context:
space:
mode:
Diffstat (limited to 'native')
-rw-r--r--native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp87
-rw-r--r--native/jni/src/suggest/core/dictionary/property/word_property.h8
-rw-r--r--native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h2
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h4
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h4
5 files changed, 103 insertions, 2 deletions
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 9016cae69..3ac424fea 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -32,6 +32,7 @@
#include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h"
#include "utils/char_utils.h"
#include "utils/jni_data_utils.h"
+#include "utils/log_utils.h"
#include "utils/time_keeper.h"
namespace latinime {
@@ -489,6 +490,87 @@ static bool latinime_BinaryDictionary_isCorruptedNative(JNIEnv *env, jclass claz
return dictionary->getDictionaryStructurePolicy()->isCorrupted();
}
+static DictionaryStructureWithBufferPolicy::StructurePolicyPtr runGCAndGetNewStructurePolicy(
+ DictionaryStructureWithBufferPolicy::StructurePolicyPtr structurePolicy,
+ const char *const dictFilePath) {
+ structurePolicy->flushWithGC(dictFilePath);
+ structurePolicy.release();
+ return DictionaryStructureWithBufferPolicyFactory::newPolicyForExistingDictFile(
+ dictFilePath, 0 /* offset */, 0 /* size */, true /* isUpdatable */);
+}
+
+static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, jlong dict,
+ jstring dictFilePath, jlong newFormatVersion) {
+ Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
+ if (!dictionary) {
+ return false;
+ }
+ const jsize filePathUtf8Length = env->GetStringUTFLength(dictFilePath);
+ char dictFilePathChars[filePathUtf8Length + 1];
+ env->GetStringUTFRegion(dictFilePath, 0, env->GetStringLength(dictFilePath), dictFilePathChars);
+ dictFilePathChars[filePathUtf8Length] = '\0';
+
+ const DictionaryHeaderStructurePolicy *const headerPolicy =
+ dictionary->getDictionaryStructurePolicy()->getHeaderStructurePolicy();
+ DictionaryStructureWithBufferPolicy::StructurePolicyPtr dictionaryStructureWithBufferPolicy =
+ DictionaryStructureWithBufferPolicyFactory::newPolicyForOnMemoryDict(
+ newFormatVersion, *headerPolicy->getLocale(), headerPolicy->getAttributeMap());
+ if (!dictionaryStructureWithBufferPolicy) {
+ LogUtils::logToJava(env, "Cannot migrate header.");
+ return false;
+ }
+
+ // TODO: Migrate historical information.
+ int wordCodePoints[MAX_WORD_LENGTH];
+ int token = 0;
+ // Add unigrams.
+ do {
+ token = dictionary->getNextWordAndNextToken(token, wordCodePoints);
+ const int wordLength = CharUtils::getCodePointCount(MAX_WORD_LENGTH, wordCodePoints);
+ const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, wordLength);
+ if (dictionaryStructureWithBufferPolicy->needsToRunGC(true /* mindsBlockByGC */)) {
+ dictionaryStructureWithBufferPolicy = runGCAndGetNewStructurePolicy(
+ std::move(dictionaryStructureWithBufferPolicy), dictFilePathChars);
+ if (!dictionaryStructureWithBufferPolicy) {
+ LogUtils::logToJava(env, "Cannot open dict after GC.");
+ return false;
+ }
+ }
+ if (!dictionaryStructureWithBufferPolicy->addUnigramWord(wordCodePoints, wordLength,
+ wordProperty.getUnigramProperty())) {
+ LogUtils::logToJava(env, "Cannot add unigram to the new dict.");
+ return false;
+ }
+ } while (token != 0);
+
+ // Add bigrams.
+ do {
+ token = dictionary->getNextWordAndNextToken(token, wordCodePoints);
+ const int wordLength = CharUtils::getCodePointCount(MAX_WORD_LENGTH, wordCodePoints);
+ const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, wordLength);
+ if (dictionaryStructureWithBufferPolicy->needsToRunGC(true /* mindsBlockByGC */)) {
+ dictionaryStructureWithBufferPolicy = runGCAndGetNewStructurePolicy(
+ std::move(dictionaryStructureWithBufferPolicy), dictFilePathChars);
+ if (!dictionaryStructureWithBufferPolicy) {
+ LogUtils::logToJava(env, "Cannot open dict after GC.");
+ return false;
+ }
+ }
+ for (const BigramProperty &bigarmProperty : *wordProperty.getBigramProperties()) {
+ const std::vector<int> *targetCodePoints = bigarmProperty.getTargetCodePoints();
+ if (!dictionaryStructureWithBufferPolicy->addBigramWords(wordCodePoints, wordLength,
+ targetCodePoints->data(), targetCodePoints->size(),
+ bigarmProperty.getProbability(), bigarmProperty.getTimestamp())) {
+ LogUtils::logToJava(env, "Cannot add bigram to the new dict.");
+ return false;
+ }
+ }
+ } while (token != 0);
+ // Save to File.
+ dictionaryStructureWithBufferPolicy->flushWithGC(dictFilePathChars);
+ return true;
+}
+
static const JNINativeMethod sMethods[] = {
{
const_cast<char *>("openNative"),
@@ -591,6 +673,11 @@ static const JNINativeMethod sMethods[] = {
const_cast<char *>("isCorruptedNative"),
const_cast<char *>("(J)Z"),
reinterpret_cast<void *>(latinime_BinaryDictionary_isCorruptedNative)
+ },
+ {
+ const_cast<char *>("migrateNative"),
+ const_cast<char *>("(JLjava/lang/String;J)Z"),
+ reinterpret_cast<void *>(latinime_BinaryDictionary_migrateNative)
}
};
diff --git a/native/jni/src/suggest/core/dictionary/property/word_property.h b/native/jni/src/suggest/core/dictionary/property/word_property.h
index 5519a917c..aa3e0b68a 100644
--- a/native/jni/src/suggest/core/dictionary/property/word_property.h
+++ b/native/jni/src/suggest/core/dictionary/property/word_property.h
@@ -42,6 +42,14 @@ class WordProperty {
jintArray outProbabilityInfo, jobject outBigramTargets, jobject outBigramProbabilities,
jobject outShortcutTargets, jobject outShortcutProbabilities) const;
+ const UnigramProperty *getUnigramProperty() const {
+ return &mUnigramProperty;
+ }
+
+ const std::vector<BigramProperty> *getBigramProperties() const {
+ return &mBigrams;
+ }
+
private:
// Default copy constructor is used for using as a return value.
DISALLOW_ASSIGNMENT_OPERATOR(WordProperty);
diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
index a8dab9fcd..845e629e6 100644
--- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
+++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h
@@ -49,6 +49,8 @@ class DictionaryHeaderStructurePolicy {
virtual bool shouldBoostExactMatches() const = 0;
+ virtual const std::vector<int> *getLocale() const = 0;
+
protected:
DictionaryHeaderStructurePolicy() {}
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
index 251a71941..e4a6dc594 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h
@@ -238,6 +238,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
const int unigramCount, const int bigramCount, const int extendedRegionSize,
DictionaryHeaderStructurePolicy::AttributeMap *outAttributeMap) const;
+ AK_FORCE_INLINE const std::vector<int> *getLocale() const {
+ return &mLocale;
+ }
+
private:
DISALLOW_COPY_AND_ASSIGN(HeaderPolicy);
diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
index 23cbe3aa3..a2e88a46c 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
+++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h
@@ -37,13 +37,13 @@ class BufferWithExtendableBuffer {
BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize,
const int maxAdditionalBufferSize)
: mOriginalBuffer(originalBuffer), mOriginalBufferSize(originalBufferSize),
- mAdditionalBuffer(EXTEND_ADDITIONAL_BUFFER_SIZE_STEP), mUsedAdditionalBufferSize(0),
+ mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
// Without original buffer.
BufferWithExtendableBuffer(const int maxAdditionalBufferSize)
: mOriginalBuffer(0), mOriginalBufferSize(0),
- mAdditionalBuffer(EXTEND_ADDITIONAL_BUFFER_SIZE_STEP), mUsedAdditionalBufferSize(0),
+ mAdditionalBuffer(0), mUsedAdditionalBufferSize(0),
mMaxAdditionalBufferSize(maxAdditionalBufferSize) {}
AK_FORCE_INLINE int getTailPosition() const {
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-05-07 09:35:03 +0000