diff options
Diffstat (limited to 'native/jni/src')
6 files changed, 98 insertions, 19 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index f950cade3..251a71941 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -69,7 +69,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { // Constructs header information using an attribute map. HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion, - const std::vector<int> locale, + const std::vector<int> &locale, const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) : mDictFormatVersion(dictFormatVersion), mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( @@ -99,6 +99,27 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mMaxBigramCount(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, MAX_BIGRAM_COUNT_KEY, DEFAULT_MAX_BIGRAM_COUNT)) {} + // Copy header information + HeaderPolicy(const HeaderPolicy *const headerPolicy) + : mDictFormatVersion(headerPolicy->mDictFormatVersion), + mDictionaryFlags(headerPolicy->mDictionaryFlags), mSize(headerPolicy->mSize), + mAttributeMap(headerPolicy->mAttributeMap), mLocale(headerPolicy->mLocale), + mMultiWordCostMultiplier(headerPolicy->mMultiWordCostMultiplier), + mRequiresGermanUmlautProcessing(headerPolicy->mRequiresGermanUmlautProcessing), + mIsDecayingDict(headerPolicy->mIsDecayingDict), + mDate(headerPolicy->mDate), mLastDecayedTime(headerPolicy->mLastDecayedTime), + mUnigramCount(headerPolicy->mUnigramCount), mBigramCount(headerPolicy->mBigramCount), + mExtendedRegionSize(headerPolicy->mExtendedRegionSize), + mHasHistoricalInfoOfWords(headerPolicy->mHasHistoricalInfoOfWords), + mForgettingCurveOccurrencesToLevelUp( + headerPolicy->mForgettingCurveOccurrencesToLevelUp), + mForgettingCurveProbabilityValuesTableId( + headerPolicy->mForgettingCurveProbabilityValuesTableId), + mForgettingCurveDurationToLevelDown( + headerPolicy->mForgettingCurveDurationToLevelDown), + mMaxUnigramCount(headerPolicy->mMaxUnigramCount), + mMaxBigramCount(headerPolicy->mMaxBigramCount) {} + // Temporary dummy header. HeaderPolicy() : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0), diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index 5f19534e2..be7e43b98 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -19,10 +19,12 @@ #include <climits> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h" +#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" #include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" @@ -30,24 +32,50 @@ namespace latinime { /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr - DictionaryStructureWithBufferPolicyFactory - ::newDictionaryStructureWithBufferPolicy(const char *const path, - const int bufOffset, const int size, const bool isUpdatable) { + DictionaryStructureWithBufferPolicyFactory::newPolicyForExistingDictFile( + const char *const path, const int bufOffset, const int size, + const bool isUpdatable) { if (FileUtils::existsDir(path)) { // Given path represents a directory. - return newPolicyforDirectoryDict(path, isUpdatable); + return newPolicyForDirectoryDict(path, isUpdatable); } else { if (isUpdatable) { AKLOGE("One file dictionaries don't support updating. path: %s", path); ASSERT(false); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); } - return newPolicyforFileDict(path, bufOffset, size); + return newPolicyForFileDict(path, bufOffset, size); } } /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr - DictionaryStructureWithBufferPolicyFactory::newPolicyforDirectoryDict( + DictionaryStructureWithBufferPolicyFactory:: newPolicyForOnMemoryDict( + const int formatVersion, const std::vector<int> &locale, + const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap) { + switch (formatVersion) { + case FormatUtils::VERSION_4: { + HeaderPolicy headerPolicy(FormatUtils::VERSION_4, locale, attributeMap); + Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = + Ver4DictBuffers::createVer4DictBuffers(&headerPolicy, + Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE); + if (!DynamicPtWritingUtils::writeEmptyDictionary( + dictBuffers->getWritableTrieBuffer(), 0 /* rootPos */)) { + AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); + } + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( + new Ver4PatriciaTriePolicy(std::move(dictBuffers))); + } + default: + AKLOGE("DICT: dictionary format %d is not supported for on memory dictionary", + formatVersion); + break; + } + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(nullptr); +} + +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyForDirectoryDict( const char *const path, const bool isUpdatable) { const int headerFilePathBufSize = PATH_MAX + 1 /* terminator */; char headerFilePath[headerFilePathBufSize]; @@ -93,7 +121,7 @@ namespace latinime { } /* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr - DictionaryStructureWithBufferPolicyFactory::newPolicyforFileDict( + DictionaryStructureWithBufferPolicyFactory::newPolicyForFileDict( const char *const path, const int bufOffset, const int size) { // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of // MmappedBufferPtr if the instance has the responsibility. diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h index 6053b7ed3..f71447e23 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h @@ -17,7 +17,10 @@ #ifndef LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H #define LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H +#include <vector> + #include "defines.h" +#include "suggest/core/policy/dictionary_header_structure_policy.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" namespace latinime { @@ -25,17 +28,22 @@ namespace latinime { class DictionaryStructureWithBufferPolicyFactory { public: static DictionaryStructureWithBufferPolicy::StructurePolicyPtr - newDictionaryStructureWithBufferPolicy(const char *const path, const int bufOffset, + newPolicyForExistingDictFile(const char *const path, const int bufOffset, const int size, const bool isUpdatable); + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr + newPolicyForOnMemoryDict(const int formatVersion, + const std::vector<int> &locale, + const DictionaryHeaderStructurePolicy::AttributeMap *const attributeMap); + private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory); static DictionaryStructureWithBufferPolicy::StructurePolicyPtr - newPolicyforDirectoryDict(const char *const path, const bool isUpdatable); + newPolicyForDirectoryDict(const char *const path, const bool isUpdatable); static DictionaryStructureWithBufferPolicy::StructurePolicyPtr - newPolicyforFileDict(const char *const path, const int bufOffset, const int size); + newPolicyForFileDict(const char *const path, const int bufOffset, const int size); static void getHeaderFilePathInDictDir(const char *const dirPath, const int outHeaderFileBufSize, char *const outHeaderFilePath); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index eda882d46..95f654498 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -131,7 +131,7 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath, mIsUpdatable(isUpdatable) {} Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy, const int maxTrieSize) - : mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(), + : mHeaderBuffer(nullptr), mDictBuffer(nullptr), mHeaderPolicy(headerPolicy), mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableTrieBuffer(maxTrieSize), mTerminalPositionLookupTable(), mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()), diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h index 98b8966df..239419d5b 100644 --- a/native/jni/src/utils/char_utils.h +++ b/native/jni/src/utils/char_utils.h @@ -86,13 +86,6 @@ class CharUtils { return spaceCount; } - static AK_FORCE_INLINE std::vector<int> convertShortArrayToIntVector( - const unsigned short *const source, const int length) { - std::vector<int> destination; - destination.insert(destination.end(), source, source + length); - return destination; // Copies the vector - } - static unsigned short latin_tolower(const unsigned short c); static const std::vector<int> EMPTY_STRING; diff --git a/native/jni/src/utils/jni_data_utils.h b/native/jni/src/utils/jni_data_utils.h index e0bbdfdc3..2ce02dc05 100644 --- a/native/jni/src/utils/jni_data_utils.h +++ b/native/jni/src/utils/jni_data_utils.h @@ -21,6 +21,8 @@ #include "defines.h" #include "jni.h" +#include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h" namespace latinime { @@ -36,6 +38,33 @@ class JniDataUtils { env->GetIntArrayRegion(array, 0 /* start */, arrayLength, outVector->data()); } + static DictionaryHeaderStructurePolicy::AttributeMap constructAttributeMap(JNIEnv *env, + jobjectArray attributeKeyStringArray, jobjectArray attributeValueStringArray) { + DictionaryHeaderStructurePolicy::AttributeMap attributeMap; + const int keyCount = env->GetArrayLength(attributeKeyStringArray); + for (int i = 0; i < keyCount; i++) { + jstring keyString = static_cast<jstring>( + env->GetObjectArrayElement(attributeKeyStringArray, i)); + const jsize keyUtf8Length = env->GetStringUTFLength(keyString); + char keyChars[keyUtf8Length + 1]; + env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars); + keyChars[keyUtf8Length] = '\0'; + DictionaryHeaderStructurePolicy::AttributeMap::key_type key; + HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key); + + jstring valueString = static_cast<jstring>( + env->GetObjectArrayElement(attributeValueStringArray, i)); + const jsize valueUtf8Length = env->GetStringUTFLength(valueString); + char valueChars[valueUtf8Length + 1]; + env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars); + valueChars[valueUtf8Length] = '\0'; + DictionaryHeaderStructurePolicy::AttributeMap::mapped_type value; + HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value); + attributeMap[key] = value; + } + return attributeMap; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(JniDataUtils); }; |