diff options
Diffstat (limited to 'native/jni/src')
20 files changed, 126 insertions, 174 deletions
diff --git a/native/jni/src/suggest/core/dictionary/digraph_utils.cpp b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp index 3271c1bfb..5f9b8f3e2 100644 --- a/native/jni/src/suggest/core/dictionary/digraph_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp @@ -28,11 +28,8 @@ const DigraphUtils::digraph_t DigraphUtils::GERMAN_UMLAUT_DIGRAPHS[] = { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS { 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS -const DigraphUtils::digraph_t DigraphUtils::FRENCH_LIGATURES_DIGRAPHS[] = - { { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE - { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = - { DIGRAPH_TYPE_GERMAN_UMLAUT, DIGRAPH_TYPE_FRENCH_LIGATURES }; + { DIGRAPH_TYPE_GERMAN_UMLAUT }; /* static */ bool DigraphUtils::hasDigraphForCodePoint( const DictionaryHeaderStructurePolicy *const headerPolicy, @@ -50,9 +47,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = if (headerPolicy->requiresGermanUmlautProcessing()) { return DIGRAPH_TYPE_GERMAN_UMLAUT; } - if (headerPolicy->requiresFrenchLigatureProcessing()) { - return DIGRAPH_TYPE_FRENCH_LIGATURES; - } return DIGRAPH_TYPE_NONE; } @@ -86,10 +80,6 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = *digraphs = GERMAN_UMLAUT_DIGRAPHS; return NELEMS(GERMAN_UMLAUT_DIGRAPHS); } - if (digraphType == DIGRAPH_TYPE_FRENCH_LIGATURES) { - *digraphs = FRENCH_LIGATURES_DIGRAPHS; - return NELEMS(FRENCH_LIGATURES_DIGRAPHS); - } return 0; } diff --git a/native/jni/src/suggest/core/dictionary/digraph_utils.h b/native/jni/src/suggest/core/dictionary/digraph_utils.h index 6ae16e390..bec2cd6e2 100644 --- a/native/jni/src/suggest/core/dictionary/digraph_utils.h +++ b/native/jni/src/suggest/core/dictionary/digraph_utils.h @@ -34,7 +34,6 @@ class DigraphUtils { typedef enum { DIGRAPH_TYPE_NONE, DIGRAPH_TYPE_GERMAN_UMLAUT, - DIGRAPH_TYPE_FRENCH_LIGATURES } DigraphType; typedef struct { int first; int second; int compositeGlyph; } digraph_t; @@ -55,7 +54,6 @@ class DigraphUtils { const DigraphType digraphType, const int compositeGlyphCodePoint); static const digraph_t GERMAN_UMLAUT_DIGRAPHS[]; - static const digraph_t FRENCH_LIGATURES_DIGRAPHS[]; static const DigraphType USED_DIGRAPH_TYPES[]; }; } // namespace latinime diff --git a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h index b05b7c334..b76b13971 100644 --- a/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_header_structure_policy.h @@ -31,12 +31,8 @@ class DictionaryHeaderStructurePolicy { virtual int getFormatVersionNumber() const = 0; - virtual bool supportsDynamicUpdate() const = 0; - virtual bool requiresGermanUmlautProcessing() const = 0; - virtual bool requiresFrenchLigatureProcessing() const = 0; - virtual float getMultiWordCostMultiplier() const = 0; virtual int getLastDecayedTime() const = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 133ae9f47..be7a3c228 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -20,6 +20,8 @@ namespace latinime { // Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE"; +const char *const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = + "REQUIRES_GERMAN_UMLAUT_PROCESSING"; // TODO: Change attribute string to "IS_DECAYING_DICT". const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE"; const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date"; @@ -66,6 +68,11 @@ float HeaderPolicy::readMultipleWordCostMultiplier() const { return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate); } +bool HeaderPolicy::readRequiresGermanUmlautProcessing() const { + return HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, + REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false); +} + bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime, const int unigramCount, const int bigramCount, const int extendedRegionSize) const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index d71e0a068..5e8b5f11e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -36,6 +36,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)), mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), + mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()), mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, IS_DECAYING_DICT_KEY, false /* defaultValue */)), mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, @@ -58,6 +59,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( attributeMap)), mSize(0), mAttributeMap(*attributeMap), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), + mRequiresGermanUmlautProcessing(readRequiresGermanUmlautProcessing()), mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, IS_DECAYING_DICT_KEY, false /* defaultValue */)), mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, @@ -71,7 +73,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { // Temporary dummy header. HeaderPolicy() : mDictFormatVersion(FormatUtils::UNKNOWN_VERSION), mDictionaryFlags(0), mSize(0), - mAttributeMap(), mMultiWordCostMultiplier(0.0f), mIsDecayingDict(false), + mAttributeMap(), mMultiWordCostMultiplier(0.0f), + mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false), mLastUpdatedTime(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false) {} @@ -109,18 +112,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mSize; } - AK_FORCE_INLINE bool supportsDynamicUpdate() const { - return HeaderReadWriteUtils::supportsDynamicUpdate(mDictionaryFlags); - } - - AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const { - return HeaderReadWriteUtils::requiresGermanUmlautProcessing(mDictionaryFlags); - } - - AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const { - return HeaderReadWriteUtils::requiresFrenchLigatureProcessing(mDictionaryFlags); - } - AK_FORCE_INLINE float getMultiWordCostMultiplier() const { return mMultiWordCostMultiplier; } @@ -129,6 +120,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mIsDecayingDict; } + AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const { + return mRequiresGermanUmlautProcessing; + } + AK_FORCE_INLINE int getLastUpdatedTime() const { return mLastUpdatedTime; } @@ -164,6 +159,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { DISALLOW_COPY_AND_ASSIGN(HeaderPolicy); static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; + static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY; static const char *const IS_DECAYING_DICT_KEY; static const char *const LAST_UPDATED_TIME_KEY; static const char *const LAST_DECAYED_TIME_KEY; @@ -179,6 +175,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const int mSize; HeaderReadWriteUtils::AttributeMap mAttributeMap; const float mMultiWordCostMultiplier; + const bool mRequiresGermanUmlautProcessing; const bool mIsDecayingDict; const int mLastUpdatedTime; const int mLastDecayedTime; @@ -188,6 +185,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const bool mHasHistoricalInfoOfWords; float readMultipleWordCostMultiplier() const; + bool readRequiresGermanUmlautProcessing() const; static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes( const uint8_t *const dictBuf); diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index f5eee99c3..2d4547066 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -35,22 +35,6 @@ const int HeaderReadWriteUtils::HEADER_FLAG_SIZE = 2; const int HeaderReadWriteUtils::HEADER_SIZE_FIELD_SIZE = 4; const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0; -// Flags for special processing -// Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or -// something very bad (like, the apocalypse) will happen. Please update both at the same time. -const HeaderReadWriteUtils::DictionaryFlags - HeaderReadWriteUtils::GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; -const HeaderReadWriteUtils::DictionaryFlags - HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_FLAG = 0x2; -const HeaderReadWriteUtils::DictionaryFlags - HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; - -// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. -const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE"; -const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = - "REQUIRES_GERMAN_UMLAUT_PROCESSING"; -const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY = - "REQUIRES_FRENCH_LIGATURE_PROCESSING"; /* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) { // See the format of the header in the comment in @@ -68,17 +52,7 @@ const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY /* static */ HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( const HeaderReadWriteUtils::AttributeMap *const attributeMap) { - const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap, - REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */); - const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap, - REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */); - const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap, - SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */); - DictionaryFlags dictflags = NO_FLAGS; - dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0; - dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0; - dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0; - return dictflags; + return NO_FLAGS; } /* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf, diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h index 225968323..fc24bbdd5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h @@ -37,18 +37,6 @@ class HeaderReadWriteUtils { static DictionaryFlags getFlags(const uint8_t *const dictBuf); - static AK_FORCE_INLINE bool supportsDynamicUpdate(const DictionaryFlags flags) { - return (flags & SUPPORTS_DYNAMIC_UPDATE_FLAG) != 0; - } - - static AK_FORCE_INLINE bool requiresGermanUmlautProcessing(const DictionaryFlags flags) { - return (flags & GERMAN_UMLAUT_PROCESSING_FLAG) != 0; - } - - static AK_FORCE_INLINE bool requiresFrenchLigatureProcessing(const DictionaryFlags flags) { - return (flags & FRENCH_LIGATURE_PROCESSING_FLAG) != 0; - } - static AK_FORCE_INLINE int getHeaderOptionsPosition() { return HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE + HEADER_FLAG_SIZE + HEADER_SIZE_FIELD_SIZE; @@ -101,17 +89,8 @@ class HeaderReadWriteUtils { static const int HEADER_FLAG_SIZE; static const int HEADER_SIZE_FIELD_SIZE; + // Value for the "flags" field. It's unused at the moment. static const DictionaryFlags NO_FLAGS; - // Flags for special processing - // Those *must* match the flags in makedict (FormatSpec#*_PROCESSING_FLAGS) or - // something very bad (like, the apocalypse) will happen. Please update both at the same time. - static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG; - static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG; - static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG; - - static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY; - static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY; - static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY; static void setIntAttributeInner(AttributeMap *const headerAttributes, const AttributeMap::key_type *const key, const int value); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index f1b733a9c..dfb110cdd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -47,7 +47,7 @@ namespace latinime { case FormatUtils::VERSION_4: { const int dictDirPathBufSize = strlen(path) + 1 /* terminator */; char dictDirPath[dictDirPathBufSize]; - if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::TRIE_FILE_EXTENSION, + if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictDirPath)) { // Dictionary file name is not valid as a version 4 dictionary. return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index bf833239b..9512bdbb0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -60,9 +60,8 @@ class SingleDictContent : public DictContent { } bool flush(const char *const dictDirPath, const char *const contentFileName) const { - const BufferWithExtendableBuffer *bufferPtr = &mExpandableContentBuffer; - return DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName, - &bufferPtr, 1 /* bufferCount */); + return DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName, + &mExpandableContentBuffer); } private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp index c65420614..84aceeffe 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp @@ -21,19 +21,16 @@ namespace latinime { bool SparseTableDictContent::flush(const char *const dictDirPath, const char *const lookupTableFileName, const char *const addressTableFileName, const char *const contentFileName) const { - const BufferWithExtendableBuffer *lookupTableBufferPtr = &mExpandableLookupTableBuffer; - if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, lookupTableFileName, - &lookupTableBufferPtr, 1 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, lookupTableFileName, + &mExpandableLookupTableBuffer)){ return false; } - const BufferWithExtendableBuffer *addressTableBufferPtr = &mExpandableAddressTableBuffer; - if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, addressTableFileName, - &addressTableBufferPtr, 1 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, addressTableFileName, + &mExpandableAddressTableBuffer)) { return false; } - const BufferWithExtendableBuffer *contentBufferPtr = &mExpandableContentBuffer; - if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName, - &contentBufferPtr, 1 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName, + &mExpandableContentBuffer)) { return false; } return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp index d0853a5d3..24f62cd4b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp @@ -28,7 +28,7 @@ int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const int terminalPos = getBuffer()->readUint( Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); return (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) ? - NOT_A_DICT_POS : terminalPos - mHeaderRegionSize; + NOT_A_DICT_POS : terminalPos; } bool TerminalPositionLookupTable::setTerminalPtNodePosition( @@ -45,18 +45,16 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition( mSize++; } const int terminalPos = (terminalPtNodePos != NOT_A_DICT_POS) ? - terminalPtNodePos + mHeaderRegionSize : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS; + terminalPtNodePos : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS; return getWritableBuffer()->writeUint(terminalPos, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); } -bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath, - const int newHeaderRegionSize) const { - const int headerRegionSizeDiff = newHeaderRegionSize - mHeaderRegionSize; - // If header region size has been changed or used buffer size is smaller than actual buffer - // size, regenerate lookup table and write the new table to file. - if (headerRegionSizeDiff != 0 || getEntryPos(mSize) < getBuffer()->getTailPosition()) { - TerminalPositionLookupTable lookupTableToWrite(newHeaderRegionSize); +bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath) const { + // If the used buffer size is smaller than the actual buffer size, regenerate the lookup + // table and write the new table to the file. + if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { + TerminalPositionLookupTable lookupTableToWrite; for (int i = 0; i < mSize; ++i) { const int terminalPtNodePosition = getTerminalPtNodePosition(i); if (!lookupTableToWrite.setTerminalPtNodePosition(i, terminalPtNodePosition)) { @@ -68,7 +66,7 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath, return lookupTableToWrite.flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } else { - // We can simply use this lookup table because the header region size has not been + // We can simply use this lookup table because the buffer size has not been // changed. return flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h index 80b358c43..283b40237 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h @@ -28,19 +28,13 @@ class TerminalPositionLookupTable : public SingleDictContent { public: typedef hash_map_compat<int, int> TerminalIdMap; - // TODO: Quit using headerRegionSize. - TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable, - const int headerRegionSize) + TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable) : SingleDictContent(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), mSize(getBuffer()->getTailPosition() - / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE), - mHeaderRegionSize(headerRegionSize) {} + / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {} - explicit TerminalPositionLookupTable(const int headerRegionSize) - : mSize(0), mHeaderRegionSize(headerRegionSize) {} - - TerminalPositionLookupTable() : mSize(0), mHeaderRegionSize(0) {} + TerminalPositionLookupTable() : mSize(0) {} int getTerminalPtNodePosition(const int terminalId) const; @@ -50,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent { return mSize; } - bool flushToFile(const char *const dictDirPath, const int newHeaderRegionSize) const; + bool flushToFile(const char *const dictDirPath) const; bool runGCTerminalIds(TerminalIdMap *const terminalIdMap); @@ -62,7 +56,6 @@ class TerminalPositionLookupTable : public SingleDictContent { } int mSize; - const int mHeaderRegionSize; }; } // namespace latinime #endif // LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index 40c9b5e78..d17d2d597 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -25,6 +25,13 @@ namespace latinime { +/* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( + const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) { + const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false; + // TODO: take only dictDirPath, and open both header and trie files in the constructor below + return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, headerBuffer, isUpdatable)); +} + bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, const BufferWithExtendableBuffer *const headerBuffer) const { // Create temporary directory. @@ -45,16 +52,22 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno); return false; } + // Write header file. + if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath, + Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) { + AKLOGE("Dictionary header file %s/%s cannot be written.", tmpDirPath, + Ver4DictConstants::HEADER_FILE_EXTENSION); + return false; + } // Write trie file. - const BufferWithExtendableBuffer *buffers[] = {headerBuffer, &mExpandableTrieBuffer}; - if (!DictFileWritingUtils::flushBuffersToFileInDir(tmpDirPath, - Ver4DictConstants::TRIE_FILE_EXTENSION, buffers, 2 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath, + Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) { AKLOGE("Dictionary trie file %s/%s cannot be written.", tmpDirPath, Ver4DictConstants::TRIE_FILE_EXTENSION); return false; } // Write dictionary contents. - if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath, headerBuffer->getTailPosition())) { + if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath)) { AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); return false; } @@ -85,4 +98,32 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, return true; } +Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath, + const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable) + : mHeaderBuffer(headerBuffer), + mDictBuffer(MmappedBuffer::openBuffer(dictDirPath, + Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), + mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4), + mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), + mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(), + mDictBuffer.get()->getBufferSize(), + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), + mTerminalPositionLookupTable(dictDirPath, isUpdatable), + mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), + isUpdatable), + mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), + isUpdatable), + mShortcutDictContent(dictDirPath, isUpdatable), + mIsUpdatable(isUpdatable) {} + +Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy) + : mHeaderBuffer(0), mDictBuffer(0), mHeaderPolicy(), + mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), + mTerminalPositionLookupTable(), + mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()), + mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(), + mIsUpdatable(true) {} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index 153d8990a..a0c219e4d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -33,11 +33,8 @@ class Ver4DictBuffers { public: typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr; - static AK_FORCE_INLINE Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, - const MmappedBuffer::MmappedBufferPtr &dictBuffer) { - const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false; - return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable)); - } + static Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, + const MmappedBuffer::MmappedBufferPtr &headerBuffer); static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers( const HeaderPolicy *const headerPolicy) { @@ -120,33 +117,12 @@ class Ver4DictBuffers { private: DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers); - AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath, - const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable) - : mDictBuffer(dictBuffer), - mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4), - mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), - BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(), - dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(), - BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - // TODO: Quit using header size. - mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()), - mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), - isUpdatable), - mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), - isUpdatable), - mShortcutDictContent(dictDirPath, isUpdatable), - mIsUpdatable(isUpdatable) {} - - AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy) - : mDictBuffer(0), mHeaderPolicy(), - mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), - mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), - mTerminalPositionLookupTable(), - mProbabilityDictContent(headerPolicy->hasHistoricalInfoOfWords()), - mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(), - mIsUpdatable(true) {} + Ver4DictBuffers(const char *const dictDirPath, + const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable); + + Ver4DictBuffers(const HeaderPolicy *const headerPolicy); + const MmappedBuffer::MmappedBufferPtr mHeaderBuffer; const MmappedBuffer::MmappedBufferPtr mDictBuffer; const HeaderPolicy mHeaderPolicy; BufferWithExtendableBuffer mExpandableHeaderBuffer; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index 066d6ed80..34fecc25f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -18,7 +18,9 @@ namespace latinime { +// These values MUST match the definitions in FormatSpec.java. const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie"; +const char *const Ver4DictConstants::HEADER_FILE_EXTENSION = ".header"; const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq"; // tat = Terminal Address Table const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index 7afdc86ed..c16e0bdcb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -25,6 +25,7 @@ namespace latinime { class Ver4DictConstants { public: static const char *const TRIE_FILE_EXTENSION; + static const char *const HEADER_FILE_EXTENSION; static const char *const FREQ_FILE_EXTENSION; static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION; static const char *const BIGRAM_FILE_EXTENSION; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index b463d4149..9223356c1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -68,42 +68,43 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = char tmpFileName[tmpFileNameBufSize]; FileUtils::getFilePathWithSuffix(filePath, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpFileNameBufSize, tmpFileName); - const BufferWithExtendableBuffer *buffers[] = {dictHeader, dictBody}; - if (!DictFileWritingUtils::flushBuffersToFile(tmpFileName, buffers, 2 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFile(tmpFileName, dictHeader)) { + AKLOGE("Dictionary header cannot be written to %s.", tmpFileName); + return false; + } + if (!DictFileWritingUtils::flushBufferToFile(tmpFileName, dictBody)) { AKLOGE("Dictionary structure cannot be written to %s.", tmpFileName); return false; } if (rename(tmpFileName, filePath) != 0) { AKLOGE("Dictionary file %s cannot be renamed to %s", tmpFileName, filePath);; + return false; } return true; } -/* static */ bool DictFileWritingUtils::flushBuffersToFileInDir(const char *const dirPath, - const char *const fileName, const BufferWithExtendableBuffer **const buffers, - const int bufferCount) { +/* static */ bool DictFileWritingUtils::flushBufferToFileInDir(const char *const dirPath, + const char *const fileName, const BufferWithExtendableBuffer *const buffer) { const int filePathBufSize = FileUtils::getFilePathBufSize(dirPath, fileName); char filePath[filePathBufSize]; FileUtils::getFilePath(dirPath, fileName, filePathBufSize, filePath); - return flushBuffersToFile(filePath, buffers, bufferCount); + return flushBufferToFile(filePath, buffer); } -/* static */ bool DictFileWritingUtils::flushBuffersToFile(const char *const filePath, - const BufferWithExtendableBuffer **const buffers, const int bufferCount) { +/* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath, + const BufferWithExtendableBuffer *const buffer) { FILE *const file = fopen(filePath, "wb"); if (!file) { AKLOGE("File %s cannot be opened.", filePath); ASSERT(false); return false; } - for (int i = 0; i < bufferCount; ++i) { - if (!writeBufferToFile(file, buffers[i])) { - remove(filePath); - AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath, - buffers[i]->getTailPosition()); - ASSERT(false); - return false; - } + if (!writeBufferToFile(file, buffer)) { + remove(filePath); + AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath, + buffer->getTailPosition()); + ASSERT(false); + return false; } fclose(file); return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h index 612d75445..ffd9db623 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h @@ -37,8 +37,8 @@ class DictFileWritingUtils { BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody); - static bool flushBuffersToFileInDir(const char *const dirPath, const char *const fileName, - const BufferWithExtendableBuffer **const buffers, const int bufferCount); + static bool flushBufferToFileInDir(const char *const dirPath, const char *const fileName, + const BufferWithExtendableBuffer *const buffer); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils); @@ -46,8 +46,8 @@ class DictFileWritingUtils { static bool createEmptyV4DictFile(const char *const filePath, const HeaderReadWriteUtils::AttributeMap *const attributeMap); - static bool flushBuffersToFile(const char *const filePath, - const BufferWithExtendableBuffer **const buffers, const int bufferCount); + static bool flushBufferToFile(const char *const filePath, + const BufferWithExtendableBuffer *const buffer); static bool writeBufferToFile(FILE *const file, const BufferWithExtendableBuffer *const buffer); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 3d14d2607..34727b8d8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -29,9 +29,10 @@ namespace latinime { class FormatUtils { public: enum FORMAT_VERSION { + // These MUST have the same values as the relevant constants in FormatSpec.java. VERSION_2 = 2, VERSION_3 = 3, - VERSION_4 = 4, + VERSION_4 = 400, UNKNOWN_VERSION = -1 }; diff --git a/native/jni/src/utils/exclusive_ownership_pointer.h b/native/jni/src/utils/exclusive_ownership_pointer.h index 6c67df28e..617b34968 100644 --- a/native/jni/src/utils/exclusive_ownership_pointer.h +++ b/native/jni/src/utils/exclusive_ownership_pointer.h @@ -25,22 +25,23 @@ template<class T> class ExclusiveOwnershipPointer { public: // This instance become an owner of the raw pointer. - ExclusiveOwnershipPointer(T *const rawPointer) + AK_FORCE_INLINE ExclusiveOwnershipPointer(T *const rawPointer) : mPointer(rawPointer), mSharedOwnerPtr(new (ExclusiveOwnershipPointer<T> *)(this)) {} // Move the ownership. - ExclusiveOwnershipPointer(const ExclusiveOwnershipPointer<T> &pointer) + AK_FORCE_INLINE ExclusiveOwnershipPointer(const ExclusiveOwnershipPointer<T> &pointer) : mPointer(pointer.mPointer), mSharedOwnerPtr(pointer.mSharedOwnerPtr) { transferOwnership(&pointer); } - ~ExclusiveOwnershipPointer() { + AK_FORCE_INLINE ~ExclusiveOwnershipPointer() { deletePointersIfHavingOwnership(); } // Move the ownership. - ExclusiveOwnershipPointer<T> &operator=(const ExclusiveOwnershipPointer<T> &pointer) { + AK_FORCE_INLINE ExclusiveOwnershipPointer<T> &operator=( + const ExclusiveOwnershipPointer<T> &pointer) { // Delete pointers when this is an owner of another pointer. deletePointersIfHavingOwnership(); mPointer = pointer.mPointer; @@ -49,7 +50,7 @@ class ExclusiveOwnershipPointer { return *this; } - T *get() const { + AK_FORCE_INLINE T *get() const { return mPointer; } |