diff options
Diffstat (limited to 'native/jni/src')
12 files changed, 72 insertions, 71 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index f1b733a9c..dfb110cdd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -47,7 +47,7 @@ namespace latinime { case FormatUtils::VERSION_4: { const int dictDirPathBufSize = strlen(path) + 1 /* terminator */; char dictDirPath[dictDirPathBufSize]; - if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::TRIE_FILE_EXTENSION, + if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictDirPath)) { // Dictionary file name is not valid as a version 4 dictionary. return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index bf833239b..9512bdbb0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -60,9 +60,8 @@ class SingleDictContent : public DictContent { } bool flush(const char *const dictDirPath, const char *const contentFileName) const { - const BufferWithExtendableBuffer *bufferPtr = &mExpandableContentBuffer; - return DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName, - &bufferPtr, 1 /* bufferCount */); + return DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName, + &mExpandableContentBuffer); } private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp index c65420614..84aceeffe 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp @@ -21,19 +21,16 @@ namespace latinime { bool SparseTableDictContent::flush(const char *const dictDirPath, const char *const lookupTableFileName, const char *const addressTableFileName, const char *const contentFileName) const { - const BufferWithExtendableBuffer *lookupTableBufferPtr = &mExpandableLookupTableBuffer; - if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, lookupTableFileName, - &lookupTableBufferPtr, 1 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, lookupTableFileName, + &mExpandableLookupTableBuffer)){ return false; } - const BufferWithExtendableBuffer *addressTableBufferPtr = &mExpandableAddressTableBuffer; - if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, addressTableFileName, - &addressTableBufferPtr, 1 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, addressTableFileName, + &mExpandableAddressTableBuffer)) { return false; } - const BufferWithExtendableBuffer *contentBufferPtr = &mExpandableContentBuffer; - if (!DictFileWritingUtils::flushBuffersToFileInDir(dictDirPath, contentFileName, - &contentBufferPtr, 1 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(dictDirPath, contentFileName, + &mExpandableContentBuffer)) { return false; } return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp index d0853a5d3..24f62cd4b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp @@ -28,7 +28,7 @@ int TerminalPositionLookupTable::getTerminalPtNodePosition(const int terminalId) const int terminalPos = getBuffer()->readUint( Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); return (terminalPos == Ver4DictConstants::NOT_A_TERMINAL_ADDRESS) ? - NOT_A_DICT_POS : terminalPos - mHeaderRegionSize; + NOT_A_DICT_POS : terminalPos; } bool TerminalPositionLookupTable::setTerminalPtNodePosition( @@ -45,18 +45,16 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition( mSize++; } const int terminalPos = (terminalPtNodePos != NOT_A_DICT_POS) ? - terminalPtNodePos + mHeaderRegionSize : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS; + terminalPtNodePos : Ver4DictConstants::NOT_A_TERMINAL_ADDRESS; return getWritableBuffer()->writeUint(terminalPos, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); } -bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath, - const int newHeaderRegionSize) const { - const int headerRegionSizeDiff = newHeaderRegionSize - mHeaderRegionSize; - // If header region size has been changed or used buffer size is smaller than actual buffer - // size, regenerate lookup table and write the new table to file. - if (headerRegionSizeDiff != 0 || getEntryPos(mSize) < getBuffer()->getTailPosition()) { - TerminalPositionLookupTable lookupTableToWrite(newHeaderRegionSize); +bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath) const { + // If the used buffer size is smaller than the actual buffer size, regenerate the lookup + // table and write the new table to the file. + if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { + TerminalPositionLookupTable lookupTableToWrite; for (int i = 0; i < mSize; ++i) { const int terminalPtNodePosition = getTerminalPtNodePosition(i); if (!lookupTableToWrite.setTerminalPtNodePosition(i, terminalPtNodePosition)) { @@ -68,7 +66,7 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictDirPath, return lookupTableToWrite.flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } else { - // We can simply use this lookup table because the header region size has not been + // We can simply use this lookup table because the buffer size has not been // changed. return flush(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h index 80b358c43..283b40237 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h @@ -28,19 +28,13 @@ class TerminalPositionLookupTable : public SingleDictContent { public: typedef hash_map_compat<int, int> TerminalIdMap; - // TODO: Quit using headerRegionSize. - TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable, - const int headerRegionSize) + TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable) : SingleDictContent(dictDirPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), mSize(getBuffer()->getTailPosition() - / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE), - mHeaderRegionSize(headerRegionSize) {} + / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {} - explicit TerminalPositionLookupTable(const int headerRegionSize) - : mSize(0), mHeaderRegionSize(headerRegionSize) {} - - TerminalPositionLookupTable() : mSize(0), mHeaderRegionSize(0) {} + TerminalPositionLookupTable() : mSize(0) {} int getTerminalPtNodePosition(const int terminalId) const; @@ -50,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent { return mSize; } - bool flushToFile(const char *const dictDirPath, const int newHeaderRegionSize) const; + bool flushToFile(const char *const dictDirPath) const; bool runGCTerminalIds(TerminalIdMap *const terminalIdMap); @@ -62,7 +56,6 @@ class TerminalPositionLookupTable : public SingleDictContent { } int mSize; - const int mHeaderRegionSize; }; } // namespace latinime #endif // LATINIME_TERMINAL_POSITION_LOOKUP_TABLE_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index 40c9b5e78..10f8d32e7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -45,16 +45,22 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, AKLOGE("Cannot create directory: %s. errno: %d.", tmpDirPath, errno); return false; } + // Write header file. + if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath, + Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) { + AKLOGE("Dictionary header file %s/%s cannot be written.", tmpDirPath, + Ver4DictConstants::HEADER_FILE_EXTENSION); + return false; + } // Write trie file. - const BufferWithExtendableBuffer *buffers[] = {headerBuffer, &mExpandableTrieBuffer}; - if (!DictFileWritingUtils::flushBuffersToFileInDir(tmpDirPath, - Ver4DictConstants::TRIE_FILE_EXTENSION, buffers, 2 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFileInDir(tmpDirPath, + Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) { AKLOGE("Dictionary trie file %s/%s cannot be written.", tmpDirPath, Ver4DictConstants::TRIE_FILE_EXTENSION); return false; } // Write dictionary contents. - if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath, headerBuffer->getTailPosition())) { + if (!mTerminalPositionLookupTable.flushToFile(tmpDirPath)) { AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h index 153d8990a..8a13671c0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h @@ -34,9 +34,10 @@ class Ver4DictBuffers { typedef ExclusiveOwnershipPointer<Ver4DictBuffers> Ver4DictBuffersPtr; static AK_FORCE_INLINE Ver4DictBuffersPtr openVer4DictBuffers(const char *const dictDirPath, - const MmappedBuffer::MmappedBufferPtr &dictBuffer) { - const bool isUpdatable = dictBuffer.get() ? dictBuffer.get()->isUpdatable() : false; - return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, dictBuffer, isUpdatable)); + const MmappedBuffer::MmappedBufferPtr &headerBuffer) { + const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false; + // TODO: take only dictDirPath, and open both header and trie files in the constructor below + return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, headerBuffer, isUpdatable)); } static AK_FORCE_INLINE Ver4DictBuffersPtr createVer4DictBuffers( @@ -121,16 +122,17 @@ class Ver4DictBuffers { DISALLOW_COPY_AND_ASSIGN(Ver4DictBuffers); AK_FORCE_INLINE Ver4DictBuffers(const char *const dictDirPath, - const MmappedBuffer::MmappedBufferPtr &dictBuffer, const bool isUpdatable) - : mDictBuffer(dictBuffer), - mHeaderPolicy(mDictBuffer.get()->getBuffer(), FormatUtils::VERSION_4), - mExpandableHeaderBuffer(dictBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), + const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable) + : mHeaderBuffer(headerBuffer), + mDictBuffer(MmappedBuffer::openBuffer(dictDirPath, + Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), + mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4), + mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mExpandableTrieBuffer(dictBuffer.get()->getBuffer() + mHeaderPolicy.getSize(), - dictBuffer.get()->getBufferSize() - mHeaderPolicy.getSize(), + mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(), + mDictBuffer.get()->getBufferSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - // TODO: Quit using header size. - mTerminalPositionLookupTable(dictDirPath, isUpdatable, mHeaderPolicy.getSize()), + mTerminalPositionLookupTable(dictDirPath, isUpdatable), mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), @@ -139,7 +141,7 @@ class Ver4DictBuffers { mIsUpdatable(isUpdatable) {} AK_FORCE_INLINE Ver4DictBuffers(const HeaderPolicy *const headerPolicy) - : mDictBuffer(0), mHeaderPolicy(), + : mHeaderBuffer(0), mDictBuffer(0), mHeaderPolicy(), mExpandableHeaderBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mExpandableTrieBuffer(Ver4DictConstants::MAX_DICTIONARY_SIZE), mTerminalPositionLookupTable(), @@ -147,6 +149,7 @@ class Ver4DictBuffers { mBigramDictContent(headerPolicy->hasHistoricalInfoOfWords()), mShortcutDictContent(), mIsUpdatable(true) {} + const MmappedBuffer::MmappedBufferPtr mHeaderBuffer; const MmappedBuffer::MmappedBufferPtr mDictBuffer; const HeaderPolicy mHeaderPolicy; BufferWithExtendableBuffer mExpandableHeaderBuffer; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp index 066d6ed80..34fecc25f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp @@ -18,7 +18,9 @@ namespace latinime { +// These values MUST match the definitions in FormatSpec.java. const char *const Ver4DictConstants::TRIE_FILE_EXTENSION = ".trie"; +const char *const Ver4DictConstants::HEADER_FILE_EXTENSION = ".header"; const char *const Ver4DictConstants::FREQ_FILE_EXTENSION = ".freq"; // tat = Terminal Address Table const char *const Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index 7afdc86ed..c16e0bdcb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -25,6 +25,7 @@ namespace latinime { class Ver4DictConstants { public: static const char *const TRIE_FILE_EXTENSION; + static const char *const HEADER_FILE_EXTENSION; static const char *const FREQ_FILE_EXTENSION; static const char *const TERMINAL_ADDRESS_TABLE_FILE_EXTENSION; static const char *const BIGRAM_FILE_EXTENSION; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index b463d4149..9223356c1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -68,42 +68,43 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = char tmpFileName[tmpFileNameBufSize]; FileUtils::getFilePathWithSuffix(filePath, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE, tmpFileNameBufSize, tmpFileName); - const BufferWithExtendableBuffer *buffers[] = {dictHeader, dictBody}; - if (!DictFileWritingUtils::flushBuffersToFile(tmpFileName, buffers, 2 /* bufferCount */)) { + if (!DictFileWritingUtils::flushBufferToFile(tmpFileName, dictHeader)) { + AKLOGE("Dictionary header cannot be written to %s.", tmpFileName); + return false; + } + if (!DictFileWritingUtils::flushBufferToFile(tmpFileName, dictBody)) { AKLOGE("Dictionary structure cannot be written to %s.", tmpFileName); return false; } if (rename(tmpFileName, filePath) != 0) { AKLOGE("Dictionary file %s cannot be renamed to %s", tmpFileName, filePath);; + return false; } return true; } -/* static */ bool DictFileWritingUtils::flushBuffersToFileInDir(const char *const dirPath, - const char *const fileName, const BufferWithExtendableBuffer **const buffers, - const int bufferCount) { +/* static */ bool DictFileWritingUtils::flushBufferToFileInDir(const char *const dirPath, + const char *const fileName, const BufferWithExtendableBuffer *const buffer) { const int filePathBufSize = FileUtils::getFilePathBufSize(dirPath, fileName); char filePath[filePathBufSize]; FileUtils::getFilePath(dirPath, fileName, filePathBufSize, filePath); - return flushBuffersToFile(filePath, buffers, bufferCount); + return flushBufferToFile(filePath, buffer); } -/* static */ bool DictFileWritingUtils::flushBuffersToFile(const char *const filePath, - const BufferWithExtendableBuffer **const buffers, const int bufferCount) { +/* static */ bool DictFileWritingUtils::flushBufferToFile(const char *const filePath, + const BufferWithExtendableBuffer *const buffer) { FILE *const file = fopen(filePath, "wb"); if (!file) { AKLOGE("File %s cannot be opened.", filePath); ASSERT(false); return false; } - for (int i = 0; i < bufferCount; ++i) { - if (!writeBufferToFile(file, buffers[i])) { - remove(filePath); - AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath, - buffers[i]->getTailPosition()); - ASSERT(false); - return false; - } + if (!writeBufferToFile(file, buffer)) { + remove(filePath); + AKLOGE("Buffer cannot be written to the file %s. size: %d", filePath, + buffer->getTailPosition()); + ASSERT(false); + return false; } fclose(file); return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h index 612d75445..ffd9db623 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h @@ -37,8 +37,8 @@ class DictFileWritingUtils { BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody); - static bool flushBuffersToFileInDir(const char *const dirPath, const char *const fileName, - const BufferWithExtendableBuffer **const buffers, const int bufferCount); + static bool flushBufferToFileInDir(const char *const dirPath, const char *const fileName, + const BufferWithExtendableBuffer *const buffer); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils); @@ -46,8 +46,8 @@ class DictFileWritingUtils { static bool createEmptyV4DictFile(const char *const filePath, const HeaderReadWriteUtils::AttributeMap *const attributeMap); - static bool flushBuffersToFile(const char *const filePath, - const BufferWithExtendableBuffer **const buffers, const int bufferCount); + static bool flushBufferToFile(const char *const filePath, + const BufferWithExtendableBuffer *const buffer); static bool writeBufferToFile(FILE *const file, const BufferWithExtendableBuffer *const buffer); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 3d14d2607..34727b8d8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -29,9 +29,10 @@ namespace latinime { class FormatUtils { public: enum FORMAT_VERSION { + // These MUST have the same values as the relevant constants in FormatSpec.java. VERSION_2 = 2, VERSION_3 = 3, - VERSION_4 = 4, + VERSION_4 = 400, UNKNOWN_VERSION = -1 }; |