diff options
Diffstat (limited to 'native/jni/src')
41 files changed, 310 insertions, 275 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp index 0ef6ccf37..7d0d09631 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp @@ -16,7 +16,6 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -78,11 +77,6 @@ const BigramListReadWriteUtils::BigramFlags offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos); break; } - if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) { - return NOT_A_DICT_POS; - } else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) { - return origin; - } if (isOffsetNegative(flags)) { return origin - offset; } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 5e8b5f11e..1208d2c2a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -87,8 +87,6 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { switch (mDictFormatVersion) { case FormatUtils::VERSION_2: return FormatUtils::VERSION_2; - case FormatUtils::VERSION_3: - return FormatUtils::VERSION_3; case FormatUtils::VERSION_4: return FormatUtils::VERSION_4; default: diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index 2d4547066..6b4598642 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -89,9 +89,6 @@ const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::NO_FLAGS = 0; case FormatUtils::VERSION_2: // Version 2 dictionary writing is not supported. return false; - case FormatUtils::VERSION_3: - return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_3 /* data */, - HEADER_DICTIONARY_VERSION_SIZE, writingPos); case FormatUtils::VERSION_4: return buffer->writeUintAndAdvancePosition(FormatUtils::VERSION_4 /* data */, HEADER_DICTIONARY_VERSION_SIZE, writingPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index dfb110cdd..c81c61d23 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -16,11 +16,13 @@ #include "suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h" +#include <climits> #include <stdint.h> #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" @@ -32,9 +34,28 @@ namespace latinime { DictionaryStructureWithBufferPolicyFactory ::newDictionaryStructureWithBufferPolicy(const char *const path, const int bufOffset, const int size, const bool isUpdatable) { - // Allocated buffer in MmapedBuffer::newBuffer() will be freed in the destructor of - // MmappedBufferWrapper if the instance has the responsibility. - MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size, + if (FileUtils::existsDir(path)) { + // Given path represents a directory. + return newPolicyforDirectoryDict(path, isUpdatable); + } else { + if (isUpdatable) { + AKLOGE("One file dictionaries don't support updating. path: %s", path); + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); + } + return newPolicyforFileDict(path, bufOffset, size); + } +} + +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyforDirectoryDict( + const char *const path, const bool isUpdatable) { + const int headerFilePathBufSize = PATH_MAX + 1 /* terminator */; + char headerFilePath[headerFilePathBufSize]; + getHeaderFilePathInDictDir(path, headerFilePathBufSize, headerFilePath); + // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of + // MmappedBufferPtr if the instance has the responsibility. + MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(headerFilePath, isUpdatable); if (!mmappedBuffer.get()) { return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); @@ -42,20 +63,22 @@ namespace latinime { switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(), mmappedBuffer.get()->getBufferSize())) { case FormatUtils::VERSION_2: - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( - new PatriciaTriePolicy(mmappedBuffer)); + AKLOGE("Given path is a directory but the format is version 2. path: %s", path); + break; case FormatUtils::VERSION_4: { - const int dictDirPathBufSize = strlen(path) + 1 /* terminator */; - char dictDirPath[dictDirPathBufSize]; - if (!FileUtils::getFilePathWithoutSuffix(path, Ver4DictConstants::HEADER_FILE_EXTENSION, - dictDirPathBufSize, dictDirPath)) { - // Dictionary file name is not valid as a version 4 dictionary. + const int dictDirPathBufSize = strlen(headerFilePath) + 1 /* terminator */; + char dictPath[dictDirPathBufSize]; + if (!FileUtils::getFilePathWithoutSuffix(headerFilePath, + Ver4DictConstants::HEADER_FILE_EXTENSION, dictDirPathBufSize, dictPath)) { + AKLOGE("Dictionary file name is not valid as a ver4 dictionary. path: %s", path); + ASSERT(false); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); } const Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers = - Ver4DictBuffers::openVer4DictBuffers(dictDirPath, mmappedBuffer); + Ver4DictBuffers::openVer4DictBuffers(dictPath, mmappedBuffer); if (!dictBuffers.get()->isValid()) { - AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements."); + AKLOGE("DICT: The dictionary doesn't satisfy ver4 format requirements. path: %s", + path); ASSERT(false); return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); } @@ -63,10 +86,47 @@ namespace latinime { new Ver4PatriciaTriePolicy(dictBuffers)); } default: - AKLOGE("DICT: dictionary format is unknown, bad magic number"); - ASSERT(false); - return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); + AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path); + break; } + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); +} + +/* static */ DictionaryStructureWithBufferPolicy::StructurePolicyPtr + DictionaryStructureWithBufferPolicyFactory::newPolicyforFileDict( + const char *const path, const int bufOffset, const int size) { + // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of + // MmappedBufferPtr if the instance has the responsibility. + MmappedBuffer::MmappedBufferPtr mmappedBuffer = MmappedBuffer::openBuffer(path, bufOffset, + size, false /* isUpdatable */); + if (!mmappedBuffer.get()) { + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); + } + switch (FormatUtils::detectFormatVersion(mmappedBuffer.get()->getBuffer(), + mmappedBuffer.get()->getBufferSize())) { + case FormatUtils::VERSION_2: + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr( + new PatriciaTriePolicy(mmappedBuffer)); + case FormatUtils::VERSION_4: + AKLOGE("Given path is a file but the format is version 4. path: %s", path); + break; + default: + AKLOGE("DICT: dictionary format is unknown, bad magic number. path: %s", path); + break; + } + ASSERT(false); + return DictionaryStructureWithBufferPolicy::StructurePolicyPtr(0); +} + +/* static */ void DictionaryStructureWithBufferPolicyFactory::getHeaderFilePathInDictDir( + const char *const dictDirPath, const int outHeaderFileBufSize, + char *const outHeaderFilePath) { + const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */; + char dictName[dictNameBufSize]; + FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName); + snprintf(outHeaderFilePath, outHeaderFileBufSize, "%s/%s%s", dictDirPath, + dictName, Ver4DictConstants::HEADER_FILE_EXTENSION); } } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h index 45237e4aa..45ab52931 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.h @@ -33,6 +33,15 @@ class DictionaryStructureWithBufferPolicyFactory { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory); + + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr + newPolicyforDirectoryDict(const char *const path, const bool isUpdatable); + + static DictionaryStructureWithBufferPolicy::StructurePolicyPtr + newPolicyforFileDict(const char *const path, const int bufOffset, const int size); + + static void getHeaderFilePathInDictDir(const char *const dirPath, + const int outHeaderFileBufSize, char *const outHeaderFilePath); }; } // namespace latinime #endif // LATINIME_DICTIONARY_STRUCTURE_WITH_BUFFER_POLICY_FACTORY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp index bcba67035..8f42df6d2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.cpp @@ -14,16 +14,16 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" #include "suggest/core/policy/dictionary_header_structure_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" namespace latinime { -bool DynamicPatriciaTrieGcEventListeners +bool DynamicPtGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless @@ -63,7 +63,7 @@ bool DynamicPatriciaTrieGcEventListeners return true; } -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability +bool DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { if (!ptNodeParams->isDeleted() && ptNodeParams->hasBigrams()) { int bigramEntryCount = 0; @@ -77,7 +77,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit } // Writes dummy PtNode array size when the head of PtNode array is read. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onDescend(const int ptNodeArrayPos) { mValidPtNodeCount = 0; int writingPos = mBufferToWrite->getTailPosition(); @@ -86,21 +86,21 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. mPtNodeArraySizeFieldPos = writingPos; - return DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( + return DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( mBufferToWrite, 0 /* arraySize */, &writingPos); } // Write PtNode array terminal and actual PtNode array size. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onReadingPtNodeArrayTail() { int writingPos = mBufferToWrite->getTailPosition(); // Write PtNode array terminal. - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition( mBufferToWrite, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } // Write actual PtNode array size. - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( mBufferToWrite, mValidPtNodeCount, &mPtNodeArraySizeFieldPos)) { return false; } @@ -108,7 +108,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo } // Write valid PtNode to buffer and memorize mapping from the old position to the new position. -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer +bool DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { if (ptNodeParams->isDeleted()) { // Current PtNode is not written in new buffer because it has been deleted. @@ -126,7 +126,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo return mPtNodeWriter->writePtNodeAndAdvancePosition(ptNodeParams, &writingPos); } -bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields +bool DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // Updates parent position. int bigramCount = 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h index 562eb69bd..d8867754d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h @@ -14,14 +14,14 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H +#ifndef LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H +#define LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H #include <vector> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "utils/hash_map_compat.h" @@ -29,14 +29,13 @@ namespace latinime { class PtNodeParams; -// TODO: Move to pt_common. -class DynamicPatriciaTrieGcEventListeners { +class DynamicPtGcEventListeners { public: // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC. // TODO: Concatenate non-terminal PtNodes. class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( PtNodeWriter *const ptNodeWriter) @@ -81,7 +80,7 @@ class DynamicPatriciaTrieGcEventListeners { // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram // entries. class TraversePolicyToUpdateBigramProbability - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter) : mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {} @@ -106,7 +105,7 @@ class DynamicPatriciaTrieGcEventListeners { }; class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite, @@ -134,7 +133,7 @@ class DynamicPatriciaTrieGcEventListeners { }; class TraversePolicyToUpdateAllPositionFields - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter, const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap) @@ -168,7 +167,7 @@ class DynamicPatriciaTrieGcEventListeners { }; private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtGcEventListeners); }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_GC_EVENT_LISTENERS_H */ +#endif /* LATINIME_DYNAMIC_PT_GC_EVENT_LISTENERS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp index b3fdbeb78..b918e0765 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.cpp @@ -14,25 +14,25 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "utils/char_utils.h" namespace latinime { // To avoid infinite loop caused by invalid or malicious forward links. -const int DynamicPatriciaTrieReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; -const int DynamicPatriciaTrieReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; -const size_t DynamicPatriciaTrieReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; +const int DynamicPtReadingHelper::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; +const int DynamicPtReadingHelper::MAX_PT_NODE_ARRAY_COUNT_TO_AVOID_INFINITE_LOOP = 100000; +const size_t DynamicPtReadingHelper::MAX_READING_STATE_STACK_SIZE = MAX_WORD_LENGTH; // Visits all PtNodes in post-order depth first manner. // For example, visits c -> b -> y -> x -> a for the following dictionary: // a _ b _ c // \ x _ y -bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( +bool DynamicPtReadingHelper::traverseAllPtNodesInPostorderDepthFirstManner( TraversingEventListener *const listener) { bool alreadyVisitedChildren = false; // Descend from the root to the root PtNode array. @@ -92,7 +92,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstMa // For example, visits a -> b -> x -> c -> y for the following dictionary: // a _ b _ c // \ x _ y -bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( +bool DynamicPtReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( TraversingEventListener *const listener) { bool alreadyVisitedAllPtNodesInArray = false; bool alreadyVisitedChildren = false; @@ -169,7 +169,7 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreor return !isError(); } -int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( +int DynamicPtReadingHelper::getCodePointsAndProbabilityAndReturnCodePointCount( const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) { // This method traverses parent nodes from the terminal by following parent pointers; thus, // node code points are stored in the buffer in the reverse order. @@ -211,7 +211,7 @@ int DynamicPatriciaTrieReadingHelper::getCodePointsAndProbabilityAndReturnCodePo return totalCodePointCount; } -int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, +int DynamicPtReadingHelper::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) { int searchCodePoints[length]; for (int i = 0; i < length; ++i) { @@ -257,7 +257,7 @@ int DynamicPatriciaTrieReadingHelper::getTerminalPtNodePositionOfWord(const int // Read node array size and process empty node arrays. Nodes and arrays are counted up in this // method to avoid an infinite loop. -void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { +void DynamicPtReadingHelper::nextPtNodeArray() { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { // Reading invalid position because of a bug or a broken dictionary. AKLOGE("Reading PtNode array info from invalid dictionary position: %d, dict size: %d", @@ -308,7 +308,7 @@ void DynamicPatriciaTrieReadingHelper::nextPtNodeArray() { } // Follow the forward link and read the next node array if exists. -void DynamicPatriciaTrieReadingHelper::followForwardLink() { +void DynamicPtReadingHelper::followForwardLink() { if (mReadingState.mPos < 0 || mReadingState.mPos >= mBuffer->getTailPosition()) { // Reading invalid position because of bug or broken dictionary. AKLOGE("Reading forward link from invalid dictionary position: %d, dict size: %d", @@ -324,12 +324,12 @@ void DynamicPatriciaTrieReadingHelper::followForwardLink() { mReadingState.mPos -= mBuffer->getOriginalBufferSize(); } const int forwardLinkPosition = - DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); + DynamicPtReadingUtils::getForwardLinkPosition(dictBuf, mReadingState.mPos); if (usesAdditionalBuffer) { mReadingState.mPos += mBuffer->getOriginalBufferSize(); } mReadingState.mPosOfLastForwardLinkField = mReadingState.mPos; - if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { + if (DynamicPtReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { // Follow the forward link. mReadingState.mPos += forwardLinkPosition; nextPtNodeArray(); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h index 197027313..a69490943 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H +#ifndef LATINIME_DYNAMIC_PT_READING_HELPER_H +#define LATINIME_DYNAMIC_PT_READING_HELPER_H #include <cstddef> #include <vector> @@ -34,8 +34,7 @@ class DictionaryShortcutsStructurePolicy; * This class is used for traversing dynamic patricia trie. This class supports iterating nodes and * dealing with additional buffer. This class counts nodes and node arrays to avoid infinite loop. */ -// TODO: Move to pt_common. -class DynamicPatriciaTrieReadingHelper { +class DynamicPtReadingHelper { public: class TraversingEventListener { public: @@ -60,12 +59,12 @@ class DynamicPatriciaTrieReadingHelper { DISALLOW_COPY_AND_ASSIGN(TraversingEventListener); }; - DynamicPatriciaTrieReadingHelper(const BufferWithExtendableBuffer *const buffer, + DynamicPtReadingHelper(const BufferWithExtendableBuffer *const buffer, const PtNodeReader *const ptNodeReader) : mIsError(false), mReadingState(), mBuffer(buffer), mPtNodeReader(ptNodeReader), mReadingStateStack() {} - ~DynamicPatriciaTrieReadingHelper() {} + ~DynamicPtReadingHelper() {} AK_FORCE_INLINE bool isError() const { return mIsError; @@ -205,7 +204,7 @@ class DynamicPatriciaTrieReadingHelper { const bool forceLowerCaseSearch); private: - DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieReadingHelper); + DISALLOW_COPY_AND_ASSIGN(DynamicPtReadingHelper); // This class encapsulates the reading state of a position in the dictionary. It points at a // specific PtNode in the dictionary. @@ -267,4 +266,4 @@ class DynamicPatriciaTrieReadingHelper { } }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_HELPER_H */ +#endif /* LATINIME_DYNAMIC_PT_READING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp index 6cd127487..3586b50ab 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.cpp @@ -14,39 +14,38 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { -typedef DynamicPatriciaTrieReadingUtils DptReadingUtils; - -const DptReadingUtils::NodeFlags DptReadingUtils::MASK_MOVED = 0xC0; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; -const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::MASK_MOVED = 0xC0; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_MOVED = 0x40; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_IS_DELETED = 0x80; +const DynamicPtReadingUtils::NodeFlags DynamicPtReadingUtils::FLAG_WILL_BECOME_NON_TERMINAL = 0x00; // TODO: Make DICT_OFFSET_ZERO_OFFSET = 0. // Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum // value of offsets, which is 0x7FFFFF is used to represent 0 offset. -const int DptReadingUtils::DICT_OFFSET_INVALID = 0; -const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; +const int DynamicPtReadingUtils::DICT_OFFSET_INVALID = 0; +const int DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; -/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, +/* static */ int DynamicPtReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } -/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( +/* static */ int DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } -/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) { +/* static */ int DynamicPtReadingUtils::getParentPtNodePos(const int parentOffset, + const int ptNodePos) { if (parentOffset == DICT_OFFSET_INVALID) { return NOT_A_DICT_POS; } else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) { @@ -56,7 +55,7 @@ const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; } } -/* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( +/* static */ int DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const int base = *pos; const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h index 25b0333a5..89ae12c0b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H +#ifndef LATINIME_DYNAMIC_PT_READING_UTILS_H +#define LATINIME_DYNAMIC_PT_READING_UTILS_H #include <stdint.h> @@ -23,7 +23,7 @@ namespace latinime { -class DynamicPatriciaTrieReadingUtils { +class DynamicPtReadingUtils { public: typedef uint8_t NodeFlags; @@ -71,7 +71,7 @@ class DynamicPatriciaTrieReadingUtils { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieReadingUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtReadingUtils); static const NodeFlags MASK_MOVED; static const NodeFlags FLAG_IS_NOT_MOVED; @@ -80,4 +80,4 @@ class DynamicPatriciaTrieReadingUtils { static const NodeFlags FLAG_WILL_BECOME_NON_TERMINAL; }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_READING_UTILS_H */ +#endif /* LATINIME_DYNAMIC_PT_READING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp index f12a8995e..2457b49c8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp @@ -14,21 +14,21 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { -const int DynamicPatriciaTrieUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; +const int DynamicPtUpdatingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; -bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( - DynamicPatriciaTrieReadingHelper *const readingHelper, +bool DynamicPtUpdatingHelper::addUnigramWord( + DynamicPtReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability, const bool isNotAWord, const bool isBlacklisted, const int timestamp, bool *const outAddedNewUnigram) { @@ -86,7 +86,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addUnigramWord( isNotAWord, isBlacklisted, probability, timestamp, &pos); } -bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, +bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, const int probability, const int timestamp, bool *const outAddedNewBigram) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); @@ -97,7 +97,7 @@ bool DynamicPatriciaTrieUpdatingHelper::addBigramWords(const int word0Pos, const } // Remove a bigram relation from word0Pos to word1Pos. -bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { +bool DynamicPtUpdatingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); const PtNodeParams targetPtNodeParams( @@ -105,7 +105,7 @@ bool DynamicPatriciaTrieUpdatingHelper::removeBigramWords(const int word0Pos, co return mPtNodeWriter->removeBigramEntry(&sourcePtNodeParams, &targetPtNodeParams); } -bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, +bool DynamicPtUpdatingHelper::addShortcutTarget(const int wordPos, const int *const targetCodePoints, const int targetCodePointCount, const int shortcutProbability) { const PtNodeParams ptNodeParams(mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(wordPos)); @@ -113,12 +113,12 @@ bool DynamicPatriciaTrieUpdatingHelper::addShortcutTarget(const int wordPos, shortcutProbability); } -bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, +bool DynamicPtUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, int *const forwardLinkFieldPos) { const int newPtNodeArrayPos = mBuffer->getTailPosition(); - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, newPtNodeArrayPos, forwardLinkFieldPos)) { return false; } @@ -126,7 +126,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createAndInsertNodeIntoPtNodeArray(const isNotAWord, isBlacklisted, probability, timestamp); } -bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( +bool DynamicPtUpdatingHelper::setPtNodeProbability( const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, bool *const outAddedNewUnigram) { @@ -154,7 +154,7 @@ bool DynamicPatriciaTrieUpdatingHelper::setPtNodeProbability( return true; } -bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( +bool DynamicPtUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode( const PtNodeParams *const parentPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp, const int *const codePoints, const int codePointCount) { @@ -166,12 +166,12 @@ bool DynamicPatriciaTrieUpdatingHelper::createChildrenPtNodeArrayAndAChildPtNode codePointCount, isNotAWord, isBlacklisted, probability, timestamp); } -bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( +bool DynamicPtUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probability, const int timestamp) { int writingPos = mBuffer->getTailPosition(); - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, 1 /* arraySize */, &writingPos)) { return false; } @@ -182,7 +182,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( &writingPos)) { return false; } - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } @@ -190,7 +190,7 @@ bool DynamicPatriciaTrieUpdatingHelper::createNewPtNodeArrayWithAChildPtNode( } // Returns whether the dictionary updating was succeeded or not. -bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( +bool DynamicPtUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( const PtNodeParams *const reallocatingPtNodeParams, const int overlappingCodePointCount, const bool isNotAWord, const bool isBlacklisted, const int probabilityOfNewPtNode, const int timestamp, const int *const newNodeCodePoints, const int newNodeCodePointCount) { @@ -227,7 +227,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( const int actualChildrenPos = writingPos; // Create new children PtNode array. const size_t newPtNodeCount = addsExtraChild ? 2 : 1; - if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, newPtNodeCount, &writingPos)) { return false; } @@ -252,7 +252,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( return false; } } - if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + if (!DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } @@ -268,7 +268,7 @@ bool DynamicPatriciaTrieUpdatingHelper::reallocatePtNodeAndAddNewPtNodes( return mPtNodeWriter->updateChildrenPosition(&ptNodeParams, actualChildrenPos); } -const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( +const PtNodeParams DynamicPtUpdatingHelper::getUpdatedPtNodeParams( const PtNodeParams *const originalPtNodeParams, const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const int parentPos, const int codePointCount, const int *const codePoints, const int probability) const { @@ -280,7 +280,7 @@ const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getUpdatedPtNodeParams( probability); } -const PtNodeParams DynamicPatriciaTrieUpdatingHelper::getPtNodeParamsForNewPtNode( +const PtNodeParams DynamicPtUpdatingHelper::getPtNodeParamsForNewPtNode( const bool isNotAWord, const bool isBlacklisted, const bool isTerminal, const int parentPos, const int codePointCount, const int *const codePoints, const int probability) const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h index f02635fe2..71f473096 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_UPDATING_HELPER_H +#ifndef LATINIME_DYNAMIC_PT_UPDATING_HELPER_H +#define LATINIME_DYNAMIC_PT_UPDATING_HELPER_H #include <stdint.h> @@ -26,21 +26,20 @@ namespace latinime { class BufferWithExtendableBuffer; -class DynamicPatriciaTrieReadingHelper; +class DynamicPtReadingHelper; class PtNodeReader; class PtNodeWriter; -// TODO: Move to pt_common. -class DynamicPatriciaTrieUpdatingHelper { +class DynamicPtUpdatingHelper { public: - DynamicPatriciaTrieUpdatingHelper(BufferWithExtendableBuffer *const buffer, + DynamicPtUpdatingHelper(BufferWithExtendableBuffer *const buffer, const PtNodeReader *const ptNodeReader, PtNodeWriter *const ptNodeWriter) : mBuffer(buffer), mPtNodeReader(ptNodeReader), mPtNodeWriter(ptNodeWriter) {} - ~DynamicPatriciaTrieUpdatingHelper() {} + ~DynamicPtUpdatingHelper() {} // Add a word to the dictionary. If the word already exists, update the probability. - bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper, + bool addUnigramWord(DynamicPtReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability, const bool isNotAWord, const bool isBlacklisted, const int timestamp, bool *const outAddedNewUnigram); @@ -57,7 +56,7 @@ class DynamicPatriciaTrieUpdatingHelper { const int targetCodePointCount, const int shortcutProbability); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieUpdatingHelper); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtUpdatingHelper); static const int CHILDREN_POSITION_FIELD_SIZE; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp index 45eeefd4c..ebbdc2ea2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include <cstddef> #include <cstdlib> @@ -24,18 +24,18 @@ namespace latinime { -const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; -const size_t DynamicPatriciaTrieWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; -const int DynamicPatriciaTrieWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; -const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; -const int DynamicPatriciaTrieWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; -const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; -const int DynamicPatriciaTrieWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; -const int DynamicPatriciaTrieWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; -const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; -const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; +const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD = 0x7F; +const size_t DynamicPtWritingUtils::MAX_PTNODE_ARRAY_SIZE = 0x7FFF; +const int DynamicPtWritingUtils::SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE = 1; +const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE = 2; +const int DynamicPtWritingUtils::LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG = 0x8000; +const int DynamicPtWritingUtils::DICT_OFFSET_FIELD_SIZE = 3; +const int DynamicPtWritingUtils::MAX_DICT_OFFSET_VALUE = 0x7FFFFF; +const int DynamicPtWritingUtils::MIN_DICT_OFFSET_VALUE = -0x7FFFFF; +const int DynamicPtWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; +const int DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE = 1; -/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( +/* static */ bool DynamicPtWritingUtils::writeEmptyDictionary( BufferWithExtendableBuffer *const buffer, const int rootPos) { int writingPos = rootPos; if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) { @@ -45,13 +45,13 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; &writingPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos) { return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writePtNodeArraySizeAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const size_t arraySize, int *const arraySizeFieldPos) { // Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to @@ -73,20 +73,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; } } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeFlagsAndAdvancePosition( BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { + const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos) { return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos); } // Note that parentOffset is offset from node's head position. -/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos, int *const parentPosFieldPos) { return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeCodePointsAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int *const codePoints, const int codePointCount, int *const codePointFieldPos) { if (codePointCount <= 0) { @@ -100,21 +100,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; hasMultipleCodePoints, codePointFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( +/* static */ bool DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int childrenPosition, int *const childrenPositionFieldPos) { return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos), childrenPositionFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset( - BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos, - int *const offsetFieldPos) { +/* static */ bool DynamicPtWritingUtils::writeDictOffset(BufferWithExtendableBuffer *const buffer, + const int targetPos, const int basePos, int *const offsetFieldPos) { int offset = targetPos - basePos; if (targetPos == NOT_A_DICT_POS) { - offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; + offset = DynamicPtReadingUtils::DICT_OFFSET_INVALID; } else if (offset == 0) { - offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; + offset = DynamicPtReadingUtils::DICT_OFFSET_ZERO_OFFSET; } if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) { AKLOGI("offset cannot be written because the offset is too large or too small: %d", diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h index 43e8ea649..362fbd1cc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h @@ -14,19 +14,19 @@ * limitations under the License. */ -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H +#ifndef LATINIME_DYNAMIC_PT_WRITING_UTILS_H +#define LATINIME_DYNAMIC_PT_WRITING_UTILS_H #include <cstddef> #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" namespace latinime { class BufferWithExtendableBuffer; -class DynamicPatriciaTrieWritingUtils { +class DynamicPtWritingUtils { public: static const int NODE_FLAG_FIELD_SIZE; @@ -40,14 +40,14 @@ class DynamicPatriciaTrieWritingUtils { const size_t arraySize, int *const arraySizeFieldPos); static bool writeFlags(BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, + const DynamicPtReadingUtils::NodeFlags nodeFlags, const int nodeFlagsFieldPos) { int writingPos = nodeFlagsFieldPos; return writeFlagsAndAdvancePosition(buffer, nodeFlags, &writingPos); } static bool writeFlagsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, - const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, + const DynamicPtReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos); static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, @@ -60,7 +60,7 @@ class DynamicPatriciaTrieWritingUtils { const int childrenPosition, int *const childrenPositionFieldPos); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPtWritingUtils); static const size_t MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD; static const size_t MAX_PTNODE_ARRAY_SIZE; @@ -76,4 +76,4 @@ class DynamicPatriciaTrieWritingUtils { const int basePos, int *const offsetFieldPos); }; } // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */ +#endif /* LATINIME_DYNAMIC_PT_WRITING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h index 2a43e39bf..84731eb17 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h @@ -20,8 +20,8 @@ #include <cstring> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" namespace latinime { @@ -111,11 +111,11 @@ class PtNodeParams { // Flags AK_FORCE_INLINE bool isDeleted() const { - return DynamicPatriciaTrieReadingUtils::isDeleted(mFlags); + return DynamicPtReadingUtils::isDeleted(mFlags); } AK_FORCE_INLINE bool willBecomeNonTerminal() const { - return DynamicPatriciaTrieReadingUtils::willBecomeNonTerminal(mFlags); + return DynamicPtReadingUtils::willBecomeNonTerminal(mFlags); } AK_FORCE_INLINE bool hasChildren() const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h index 8420ee95a..b28f58336 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h @@ -23,6 +23,7 @@ namespace latinime { +// TODO: Move to pt_common class PatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h index 95ee74f99..ba2a05209 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h @@ -27,9 +27,9 @@ namespace latinime { class BigramDictContent : public SparseTableDictContent { public: - BigramDictContent(const char *const dictDirPath, const bool hasHistoricalInfo, + BigramDictContent(const char *const dictPath, const bool hasHistoricalInfo, const bool isUpdatable) - : SparseTableDictContent(dictDirPath, + : SparseTableDictContent(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_FILE_EXTENSION, isUpdatable, @@ -73,8 +73,8 @@ class BigramDictContent : public SparseTableDictContent { bool copyBigramList(const int bigramListPos, const int toPos); - bool flushToFile(const char *const dictBasePath) const { - return flush(dictBasePath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, + bool flushToFile(const char *const dictPath) const { + return flush(dictPath, Ver4DictConstants::BIGRAM_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::BIGRAM_FILE_EXTENSION); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp index 749e3fe8c..3b7c70efd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.cpp @@ -71,7 +71,7 @@ bool ProbabilityDictContent::setProbabilityEntry(const int terminalId, return writeEntry(probabilityEntry, entryPos); } -bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const { +bool ProbabilityDictContent::flushToFile(const char *const dictPath) const { if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { ProbabilityDictContent probabilityDictContentToWrite(mHasHistoricalInfo); for (int i = 0; i < mSize; ++i) { @@ -81,10 +81,10 @@ bool ProbabilityDictContent::flushToFile(const char *const dictBasePath) const { return false; } } - return probabilityDictContentToWrite.flush(dictBasePath, + return probabilityDictContentToWrite.flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION); } else { - return flush(dictBasePath, Ver4DictConstants::FREQ_FILE_EXTENSION); + return flush(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h index db96f9082..b065bc954 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h @@ -29,9 +29,9 @@ class ProbabilityEntry; class ProbabilityDictContent : public SingleDictContent { public: - ProbabilityDictContent(const char *const dictDirPath, const bool hasHistoricalInfo, + ProbabilityDictContent(const char *const dictPath, const bool hasHistoricalInfo, const bool isUpdatable) - : SingleDictContent(dictDirPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable), + : SingleDictContent(dictPath, Ver4DictConstants::FREQ_FILE_EXTENSION, isUpdatable), mHasHistoricalInfo(hasHistoricalInfo), mSize(getBuffer()->getTailPosition() / getEntrySize()) {} @@ -42,7 +42,7 @@ class ProbabilityDictContent : public SingleDictContent { bool setProbabilityEntry(const int terminalId, const ProbabilityEntry *const probabilityEntry); - bool flushToFile(const char *const dictDirPath) const; + bool flushToFile(const char *const dictPath) const; bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ProbabilityDictContent *const originalProbabilityDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp index 555217837..29972a4e8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.cpp @@ -46,8 +46,8 @@ int ShortcutDictContent::getShortcutListHeadPos(const int terminalId) const { return addressLookupTable->get(terminalId); } -bool ShortcutDictContent::flushToFile(const char *const dictBasePath) const { - return flush(dictBasePath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, +bool ShortcutDictContent::flushToFile(const char *const dictPath) const { + return flush(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_FILE_EXTENSION); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h index a52214ca2..eaafc27bc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h @@ -26,8 +26,8 @@ namespace latinime { class ShortcutDictContent : public SparseTableDictContent { public: - ShortcutDictContent(const char *const dictDirPath, const bool isUpdatable) - : SparseTableDictContent(dictDirPath, + ShortcutDictContent(const char *const dictPath, const bool isUpdatable) + : SparseTableDictContent(dictPath, Ver4DictConstants::SHORTCUT_LOOKUP_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_CONTENT_TABLE_FILE_EXTENSION, Ver4DictConstants::SHORTCUT_FILE_EXTENSION, isUpdatable, @@ -53,7 +53,7 @@ class ShortcutDictContent : public SparseTableDictContent { // Returns head position of shortcut list for a PtNode specified by terminalId. int getShortcutListHeadPos(const int terminalId) const; - bool flushToFile(const char *const dictBasePath) const; + bool flushToFile(const char *const dictPath) const; bool runGC(const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap, const ShortcutDictContent *const originalShortcutDictContent); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h index d8eedf36e..9064b7e72 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/single_dict_content.h @@ -28,9 +28,9 @@ namespace latinime { class SingleDictContent : public DictContent { public: - SingleDictContent(const char *const dictDirPath, const char *const contentFileName, + SingleDictContent(const char *const dictPath, const char *const contentFileName, const bool isUpdatable) - : mMmappedBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)), + : mMmappedBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), mExpandableContentBuffer(mMmappedBuffer.get() ? mMmappedBuffer.get()->getBuffer() : 0, mMmappedBuffer.get() ? mMmappedBuffer.get()->getBufferSize() : 0, BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), @@ -59,8 +59,8 @@ class SingleDictContent : public DictContent { return &mExpandableContentBuffer; } - bool flush(const char *const dictBasePath, const char *const contentFileNameSuffix) const { - return DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, + bool flush(const char *const dictPath, const char *const contentFileNameSuffix) const { + return DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix, &mExpandableContentBuffer); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp index abb7d5fd2..63c6ea3a4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.cpp @@ -18,18 +18,18 @@ namespace latinime { -bool SparseTableDictContent::flush(const char *const dictBasePath, +bool SparseTableDictContent::flush(const char *const dictPath, const char *const lookupTableFileNameSuffix, const char *const addressTableFileNameSuffix, const char *const contentFileNameSuffix) const { - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, lookupTableFileNameSuffix, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, lookupTableFileNameSuffix, &mExpandableLookupTableBuffer)){ return false; } - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, addressTableFileNameSuffix, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, addressTableFileNameSuffix, &mExpandableAddressTableBuffer)) { return false; } - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, contentFileNameSuffix, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, contentFileNameSuffix, &mExpandableContentBuffer)) { return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h index 9a4f1e1c0..a82e3f50a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/sparse_table_dict_content.h @@ -30,15 +30,15 @@ namespace latinime { // TODO: Support multiple contents. class SparseTableDictContent : public DictContent { public: - AK_FORCE_INLINE SparseTableDictContent(const char *const dictDirPath, + AK_FORCE_INLINE SparseTableDictContent(const char *const dictPath, const char *const lookupTableFileName, const char *const addressTableFileName, const char *const contentFileName, const bool isUpdatable, const int sparseTableBlockSize, const int sparseTableDataSize) : mLookupTableBuffer( - MmappedBuffer::openBuffer(dictDirPath, lookupTableFileName, isUpdatable)), + MmappedBuffer::openBuffer(dictPath, lookupTableFileName, isUpdatable)), mAddressTableBuffer( - MmappedBuffer::openBuffer(dictDirPath, addressTableFileName, isUpdatable)), - mContentBuffer(MmappedBuffer::openBuffer(dictDirPath, contentFileName, isUpdatable)), + MmappedBuffer::openBuffer(dictPath, addressTableFileName, isUpdatable)), + mContentBuffer(MmappedBuffer::openBuffer(dictPath, contentFileName, isUpdatable)), mExpandableLookupTableBuffer( mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBuffer() : 0, mLookupTableBuffer.get() ? mLookupTableBuffer.get()->getBufferSize() : 0, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp index c889cf5d1..0b17a009d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.cpp @@ -50,7 +50,7 @@ bool TerminalPositionLookupTable::setTerminalPtNodePosition( Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, getEntryPos(terminalId)); } -bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) const { +bool TerminalPositionLookupTable::flushToFile(const char *const dictPath) const { // If the used buffer size is smaller than the actual buffer size, regenerate the lookup // table and write the new table to the file. if (getEntryPos(mSize) < getBuffer()->getTailPosition()) { @@ -63,12 +63,12 @@ bool TerminalPositionLookupTable::flushToFile(const char *const dictBasePath) co return false; } } - return lookupTableToWrite.flush(dictBasePath, + return lookupTableToWrite.flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } else { // We can simply use this lookup table because the buffer size has not been // changed. - return flush(dictBasePath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + return flush(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h index 5a28f52fd..f73e22754 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h @@ -28,8 +28,8 @@ class TerminalPositionLookupTable : public SingleDictContent { public: typedef hash_map_compat<int, int> TerminalIdMap; - TerminalPositionLookupTable(const char *const dictDirPath, const bool isUpdatable) - : SingleDictContent(dictDirPath, + TerminalPositionLookupTable(const char *const dictPath, const bool isUpdatable) + : SingleDictContent(dictPath, Ver4DictConstants::TERMINAL_ADDRESS_TABLE_FILE_EXTENSION, isUpdatable), mSize(getBuffer()->getTailPosition() / Ver4DictConstants::TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE) {} @@ -44,7 +44,7 @@ class TerminalPositionLookupTable : public SingleDictContent { return mSize; } - bool flushToFile(const char *const dictBasePath) const; + bool flushToFile(const char *const dictPath) const; bool runGCTerminalIds(TerminalIdMap *const terminalIdMap); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp index e2355407a..918c02ba2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp @@ -27,10 +27,10 @@ namespace latinime { /* static */ Ver4DictBuffers::Ver4DictBuffersPtr Ver4DictBuffers::openVer4DictBuffers( - const char *const dictDirPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) { + const char *const dictPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer) { const bool isUpdatable = headerBuffer.get() ? headerBuffer.get()->isUpdatable() : false; // TODO: take only dictDirPath, and open both header and trie files in the constructor below - return Ver4DictBuffersPtr(new Ver4DictBuffers(dictDirPath, headerBuffer, isUpdatable)); + return Ver4DictBuffersPtr(new Ver4DictBuffers(dictPath, headerBuffer, isUpdatable)); } bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, @@ -57,38 +57,38 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, const int dictNameBufSize = strlen(dictDirPath) + 1 /* terminator */; char dictName[dictNameBufSize]; FileUtils::getBasename(dictDirPath, dictNameBufSize, dictName); - const int dictBasePathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName); - char dictBasePath[dictBasePathBufSize]; - FileUtils::getFilePath(tmpDirPath, dictName, dictBasePathBufSize, dictBasePath); + const int dictPathBufSize = FileUtils::getFilePathBufSize(tmpDirPath, dictName); + char dictPath[dictPathBufSize]; + FileUtils::getFilePath(tmpDirPath, dictName, dictPathBufSize, dictPath); // Write header file. - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, Ver4DictConstants::HEADER_FILE_EXTENSION, headerBuffer)) { AKLOGE("Dictionary header file %s%s cannot be written.", tmpDirPath, Ver4DictConstants::HEADER_FILE_EXTENSION); return false; } // Write trie file. - if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictBasePath, + if (!DictFileWritingUtils::flushBufferToFileWithSuffix(dictPath, Ver4DictConstants::TRIE_FILE_EXTENSION, &mExpandableTrieBuffer)) { AKLOGE("Dictionary trie file %s%s cannot be written.", tmpDirPath, Ver4DictConstants::TRIE_FILE_EXTENSION); return false; } // Write dictionary contents. - if (!mTerminalPositionLookupTable.flushToFile(dictBasePath)) { + if (!mTerminalPositionLookupTable.flushToFile(dictPath)) { AKLOGE("Terminal position lookup table cannot be written. %s", tmpDirPath); return false; } - if (!mProbabilityDictContent.flushToFile(dictBasePath)) { + if (!mProbabilityDictContent.flushToFile(dictPath)) { AKLOGE("Probability dict content cannot be written. %s", tmpDirPath); return false; } - if (!mBigramDictContent.flushToFile(dictBasePath)) { + if (!mBigramDictContent.flushToFile(dictPath)) { AKLOGE("Bigram dict content cannot be written. %s", tmpDirPath); return false; } - if (!mShortcutDictContent.flushToFile(dictBasePath)) { + if (!mShortcutDictContent.flushToFile(dictPath)) { AKLOGE("Shortcut dict content cannot be written. %s", tmpDirPath); return false; } @@ -107,10 +107,10 @@ bool Ver4DictBuffers::flushHeaderAndDictBuffers(const char *const dictDirPath, return true; } -Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath, +Ver4DictBuffers::Ver4DictBuffers(const char *const dictPath, const MmappedBuffer::MmappedBufferPtr &headerBuffer, const bool isUpdatable) : mHeaderBuffer(headerBuffer), - mDictBuffer(MmappedBuffer::openBuffer(dictDirPath, + mDictBuffer(MmappedBuffer::openBuffer(dictPath, Ver4DictConstants::TRIE_FILE_EXTENSION, isUpdatable)), mHeaderPolicy(headerBuffer.get()->getBuffer(), FormatUtils::VERSION_4), mExpandableHeaderBuffer(headerBuffer.get()->getBuffer(), mHeaderPolicy.getSize(), @@ -118,12 +118,12 @@ Ver4DictBuffers::Ver4DictBuffers(const char *const dictDirPath, mExpandableTrieBuffer(mDictBuffer.get()->getBuffer(), mDictBuffer.get()->getBufferSize(), BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mTerminalPositionLookupTable(dictDirPath, isUpdatable), - mProbabilityDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), + mTerminalPositionLookupTable(dictPath, isUpdatable), + mProbabilityDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), - mBigramDictContent(dictDirPath, mHeaderPolicy.hasHistoricalInfoOfWords(), + mBigramDictContent(dictPath, mHeaderPolicy.hasHistoricalInfoOfWords(), isUpdatable), - mShortcutDictContent(dictDirPath, isUpdatable), + mShortcutDictContent(dictPath, isUpdatable), mIsUpdatable(isUpdatable) {} Ver4DictBuffers::Ver4DictBuffers(const HeaderPolicy *const headerPolicy) diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h index c16e0bdcb..d6d22c5c1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h @@ -21,6 +21,7 @@ namespace latinime { +// TODO: Create PtConstants under the pt_common and move some constant values there. // Note that there are corresponding definitions in FormatSpec.java. class Ver4DictConstants { public: diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 8ea029076..17fc9483b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -16,8 +16,8 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_reading_utils.h" @@ -45,10 +45,10 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce const PatriciaTrieReadingUtils::NodeFlags flags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const int parentPosOffset = - DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( + DynamicPtReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( dictBuf, &pos); const int parentPos = - DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, headPos); + DynamicPtReadingUtils::getParentPtNodePos(parentPosOffset, headPos); int codePoints[MAX_WORD_LENGTH]; const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos); @@ -74,7 +74,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce if (usesAdditionalBuffer) { childrenPosFieldPos += mBuffer->getOriginalBufferSize(); } - int childrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( + int childrenPos = DynamicPtReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, &pos); if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) { childrenPos += mBuffer->getOriginalBufferSize(); @@ -85,7 +85,7 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce // Sibling position is the tail position of original PtNode. int newSiblingNodePos = (siblingNodePos == NOT_A_DICT_POS) ? pos : siblingNodePos; // Read destination node if the read node is a moved node. - if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) { + if (DynamicPtReadingUtils::isMoved(flags)) { // The destination position is stored at the same place as the parent position. return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos); } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 10f90523a..32576cf0a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -19,11 +19,11 @@ #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" @@ -44,11 +44,11 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsDeleted( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, true /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos)) { return false; } @@ -74,16 +74,16 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, false /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos)) { return false; } // Update moved position, which is stored in the parent offset field. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( mTrieBuffer, movedPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { return false; } @@ -93,8 +93,8 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( while (!mReadingHelper.isEnd()) { const PtNodeParams childPtNodeParams(mReadingHelper.getPtNodeParams()); int parentOffsetFieldPos = childPtNodeParams.getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition( mTrieBuffer, bigramLinkedNodePos, childPtNodeParams.getHeadPos(), &parentOffsetFieldPos)) { // Parent offset cannot be written because of a bug or a broken dictionary; thus, @@ -119,7 +119,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( const PatriciaTrieReadingUtils::NodeFlags originalFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = - DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, + DynamicPtReadingUtils::updateAndGetFlags(originalFlags, false /* isMoved */, false /* isDeleted */, true /* willBecomeNonTerminal */); if (!mBuffers->getMutableTerminalPositionLookupTable()->setTerminalPtNodePosition( toBeUpdatedPtNodeParams->getTerminalId(), NOT_A_DICT_POS /* ptNodePos */)) { @@ -129,7 +129,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsWillBecomeNonTerminal( } // Update flags. int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); - return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, + return DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, &writingPos); } @@ -186,7 +186,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA bool Ver4PatriciaTrieNodeWriter::updateChildrenPosition( const PtNodeParams *const toBeUpdatedPtNodeParams, const int newChildrenPosition) { int childrenPosFieldPos = toBeUpdatedPtNodeParams->getChildrenPosFieldPos(); - return DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, + return DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, newChildrenPosition, &childrenPosFieldPos); } @@ -264,9 +264,9 @@ bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields( } } int writingPos = toBeUpdatedPtNodeParams->getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + + DynamicPtWritingUtils::NODE_FLAG_FIELD_SIZE; // Write updated parent offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { return false; } @@ -328,17 +328,17 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( const int nodePos = *ptNodeWritingPos; // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // PtNode writing. - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, 0 /* nodeFlags */, ptNodeWritingPos)) { return false; } // Calculate a parent offset and write the offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeParentPosOffsetAndAdvancePosition(mTrieBuffer, ptNodeParams->getParentPos(), nodePos, ptNodeWritingPos)) { return false; } // Write code points - if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeCodePointsAndAdvancePosition(mTrieBuffer, ptNodeParams->getCodePoints(), ptNodeParams->getCodePointCount(), ptNodeWritingPos)) { return false; } @@ -369,7 +369,7 @@ bool Ver4PatriciaTrieNodeWriter::writePtNodeAndGetTerminalIdAndAdvancePosition( } } // Write children position - if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, + if (!DynamicPtWritingUtils::writeChildrenPositionAndAdvancePosition(mTrieBuffer, ptNodeParams->getChildrenPos(), ptNodeWritingPos)) { return false; } @@ -401,7 +401,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos, PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, isTerminal, hasShortcutTargets, hasBigrams, hasMultipleChars, CHILDREN_POSITION_FIELD_SIZE); - if (!DynamicPatriciaTrieWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { + if (!DynamicPtWritingUtils::writeFlags(mTrieBuffer, nodeFlags, ptNodePos)) { AKLOGE("Cannot write PtNode flags. flags: %x, pos: %d", nodeFlags, ptNodePos); return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index 1df853a30..69576d8e5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -20,9 +20,9 @@ #include <stdint.h> #include "defines.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" @@ -115,7 +115,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { BufferWithExtendableBuffer *const mTrieBuffer; Ver4DictBuffers *const mBuffers; const Ver4PatriciaTrieNodeReader *const mPtNodeReader; - DynamicPatriciaTrieReadingHelper mReadingHelper; + DynamicPtReadingHelper mReadingHelper; Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 7b5461d74..96bb8128e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -21,7 +21,7 @@ #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_vector.h" #include "suggest/core/dictionary/unigram_property.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" @@ -48,7 +48,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d if (!dicNode->hasChildren()) { return; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos()); while (!readingHelper.isEnd()) { const PtNodeParams ptNodeParams = readingHelper.getPtNodeParams(); @@ -75,7 +75,7 @@ void Ver4PatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const d int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) const { - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodePos(ptNodePos); return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount( maxCodePointCount, outCodePoints, outUnigramProbability); @@ -83,7 +83,7 @@ int Ver4PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( int Ver4PatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) const { - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); } @@ -154,7 +154,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len mDictBuffer->getTailPosition()); return false; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictBuffer, &mNodeReader); + DynamicPtReadingHelper readingHelper(mDictBuffer, &mNodeReader); readingHelper.initWithPtNodeArrayPos(getRootPosition()); bool addedNewUnigram = false; if (mUpdatingHelper.addUnigramWord(&readingHelper, word, length, probability, isNotAWord, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 2d46c0ae2..8187b7a39 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -22,7 +22,7 @@ #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" @@ -131,7 +131,7 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { Ver4ShortcutListPolicy mShortcutPolicy; Ver4PatriciaTrieNodeReader mNodeReader; Ver4PatriciaTrieNodeWriter mNodeWriter; - DynamicPatriciaTrieUpdatingHelper mUpdatingHelper; + DynamicPtUpdatingHelper mUpdatingHelper; Ver4PatriciaTrieWritingHelper mWritingHelper; int mUnigramCount; int mBigramCount; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 2ee9483f7..43227635c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -32,12 +32,9 @@ namespace latinime { -void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePath, +void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const dictDirPath, const int unigramCount, const int bigramCount) const { const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy(); - const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */; - char dirPath[dirPathBufSize]; - FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath); BufferWithExtendableBuffer headerBuffer( BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE); const int extendedRegionSize = headerPolicy->getExtendedRegionSize() @@ -50,11 +47,11 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa extendedRegionSize); return; } - mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer); + mBuffers->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer); } void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, - const char *const trieFilePath) { + const char *const dictDirPath) { const HeaderPolicy *const headerPolicy = mBuffers->getHeaderPolicy(); Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers( Ver4DictBuffers::createVer4DictBuffers(headerPolicy)); @@ -70,10 +67,7 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeAr 0 /* extendedRegionSize */)) { return; } - const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */; - char dirPath[dirPathBufSize]; - FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath); - dictBuffers.get()->flushHeaderAndDictBuffers(dirPath, &headerBuffer); + dictBuffers.get()->flushHeaderAndDictBuffers(dictDirPath, &headerBuffer); } bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, @@ -88,9 +82,9 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy); - DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); + DynamicPtReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners + DynamicPtGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( &ptNodeWriter); @@ -111,7 +105,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, } readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability + DynamicPtGcEventListeners::TraversePolicyToUpdateBigramProbability traversePolicyToUpdateBigramProbability(&ptNodeWriter); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( &traversePolicyToUpdateBigramProbability)) { @@ -132,7 +126,7 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); Ver4PatriciaTrieNodeWriter ptNodeWriterForNewBuffers(buffersToWrite->getWritableTrieBuffer(), buffersToWrite, &ptNodeReader, &bigramPolicy, &shortcutPolicy); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer + DynamicPtGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&ptNodeWriterForNewBuffers, buffersToWrite->getWritableTrieBuffer(), &dictPositionRelocationMap); if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( @@ -170,10 +164,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, mBuffers->getShortcutDictContent())) { return false; } - DynamicPatriciaTrieReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), + DynamicPtReadingHelper newDictReadingHelper(buffersToWrite->getTrieBuffer(), &newPtNodeReader); newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields + DynamicPtGcEventListeners::TraversePolicyToUpdateAllPositionFields traversePolicyToUpdateAllPositionFields(&newPtNodeWriter, &dictPositionRelocationMap); if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( &traversePolicyToUpdateAllPositionFields)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h index 73e63ef45..c3a155e0e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h @@ -18,7 +18,7 @@ #define LATINIME_VER4_PATRICIA_TRIE_WRITING_HELPER_H #include "defines.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_gc_event_listeners.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" namespace latinime { @@ -33,17 +33,16 @@ class Ver4PatriciaTrieWritingHelper { Ver4PatriciaTrieWritingHelper(Ver4DictBuffers *const buffers) : mBuffers(buffers) {} - void writeToDictFile(const char *const trieFilePath, const int unigramCount, + void writeToDictFile(const char *const dictDirPath, const int unigramCount, const int bigramCount) const; - void writeToDictFileWithGC(const int rootPtNodeArrayPos, - const char *const trieFilePath); + void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const dictDirPath); private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); class TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds - : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + : public DynamicPtReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateAllPtNodeFlagsAndTerminalIds( Ver4PatriciaTrieNodeWriter *const ptNodeWriter, diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index b7db21365..442373b29 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -19,7 +19,7 @@ #include <cstdio> #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" @@ -51,7 +51,7 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = headerPolicy.writeHeaderToBuffer(dictBuffers.get()->getWritableHeaderBuffer(), true /* updatesLastUpdatedTime */, true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); - if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( + if (!DynamicPtWritingUtils::writeEmptyDictionary( dictBuffers.get()->getWritableTrieBuffer(), 0 /* rootPos */)) { AKLOGE("Empty ver4 dictionary structure cannot be created on memory."); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp index 49ae7f156..1f25cfa1e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/file_utils.cpp @@ -147,7 +147,7 @@ namespace latinime { const char *const baseName = basename(filePathBuf); const int baseNameLength = strlen(baseName); if (baseNameLength >= outNameBufSize) { - AKLOGE("outNameBufSize is too small. dirPath: %s, outNameBufSize: %d", + AKLOGE("outNameBufSize is too small. filePath: %s, outNameBufSize: %d", filePath, outNameBufSize); return; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp index e3d783835..cd3c403fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp @@ -46,8 +46,6 @@ const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 12; // same so we use them for both here. if (ByteArrayUtils::readUint16(dict, 4) == VERSION_2) { return VERSION_2; - } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_3) { - return VERSION_3; } else if (ByteArrayUtils::readUint16(dict, 4) == VERSION_4) { return VERSION_4; } else { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 34727b8d8..eb2227d60 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -31,7 +31,6 @@ class FormatUtils { enum FORMAT_VERSION { // These MUST have the same values as the relevant constants in FormatSpec.java. VERSION_2 = 2, - VERSION_3 = 3, VERSION_4 = 400, UNKNOWN_VERSION = -1 }; diff --git a/native/jni/src/utils/exclusive_ownership_pointer.h b/native/jni/src/utils/exclusive_ownership_pointer.h index 617b34968..081802e8b 100644 --- a/native/jni/src/utils/exclusive_ownership_pointer.h +++ b/native/jni/src/utils/exclusive_ownership_pointer.h @@ -39,25 +39,15 @@ class ExclusiveOwnershipPointer { deletePointersIfHavingOwnership(); } - // Move the ownership. - AK_FORCE_INLINE ExclusiveOwnershipPointer<T> &operator=( - const ExclusiveOwnershipPointer<T> &pointer) { - // Delete pointers when this is an owner of another pointer. - deletePointersIfHavingOwnership(); - mPointer = pointer.mPointer; - mSharedOwnerPtr = pointer.mSharedOwnerPtr; - transferOwnership(pointer); - return *this; - } - AK_FORCE_INLINE T *get() const { return mPointer; } private: - // This class allows to copy and assign and ensures only one instance has the ownership of the + // This class allows to copy and ensures only one instance has the ownership of the // managed pointer. DISALLOW_DEFAULT_CONSTRUCTOR(ExclusiveOwnershipPointer); + DISALLOW_ASSIGNMENT_OPERATOR(ExclusiveOwnershipPointer); void transferOwnership(const ExclusiveOwnershipPointer<T> *const src) { if (*mSharedOwnerPtr != src) { |