diff options
13 files changed, 198 insertions, 98 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index 83a32fb0b..b21a606cf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -182,8 +182,8 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( // Updates bigram target PtNode positions in the list after the placing step in GC. bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos, - const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const - ptNodePositionRelocationMap, int *const outBigramEntryCount) { + const PtNodeWriter::PtNodePositionRelocationMap *const ptNodePositionRelocationMap, + int *const outBigramEntryCount) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); if (usesAdditionalBuffer) { *bigramListPos -= mBuffer->getOriginalBufferSize(); @@ -213,7 +213,7 @@ bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bi bigramTargetPtNodePos += mBuffer->getOriginalBufferSize(); } - DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it = + PtNodeWriter::PtNodePositionRelocationMap::const_iterator it = ptNodePositionRelocationMap->find(bigramTargetPtNodePos); if (it != ptNodePositionRelocationMap->end()) { bigramTargetPtNodePos = it->second; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index 5de456656..6cdf74a08 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -22,7 +22,8 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" +#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" namespace latinime { @@ -59,7 +60,7 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { int *const outBigramEntryCount); bool updateAllBigramTargetPtNodePositions(int *const bigramListPos, - const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const + const PtNodeWriter::PtNodePositionRelocationMap *const ptNodePositionRelocationMap, int *const outValidBigramEntryCount); bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h index 0f1e635bc..609b8ab07 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h @@ -20,12 +20,27 @@ #include "defines.h" #include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" +#include "utils/hash_map_compat.h" namespace latinime { // Interface class used to write PtNode information. class PtNodeWriter { public: + typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap; + typedef hash_map_compat<int, int> PtNodePositionRelocationMap; + struct DictPositionRelocationMap { + public: + DictPositionRelocationMap() + : mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {} + + PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap; + PtNodePositionRelocationMap mPtNodePositionRelocationMap; + + private: + DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap); + }; + virtual ~PtNodeWriter() {} virtual bool markPtNodeAsDeleted(const PtNodeParams *const toBeUpdatedPtNodeParams) = 0; @@ -49,6 +64,13 @@ class PtNodeWriter { virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam) = 0; + virtual bool updateAllBigramEntriesAndDeleteUselessEntries( + const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) = 0; + + virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams, + const DictPositionRelocationMap *const dictPositionRelocationMap, + int *const outBigramEntryCount) = 0; + protected: PtNodeWriter() {}; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp index a252c6fb2..173017dc7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.cpp @@ -34,10 +34,8 @@ bool DynamicPatriciaTrieGcEventListeners const int newProbability = ForgettingCurveUtils::getEncodedProbabilityToSave(ptNodeParams->getProbability(), mHeaderPolicy); - int writingPos = ptNodeParams->getProbabilityFieldPos(); // Update probability. - if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition( - mBuffer, newProbability, &writingPos)) { + if (!mPtNodeWriter->updatePtNodeProbability(ptNodeParams, newProbability)) { return false; } if (!ForgettingCurveUtils::isValidEncodedProbability(newProbability)) { @@ -48,9 +46,8 @@ bool DynamicPatriciaTrieGcEventListeners isUselessPtNode = false; } else if (ptNodeParams->isTerminal()) { // Remove children as all children are useless. - int writingPos = ptNodeParams->getChildrenPosFieldPos(); - if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( - mBuffer, NOT_A_DICT_POS /* childrenPosition */, &writingPos)) { + if (!mPtNodeWriter->updateChildrenPosition(ptNodeParams, + NOT_A_DICT_POS /* newChildrenPosition */)) { return false; } } @@ -74,7 +71,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbabilit int pos = ptNodeParams->getBigramsPos(); if (pos != NOT_A_DICT_POS) { int bigramEntryCount = 0; - if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos, + if (!mPtNodeWriter->updateAllBigramEntriesAndDeleteUselessEntries(ptNodeParams, &bigramEntryCount)) { return false; } @@ -90,8 +87,7 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo mValidPtNodeCount = 0; int writingPos = mBufferToWrite->getTailPosition(); mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert( - DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type( - ptNodeArrayPos, writingPos)); + PtNodeWriter::PtNodeArrayPositionRelocationMap::value_type(ptNodeArrayPos, writingPos)); // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. mPtNodeArraySizeFieldPos = writingPos; @@ -122,13 +118,13 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo if (ptNodeParams->isDeleted()) { // Current PtNode is not written in new buffer because it has been deleted. mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert( - DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type( + PtNodeWriter::PtNodePositionRelocationMap::value_type( ptNodeParams->getHeadPos(), NOT_A_DICT_POS)); return true; } int writingPos = mBufferToWrite->getTailPosition(); mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert( - DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type( + PtNodeWriter::PtNodePositionRelocationMap::value_type( ptNodeParams->getHeadPos(), writingPos)); mValidPtNodeCount++; // Writes current PtNode. @@ -138,51 +134,15 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNo bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields ::onVisitingPtNode(const PtNodeParams *const ptNodeParams) { // Updates parent position. - int parentPos = ptNodeParams->getParentPos(); - if (parentPos != NOT_A_DICT_POS) { - DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it = - mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos); - if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) { - parentPos = it->second; - } - } - int writingPos = ptNodeParams->getHeadPos() - + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; - // Write updated parent offset. - if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBufferToWrite, - parentPos, ptNodeParams->getHeadPos(), &writingPos)) { - return false; - } - - // Updates children position. - int childrenPos = ptNodeParams->getChildrenPos(); - if (childrenPos != NOT_A_DICT_POS) { - DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it = - mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos); - if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) { - childrenPos = it->second; - } - } - writingPos = ptNodeParams->getChildrenPosFieldPos(); - if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite, - childrenPos, &writingPos)) { + int bigramCount = 0; + if (!mPtNodeWriter->updateAllPositionFields(ptNodeParams, mDictPositionRelocationMap, + &bigramCount)) { return false; } - - // Updates bigram target PtNode positions in the bigram list. - int bigramsPos = ptNodeParams->getBigramsPos(); - if (bigramsPos != NOT_A_DICT_POS) { - int bigramEntryCount; - if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos, - &mDictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) { - return false; - } - mBigramCount += bigramEntryCount; - } + mBigramCount += bigramCount; if (ptNodeParams->isTerminal()) { mUnigramCount++; } - return true; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h index a0b3b4d3a..fe82910a7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h @@ -20,9 +20,8 @@ #include <vector> #include "defines.h" -#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h" #include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "utils/hash_map_compat.h" @@ -32,6 +31,7 @@ class DictionaryHeaderStructurePolicy; class PtNodeWriter; class PtNodeParams; +// TODO: Move to pt_common. class DynamicPatriciaTrieGcEventListeners { public: // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or @@ -91,9 +91,8 @@ class DynamicPatriciaTrieGcEventListeners { class TraversePolicyToUpdateBigramProbability : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { public: - TraversePolicyToUpdateBigramProbability( - DynamicBigramListPolicy *const bigramPolicy) - : mBigramPolicy(bigramPolicy), mValidBigramEntryCount(0) {} + TraversePolicyToUpdateBigramProbability(PtNodeWriter *const ptNodeWriter) + : mPtNodeWriter(ptNodeWriter), mValidBigramEntryCount(0) {} bool onAscend() { return true; } @@ -110,7 +109,7 @@ class DynamicPatriciaTrieGcEventListeners { private: DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability); - DynamicBigramListPolicy *const mBigramPolicy; + PtNodeWriter *const mPtNodeWriter; int mValidBigramEntryCount; }; @@ -119,8 +118,7 @@ class DynamicPatriciaTrieGcEventListeners { public: TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( PtNodeWriter *const ptNodeWriter, BufferWithExtendableBuffer *const bufferToWrite, - DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const - dictPositionRelocationMap) + PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap) : mPtNodeWriter(ptNodeWriter), mBufferToWrite(bufferToWrite), mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0), mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {}; @@ -138,8 +136,7 @@ class DynamicPatriciaTrieGcEventListeners { PtNodeWriter *const mPtNodeWriter; BufferWithExtendableBuffer *const mBufferToWrite; - DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const - mDictPositionRelocationMap; + PtNodeWriter::DictPositionRelocationMap *const mDictPositionRelocationMap; int mValidPtNodeCount; int mPtNodeArraySizeFieldPos; }; @@ -147,12 +144,9 @@ class DynamicPatriciaTrieGcEventListeners { class TraversePolicyToUpdateAllPositionFields : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { public: - TraversePolicyToUpdateAllPositionFields( - DynamicBigramListPolicy *const bigramPolicy, - BufferWithExtendableBuffer *const bufferToWrite, - const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const - dictPositionRelocationMap) - : mBigramPolicy(bigramPolicy), mBufferToWrite(bufferToWrite), + TraversePolicyToUpdateAllPositionFields(PtNodeWriter *const ptNodeWriter, + const PtNodeWriter::DictPositionRelocationMap *const dictPositionRelocationMap) + : mPtNodeWriter(ptNodeWriter), mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0), mBigramCount(0) {}; @@ -175,10 +169,8 @@ class DynamicPatriciaTrieGcEventListeners { private: DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields); - DynamicBigramListPolicy *const mBigramPolicy; - BufferWithExtendableBuffer *const mBufferToWrite; - const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const - mDictPositionRelocationMap; + PtNodeWriter *const mPtNodeWriter; + const PtNodeWriter::DictPositionRelocationMap *const mDictPositionRelocationMap; int mUnigramCount; int mBigramCount; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.cpp index 3b1e16cc7..36b2049fe 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.cpp @@ -233,4 +233,60 @@ bool DynamicPatriciaTrieNodeWriter::removeBigramEntry( targetPtNodeParam->getHeadPos()); } +bool DynamicPatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries( + const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) { + int bigramListPos = sourcePtNodeParams->getBigramsPos(); + return mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&bigramListPos, + outBigramEntryCount); +} + +bool DynamicPatriciaTrieNodeWriter::updateAllPositionFields( + const PtNodeParams *const toBeUpdatedPtNodeParams, + const DictPositionRelocationMap *const dictPositionRelocationMap, + int *const outBigramEntryCount) { + int parentPos = toBeUpdatedPtNodeParams->getParentPos(); + if (parentPos != NOT_A_DICT_POS) { + PtNodeWriter::PtNodePositionRelocationMap::const_iterator it = + dictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos); + if (it != dictPositionRelocationMap->mPtNodePositionRelocationMap.end()) { + parentPos = it->second; + } + } + int writingPos = toBeUpdatedPtNodeParams->getHeadPos() + + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + // Write an updated offset to the parent PtNode. + if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBuffer, + parentPos, toBeUpdatedPtNodeParams->getHeadPos(), &writingPos)) { + return false; + } + + // Updates children position that is a PtNodeArray position. + int childrenPos = toBeUpdatedPtNodeParams->getChildrenPos(); + if (childrenPos != NOT_A_DICT_POS) { + PtNodeWriter::PtNodeArrayPositionRelocationMap::const_iterator it = + dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos); + if (it != dictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) { + childrenPos = it->second; + } + } + if (!updateChildrenPosition(toBeUpdatedPtNodeParams, childrenPos)) { + return false; + } + // Updates bigram target PtNode positions in the bigram list. + int bigramsPos = toBeUpdatedPtNodeParams->getBigramsPos(); + int bigramCount = 0; + if (bigramsPos != NOT_A_DICT_POS) { + int bigramEntryCount; + if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos, + &dictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) { + return false; + } + bigramCount += bigramEntryCount; + } + if (outBigramEntryCount) { + *outBigramEntryCount = bigramCount; + } + return true; +} + } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.h index 87ed1b159..e1d08fb74 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_writer.h @@ -66,6 +66,13 @@ class DynamicPatriciaTrieNodeWriter : public PtNodeWriter { virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam); + virtual bool updateAllBigramEntriesAndDeleteUselessEntries( + const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount); + + virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams, + const DictPositionRelocationMap *const dictPositionRelocationMap, + int *const outBigramEntryCount); + private: DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeWriter); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.cpp index d97cec53b..fe5e2c626 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.cpp @@ -92,7 +92,7 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability - traversePolicyToUpdateBigramProbability(mBigramPolicy); + traversePolicyToUpdateBigramProbability(mPtNodeWriter); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( &traversePolicyToUpdateBigramProbability)) { return false; @@ -103,10 +103,10 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, } // Mapping from positions in mBuffer to positions in bufferToWrite. - DictPositionRelocationMap dictPositionRelocationMap; + PtNodeWriter::DictPositionRelocationMap dictPositionRelocationMap; readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieNodeWriter newPtNodeWriter(bufferToWrite, - &ptNodeReader, mBigramPolicy, mShortcutPolicy); + DynamicPatriciaTrieNodeWriter newPtNodeWriter(bufferToWrite, &ptNodeReader, mBigramPolicy, + mShortcutPolicy); DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer traversePolicyToPlaceAndWriteValidPtNodesToBuffer(&newPtNodeWriter, bufferToWrite, &dictPositionRelocationMap); @@ -123,10 +123,11 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, DynamicPatriciaTrieNodeReader newDictNodeReader(bufferToWrite, &newDictBigramPolicy, &newDictShortcutPolicy); DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictNodeReader); + DynamicPatriciaTrieNodeWriter newDictNodeWriter(bufferToWrite, &newDictNodeReader, + &newDictBigramPolicy, &newDictShortcutPolicy); newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields - traversePolicyToUpdateAllPositionFields(&newDictBigramPolicy, bufferToWrite, - &dictPositionRelocationMap); + traversePolicyToUpdateAllPositionFields(&newDictNodeWriter, &dictPositionRelocationMap); if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( &traversePolicyToUpdateAllPositionFields)) { return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h index 7223dea8b..4a6b964ef 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_writing_helper.h @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" -#include "utils/hash_map_compat.h" namespace latinime { @@ -35,20 +34,6 @@ class PtNodeWriter; // TODO: Make it independent from a particular format and move to pt_common. class DynamicPatriciaTrieWritingHelper { public: - typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap; - typedef hash_map_compat<int, int> PtNodePositionRelocationMap; - struct DictPositionRelocationMap { - public: - DictPositionRelocationMap() - : mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {} - - PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap; - PtNodePositionRelocationMap mPtNodePositionRelocationMap; - - private: - DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap); - }; - static const size_t MAX_DICTIONARY_SIZE; DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index b572ee87f..5b1ddddf5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -196,4 +196,18 @@ bool Ver4PatriciaTrieNodeWriter::removeBigramEntry( targetPtNodeParam->getTerminalId()); } +bool Ver4PatriciaTrieNodeWriter::updateAllBigramEntriesAndDeleteUselessEntries( + const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount) { + // TODO: Implement. + return false; +} + +bool Ver4PatriciaTrieNodeWriter::updateAllPositionFields( + const PtNodeParams *const toBeUpdatedPtNodeParams, + const DictPositionRelocationMap *const dictPositionRelocationMap, + int *const outBigramEntryCount) { + // TODO: Implement. + return false; +} + } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index d11952304..23e54cc09 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -67,6 +67,13 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam); + virtual bool updateAllBigramEntriesAndDeleteUselessEntries( + const PtNodeParams *const sourcePtNodeParams, int *const outBigramEntryCount); + + virtual bool updateAllPositionFields(const PtNodeParams *const toBeUpdatedPtNodeParams, + const DictPositionRelocationMap *const dictPositionRelocationMap, + int *const outBigramEntryCount); + private: DISALLOW_COPY_AND_ASSIGN(Ver4PatriciaTrieNodeWriter); @@ -78,7 +85,6 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { DynamicPatriciaTrieReadingHelper mReadingHelper; Ver4BigramListPolicy *const mBigramPolicy; Ver4ShortcutListPolicy *const mShortcutPolicy; - }; } // namespace latinime #endif /* LATINIME_VER4_PATRICIA_TRIE_NODE_WRITER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index c85a632d3..853cf38ad 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -18,11 +18,16 @@ #include <cstring> +#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" +#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" +#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" namespace latinime { @@ -47,4 +52,47 @@ void Ver4PatriciaTrieWritingHelper::writeToDictFile(const char *const trieFilePa mBuffers->flushHeaderAndDictBuffers(dirPath, &headerBuffer); } +void Ver4PatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, + const char *const trieFilePath, const HeaderPolicy *const headerPolicy, + const bool needsToDecay) { + Ver4DictBuffers::Ver4DictBuffersPtr dictBuffers(Ver4DictBuffers::createVer4DictBuffers()); + int unigramCount = 0; + int bigramCount = 0; + if (needsToDecay) { + ForgettingCurveUtils::sTimeKeeper.setCurrentTime(); + } + if (!runGC(rootPtNodeArrayPos, headerPolicy, dictBuffers.get(), &unigramCount, &bigramCount, + needsToDecay)) { + return; + } + BufferWithExtendableBuffer headerBuffer( + BufferWithExtendableBuffer::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE); + if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */, + needsToDecay, unigramCount, bigramCount, 0 /* extendedRegionSize */)) { + return; + } + const int dirPathBufSize = strlen(trieFilePath) + 1 /* terminator */; + char dirPath[dirPathBufSize]; + FileUtils::getDirPath(trieFilePath, dirPathBufSize, dirPath); + dictBuffers.get()->flushHeaderAndDictBuffers(dirPath, &headerBuffer); +} + +bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, + const HeaderPolicy *const headerPolicy, Ver4DictBuffers *const buffersToWrite, + int *const outUnigramCount, int *const outBigramCount, const bool needsToDecay) { + Ver4PatriciaTrieNodeReader ptNodeReader(mBuffers->getTrieBuffer(), + mBuffers->getProbabilityDictContent()); + Ver4BigramListPolicy bigramPolicy(mBuffers->getUpdatableBigramDictContent(), + mBuffers->getTerminalPositionLookupTable()); + Ver4ShortcutListPolicy shortcutPolicy(mBuffers->getShortcutDictContent(), + mBuffers->getTerminalPositionLookupTable()); + Ver4PatriciaTrieNodeWriter ptNodeWriter(mBuffers->getWritableTrieBuffer(), + mBuffers, &ptNodeReader, &bigramPolicy, &shortcutPolicy); + + DynamicPatriciaTrieReadingHelper readingHelper(mBuffers->getTrieBuffer(), &ptNodeReader); + readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); + + return true; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h index 80d631527..8072aa376 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.h @@ -32,9 +32,17 @@ class Ver4PatriciaTrieWritingHelper { void writeToDictFile(const char *const trieFilePath, const HeaderPolicy *const headerPolicy, const int unigramCount, const int bigramCount) const; + void writeToDictFileWithGC(const int rootPtNodeArrayPos, + const char *const trieFilePath, const HeaderPolicy *const headerPolicy, + const bool needsToDecay); + private: DISALLOW_IMPLICIT_CONSTRUCTORS(Ver4PatriciaTrieWritingHelper); + bool runGC(const int rootPtNodeArrayPos, const HeaderPolicy *const headerPolicy, + Ver4DictBuffers *const buffersToWrite, int *const outUnigramCount, + int *const outBigramCount, const bool needsToDecay); + Ver4DictBuffers *const mBuffers; }; } // namespace latinime |