diff options
10 files changed, 118 insertions, 125 deletions
diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index 513878e1d..5fb3500d6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -31,50 +31,48 @@ namespace latinime { */ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { public: - DynamicBigramListPolicy(const uint8_t *const bigramsBuf, const int bufSize, - const ExtendableBuffer *const additionalBuffer) - : mDictRoot(bigramsBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {} + DynamicBigramListPolicy(const BufferWithExtendableBuffer *const buffer) + : mBuffer(buffer) {} ~DynamicBigramListPolicy() {} void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, int *const pos) const { - const bool usesAdditionalBuffer = *pos >= mBufSize; - const uint8_t *const buffer = (usesAdditionalBuffer) ? - mAdditionalBuffer->getBuffer() : mDictRoot; + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { - *pos -= mBufSize; + *pos -= mBuffer->getOriginalBufferSize(); } const BigramListReadingUtils::BigramFlags flags = BigramListReadingUtils::getFlagsAndForwardPointer(buffer, pos); *outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer( buffer, flags, pos); if (usesAdditionalBuffer) { - *outBigramPos += mBufSize; + *outBigramPos += mBuffer->getOriginalBufferSize(); } *outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags); *outHasNext = BigramListReadingUtils::hasNext(flags); if (usesAdditionalBuffer) { - *pos += mBufSize; + *pos += mBuffer->getOriginalBufferSize(); } } void skipAllBigrams(int *const pos) const { - if (*pos >= mBufSize) { - *pos -= mBufSize; - BigramListReadingUtils::skipExistingBigrams(mAdditionalBuffer->getBuffer(), pos); - *pos += mBufSize; - } else { - BigramListReadingUtils::skipExistingBigrams(mDictRoot, pos); + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + *pos -= mBuffer->getOriginalBufferSize(); + } + BigramListReadingUtils::skipExistingBigrams(buffer, pos); + if (usesAdditionalBuffer) { + *pos += mBuffer->getOriginalBufferSize(); } } private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy); - const uint8_t *const mDictRoot; - const int mBufSize; - const ExtendableBuffer *const mAdditionalBuffer; + const BufferWithExtendableBuffer *const mBuffer; }; } // namespace latinime #endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index 2b4490701..2b5aa26b7 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -25,10 +25,12 @@ namespace latinime { void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(const int nodePos, const int maxCodePointCount, int *const outCodePoints) { - const bool usesAdditionalBuffer = nodePos >= mOriginalDictSize; - const uint8_t *const dictBuf = - usesAdditionalBuffer ? mExtendableBuffer->getBuffer() : mDictRoot; - int pos = (usesAdditionalBuffer) ? nodePos - mOriginalDictSize : nodePos; + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(nodePos); + const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); + int pos = nodePos; + if (usesAdditionalBuffer) { + pos -= mBuffer->getOriginalBufferSize(); + } mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const int parentPos = DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictBuf, &pos); @@ -48,10 +50,10 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, mFlags, &pos); if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { - mChildrenPos += mOriginalDictSize; + mChildrenPos += mBuffer->getOriginalBufferSize(); } if (usesAdditionalBuffer) { - pos += mOriginalDictSize; + pos += mBuffer->getOriginalBufferSize(); } if (PatriciaTrieReadingUtils::hasShortcutTargets(mFlags)) { mShortcutPos = pos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h index 8e7db3568..acc68b321 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h @@ -25,9 +25,9 @@ namespace latinime { +class BufferWithExtendableBuffer; class DictionaryBigramsStructurePolicy; class DictionaryShortcutsStructurePolicy; -class ExtendableBuffer; /* * This class is used for helping to read nodes of dynamic patricia trie. This class handles moved @@ -35,12 +35,10 @@ class ExtendableBuffer; */ class DynamicPatriciaTrieNodeReader { public: - DynamicPatriciaTrieNodeReader(const uint8_t *const dictRoot, const int originalDictSize, - const ExtendableBuffer *const extendableBuffer, + DynamicPatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer, const DictionaryBigramsStructurePolicy *const bigramsPolicy, const DictionaryShortcutsStructurePolicy *const shortcutsPolicy) - : mDictRoot(dictRoot), mOriginalDictSize(originalDictSize), - mExtendableBuffer(extendableBuffer), mBigramsPolicy(bigramsPolicy), + : mBuffer(buffer), mBigramsPolicy(bigramsPolicy), mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbability(NOT_A_PROBABILITY), mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS), @@ -124,10 +122,7 @@ class DynamicPatriciaTrieNodeReader { private: DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeReader); - // TODO: Consolidate mDictRoot. - const uint8_t *const mDictRoot; - const int mOriginalDictSize; - const ExtendableBuffer *const mExtendableBuffer; + const BufferWithExtendableBuffer *const mBuffer; const DictionaryBigramsStructurePolicy *const mBigramsPolicy; const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy; int mNodePos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index a26b8ffdc..a90e5a0bd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -31,8 +31,8 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d if (!dicNode->hasChildren()) { return; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictRoot, mOriginalDictSize, - &mExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); + DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, + getBigramsStructurePolicy(), getShortcutsStructurePolicy()); readingHelper.initWithNodeArrayPos(dicNode->getChildrenPos()); const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); while (!readingHelper.isEnd()) { @@ -51,8 +51,8 @@ int DynamicPatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCoun // This method traverses parent nodes from the terminal by following parent pointers; thus, // node code points are stored in the buffer in the reverse order. int reverseCodePoints[maxCodePointCount]; - DynamicPatriciaTrieReadingHelper readingHelper(mDictRoot, mOriginalDictSize, - &mExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); + DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, + getBigramsStructurePolicy(), getShortcutsStructurePolicy()); // First, read the terminal node and get its probability. readingHelper.initWithNodePos(nodePos); if (!readingHelper.isValidTerminalNode()) { @@ -94,8 +94,8 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in for (int i = 0; i < length; ++i) { searchCodePoints[i] = forceLowerCaseSearch ? CharUtils::toLowerCase(inWord[i]) : inWord[i]; } - DynamicPatriciaTrieReadingHelper readingHelper(mDictRoot, mOriginalDictSize, - &mExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); + DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, + getBigramsStructurePolicy(), getShortcutsStructurePolicy()); readingHelper.initWithNodeArrayPos(getRootPosition()); const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); while (!readingHelper.isEnd()) { @@ -137,7 +137,7 @@ int DynamicPatriciaTriePolicy::getUnigramProbability(const int nodePos) const { if (nodePos == NOT_A_VALID_WORD_POS) { return NOT_A_PROBABILITY; } - DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, mOriginalDictSize, &mExtendableBuffer, + DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); nodeReader.fetchNodeInfoFromBuffer(nodePos); if (nodeReader.isDeleted() || nodeReader.isBlacklisted() || nodeReader.isNotAWord()) { @@ -150,7 +150,7 @@ int DynamicPatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) cons if (nodePos == NOT_A_VALID_WORD_POS) { return NOT_A_DICT_POS; } - DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, mOriginalDictSize, &mExtendableBuffer, + DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); nodeReader.fetchNodeInfoFromBuffer(nodePos); if (nodeReader.isDeleted()) { @@ -163,7 +163,7 @@ int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const if (nodePos == NOT_A_VALID_WORD_POS) { return NOT_A_DICT_POS; } - DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, mOriginalDictSize, &mExtendableBuffer, + DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); nodeReader.fetchNodeInfoFromBuffer(nodePos); if (nodeReader.isDeleted()) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index 1267858a3..63c5ac874 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -17,8 +17,6 @@ #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H -#include <stdint.h> - #include "defines.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" #include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" @@ -35,11 +33,11 @@ class DicNodeVector; class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: DynamicPatriciaTriePolicy(const MmappedBuffer *const buffer) - : mBuffer(buffer), mExtendableBuffer(), mHeaderPolicy(mBuffer->getBuffer()), - mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()), - mOriginalDictSize(mBuffer->getBufferSize() - mHeaderPolicy.getSize()), - mBigramListPolicy(mDictRoot, mOriginalDictSize, &mExtendableBuffer), - mShortcutListPolicy(mDictRoot, mOriginalDictSize, &mExtendableBuffer) {} + : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), + mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(), + mBuffer->getBufferSize() - mHeaderPolicy.getSize()), + mBigramListPolicy(&mBufferWithExtendableBuffer), + mShortcutListPolicy(&mBufferWithExtendableBuffer) {} ~DynamicPatriciaTriePolicy() { delete mBuffer; @@ -89,12 +87,8 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); const MmappedBuffer *const mBuffer; - const ExtendableBuffer mExtendableBuffer; const HeaderPolicy mHeaderPolicy; - // TODO: Consolidate mDictRoot. - // CAVEAT!: Be careful about array out of bound access with mDictRoot - const uint8_t *const mDictRoot; - const int mOriginalDictSize; + const BufferWithExtendableBuffer mBufferWithExtendableBuffer; const DynamicBigramListPolicy mBigramListPolicy; const DynamicShortcutListPolicy mShortcutListPolicy; }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp index 7b7adebe4..d5c8082c1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp @@ -25,16 +25,15 @@ const int DynamicPatriciaTrieReadingHelper::MAX_NODE_ARRAY_COUNT_TO_AVOID_INFINI // Read node array size and process empty node arrays. Nodes and arrays are counted up in this // method to avoid an infinite loop. void DynamicPatriciaTrieReadingHelper::nextNodeArray() { - const bool usesAdditionalBuffer = mPos >= mOriginalDictSize; - const uint8_t *const dictBuf = (usesAdditionalBuffer) - ? mExtendableBuffer->getBuffer() : mDictRoot; + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(mPos); + const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { - mPos -= mOriginalDictSize; + mPos -= mBuffer->getOriginalBufferSize(); } mNodeCount = PatriciaTrieReadingUtils::getPtNodeArraySizeAndAdvancePosition(dictBuf, &mPos); if (usesAdditionalBuffer) { - mPos += mOriginalDictSize; + mPos += mBuffer->getOriginalBufferSize(); } // Count up nodes and node arrays to avoid infinite loop. mTotalNodeCount += mNodeCount; @@ -59,16 +58,15 @@ void DynamicPatriciaTrieReadingHelper::nextNodeArray() { // Follow the forward link and read the next node array if exists. void DynamicPatriciaTrieReadingHelper::followForwardLink() { - const bool usesAdditionalBuffer = mPos >= mOriginalDictSize; - const uint8_t *const dictBuf = (usesAdditionalBuffer) - ? mExtendableBuffer->getBuffer() : mDictRoot; + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(mPos); + const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { - mPos -= mOriginalDictSize; + mPos -= mBuffer->getOriginalBufferSize(); } const int forwardLinkPosition = DynamicPatriciaTrieReadingUtils::getForwardLinkPosition(dictBuf, mPos); if (usesAdditionalBuffer) { - mPos += mOriginalDictSize; + mPos += mBuffer->getOriginalBufferSize(); } if (DynamicPatriciaTrieReadingUtils::isValidForwardLinkPosition(forwardLinkPosition)) { // Follow the forward link. diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h index 678b2ca11..518b232fb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h @@ -31,15 +31,12 @@ namespace latinime { */ class DynamicPatriciaTrieReadingHelper { public: - DynamicPatriciaTrieReadingHelper(const uint8_t *const dictRoot, const int originalDictSize, - const ExtendableBuffer *const extendableBuffer, + DynamicPatriciaTrieReadingHelper(const BufferWithExtendableBuffer *const buffer, const DictionaryBigramsStructurePolicy *const bigramsPolicy, const DictionaryShortcutsStructurePolicy *const shortcutsPolicy) : mIsError(false), mPos(NOT_A_DICT_POS), mNodeCount(0), mPrevTotalCodePointCount(0), - mTotalNodeCount(0), mNodeArrayCount(0), mDictRoot(dictRoot), - mOriginalDictSize(originalDictSize), mExtendableBuffer(extendableBuffer), - mNodeReader(mDictRoot, mOriginalDictSize, mExtendableBuffer, bigramsPolicy, - shortcutsPolicy) {} + mTotalNodeCount(0), mNodeArrayCount(0), mBuffer(buffer), + mNodeReader(mBuffer, bigramsPolicy, shortcutsPolicy) {} ~DynamicPatriciaTrieReadingHelper() {} @@ -177,9 +174,7 @@ class DynamicPatriciaTrieReadingHelper { int mPrevTotalCodePointCount; int mTotalNodeCount; int mNodeArrayCount; - const uint8_t *const mDictRoot; - const int mOriginalDictSize; - const ExtendableBuffer *const mExtendableBuffer; + const BufferWithExtendableBuffer *const mBuffer; DynamicPatriciaTrieNodeReader mNodeReader; int mMergedNodeCodePoints[MAX_WORD_LENGTH]; diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h index fdaf18f7c..0bc44dd4f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h @@ -31,9 +31,8 @@ namespace latinime { */ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { public: - DynamicShortcutListPolicy(const uint8_t *const shortcutBuf, const int bufSize, - const ExtendableBuffer *const additionalBuffer) - : mShortcutsBuf(shortcutBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {} + explicit DynamicShortcutListPolicy(const BufferWithExtendableBuffer *const buffer) + : mBuffer(buffer) {} ~DynamicShortcutListPolicy() {} @@ -47,11 +46,10 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { void getNextShortcut(const int maxCodePointCount, int *const outCodePoint, int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext, int *const pos) const { - const bool usesAdditionalBuffer = *pos >= mBufSize; - const uint8_t *const buffer = usesAdditionalBuffer - ? mAdditionalBuffer->getBuffer() : mShortcutsBuf; + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { - *pos -= mBufSize; + *pos -= mBuffer->getOriginalBufferSize(); } const ShortcutListReadingUtils::ShortcutFlags flags = ShortcutListReadingUtils::getFlagsAndForwardPointer(buffer, pos); @@ -66,29 +64,28 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { buffer, maxCodePointCount, outCodePoint, pos); } if (usesAdditionalBuffer) { - *pos += mBufSize; + *pos += mBuffer->getOriginalBufferSize(); } } void skipAllShortcuts(int *const pos) const { - if (*pos >= mBufSize) { - *pos -= mBufSize; - const int shortcutListSize = ShortcutListReadingUtils - ::getShortcutListSizeAndForwardPointer(mAdditionalBuffer->getBuffer(), pos); - *pos += mBufSize + shortcutListSize; - } else { - const int shortcutListSize = ShortcutListReadingUtils - ::getShortcutListSizeAndForwardPointer(mShortcutsBuf, pos); - *pos += shortcutListSize; + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + *pos -= mBuffer->getOriginalBufferSize(); + } + const int shortcutListSize = ShortcutListReadingUtils + ::getShortcutListSizeAndForwardPointer(buffer, pos); + *pos += shortcutListSize; + if (usesAdditionalBuffer) { + *pos += mBuffer->getOriginalBufferSize(); } } private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicShortcutListPolicy); - const uint8_t *const mShortcutsBuf; - const int mBufSize; - const ExtendableBuffer *const mAdditionalBuffer; + const BufferWithExtendableBuffer *const mBuffer; }; } // namespace latinime #endif // LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.cpp index e55cc2410..7f74adc45 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.cpp @@ -18,8 +18,8 @@ namespace latinime { -const size_t ExtendableBuffer::INITIAL_BUFFER_SIZE = 16 * 1024; -const size_t ExtendableBuffer::MAX_BUFFER_SIZE = 1024 * 1024; -const size_t ExtendableBuffer::EXTEND_BUFFER_SIZE_STEP = 16 * 1024; +const size_t BufferWithExtendableBuffer::INITIAL_ADDITIONAL_BUFFER_SIZE = 16 * 1024; +const size_t BufferWithExtendableBuffer::MAX_ADDITIONAL_BUFFER_SIZE = 1024 * 1024; +const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 16 * 1024; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h index 5c75027d2..859894d98 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/extendable_buffer.h @@ -14,57 +14,71 @@ * limitations under the License. */ -#ifndef LATINIME_EXTENDABLE_BUFFER_H -#define LATINIME_EXTENDABLE_BUFFER_H +#ifndef LATINIME_BUFFER_WITH_EXTENDABLE_BUFFER_H +#define LATINIME_BUFFER_WITH_EXTENDABLE_BUFFER_H #include <cstddef> #include <stdint.h> #include <vector> #include "defines.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { +// TODO: change file name according to the class name. // This is used as a buffer that can be extended for updatable dictionaries. -class ExtendableBuffer { +// To optimize performance, raw pointer is directly used for reading buffer. The position has to be +// adjusted to access additional buffer. On the other hand, this class does not provide writable +// raw pointer but provides several methods that handle boundary checking for writing data. +class BufferWithExtendableBuffer { public: - ExtendableBuffer() : mBuffer(INITIAL_BUFFER_SIZE), mUsedSize(0) {} + BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize) + : mOriginalBuffer(originalBuffer), mOriginalBufferSize(originalBufferSize), + mAdditionalBuffer(INITIAL_ADDITIONAL_BUFFER_SIZE), mUsedAdditionalBufferSize(0) {} - AK_FORCE_INLINE const uint8_t *getBuffer() const { - return &mBuffer[0]; + /** + * For reading. + */ + AK_FORCE_INLINE bool isInAdditionalBuffer(const int position) const { + return position >= mOriginalBufferSize; } - // Return if the buffer is successfully extended or not. - AK_FORCE_INLINE bool extendBuffer() { - if (mBuffer.size() + EXTEND_BUFFER_SIZE_STEP > MAX_BUFFER_SIZE) { - return false; + // CAVEAT!: Be careful about array out of bound access with buffers + AK_FORCE_INLINE const uint8_t *getBuffer(const bool usesAdditionalBuffer) const { + if (usesAdditionalBuffer) { + return &mAdditionalBuffer[0]; + } else { + return mOriginalBuffer; } - mBuffer.resize(mBuffer.size() + EXTEND_BUFFER_SIZE_STEP); - return true; } - AK_FORCE_INLINE int getAllocatedSize() const { - return mBuffer.size(); + AK_FORCE_INLINE int getOriginalBufferSize() const { + return mOriginalBufferSize; } - AK_FORCE_INLINE int getUsedSize() const { - return mUsedSize; - } - - AK_FORCE_INLINE void clear() { - mUsedSize = 0; - mBuffer.clear(); - } private: - DISALLOW_COPY_AND_ASSIGN(ExtendableBuffer); + DISALLOW_COPY_AND_ASSIGN(BufferWithExtendableBuffer); - static const size_t INITIAL_BUFFER_SIZE; - static const size_t MAX_BUFFER_SIZE; - static const size_t EXTEND_BUFFER_SIZE_STEP; + static const size_t INITIAL_ADDITIONAL_BUFFER_SIZE; + static const size_t MAX_ADDITIONAL_BUFFER_SIZE; + static const size_t EXTEND_ADDITIONAL_BUFFER_SIZE_STEP; - std::vector<uint8_t> mBuffer; - int mUsedSize; + uint8_t *const mOriginalBuffer; + const int mOriginalBufferSize; + std::vector<uint8_t> mAdditionalBuffer; + int mUsedAdditionalBufferSize; + + // Return if the buffer is successfully extended or not. + AK_FORCE_INLINE bool extendBuffer() { + if (mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP + > MAX_ADDITIONAL_BUFFER_SIZE) { + return false; + } + mAdditionalBuffer.resize(mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP); + return true; + } }; } -#endif /* LATINIME_MMAPED_BUFFER_H */ +#endif /* LATINIME_BUFFER_WITH_EXTENDABLE_BUFFER_H */ |