diff options
Diffstat (limited to 'native')
11 files changed, 242 insertions, 57 deletions
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 86c2394d1..8da1859c4 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -204,6 +204,7 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, } jsize wordLength = env->GetArrayLength(word); int codePoints[wordLength]; + env->GetIntArrayRegion(word, 0, wordLength, codePoints); dictionary->addUnigramWord(codePoints, wordLength, probability); } @@ -215,8 +216,10 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, } jsize word0Length = env->GetArrayLength(word0); int word0CodePoints[word0Length]; + env->GetIntArrayRegion(word0, 0, word0Length, word0CodePoints); jsize word1Length = env->GetArrayLength(word1); int word1CodePoints[word1Length]; + env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length, probability); } @@ -229,8 +232,10 @@ static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass claz } jsize word0Length = env->GetArrayLength(word0); int word0CodePoints[word0Length]; + env->GetIntArrayRegion(word0, 0, word0Length, word0CodePoints); jsize word1Length = env->GetArrayLength(word1); int word1CodePoints[word1Length]; + env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); dictionary->removeBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index e31a91069..936dc9c5d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -20,12 +20,13 @@ namespace latinime { bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); } BigramListReadWriteUtils::BigramFlags flags; do { + // The buffer address can be changed after calling buffer writing methods. + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, fromPos); int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( buffer, flags, fromPos); @@ -63,7 +64,6 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int probability, int *const pos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { *pos -= mBuffer->getOriginalBufferSize(); } @@ -73,6 +73,8 @@ bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int prob if (usesAdditionalBuffer) { entryPos += mBuffer->getOriginalBufferSize(); } + // The buffer address can be changed after calling buffer writing methods. + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); BigramListReadWriteUtils::getBigramAddressAndForwardPointer(buffer, flags, pos); if (BigramListReadWriteUtils::hasNext(flags)) { @@ -118,13 +120,14 @@ bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int prob bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int targetBigramPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(bigramListPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); int pos = bigramListPos; if (usesAdditionalBuffer) { pos -= mBuffer->getOriginalBufferSize(); } BigramListReadWriteUtils::BigramFlags flags; do { + // The buffer address can be changed after calling buffer writing methods. + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, &pos); int bigramOffsetFieldPos = pos; if (usesAdditionalBuffer) { @@ -139,8 +142,7 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta continue; } // Target entry is found. Write 0 into the bigram pos field to mark the bigram invalid. - const int bigramOffsetFieldSize = - BigramListReadWriteUtils::attributeAddressSize(flags); + const int bigramOffsetFieldSize = BigramListReadWriteUtils::attributeAddressSize(flags); if (!mBuffer->writeUintAndAdvancePosition(0 /* data */, bigramOffsetFieldSize, &bigramOffsetFieldPos)) { return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index 6bb90fc2d..5674cb48e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -34,7 +34,7 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const int parentPos = DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictBuf, &pos); - mParentPos = (parentPos != 0) ? mNodePos + parentPos : NOT_A_DICT_POS; + mParentPos = (parentPos != 0) ? nodePos + parentPos : NOT_A_DICT_POS; if (outCodePoints != 0) { mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( dictBuf, mFlags, maxCodePointCount, outCodePoints, &pos); @@ -43,10 +43,19 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c dictBuf, mFlags, MAX_WORD_LENGTH, &pos); } if (isTerminal()) { + mProbabilityFieldPos = pos; + if (usesAdditionalBuffer) { + mProbabilityFieldPos += mBuffer->getOriginalBufferSize(); + } mProbability = PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(dictBuf, &pos); } else { + mProbabilityFieldPos = NOT_A_DICT_POS; mProbability = NOT_A_PROBABILITY; } + mChildrenPosFieldPos = pos; + if (usesAdditionalBuffer) { + mChildrenPosFieldPos += mBuffer->getOriginalBufferSize(); + } mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, mFlags, &pos); if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h index acc68b321..2ee7c2495 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h @@ -40,9 +40,11 @@ class DynamicPatriciaTrieNodeReader { const DictionaryShortcutsStructurePolicy *const shortcutsPolicy) : mBuffer(buffer), mBigramsPolicy(bigramsPolicy), mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), - mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbability(NOT_A_PROBABILITY), - mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS), - mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {} + mParentPos(NOT_A_DICT_POS), mCodePointCount(0), + mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), + mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS), + mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), + mSiblingPos(NOT_A_VALID_WORD_POS) {} ~DynamicPatriciaTrieNodeReader() {} @@ -95,11 +97,19 @@ class DynamicPatriciaTrieNodeReader { } // Probability + AK_FORCE_INLINE int getProbabilityFieldPos() const { + return mProbabilityFieldPos; + } + AK_FORCE_INLINE int getProbability() const { return mProbability; } - // Children node group position + // Children PtNode array position + AK_FORCE_INLINE int getChildrenPosFieldPos() const { + return mChildrenPosFieldPos; + } + AK_FORCE_INLINE int getChildrenPos() const { return mChildrenPos; } @@ -129,7 +139,9 @@ class DynamicPatriciaTrieNodeReader { DynamicPatriciaTrieReadingUtils::NodeFlags mFlags; int mParentPos; uint8_t mCodePointCount; + int mProbabilityFieldPos; int mProbability; + int mChildrenPosFieldPos; int mChildrenPos; int mShortcutPos; int mBigramPos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index 03dc57628..99a983f21 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -44,28 +44,23 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( const int nodeCodePointCount = nodeReader->getCodePointCount(); for (int j = 1; j < nodeCodePointCount; ++j) { const int nextIndex = matchedCodePointCount + j; - if (nextIndex >= codePointCount) { - // TODO: split current node after j - 1, create child and make this terminal. - return false; - } - if (!readingHelper->isMatchedCodePoint(j, + if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(j, wordCodePoints[matchedCodePointCount + j])) { - // TODO: split current node after j - 1 and create two children. - return false; + return reallocatePtNodeAndAddNewPtNodes(nodeReader, + readingHelper->getMergedNodeCodePoints(), j, probability, + wordCodePoints + matchedCodePointCount, + codePointCount - matchedCodePointCount); } } // All characters are matched. if (codePointCount == readingHelper->getTotalCodePointCount()) { - if (nodeReader->isTerminal()) { - // TODO: Update probability. - } else { - // TODO: Make it terminal and update probability. - } - return false; + return setPtNodeProbability(nodeReader, probability, + readingHelper->getMergedNodeCodePoints()); } if (!nodeReader->hasChildren()) { - // TODO: Create children node array and add new node as a child. - return false; + return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability, + wordCodePoints + readingHelper->getTotalCodePointCount(), + codePointCount - readingHelper->getTotalCodePointCount()); } // Advance to the children nodes. parentPos = nodeReader->getNodePos(); @@ -76,8 +71,6 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( return false; } int pos = readingHelper->getPosOfLastForwardLinkField(); - // TODO: Remove. - return false; return createAndInsertNodeIntoPtNodeArray(parentPos, wordCodePoints + readingHelper->getPrevTotalCodePointCount(), codePointCount - readingHelper->getPrevTotalCodePointCount(), @@ -126,20 +119,22 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( &writingPos)) { return false; } - // Update moved position, which is stored in the parent position field. - if (!DynamicPatriciaTrieWritingUtils::writeParentPositionAndAdvancePosition( - mBuffer, movedPos, &writingPos)) { + // Update moved position, which is stored in the parent offset field. + const int movedPosOffset = movedPos - originalNode->getNodePos(); + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( + mBuffer, movedPosOffset, &writingPos)) { return false; } return true; } -// Write new node at writingPos. -bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlacklisted, +// Write new PtNode at writingPos. +bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos) { + const int nodePos = *writingPos; // Create node flags and write them. const PatriciaTrieReadingUtils::NodeFlags nodeFlags = PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, @@ -150,9 +145,10 @@ bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlackliste writingPos)) { return false; } - // Write parent position - if (!DynamicPatriciaTrieWritingUtils::writeParentPositionAndAdvancePosition(mBuffer, parentPos, - writingPos)) { + // Calculate a parent offset and write the offset. + const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBuffer, + parentOffset, writingPos)) { return false; } // Write code points @@ -176,7 +172,9 @@ bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlackliste // Copy shortcut list when the originalShortcutListPos is valid dictionary position. if (originalShortcutListPos != NOT_A_DICT_POS) { int fromPos = originalShortcutListPos; - mShortcutPolicy->copyAllShortcuts(&fromPos, writingPos); + if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(&fromPos, writingPos)) { + return false; + } } // Copy bigram list when the originalBigramListPos is valid dictionary position. if (originalBigramListPos != NOT_A_DICT_POS) { @@ -188,6 +186,25 @@ bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlackliste return true; } +bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(const int parentPos, + const int *const codePoints, const int codePointCount, const int probability, + int *const writingPos) { + return writePtNodeWithFullInfoToBuffer(false /* isBlacklisted */, false /* isNotAWord */, + parentPos, codePoints, codePointCount, probability, + NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, + NOT_A_DICT_POS /* originalShortcutPos */, writingPos); +} + +bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( + const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, + const int *const codePoints, const int codePointCount, const int probability, + int *const writingPos) { + return writePtNodeWithFullInfoToBuffer(originalNode->isBlacklisted(), + originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, + originalNode->getChildrenPos(), originalNode->getBigramsPos(), + originalNode->getShortcutPos(), writingPos); +} + bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos) { @@ -196,14 +213,56 @@ bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const newPtNodeArrayPos, forwardLinkFieldPos)) { return false; } - int writingPos = newPtNodeArrayPos; + return createNewPtNodeArrayWithAChildPtNode(parentPos, nodeCodePoints, nodeCodePointCount, + probability); +} + +bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( + const DynamicPatriciaTrieNodeReader *const originalPtNode, const int probability, + const int *const codePoints) { + if (originalPtNode->isTerminal()) { + // Overwrites the probability. + int probabilityFieldPos = originalPtNode->getProbabilityFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer, + probability, &probabilityFieldPos)) { + return false; + } + } else { + // Make the node terminal and write the probability. + int movedPos = mBuffer->getTailPosition(); + if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos)) { + return false; + } + if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), + codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { + return false; + } + } + return true; +} + +bool DynamicPatriciaTrieWritingHelper::createChildrenPtNodeArrayAndAChildPtNode( + const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, + const int *const codePoints, const int codePointCount) { + const int newPtNodeArrayPos = mBuffer->getTailPosition(); + int childrenPosFieldPos = parentNode->getChildrenPosFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, + newPtNodeArrayPos, &childrenPosFieldPos)) { + return false; + } + return createNewPtNodeArrayWithAChildPtNode(parentNode->getNodePos(), codePoints, + codePointCount, probability); +} + +bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( + const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, + const int probability) { + int writingPos = mBuffer->getTailPosition(); if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, 1 /* arraySize */, &writingPos)) { return false; } - if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPos, - nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */, - NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, + if (!writePtNodeToBuffer(parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, &writingPos)) { return false; } @@ -214,4 +273,69 @@ bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const return true; } +// Returns whether the dictionary updating was succeeded or not. +bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( + const DynamicPatriciaTrieNodeReader *const reallocatingPtNode, + const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, + const int probabilityOfNewPtNode, const int *const newNodeCodePoints, + const int newNodeCodePointCount) { + // When addsExtraChild is true, split the reallocating PtNode and add new child. + // Reallocating PtNode: abcde, newNode: abcxy. + // abc (1st, not terminal) __ de (2nd) + // \_ xy (extra child, terminal) + // Otherwise, this method makes 1st part terminal and write probabilityOfNewPtNode. + // Reallocating PtNode: abcde, newNode: abc. + // abc (1st, terminal) __ de (2nd) + const bool addsExtraChild = newNodeCodePointCount > overlappingCodePointCount; + const int firstPtNodePos = mBuffer->getTailPosition(); + if (!markNodeAsMovedAndSetPosition(reallocatingPtNode, firstPtNodePos)) { + return false; + } + int writingPos = firstPtNodePos; + // Write the 1st part of the reallocating node. The children position will be updated later + // with actual children position. + const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; + if (!writePtNodeToBuffer(reallocatingPtNode->getParentPos(), reallocatingPtNodeCodePoints, + overlappingCodePointCount, newProbability, &writingPos)) { + return false; + } + const int actualChildrenPos = writingPos; + // Create new children PtNode array. + const size_t newPtNodeCount = addsExtraChild ? 2 : 1; + if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + newPtNodeCount, &writingPos)) { + return false; + } + // Write the 2nd part of the reallocating node. + if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, + reallocatingPtNode->getNodePos(), + reallocatingPtNodeCodePoints + overlappingCodePointCount, + reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, + reallocatingPtNode->getProbability(), &writingPos)) { + return false; + } + if (addsExtraChild) { + if (!writePtNodeToBuffer(reallocatingPtNode->getNodePos(), + newNodeCodePoints + overlappingCodePointCount, + newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, + &writingPos)) { + return false; + } + } + if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { + return false; + } + // Load node info. Information of the 1st part will be fetched. + DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); + nodeReader.fetchNodeInfoFromBuffer(firstPtNodePos); + // Update children position. + int childrenPosFieldPos = nodeReader.getChildrenPosFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, + actualChildrenPos, &childrenPosFieldPos)) { + return false; + } + return true; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index 524f361d3..ada634a54 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -56,13 +56,37 @@ class DynamicPatriciaTrieWritingHelper { bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos); - bool writeNodeToBuffer(const bool isBlacklisted, const bool isNotAWord, const int parentPos, - const int *const codePoints, const int codePointCount, const int probability, - const int childrenPos, const int originalBigramListPos, + bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, + const int parentPos, const int *const codePoints, const int codePointCount, + const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos); + bool writePtNodeToBuffer(const int parentPos, const int *const codePoints, + const int codePointCount, const int probability, int *const writingPos); + + bool writePtNodeToBufferByCopyingPtNodeInfo( + const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, + const int *const codePoints, const int codePointCount, const int probability, + int *const writingPos); + bool createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos); + + bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode, + const int probability, const int *const codePoints); + + bool createChildrenPtNodeArrayAndAChildPtNode( + const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, + const int *const codePoints, const int codePointCount); + + bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints, + const int nodeCodePointCount, const int probability); + + bool reallocatePtNodeAndAddNewPtNodes( + const DynamicPatriciaTrieNodeReader *const reallocatingPtNode, + const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, + const int probabilityOfNewPtNode, const int *const newNodeCodePoints, + const int newNodeCodePointCount); }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp index 4187504b4..b261e594d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp @@ -68,11 +68,11 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPositionAndAdvancePosition( - BufferWithExtendableBuffer *const buffer, const int parentPosition, +// Note that parentOffset is offset from node's head position. +/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( + BufferWithExtendableBuffer *const buffer, const int parentOffset, int *const parentPosFieldPos) { - // Note that parentPosition is offset from node's head position. - int offset = (parentPosition != NOT_A_DICT_POS) ? parentPosition : 0; + int offset = (parentOffset != NOT_A_DICT_POS) ? parentOffset : 0; return writeDictOffset(buffer, offset, parentPosFieldPos); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h index 801042ddf..183ede444 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h @@ -39,7 +39,7 @@ class DynamicPatriciaTrieWritingUtils { const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos); - static bool writeParentPositionAndAdvancePosition(BufferWithExtendableBuffer *const buffer, + static bool writeParentOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, const int parentPosition, int *const parentPosFieldPos); static bool writeCodePointsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h index 5e9c52950..1803c09cb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h @@ -83,8 +83,8 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { } // Copy shortcuts from the shortcut list that starts at fromPos to toPos and advance these - // positions after the shortcut lists. - void copyAllShortcuts(int *const fromPos, int *const toPos) { + // positions after the shortcut lists. This returns whether the copy was succeeded or not. + bool copyAllShortcutsAndReturnIfSucceededOrNot(int *const fromPos, int *const toPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { @@ -93,16 +93,23 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { const int shortcutListSize = ShortcutListReadingUtils ::getShortcutListSizeAndForwardPointer(buffer, fromPos); // Copy shortcut list size. - mBuffer->writeUintAndAdvancePosition( + if (!mBuffer->writeUintAndAdvancePosition( shortcutListSize + ShortcutListReadingUtils::getShortcutListSizeFieldSize(), - ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos); + ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos)) { + return false; + } + // Copy shortcut list. for (int i = 0; i < shortcutListSize; ++i) { - const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition(buffer, fromPos); - mBuffer->writeUintAndAdvancePosition(data, 1 /* size */, toPos); + const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition( + mBuffer->getBuffer(usesAdditionalBuffer), fromPos); + if (!mBuffer->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) { + return false; + } } if (usesAdditionalBuffer) { *fromPos += mBuffer->getOriginalBufferSize(); } + return true; } private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp index 6326754c2..dfdaebd18 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp @@ -66,16 +66,17 @@ bool BufferWithExtendableBuffer::writeCodePointsAndAdvancePosition(const int *co bool BufferWithExtendableBuffer::checkAndPrepareWriting(const int pos, const int size) { if (isInAdditionalBuffer(pos)) { - if (pos == mUsedAdditionalBufferSize) { + const int tailPosition = getTailPosition(); + if (pos == tailPosition) { // Append data to the tail. - if (pos + size > static_cast<int>(mAdditionalBuffer.size())) { + if (pos + size > static_cast<int>(mAdditionalBuffer.size()) + mOriginalBufferSize) { // Need to extend buffer. if (!extendBuffer()) { return false; } } mUsedAdditionalBufferSize += size; - } else if (pos + size >= mUsedAdditionalBufferSize) { + } else if (pos + size >= tailPosition) { // The access will beyond the tail of used region. return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h index b35b47d7a..c6a484131 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h @@ -47,6 +47,7 @@ class BufferWithExtendableBuffer { return position >= mOriginalBufferSize; } + // TODO: Resolve the issue that the address can be changed when the vector is resized. // CAVEAT!: Be careful about array out of bound access with buffers AK_FORCE_INLINE const uint8_t *getBuffer(const bool usesAdditionalBuffer) const { if (usesAdditionalBuffer) { |