diff options
Diffstat (limited to 'native/jni/src')
10 files changed, 100 insertions, 66 deletions
diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h index 47f5ec0d7..e69d2c46b 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node.h +++ b/native/jni/src/suggest/core/dicnode/dic_node.h @@ -103,10 +103,10 @@ class DicNode { PROF_NODE_COPY(&dicNode->mProfiler, mProfiler); } - // Init for root with prevWordPtNodePos which is used for bigram - void initAsRoot(const int rootPtNodeArrayPos, const int prevWordPtNodePos) { + // Init for root with prevWordsPtNodePos which is used for n-gram + void initAsRoot(const int rootPtNodeArrayPos, const int *const prevWordsPtNodePos) { mIsCachedForNextSuggestion = false; - mDicNodeProperties.init(rootPtNodeArrayPos, prevWordPtNodePos); + mDicNodeProperties.init(rootPtNodeArrayPos, prevWordsPtNodePos); mDicNodeState.init(); PROF_NODE_RESET(mProfiler); } @@ -114,7 +114,12 @@ class DicNode { // Init for root with previous word void initAsRootWithPreviousWord(const DicNode *const dicNode, const int rootPtNodeArrayPos) { mIsCachedForNextSuggestion = dicNode->mIsCachedForNextSuggestion; - mDicNodeProperties.init(rootPtNodeArrayPos, dicNode->mDicNodeProperties.getPtNodePos()); + int newPrevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM]; + newPrevWordsPtNodePos[0] = dicNode->mDicNodeProperties.getPtNodePos(); + for (size_t i = 1; i < NELEMS(newPrevWordsPtNodePos); ++i) { + newPrevWordsPtNodePos[i] = dicNode->getNthPrevWordTerminalPtNodePos(i); + } + mDicNodeProperties.init(rootPtNodeArrayPos, newPrevWordsPtNodePos); mDicNodeState.initAsRootWithPreviousWord(&dicNode->mDicNodeState, dicNode->mDicNodeProperties.getDepth()); PROF_NODE_COPY(&dicNode->mProfiler, mProfiler); @@ -140,7 +145,7 @@ class DicNode { dicNode->mDicNodeProperties.getLeavingDepth() + mergedNodeCodePointCount); mDicNodeProperties.init(ptNodePos, childrenPtNodeArrayPos, mergedNodeCodePoints[0], probability, isTerminal, hasChildren, isBlacklistedOrNotAWord, newDepth, - newLeavingDepth, dicNode->mDicNodeProperties.getPrevWordTerminalPtNodePos()); + newLeavingDepth, dicNode->mDicNodeProperties.getPrevWordsTerminalPtNodePos()); mDicNodeState.init(&dicNode->mDicNodeState, mergedNodeCodePointCount, mergedNodeCodePoints); PROF_NODE_COPY(&dicNode->mProfiler, mProfiler); @@ -198,14 +203,17 @@ class DicNode { return mDicNodeState.mDicNodeStateInput.getInputIndex(0) < inputSize - 1; } - // Used to get bigram probability in DicNodeUtils + // Used to get n-gram probability in DicNodeUtils int getPtNodePos() const { return mDicNodeProperties.getPtNodePos(); } - // Used to get bigram probability in DicNodeUtils - int getPrevWordTerminalPtNodePos() const { - return mDicNodeProperties.getPrevWordTerminalPtNodePos(); + // Used to get n-gram probability in DicNodeUtils + int getNthPrevWordTerminalPtNodePos(const int n) const { + if (n <= 0 || n > MAX_PREV_WORD_COUNT_FOR_N_GRAM) { + return NOT_A_DICT_POS; + } + return mDicNodeProperties.getPrevWordsTerminalPtNodePos()[n - 1]; } // Used in DicNodeUtils diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp index 2d02a7d9c..bf2a0000d 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp @@ -29,8 +29,8 @@ namespace latinime { /* static */ void DicNodeUtils::initAsRoot( const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, - const int prevWordPtNodePos, DicNode *const newRootDicNode) { - newRootDicNode->initAsRoot(dictionaryStructurePolicy->getRootPosition(), prevWordPtNodePos); + const int *const prevWordsPtNodePos, DicNode *const newRootDicNode) { + newRootDicNode->initAsRoot(dictionaryStructurePolicy->getRootPosition(), prevWordsPtNodePos); } /*static */ void DicNodeUtils::initAsRootWithPreviousWord( @@ -86,7 +86,7 @@ namespace latinime { const DicNode *const dicNode, MultiBigramMap *const multiBigramMap) { const int unigramProbability = dicNode->getProbability(); const int ptNodePos = dicNode->getPtNodePos(); - const int prevWordTerminalPtNodePos = dicNode->getPrevWordTerminalPtNodePos(); + const int prevWordTerminalPtNodePos = dicNode->getNthPrevWordTerminalPtNodePos(1 /* n */); if (NOT_A_DICT_POS == ptNodePos || NOT_A_DICT_POS == prevWordTerminalPtNodePos) { // Note: Normally wordPos comes from the dictionary and should never equal // NOT_A_VALID_WORD_POS. diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.h b/native/jni/src/suggest/core/dicnode/dic_node_utils.h index 4c0f1f15d..0d60e5796 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.h @@ -30,7 +30,7 @@ class DicNodeUtils { public: static void initAsRoot( const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, - const int prevWordPtNodePos, DicNode *const newRootDicNode); + const int *const prevWordPtNodePos, DicNode *const newRootDicNode); static void initAsRootWithPreviousWord( const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, const DicNode *const prevWordLastDicNode, DicNode *const newRootDicNode); diff --git a/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h b/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h index 11f8c2905..8202176f7 100644 --- a/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h +++ b/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h @@ -32,15 +32,14 @@ class DicNodeProperties { : mPtNodePos(NOT_A_DICT_POS), mChildrenPtNodeArrayPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), mDicNodeCodePoint(NOT_A_CODE_POINT), mIsTerminal(false), mHasChildrenPtNodes(false), - mIsBlacklistedOrNotAWord(false), mDepth(0), mLeavingDepth(0), - mPrevWordTerminalPtNodePos(NOT_A_DICT_POS) {} + mIsBlacklistedOrNotAWord(false), mDepth(0), mLeavingDepth(0) {} ~DicNodeProperties() {} // Should be called only once per DicNode is initialized. void init(const int pos, const int childrenPos, const int nodeCodePoint, const int probability, const bool isTerminal, const bool hasChildren, const bool isBlacklistedOrNotAWord, - const uint16_t depth, const uint16_t leavingDepth, const int prevWordNodePos) { + const uint16_t depth, const uint16_t leavingDepth, const int *const prevWordsNodePos) { mPtNodePos = pos; mChildrenPtNodeArrayPos = childrenPos; mDicNodeCodePoint = nodeCodePoint; @@ -50,11 +49,11 @@ class DicNodeProperties { mIsBlacklistedOrNotAWord = isBlacklistedOrNotAWord; mDepth = depth; mLeavingDepth = leavingDepth; - mPrevWordTerminalPtNodePos = prevWordNodePos; + memmove(mPrevWordsTerminalPtNodePos, prevWordsNodePos, sizeof(mPrevWordsTerminalPtNodePos)); } - // Init for root with prevWordPtNodePos which is used for bigram - void init(const int rootPtNodeArrayPos, const int prevWordNodePos) { + // Init for root with prevWordsPtNodePos which is used for n-gram + void init(const int rootPtNodeArrayPos, const int *const prevWordsNodePos) { mPtNodePos = NOT_A_DICT_POS; mChildrenPtNodeArrayPos = rootPtNodeArrayPos; mDicNodeCodePoint = NOT_A_CODE_POINT; @@ -64,7 +63,7 @@ class DicNodeProperties { mIsBlacklistedOrNotAWord = false; mDepth = 0; mLeavingDepth = 0; - mPrevWordTerminalPtNodePos = prevWordNodePos; + memmove(mPrevWordsTerminalPtNodePos, prevWordsNodePos, sizeof(mPrevWordsTerminalPtNodePos)); } void initByCopy(const DicNodeProperties *const dicNodeProp) { @@ -77,7 +76,8 @@ class DicNodeProperties { mIsBlacklistedOrNotAWord = dicNodeProp->mIsBlacklistedOrNotAWord; mDepth = dicNodeProp->mDepth; mLeavingDepth = dicNodeProp->mLeavingDepth; - mPrevWordTerminalPtNodePos = dicNodeProp->mPrevWordTerminalPtNodePos; + memmove(mPrevWordsTerminalPtNodePos, dicNodeProp->mPrevWordsTerminalPtNodePos, + sizeof(mPrevWordsTerminalPtNodePos)); } // Init as passing child @@ -91,7 +91,8 @@ class DicNodeProperties { mIsBlacklistedOrNotAWord = dicNodeProp->mIsBlacklistedOrNotAWord; mDepth = dicNodeProp->mDepth + 1; // Increment the depth of a passing child mLeavingDepth = dicNodeProp->mLeavingDepth; - mPrevWordTerminalPtNodePos = dicNodeProp->mPrevWordTerminalPtNodePos; + memmove(mPrevWordsTerminalPtNodePos, dicNodeProp->mPrevWordsTerminalPtNodePos, + sizeof(mPrevWordsTerminalPtNodePos)); } int getPtNodePos() const { @@ -131,8 +132,8 @@ class DicNodeProperties { return mIsBlacklistedOrNotAWord; } - int getPrevWordTerminalPtNodePos() const { - return mPrevWordTerminalPtNodePos; + const int *getPrevWordsTerminalPtNodePos() const { + return mPrevWordsTerminalPtNodePos; } private: @@ -148,7 +149,7 @@ class DicNodeProperties { bool mIsBlacklistedOrNotAWord; uint16_t mDepth; uint16_t mLeavingDepth; - int mPrevWordTerminalPtNodePos; + int mPrevWordsTerminalPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM]; }; } // namespace latinime #endif // LATINIME_DIC_NODE_PROPERTIES_H diff --git a/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp index 847fa1b02..295e760d6 100644 --- a/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp @@ -48,21 +48,10 @@ BigramDictionary::~BigramDictionary() { */ void BigramDictionary::getPredictions(const PrevWordsInfo *const prevWordsInfo, SuggestionResults *const outSuggestionResults) const { - int pos = getBigramListPositionForWord(prevWordsInfo->getPrevWordCodePoints(), - prevWordsInfo->getPrevWordCodePointCount(), false /* forceLowerCaseSearch */); - // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams - if (NOT_A_DICT_POS == pos) { - // If no bigrams for this exact word, search again in lower case. - pos = getBigramListPositionForWord(prevWordsInfo->getPrevWordCodePoints(), - prevWordsInfo->getPrevWordCodePointCount(), true /* forceLowerCaseSearch */); - } - // If still no bigrams, we really don't have them! - if (NOT_A_DICT_POS == pos) return; - int unigramProbability = 0; int bigramCodePoints[MAX_WORD_LENGTH]; - BinaryDictionaryBigramsIterator bigramsIt( - mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos); + BinaryDictionaryBigramsIterator bigramsIt = + prevWordsInfo->getBigramsIteratorForPrediction(mDictionaryStructurePolicy); while (bigramsIt.hasNext()) { bigramsIt.next(); if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) { @@ -98,16 +87,11 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in int BigramDictionary::getBigramProbability(const PrevWordsInfo *const prevWordsInfo, const int *word1, int length1) const { - int pos = getBigramListPositionForWord(prevWordsInfo->getPrevWordCodePoints(), - prevWordsInfo->getPrevWordCodePointCount(), false /* forceLowerCaseSearch */); - // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams - if (NOT_A_DICT_POS == pos) return NOT_A_PROBABILITY; int nextWordPos = mDictionaryStructurePolicy->getTerminalPtNodePositionOfWord(word1, length1, false /* forceLowerCaseSearch */); if (NOT_A_DICT_POS == nextWordPos) return NOT_A_PROBABILITY; - - BinaryDictionaryBigramsIterator bigramsIt( - mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos); + BinaryDictionaryBigramsIterator bigramsIt = + prevWordsInfo->getBigramsIteratorForPrediction(mDictionaryStructurePolicy); while (bigramsIt.hasNext()) { bigramsIt.next(); if (bigramsIt.getBigramPos() == nextWordPos diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h index d16ac47fe..bc9d57671 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h @@ -30,6 +30,11 @@ class BinaryDictionaryBigramsIterator { mBigramPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), mHasNext(pos != NOT_A_DICT_POS) {} + BinaryDictionaryBigramsIterator(BinaryDictionaryBigramsIterator &&bigramsIterator) + : mBigramsStructurePolicy(bigramsIterator.mBigramsStructurePolicy), + mPos(bigramsIterator.mPos), mBigramPos(bigramsIterator.mBigramPos), + mProbability(bigramsIterator.mProbability), mHasNext(bigramsIterator.mHasNext) {} + AK_FORCE_INLINE bool hasNext() const { return mHasNext; } diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.cpp b/native/jni/src/suggest/core/session/dic_traverse_session.cpp index b9e9db719..dc2b66a2c 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.cpp +++ b/native/jni/src/suggest/core/session/dic_traverse_session.cpp @@ -35,21 +35,8 @@ void DicTraverseSession::init(const Dictionary *const dictionary, mMultiWordCostMultiplier = getDictionaryStructurePolicy()->getHeaderStructurePolicy() ->getMultiWordCostMultiplier(); mSuggestOptions = suggestOptions; - if (!prevWordsInfo->getPrevWordCodePoints()) { - mPrevWordsPtNodePos[0] = NOT_A_DICT_POS; - return; - } - // TODO: merge following similar calls to getTerminalPosition into one case-insensitive call. - mPrevWordsPtNodePos[0] = getDictionaryStructurePolicy()->getTerminalPtNodePositionOfWord( - prevWordsInfo->getPrevWordCodePoints(), prevWordsInfo->getPrevWordCodePointCount(), - false /* forceLowerCaseSearch */); - if (mPrevWordsPtNodePos[0] == NOT_A_DICT_POS) { - // Check bigrams for lower-cased previous word if original was not found. Useful for - // auto-capitalized words like "The [current_word]". - mPrevWordsPtNodePos[0] = getDictionaryStructurePolicy()->getTerminalPtNodePositionOfWord( - prevWordsInfo->getPrevWordCodePoints(), prevWordsInfo->getPrevWordCodePointCount(), - true /* forceLowerCaseSearch */); - } + prevWordsInfo->getPrevWordsTerminalPtNodePos( + getDictionaryStructurePolicy(), mPrevWordsPtNodePos); } void DicTraverseSession::setupForGetSuggestions(const ProximityInfo *pInfo, diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h index 90aff06c3..5a51a112d 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.h +++ b/native/jni/src/suggest/core/session/dic_traverse_session.h @@ -79,7 +79,7 @@ class DicTraverseSession { //-------------------- const ProximityInfo *getProximityInfo() const { return mProximityInfo; } const SuggestOptions *getSuggestOptions() const { return mSuggestOptions; } - int getPrevWordPtNodePos() const { return mPrevWordsPtNodePos[0]; } + const int *getPrevWordsPtNodePos() const { return mPrevWordsPtNodePos; } DicNodesCache *getDicTraverseCache() { return &mDicNodesCache; } MultiBigramMap *getMultiBigramMap() { return &mMultiBigramMap; } const ProximityInfoState *getProximityInfoState(int id) const { diff --git a/native/jni/src/suggest/core/session/prev_words_info.h b/native/jni/src/suggest/core/session/prev_words_info.h index bc685945e..70a99ef38 100644 --- a/native/jni/src/suggest/core/session/prev_words_info.h +++ b/native/jni/src/suggest/core/session/prev_words_info.h @@ -18,6 +18,8 @@ #define LATINIME_PREV_WORDS_INFO_H #include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h" +#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" namespace latinime { @@ -38,17 +40,64 @@ class PrevWordsInfo { mPrevWordCodePointCount[0] = prevWordCodePointCount; mIsBeginningOfSentence[0] = isBeginningOfSentence; } - const int *getPrevWordCodePoints() const { - return mPrevWordCodePoints[0]; + + void getPrevWordsTerminalPtNodePos( + const DictionaryStructureWithBufferPolicy *const dictStructurePolicy, + int *const outPrevWordsTerminalPtNodePos) const { + for (size_t i = 0; i < NELEMS(mPrevWordCodePoints); ++i) { + outPrevWordsTerminalPtNodePos[i] = getTerminalPtNodePosOfWord(dictStructurePolicy, + mPrevWordCodePoints[i], mPrevWordCodePointCount[i], + mIsBeginningOfSentence[i]); + } } - int getPrevWordCodePointCount() const { - return mPrevWordCodePointCount[0]; + BinaryDictionaryBigramsIterator getBigramsIteratorForPrediction( + const DictionaryStructureWithBufferPolicy *const dictStructurePolicy) const { + int pos = getBigramListPositionForWord(dictStructurePolicy, mPrevWordCodePoints[0], + mPrevWordCodePointCount[0], false /* forceLowerCaseSearch */); + // getBigramListPositionForWord returns NOT_A_DICT_POS if this word isn't in the + // dictionary or has no bigrams + if (NOT_A_DICT_POS == pos) { + // If no bigrams for this exact word, search again in lower case. + pos = getBigramListPositionForWord(dictStructurePolicy, mPrevWordCodePoints[0], + mPrevWordCodePointCount[0], true /* forceLowerCaseSearch */); + } + return BinaryDictionaryBigramsIterator( + dictStructurePolicy->getBigramsStructurePolicy(), pos); } private: DISALLOW_COPY_AND_ASSIGN(PrevWordsInfo); + static int getTerminalPtNodePosOfWord( + const DictionaryStructureWithBufferPolicy *const dictStructurePolicy, + const int *const wordCodePoints, const int wordCodePointCount, + const bool isBeginningOfSentence) { + if (!dictStructurePolicy || !wordCodePoints) { + return NOT_A_DICT_POS; + } + const int wordPtNodePos = dictStructurePolicy->getTerminalPtNodePositionOfWord( + wordCodePoints, wordCodePointCount, false /* forceLowerCaseSearch */); + if (wordPtNodePos != NOT_A_DICT_POS) { + return wordPtNodePos; + } + // Check bigrams for lower-cased previous word if original was not found. Useful for + // auto-capitalized words like "The [current_word]". + return dictStructurePolicy->getTerminalPtNodePositionOfWord( + wordCodePoints, wordCodePointCount, true /* forceLowerCaseSearch */); + } + + static int getBigramListPositionForWord( + const DictionaryStructureWithBufferPolicy *const dictStructurePolicy, + const int *wordCodePoints, const int wordCodePointCount, + const bool forceLowerCaseSearch) { + if (!wordCodePoints || wordCodePointCount <= 0) return NOT_A_DICT_POS; + const int terminalPtNodePos = dictStructurePolicy->getTerminalPtNodePositionOfWord( + wordCodePoints, wordCodePointCount, forceLowerCaseSearch); + if (NOT_A_DICT_POS == terminalPtNodePos) return NOT_A_DICT_POS; + return dictStructurePolicy->getBigramsPositionOfPtNode(terminalPtNodePos); + } + void clear() { for (size_t i = 0; i < NELEMS(mPrevWordCodePoints); ++i) { mPrevWordCodePoints[i] = nullptr; diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index e675e0bb3..0cd305f5a 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -92,7 +92,7 @@ void Suggest::initializeSearch(DicTraverseSession *traverseSession) const { // Create a new dic node here DicNode rootNode; DicNodeUtils::initAsRoot(traverseSession->getDictionaryStructurePolicy(), - traverseSession->getPrevWordPtNodePos(), &rootNode); + traverseSession->getPrevWordsPtNodePos(), &rootNode); traverseSession->getDicTraverseCache()->copyPushActive(&rootNode); } } |