diff options
Diffstat (limited to 'native/jni/src/unigram_dictionary.cpp')
| -rw-r--r-- | native/jni/src/unigram_dictionary.cpp | 327 |
1 files changed, 176 insertions, 151 deletions
diff --git a/native/jni/src/unigram_dictionary.cpp b/native/jni/src/unigram_dictionary.cpp index ea9f11b2c..ba3c2db6b 100644 --- a/native/jni/src/unigram_dictionary.cpp +++ b/native/jni/src/unigram_dictionary.cpp @@ -1,32 +1,33 @@ /* -** -** Copyright 2010, The Android Open Source Project -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -*/ - -#include <assert.h> -#include <string.h> + * Copyright (C) 2010, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cassert> +#include <cstring> #define LOG_TAG "LatinIME: unigram_dictionary.cpp" +#include "binary_format.h" #include "char_utils.h" #include "defines.h" #include "dictionary.h" -#include "unigram_dictionary.h" - -#include "binary_format.h" +#include "proximity_info.h" #include "terminal_attributes.h" +#include "unigram_dictionary.h" +#include "words_priority_queue.h" +#include "words_priority_queue_pool.h" namespace latinime { @@ -40,7 +41,7 @@ const UnigramDictionary::digraph_t UnigramDictionary::FRENCH_LIGATURES_DIGRAPHS[ { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE // TODO: check the header -UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier, +UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, int typedLetterMultiplier, int fullWordMultiplier, int maxWordLength, int maxWords, const unsigned int flags) : DICT_ROOT(streamStart), MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords), TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier), @@ -62,13 +63,13 @@ static inline unsigned int getCodesBufferSize(const int *codes, const int codesS // TODO: This needs to take a const unsigned short* and not tinker with its contents static inline void addWord( - unsigned short *word, int length, int frequency, WordsPriorityQueue *queue) { - queue->push(frequency, word, length); + unsigned short *word, int length, int frequency, WordsPriorityQueue *queue, int type) { + queue->push(frequency, word, length, type); } // Return the replacement code point for a digraph, or 0 if none. int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int codesSize, - const digraph_t* const digraphs, const unsigned int digraphsSize) const { + const digraph_t *const digraphs, const unsigned int digraphsSize) const { // There can't be a digraph if we don't have at least 2 characters to examine if (i + 2 > codesSize) return false; @@ -103,7 +104,7 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit const bool useFullEditDistance, const int *codesSrc, const int codesRemain, const int currentDepth, int *codesDest, Correction *correction, WordsPriorityQueuePool *queuePool, - const digraph_t* const digraphs, const unsigned int digraphsSize) { + const digraph_t *const digraphs, const unsigned int digraphsSize) const { const int startIndex = codesDest - codesBuffer; if (currentDepth < MAX_DIGRAPH_SEARCH_DEPTH) { @@ -170,14 +171,16 @@ void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximit // bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter // in bigram_dictionary.cpp int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, - WordsPriorityQueuePool *queuePool, Correction *correction, const int *xcoordinates, + const int *xcoordinates, const int *ycoordinates, const int *codes, const int codesSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, unsigned short *outWords, int *frequencies) { + const bool useFullEditDistance, unsigned short *outWords, int *frequencies, + int *outputTypes) const { - queuePool->clearAll(); - Correction* masterCorrection = correction; - correction->resetCorrection(); + WordsPriorityQueuePool queuePool(MAX_WORDS, SUB_QUEUE_MAX_WORDS, MAX_WORD_LENGTH); + queuePool.clearAll(); + Correction masterCorrection; + masterCorrection.resetCorrection(); if (BinaryFormat::REQUIRES_GERMAN_UMLAUT_PROCESSING & FLAGS) { // Incrementally tune the word and try all possibilities int codesBuffer[getCodesBufferSize(codes, codesSize)]; @@ -185,8 +188,8 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, int yCoordinatesBuffer[codesSize]; getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter, - useFullEditDistance, codes, codesSize, 0, codesBuffer, masterCorrection, - queuePool, GERMAN_UMLAUT_DIGRAPHS, + useFullEditDistance, codes, codesSize, 0, codesBuffer, &masterCorrection, + &queuePool, GERMAN_UMLAUT_DIGRAPHS, sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0])); } else if (BinaryFormat::REQUIRES_FRENCH_LIGATURES_PROCESSING & FLAGS) { int codesBuffer[getCodesBufferSize(codes, codesSize)]; @@ -194,33 +197,33 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, int yCoordinatesBuffer[codesSize]; getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter, - useFullEditDistance, codes, codesSize, 0, codesBuffer, masterCorrection, - queuePool, FRENCH_LIGATURES_DIGRAPHS, + useFullEditDistance, codes, codesSize, 0, codesBuffer, &masterCorrection, + &queuePool, FRENCH_LIGATURES_DIGRAPHS, sizeof(FRENCH_LIGATURES_DIGRAPHS) / sizeof(FRENCH_LIGATURES_DIGRAPHS[0])); } else { // Normal processing getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, - bigramMap, bigramFilter, useFullEditDistance, masterCorrection, queuePool); + bigramMap, bigramFilter, useFullEditDistance, &masterCorrection, &queuePool); } PROF_START(20); if (DEBUG_DICT) { - float ns = queuePool->getMasterQueue()->getHighestNormalizedScore( - proximityInfo->getPrimaryInputWord(), codesSize, 0, 0, 0); + float ns = queuePool.getMasterQueue()->getHighestNormalizedScore( + masterCorrection.getPrimaryInputWord(), codesSize, 0, 0, 0); ns += 0; AKLOGI("Max normalized score = %f", ns); } const int suggestedWordsCount = - queuePool->getMasterQueue()->outputSuggestions( - proximityInfo->getPrimaryInputWord(), codesSize, frequencies, outWords); + queuePool.getMasterQueue()->outputSuggestions(masterCorrection.getPrimaryInputWord(), + codesSize, frequencies, outWords, outputTypes); if (DEBUG_DICT) { - float ns = queuePool->getMasterQueue()->getHighestNormalizedScore( - proximityInfo->getPrimaryInputWord(), codesSize, 0, 0, 0); + float ns = queuePool.getMasterQueue()->getHighestNormalizedScore( + masterCorrection.getPrimaryInputWord(), codesSize, 0, 0, 0); ns += 0; AKLOGI("Returning %d words", suggestedWordsCount); /// Print the returned words for (int j = 0; j < suggestedWordsCount; ++j) { - short unsigned int* w = outWords + j * MAX_WORD_LENGTH; + short unsigned int *w = outWords + j * MAX_WORD_LENGTH; char s[MAX_WORD_LENGTH]; for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i]; (void)s; @@ -234,8 +237,9 @@ int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, const int *codes, - const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, Correction *correction, WordsPriorityQueuePool *queuePool) { + const int inputSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, Correction *correction, + WordsPriorityQueuePool *queuePool) const { PROF_OPEN; PROF_START(0); @@ -243,7 +247,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, PROF_START(1); getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, bigramMap, bigramFilter, - useFullEditDistance, inputLength, correction, queuePool); + useFullEditDistance, inputSize, correction, queuePool); PROF_END(1); PROF_START(2); @@ -256,10 +260,10 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, PROF_START(4); bool hasAutoCorrectionCandidate = false; - WordsPriorityQueue* masterQueue = queuePool->getMasterQueue(); + WordsPriorityQueue *masterQueue = queuePool->getMasterQueue(); if (masterQueue->size() > 0) { float nsForMaster = masterQueue->getHighestNormalizedScore( - proximityInfo->getPrimaryInputWord(), inputLength, 0, 0, 0); + correction->getPrimaryInputWord(), inputSize, 0, 0, 0); hasAutoCorrectionCandidate = (nsForMaster > START_TWO_WORDS_CORRECTION_THRESHOLD); } PROF_END(4); @@ -267,9 +271,9 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, PROF_START(5); // Multiple word suggestions if (SUGGEST_MULTIPLE_WORDS - && inputLength >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) { + && inputSize >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) { getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, inputLength, correction, queuePool, + useFullEditDistance, inputSize, correction, queuePool, hasAutoCorrectionCandidate); } PROF_END(5); @@ -281,18 +285,18 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, if (DEBUG_DICT) { queuePool->dumpSubQueue1TopSuggestions(); for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { - WordsPriorityQueue* queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i); + WordsPriorityQueue *queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i); if (queue->size() > 0) { - WordsPriorityQueue::SuggestedWord* sw = queue->top(); + WordsPriorityQueue::SuggestedWord *sw = queue->top(); const int score = sw->mScore; - const unsigned short* word = sw->mWord; + const unsigned short *word = sw->mWord; const int wordLength = sw->mWordLength; float ns = Correction::RankingAlgorithm::calcNormalizedScore( - proximityInfo->getPrimaryInputWord(), i, word, wordLength, score); + correction->getPrimaryInputWord(), i, word, wordLength, score); ns += 0; AKLOGI("--- TOP SUB WORDS for %d --- %d %f [%d]", i, score, ns, (ns > TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD)); - DUMP_WORD(proximityInfo->getPrimaryInputWord(), i); + DUMP_WORD(correction->getPrimaryInputWord(), i); DUMP_WORD(word, wordLength); } } @@ -300,14 +304,15 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, } void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates, - const int *yCoordinates, const int *codes, const int inputLength, Correction *correction) { + const int *yCoordinates, const int *codes, const int inputSize, + Correction *correction) const { if (DEBUG_DICT) { AKLOGI("initSuggest"); - DUMP_WORD_INT(codes, inputLength); + DUMP_WORD_INT(codes, inputSize); } - proximityInfo->setInputParams(codes, inputLength, xCoordinates, yCoordinates); - const int maxDepth = min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH); - correction->initCorrection(proximityInfo, inputLength, maxDepth); + correction->initInputParams(proximityInfo, codes, inputSize, xCoordinates, yCoordinates); + const int maxDepth = min(inputSize * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH); + correction->initCorrection(proximityInfo, inputSize, maxDepth); } static const char QUOTE = '\''; @@ -316,17 +321,17 @@ static const char SPACE = ' '; void UnigramDictionary::getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, const int *codes, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, const int inputLength, - Correction *correction, WordsPriorityQueuePool *queuePool) { - initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputLength, correction); - getSuggestionCandidates(useFullEditDistance, inputLength, bigramMap, bigramFilter, correction, + const bool useFullEditDistance, const int inputSize, + Correction *correction, WordsPriorityQueuePool *queuePool) const { + initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputSize, correction); + getSuggestionCandidates(useFullEditDistance, inputSize, bigramMap, bigramFilter, correction, queuePool, true /* doAutoCompletion */, DEFAULT_MAX_ERRORS, FIRST_WORD_INDEX); } void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, - const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const int inputSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, Correction *correction, WordsPriorityQueuePool *queuePool, - const bool doAutoCompletion, const int maxErrors, const int currentWordIndex) { + const bool doAutoCompletion, const int maxErrors, const int currentWordIndex) const { uint8_t totalTraverseCount = correction->pushAndGetTotalTraverseCount(); if (DEBUG_DICT) { AKLOGI("Traverse count %d", totalTraverseCount); @@ -346,7 +351,7 @@ void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition); int outputIndex = 0; - correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0)); + correction->initCorrectionState(rootPosition, childCount, (inputSize <= 0)); // Depth first search while (outputIndex >= 0) { @@ -374,38 +379,53 @@ void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, inline void UnigramDictionary::onTerminal(const int probability, const TerminalAttributes& terminalAttributes, Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, - const int currentWordIndex) { + const int currentWordIndex) const { const int inputIndex = correction->getInputIndex(); const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT; int wordLength; - unsigned short* wordPointer; + unsigned short *wordPointer; if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) { WordsPriorityQueue *masterQueue = queuePool->getMasterQueue(); const int finalProbability = correction->getFinalProbability(probability, &wordPointer, &wordLength); - if (finalProbability != NOT_A_PROBABILITY) { - addWord(wordPointer, wordLength, finalProbability, masterQueue); - - const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0; - // Please note that the shortcut candidates will be added to the master queue only. - TerminalAttributes::ShortcutIterator iterator = - terminalAttributes.getShortcutIterator(); - while (iterator.hasNextShortcutTarget()) { - // TODO: addWord only supports weak ordering, meaning we have no means - // to control the order of the shortcuts relative to one another or to the word. - // We need to either modulate the probability of each shortcut according - // to its own shortcut probability or to make the queue - // so that the insert order is protected inside the queue for words - // with the same score. For the moment we use -1 to make sure the shortcut will - // never be in front of the word. - uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL]; - const int shortcutTargetStringLength = iterator.getNextShortcutTarget( - MAX_WORD_LENGTH_INTERNAL, shortcutTarget); - addWord(shortcutTarget, shortcutTargetStringLength, shortcutProbability, - masterQueue); + + if (0 != finalProbability) { + // If the probability is 0, we don't want to add this word. However we still + // want to add its shortcuts (including a possible whitelist entry) if any. + addWord(wordPointer, wordLength, finalProbability, masterQueue, + Dictionary::KIND_CORRECTION); + } + + const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0; + // Please note that the shortcut candidates will be added to the master queue only. + TerminalAttributes::ShortcutIterator iterator = + terminalAttributes.getShortcutIterator(); + while (iterator.hasNextShortcutTarget()) { + // TODO: addWord only supports weak ordering, meaning we have no means + // to control the order of the shortcuts relative to one another or to the word. + // We need to either modulate the probability of each shortcut according + // to its own shortcut probability or to make the queue + // so that the insert order is protected inside the queue for words + // with the same score. For the moment we use -1 to make sure the shortcut will + // never be in front of the word. + uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL]; + int shortcutFrequency; + const int shortcutTargetStringLength = iterator.getNextShortcutTarget( + MAX_WORD_LENGTH_INTERNAL, shortcutTarget, &shortcutFrequency); + int shortcutScore; + int kind; + if (shortcutFrequency == BinaryFormat::WHITELIST_SHORTCUT_FREQUENCY + && correction->sameAsTyped()) { + shortcutScore = S_INT_MAX; + kind = Dictionary::KIND_WHITELIST; + } else { + shortcutScore = shortcutProbability; + kind = Dictionary::KIND_CORRECTION; } + addWord(shortcutTarget, shortcutTargetStringLength, shortcutScore, + masterQueue, kind); } } @@ -419,18 +439,18 @@ inline void UnigramDictionary::onTerminal(const int probability, } const int finalProbability = correction->getFinalProbabilityForSubQueue( probability, &wordPointer, &wordLength, inputIndex); - addWord(wordPointer, wordLength, finalProbability, subQueue); + addWord(wordPointer, wordLength, finalProbability, subQueue, Dictionary::KIND_CORRECTION); } } int UnigramDictionary::getSubStringSuggestion( ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, const int *codes, const bool useFullEditDistance, Correction *correction, - WordsPriorityQueuePool* queuePool, const int inputLength, + WordsPriorityQueuePool *queuePool, const int inputSize, const bool hasAutoCorrectionCandidate, const int currentWordIndex, const int inputWordStartPos, const int inputWordLength, const int outputWordStartPos, const bool isSpaceProximity, int *freqArray, - int*wordLengthArray, unsigned short* outputWord, int *outputWordLength) { + int*wordLengthArray, unsigned short *outputWord, int *outputWordLength) const { if (inputWordLength > MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH) { return FLAG_MULTIPLE_SUGGEST_ABORT; } @@ -473,17 +493,18 @@ int UnigramDictionary::getSubStringSuggestion( // TODO: Remove the safety net above // ////////////////////////////////////////////// - unsigned short* tempOutputWord = 0; + unsigned short *tempOutputWord = 0; int nextWordLength = 0; // TODO: Optimize init suggestion initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, - inputLength, correction); + inputSize, correction); + unsigned short word[MAX_WORD_LENGTH_INTERNAL]; int freq = getMostFrequentWordLike( - inputWordStartPos, inputWordLength, proximityInfo, mWord); + inputWordStartPos, inputWordLength, correction, word); if (freq > 0) { nextWordLength = inputWordLength; - tempOutputWord = mWord; + tempOutputWord = word; } else if (!hasAutoCorrectionCandidate) { if (inputWordStartPos > 0) { const int offset = inputWordStartPos; @@ -503,14 +524,14 @@ int UnigramDictionary::getSubStringSuggestion( } } } - WordsPriorityQueue* queue = queuePool->getSubQueue(currentWordIndex, inputWordLength); + WordsPriorityQueue *queue = queuePool->getSubQueue(currentWordIndex, inputWordLength); // TODO: Return the correct value depending on doAutoCompletion if (!queue || queue->size() <= 0) { return FLAG_MULTIPLE_SUGGEST_ABORT; } int score = 0; const float ns = queue->getHighestNormalizedScore( - proximityInfo->getPrimaryInputWord(), inputWordLength, + correction->getPrimaryInputWord(), inputWordLength, &tempOutputWord, &score, &nextWordLength); if (DEBUG_DICT) { AKLOGI("NS(%d) = %f, Score = %d", currentWordIndex, ns, score); @@ -545,7 +566,7 @@ int UnigramDictionary::getSubStringSuggestion( *outputWordLength = tempOutputWordLength; } - if ((inputWordStartPos + inputWordLength) < inputLength) { + if ((inputWordStartPos + inputWordLength) < inputSize) { if (outputWordStartPos + nextWordLength >= MAX_WORD_LENGTH) { return FLAG_MULTIPLE_SUGGEST_SKIP; } @@ -564,31 +585,32 @@ int UnigramDictionary::getSubStringSuggestion( freqArray[i], wordLengthArray[i]); } AKLOGI("Split two words: freq = %d, length = %d, %d, isSpace ? %d", pairFreq, - inputLength, tempOutputWordLength, isSpaceProximity); + inputSize, tempOutputWordLength, isSpaceProximity); } - addWord(outputWord, tempOutputWordLength, pairFreq, queuePool->getMasterQueue()); + addWord(outputWord, tempOutputWordLength, pairFreq, queuePool->getMasterQueue(), + Dictionary::KIND_CORRECTION); } return FLAG_MULTIPLE_SUGGEST_CONTINUE; } void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, const int *codes, - const bool useFullEditDistance, const int inputLength, - Correction *correction, WordsPriorityQueuePool* queuePool, + const bool useFullEditDistance, const int inputSize, + Correction *correction, WordsPriorityQueuePool *queuePool, const bool hasAutoCorrectionCandidate, const int startInputPos, const int startWordIndex, - const int outputWordLength, int *freqArray, int* wordLengthArray, - unsigned short* outputWord) { + const int outputWordLength, int *freqArray, int *wordLengthArray, + unsigned short *outputWord) const { if (startWordIndex >= (MULTIPLE_WORDS_SUGGESTION_MAX_WORDS - 1)) { // Return if the last word index return; } if (startWordIndex >= 1 && (hasAutoCorrectionCandidate - || inputLength < MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION)) { + || inputSize < MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION)) { // Do not suggest 3+ words if already has auto correction candidate return; } - for (int i = startInputPos + 1; i < inputLength; ++i) { + for (int i = startInputPos + 1; i < inputSize; ++i) { if (DEBUG_CORRECTION_FREQ) { AKLOGI("Multi words(%d), start in %d sep %d start out %d", startWordIndex, startInputPos, i, outputWordLength); @@ -599,7 +621,7 @@ void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, int inputWordStartPos = startInputPos; int inputWordLength = i - startInputPos; const int suggestionFlag = getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, - codes, useFullEditDistance, correction, queuePool, inputLength, + codes, useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate, startWordIndex, inputWordStartPos, inputWordLength, outputWordLength, true /* not used */, freqArray, wordLengthArray, outputWord, &tempOutputWordLength); @@ -616,14 +638,14 @@ void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, // Next word // Missing space inputWordStartPos = i; - inputWordLength = inputLength - i; + inputWordLength = inputSize - i; if(getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, correction, queuePool, inputLength, hasAutoCorrectionCandidate, + useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate, startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength, false /* missing space */, freqArray, wordLengthArray, outputWord, 0) != FLAG_MULTIPLE_SUGGEST_CONTINUE) { getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, inputLength, correction, queuePool, + useFullEditDistance, inputSize, correction, queuePool, hasAutoCorrectionCandidate, inputWordStartPos, startWordIndex + 1, tempOutputWordLength, freqArray, wordLengthArray, outputWord); } @@ -646,7 +668,7 @@ void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, AKLOGI("Do mistyped space correction"); } getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, correction, queuePool, inputLength, hasAutoCorrectionCandidate, + useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate, startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength, true /* mistyped space */, freqArray, wordLengthArray, outputWord, 0); } @@ -654,10 +676,10 @@ void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, const int *codes, - const bool useFullEditDistance, const int inputLength, - Correction *correction, WordsPriorityQueuePool* queuePool, - const bool hasAutoCorrectionCandidate) { - if (inputLength >= MAX_WORD_LENGTH) return; + const bool useFullEditDistance, const int inputSize, + Correction *correction, WordsPriorityQueuePool *queuePool, + const bool hasAutoCorrectionCandidate) const { + if (inputSize >= MAX_WORD_LENGTH) return; if (DEBUG_DICT) { AKLOGI("--- Suggest multiple words"); } @@ -670,7 +692,7 @@ void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximit const int startInputPos = 0; const int startWordIndex = 0; getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, inputLength, correction, queuePool, hasAutoCorrectionCandidate, + useFullEditDistance, inputSize, correction, queuePool, hasAutoCorrectionCandidate, startInputPos, startWordIndex, outputWordLength, freqArray, wordLengthArray, outputWord); } @@ -678,13 +700,13 @@ void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximit // Wrapper for getMostFrequentWordLikeInner, which matches it to the previous // interface. inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex, - const int inputLength, ProximityInfo *proximityInfo, unsigned short *word) { - uint16_t inWord[inputLength]; + const int inputSize, Correction *correction, unsigned short *word) const { + uint16_t inWord[inputSize]; - for (int i = 0; i < inputLength; ++i) { - inWord[i] = (uint16_t)proximityInfo->getPrimaryCharAt(startInputIndex + i); + for (int i = 0; i < inputSize; ++i) { + inWord[i] = (uint16_t)correction->getPrimaryCharAt(startInputIndex + i); } - return getMostFrequentWordLikeInner(inWord, inputLength, word); + return getMostFrequentWordLikeInner(inWord, inputSize, word); } // This function will take the position of a character array within a CharGroup, @@ -700,10 +722,10 @@ inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex, // In and out parameters may point to the same location. This function takes care // not to use any input parameters after it wrote into its outputs. static inline bool testCharGroupForContinuedLikeness(const uint8_t flags, - const uint8_t* const root, const int startPos, - const uint16_t* const inWord, const int startInputIndex, - int32_t* outNewWord, int* outInputIndex, int* outPos) { - const bool hasMultipleChars = (0 != (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags)); + const uint8_t *const root, const int startPos, + const uint16_t *const inWord, const int startInputIndex, + int32_t *outNewWord, int *outInputIndex, int *outPos) { + const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); int pos = startPos; int32_t character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); int32_t baseChar = toBaseLowerCase(character); @@ -738,8 +760,8 @@ static inline bool testCharGroupForContinuedLikeness(const uint8_t flags, // It will compare the frequency to the max frequency, and if greater, will // copy the word into the output buffer. In output value maxFreq, it will // write the new maximum frequency if it changed. -static inline void onTerminalWordLike(const int freq, int32_t* newWord, const int length, - short unsigned int* outWord, int* maxFreq) { +static inline void onTerminalWordLike(const int freq, int32_t *newWord, const int length, + short unsigned int *outWord, int *maxFreq) { if (freq > *maxFreq) { for (int q = 0; q < length; ++q) outWord[q] = newWord[q]; @@ -750,22 +772,25 @@ static inline void onTerminalWordLike(const int freq, int32_t* newWord, const in // Will find the highest frequency of the words like the one passed as an argument, // that is, everything that only differs by case/accents. -int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWord, - const int length, short unsigned int* outWord) { +int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t *const inWord, + const int length, short unsigned int *outWord) const { int32_t newWord[MAX_WORD_LENGTH_INTERNAL]; int depth = 0; int maxFreq = -1; - const uint8_t* const root = DICT_ROOT; + const uint8_t *const root = DICT_ROOT; + int stackChildCount[MAX_WORD_LENGTH_INTERNAL]; + int stackInputIndex[MAX_WORD_LENGTH_INTERNAL]; + int stackSiblingPos[MAX_WORD_LENGTH_INTERNAL]; int startPos = 0; - mStackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos); - mStackInputIndex[0] = 0; - mStackSiblingPos[0] = startPos; + stackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos); + stackInputIndex[0] = 0; + stackSiblingPos[0] = startPos; while (depth >= 0) { - const int charGroupCount = mStackChildCount[depth]; - int pos = mStackSiblingPos[depth]; + const int charGroupCount = stackChildCount[depth]; + int pos = stackSiblingPos[depth]; for (int charGroupIndex = charGroupCount - 1; charGroupIndex >= 0; --charGroupIndex) { - int inputIndex = mStackInputIndex[depth]; + int inputIndex = stackInputIndex[depth]; const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); // Test whether all chars in this group match with the word we are searching for. If so, // we want to traverse its children (or if the length match, evaluate its frequency). @@ -773,7 +798,7 @@ int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWor // into inputIndex if there is a match. const bool isAlike = testCharGroupForContinuedLikeness(flags, root, pos, inWord, inputIndex, newWord, &inputIndex, &pos); - if (isAlike && (FLAG_IS_TERMINAL & flags) && (inputIndex == length)) { + if (isAlike && (BinaryFormat::FLAG_IS_TERMINAL & flags) && (inputIndex == length)) { const int frequency = BinaryFormat::readFrequencyWithoutMovingPointer(root, pos); onTerminalWordLike(frequency, newWord, inputIndex, outWord, &maxFreq); } @@ -785,15 +810,15 @@ int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWor // anyway, so don't traverse unless inputIndex < length. if (isAlike && (-1 != childrenNodePos) && (inputIndex < length)) { // Save position for this depth, to get back to this once children are done - mStackChildCount[depth] = charGroupIndex; - mStackSiblingPos[depth] = siblingPos; + stackChildCount[depth] = charGroupIndex; + stackSiblingPos[depth] = siblingPos; // Prepare stack values for next depth ++depth; int childrenPos = childrenNodePos; - mStackChildCount[depth] = + stackChildCount[depth] = BinaryFormat::getGroupCountAndForwardPointer(root, &childrenPos); - mStackSiblingPos[depth] = childrenPos; - mStackInputIndex[depth] = inputIndex; + stackSiblingPos[depth] = childrenPos; + stackInputIndex[depth] = inputIndex; pos = childrenPos; // Go to the next depth level. ++depth; @@ -808,14 +833,15 @@ int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWor return maxFreq; } -int UnigramDictionary::getFrequency(const int32_t* const inWord, const int length) const { - const uint8_t* const root = DICT_ROOT; - int pos = BinaryFormat::getTerminalPosition(root, inWord, length); +int UnigramDictionary::getFrequency(const int32_t *const inWord, const int length) const { + const uint8_t *const root = DICT_ROOT; + int pos = BinaryFormat::getTerminalPosition(root, inWord, length, + false /* forceLowerCaseSearch */); if (NOT_VALID_WORD == pos) { return NOT_A_PROBABILITY; } const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); - const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags)); + const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); if (hasMultipleChars) { pos = BinaryFormat::skipOtherCharacters(root, pos); } else { @@ -848,7 +874,7 @@ int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offs inline bool UnigramDictionary::processCurrentNode(const int initialPos, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, Correction *correction, int *newCount, int *newChildrenPosition, int *nextSiblingPosition, - WordsPriorityQueuePool *queuePool, const int currentWordIndex) { + WordsPriorityQueuePool *queuePool, const int currentWordIndex) const { if (DEBUG_DICT) { correction->checkState(); } @@ -863,8 +889,8 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos, // - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children) // - FLAG_HAS_BIGRAMS: whether this node has bigrams or not const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos); - const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags)); - const bool isTerminalNode = (0 != (FLAG_IS_TERMINAL & flags)); + const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); + const bool isTerminalNode = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags)); bool needsToInvokeOnTerminal = false; @@ -982,5 +1008,4 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos, *newChildrenPosition = childrenPos; return true; } - } // namespace latinime |