aboutsummaryrefslogtreecommitdiffstats
path: root/native/src
diff options
context:
space:
mode:
Diffstat (limited to 'native/src')
-rw-r--r--native/src/bigram_dictionary.cpp202
-rw-r--r--native/src/binary_format.h147
-rw-r--r--native/src/correction_state.cpp210
-rw-r--r--native/src/correction_state.h32
-rw-r--r--native/src/dictionary.cpp8
-rw-r--r--native/src/dictionary.h2
-rw-r--r--native/src/unigram_dictionary.cpp235
-rw-r--r--native/src/unigram_dictionary.h15
8 files changed, 494 insertions, 357 deletions
diff --git a/native/src/bigram_dictionary.cpp b/native/src/bigram_dictionary.cpp
index 6ed4d0982..c340c6c1a 100644
--- a/native/src/bigram_dictionary.cpp
+++ b/native/src/bigram_dictionary.cpp
@@ -21,13 +21,14 @@
#include "bigram_dictionary.h"
#include "dictionary.h"
+#include "binary_format.h"
namespace latinime {
BigramDictionary::BigramDictionary(const unsigned char *dict, int maxWordLength,
int maxAlternatives, const bool isLatestDictVersion, const bool hasBigram,
Dictionary *parentDictionary)
- : DICT(dict), MAX_WORD_LENGTH(maxWordLength),
+ : DICT(dict + NEW_DICTIONARY_HEADER_SIZE), MAX_WORD_LENGTH(maxWordLength),
MAX_ALTERNATIVES(maxAlternatives), IS_LATEST_DICT_VERSION(isLatestDictVersion),
HAS_BIGRAM(hasBigram), mParentDictionary(parentDictionary) {
if (DEBUG_DICT) {
@@ -82,169 +83,64 @@ bool BigramDictionary::addWordBigram(unsigned short *word, int length, int frequ
return false;
}
-int BigramDictionary::getBigramAddress(int *pos, bool advance) {
- int address = 0;
-
- address += (DICT[*pos] & 0x3F) << 16;
- address += (DICT[*pos + 1] & 0xFF) << 8;
- address += (DICT[*pos + 2] & 0xFF);
-
- if (advance) {
- *pos += 3;
- }
-
- return address;
-}
-
-int BigramDictionary::getBigramFreq(int *pos) {
- int freq = DICT[(*pos)++] & FLAG_BIGRAM_FREQ;
-
- return freq;
-}
-
-
+/* Parameters :
+ * prevWord: the word before, the one for which we need to look up bigrams.
+ * prevWordLength: its length.
+ * codes: what user typed, in the same format as for UnigramDictionary::getSuggestions.
+ * codesSize: the size of the codes array.
+ * bigramChars: an array for output, at the same format as outwords for getSuggestions.
+ * bigramFreq: an array to output frequencies.
+ * maxWordLength: the maximum size of a word.
+ * maxBigrams: the maximum number of bigrams fitting in the bigramChars array.
+ * maxAlteratives: unused.
+ * This method returns the number of bigrams this word has, for backward compatibility.
+ * Note: this is not the number of bigrams output in the array, which is the number of
+ * bigrams this word has WHOSE first letter also matches the letter the user typed.
+ * TODO: this may not be a sensible thing to do. It makes sense when the bigrams are
+ * used to match the first letter of the second word, but once the user has typed more
+ * and the bigrams are used to boost unigram result scores, it makes little sense to
+ * reduce their scope to the ones that match the first letter.
+ */
int BigramDictionary::getBigrams(unsigned short *prevWord, int prevWordLength, int *codes,
int codesSize, unsigned short *bigramChars, int *bigramFreq, int maxWordLength,
int maxBigrams, int maxAlternatives) {
+ // TODO: remove unused arguments, and refrain from storing stuff in members of this class
+ // TODO: have "in" arguments before "out" ones, and make out args explicit in the name
mBigramFreq = bigramFreq;
mBigramChars = bigramChars;
mInputCodes = codes;
- mInputLength = codesSize;
mMaxBigrams = maxBigrams;
- if (HAS_BIGRAM && IS_LATEST_DICT_VERSION) {
- int pos = mParentDictionary->getBigramPosition(prevWord, prevWordLength);
- if (DEBUG_DICT) {
- LOGI("Pos -> %d", pos);
- }
- if (pos < 0) {
- return 0;
- }
-
- int bigramCount = 0;
- int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0 && bigramCount < maxBigrams) {
- int bigramAddress = getBigramAddress(&pos, true);
- int frequency = (FLAG_BIGRAM_FREQ & DICT[pos]);
- // search for all bigrams and store them
- searchForTerminalNode(bigramAddress, frequency);
- nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED);
- bigramCount++;
- }
- }
+ const uint8_t* const root = DICT;
+ int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength);
- return bigramCount;
+ if (NOT_VALID_WORD == pos) return 0;
+ const int flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
+ if (0 == (flags & UnigramDictionary::FLAG_HAS_BIGRAMS)) return 0;
+ if (0 == (flags & UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS)) {
+ BinaryFormat::getCharCodeAndForwardPointer(root, &pos);
+ } else {
+ pos = BinaryFormat::skipOtherCharacters(root, pos);
}
- return 0;
-}
-
-void BigramDictionary::searchForTerminalNode(int addressLookingFor, int frequency) {
- // track word with such address and store it in an array
- unsigned short word[MAX_WORD_LENGTH];
-
- int pos;
- int followDownBranchAddress = DICTIONARY_HEADER_SIZE;
- bool found = false;
- char followingChar = ' ';
- int depth = -1;
-
- while(!found) {
- bool followDownAddressSearchStop = false;
- bool firstAddress = true;
- bool haveToSearchAll = true;
-
- if (depth < MAX_WORD_LENGTH && depth >= 0) {
- word[depth] = (unsigned short) followingChar;
- }
- pos = followDownBranchAddress; // pos start at count
- int count = DICT[pos] & 0xFF;
- if (DEBUG_DICT) {
- LOGI("count - %d",count);
- }
- pos++;
- for (int i = 0; i < count; i++) {
- // pos at data
- pos++;
- // pos now at flag
- if (!getFirstBitOfByte(&pos)) { // non-terminal
- if (!followDownAddressSearchStop) {
- int addr = getBigramAddress(&pos, false);
- if (addr > addressLookingFor) {
- followDownAddressSearchStop = true;
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- } else if (!haveToSearchAll) {
- break;
- }
- } else {
- followDownBranchAddress = addr;
- followingChar = (char)(0xFF & DICT[pos-1]);
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = false;
- }
- }
- }
- pos += 3;
- } else if (getFirstBitOfByte(&pos)) { // terminal
- if (addressLookingFor == (pos-1)) { // found !!
- depth++;
- word[depth] = (0xFF & DICT[pos-1]);
- found = true;
- break;
- }
- if (getSecondBitOfByte(&pos)) { // address + freq (4 byte)
- if (!followDownAddressSearchStop) {
- int addr = getBigramAddress(&pos, false);
- if (addr > addressLookingFor) {
- followDownAddressSearchStop = true;
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- } else if (!haveToSearchAll) {
- break;
- }
- } else {
- followDownBranchAddress = addr;
- followingChar = (char)(0xFF & DICT[pos-1]);
- if (firstAddress) {
- firstAddress = false;
- haveToSearchAll = true;
- }
- }
- }
- pos += 4;
- } else { // freq only (2 byte)
- pos += 2;
- }
-
- // skipping bigram
- int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ);
- if (bigramExist > 0) {
- int nextBigramExist = 1;
- while (nextBigramExist > 0) {
- pos += 3;
- nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED);
- }
- } else {
- pos++;
- }
- }
+ pos = BinaryFormat::skipChildrenPosition(flags, pos);
+ pos = BinaryFormat::skipFrequency(flags, pos);
+ int bigramFlags;
+ int bigramCount = 0;
+ do {
+ bigramFlags = BinaryFormat::getFlagsAndForwardPointer(root, &pos);
+ uint16_t bigramBuffer[MAX_WORD_LENGTH];
+ const int bigramPos = BinaryFormat::getAttributeAddressAndForwardPointer(root, bigramFlags,
+ &pos);
+ const int length = BinaryFormat::getWordAtAddress(root, bigramPos, MAX_WORD_LENGTH,
+ bigramBuffer);
+
+ if (checkFirstCharacter(bigramBuffer)) {
+ const int frequency = UnigramDictionary::MASK_ATTRIBUTE_FREQUENCY & bigramFlags;
+ addWordBigram(bigramBuffer, length, frequency);
}
- depth++;
- if (followDownBranchAddress == 0) {
- if (DEBUG_DICT) {
- LOGI("ERROR!!! Cannot find bigram!!");
- }
- break;
- }
- }
- if (checkFirstCharacter(word)) {
- addWordBigram(word, depth, frequency);
- }
+ ++bigramCount;
+ } while (0 != (UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags));
+ return bigramCount;
}
bool BigramDictionary::checkFirstCharacter(unsigned short *word) {
diff --git a/native/src/binary_format.h b/native/src/binary_format.h
index a946b1ee3..6f65088db 100644
--- a/native/src/binary_format.h
+++ b/native/src/binary_format.h
@@ -50,6 +50,8 @@ public:
int *pos);
static int getTerminalPosition(const uint8_t* const root, const uint16_t* const inWord,
const int length);
+ static int getWordAtAddress(const uint8_t* const root, const int address, const int maxDepth,
+ uint16_t* outWord);
};
inline int BinaryFormat::detectFormat(const uint8_t* const dict) {
@@ -290,6 +292,151 @@ inline int BinaryFormat::getTerminalPosition(const uint8_t* const root,
}
}
+// This function searches for a terminal in the dictionary by its address.
+// Due to the fact that words are ordered in the dictionary in a strict breadth-first order,
+// it is possible to check for this with advantageous complexity. For each node, we search
+// for groups with children and compare the children address with the address we look for.
+// When we shoot the address we look for, it means the word we look for is in the children
+// of the previous group. The only tricky part is the fact that if we arrive at the end of a
+// node with the last group's children address still less than what we are searching for, we
+// must descend the last group's children (for example, if the word we are searching for starts
+// with a z, it's the last group of the root node, so all children addresses will be smaller
+// than the address we look for, and we have to descend the z node).
+/* Parameters :
+ * root: the dictionary buffer
+ * address: the byte position of the last chargroup of the word we are searching for (this is
+ * what is stored as the "bigram address" in each bigram)
+ * outword: an array to write the found word, with MAX_WORD_LENGTH size.
+ * Return value : the length of the word, of 0 if the word was not found.
+ */
+inline int BinaryFormat::getWordAtAddress(const uint8_t* const root, const int address,
+ const int maxDepth, uint16_t* outWord) {
+ int pos = 0;
+ int wordPos = 0;
+
+ // One iteration of the outer loop iterates through nodes. As stated above, we will only
+ // traverse nodes that are actually a part of the terminal we are searching, so each time
+ // we enter this loop we are one depth level further than last time.
+ // The only reason we count nodes is because we want to reduce the probability of infinite
+ // looping in case there is a bug. Since we know there is an upper bound to the depth we are
+ // supposed to traverse, it does not hurt to count iterations.
+ for (int loopCount = maxDepth; loopCount > 0; --loopCount) {
+ int lastCandidateGroupPos = 0;
+ // Let's loop through char groups in this node searching for either the terminal
+ // or one of its ascendants.
+ for (int charGroupCount = getGroupCountAndForwardPointer(root, &pos); charGroupCount > 0;
+ --charGroupCount) {
+ const int startPos = pos;
+ const uint8_t flags = getFlagsAndForwardPointer(root, &pos);
+ const int32_t character = getCharCodeAndForwardPointer(root, &pos);
+ if (address == startPos) {
+ // We found the address. Copy the rest of the word in the buffer and return
+ // the length.
+ outWord[wordPos] = character;
+ if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags) {
+ int32_t nextChar = getCharCodeAndForwardPointer(root, &pos);
+ // We count chars in order to avoid infinite loops if the file is broken or
+ // if there is some other bug
+ int charCount = maxDepth;
+ while (-1 != nextChar && --charCount > 0) {
+ outWord[++wordPos] = nextChar;
+ nextChar = getCharCodeAndForwardPointer(root, &pos);
+ }
+ }
+ return ++wordPos;
+ }
+ // We need to skip past this char group, so skip any remaining chars after the
+ // first and possibly the frequency.
+ if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags) {
+ pos = skipOtherCharacters(root, pos);
+ }
+ pos = skipFrequency(flags, pos);
+
+ // The fact that this group has children is very important. Since we already know
+ // that this group does not match, if it has no children we know it is irrelevant
+ // to what we are searching for.
+ const bool hasChildren = (UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_NOADDRESS !=
+ (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags));
+ // We will write in `found' whether we have passed the children address we are
+ // searching for. For example if we search for "beer", the children of b are less
+ // than the address we are searching for and the children of c are greater. When we
+ // come here for c, we realize this is too big, and that we should descend b.
+ bool found;
+ if (hasChildren) {
+ // Here comes the tricky part. First, read the children position.
+ const int childrenPos = readChildrenPosition(root, flags, pos);
+ if (childrenPos > address) {
+ // If the children pos is greater than address, it means the previous chargroup,
+ // which address is stored in lastCandidateGroupPos, was the right one.
+ found = true;
+ } else if (1 >= charGroupCount) {
+ // However if we are on the LAST group of this node, and we have NOT shot the
+ // address we should descend THIS node. So we trick the lastCandidateGroupPos
+ // so that we will descend this node, not the previous one.
+ lastCandidateGroupPos = startPos;
+ found = true;
+ } else {
+ // Else, we should continue looking.
+ found = false;
+ }
+ } else {
+ // Even if we don't have children here, we could still be on the last group of this
+ // node. If this is the case, we should descend the last group that had children,
+ // and their address is already in lastCandidateGroup.
+ found = (1 >= charGroupCount);
+ }
+
+ if (found) {
+ // Okay, we found the group we should descend. Its address is in
+ // the lastCandidateGroupPos variable, so we just re-read it.
+ if (0 != lastCandidateGroupPos) {
+ const uint8_t lastFlags =
+ getFlagsAndForwardPointer(root, &lastCandidateGroupPos);
+ const int32_t lastChar =
+ getCharCodeAndForwardPointer(root, &lastCandidateGroupPos);
+ // We copy all the characters in this group to the buffer
+ outWord[wordPos] = lastChar;
+ if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & lastFlags) {
+ int32_t nextChar =
+ getCharCodeAndForwardPointer(root, &lastCandidateGroupPos);
+ int charCount = maxDepth;
+ while (-1 != nextChar && --charCount > 0) {
+ outWord[++wordPos] = nextChar;
+ nextChar = getCharCodeAndForwardPointer(root, &lastCandidateGroupPos);
+ }
+ }
+ ++wordPos;
+ // Now we only need to branch to the children address. Skip the frequency if
+ // it's there, read pos, and break to resume the search at pos.
+ lastCandidateGroupPos = skipFrequency(lastFlags, lastCandidateGroupPos);
+ pos = readChildrenPosition(root, lastFlags, lastCandidateGroupPos);
+ break;
+ } else {
+ // Here is a little tricky part: we come here if we found out that all children
+ // addresses in this group are bigger than the address we are searching for.
+ // Should we conclude the word is not in the dictionary? No! It could still be
+ // one of the remaining chargroups in this node, so we have to keep looking in
+ // this node until we find it (or we realize it's not there either, in which
+ // case it's actually not in the dictionary). Pass the end of this group, ready
+ // to start the next one.
+ pos = skipChildrenPosAndAttributes(root, flags, pos);
+ }
+ } else {
+ // If we did not find it, we should record the last children address for the next
+ // iteration.
+ if (hasChildren) lastCandidateGroupPos = startPos;
+ // Now skip the end of this group (children pos and the attributes if any) so that
+ // our pos is after the end of this char group, at the start of the next one.
+ pos = skipChildrenPosAndAttributes(root, flags, pos);
+ }
+
+ }
+ }
+ // If we have looked through all the chargroups and found no match, the address is
+ // not the address of a terminal in this dictionary.
+ return 0;
+}
+
} // namespace latinime
#endif // LATINIME_BINARY_FORMAT_H
diff --git a/native/src/correction_state.cpp b/native/src/correction_state.cpp
index aa5efce40..fba947ed4 100644
--- a/native/src/correction_state.cpp
+++ b/native/src/correction_state.cpp
@@ -21,18 +21,26 @@
#define LOG_TAG "LatinIME: correction_state.cpp"
#include "correction_state.h"
+#include "proximity_info.h"
namespace latinime {
-CorrectionState::CorrectionState() {
+CorrectionState::CorrectionState(const int typedLetterMultiplier, const int fullWordMultiplier)
+ : TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier) {
}
-void CorrectionState::setCorrectionParams(const ProximityInfo *pi, const int inputLength,
- const int skipPos, const int excessivePos, const int transposedPos) {
+void CorrectionState::initCorrectionState(const ProximityInfo *pi, const int inputLength) {
mProximityInfo = pi;
+ mInputLength = inputLength;
+}
+
+void CorrectionState::setCorrectionParams(const int skipPos, const int excessivePos,
+ const int transposedPos, const int spaceProximityPos, const int missingSpacePos) {
mSkipPos = skipPos;
mExcessivePos = excessivePos;
mTransposedPos = transposedPos;
+ mSpaceProximityPos = spaceProximityPos;
+ mMissingSpacePos = missingSpacePos;
}
void CorrectionState::checkState() {
@@ -46,7 +54,203 @@ void CorrectionState::checkState() {
}
}
+int CorrectionState::getFreqForSplitTwoWords(const int firstFreq, const int secondFreq) {
+ return CorrectionState::RankingAlgorithm::calcFreqForSplitTwoWords(firstFreq, secondFreq, this);
+}
+
+int CorrectionState::getFinalFreq(const int inputIndex, const int depth, const int matchWeight,
+ const int freq, const bool sameLength) {
+ return CorrectionState::RankingAlgorithm::calculateFinalFreq(inputIndex, depth, matchWeight,
+ freq, sameLength, this);
+}
+
CorrectionState::~CorrectionState() {
}
+/////////////////////////
+// static inline utils //
+/////////////////////////
+
+static const int TWO_31ST_DIV_255 = S_INT_MAX / 255;
+static inline int capped255MultForFullMatchAccentsOrCapitalizationDifference(const int num) {
+ return (num < TWO_31ST_DIV_255 ? 255 * num : S_INT_MAX);
+}
+
+static const int TWO_31ST_DIV_2 = S_INT_MAX / 2;
+inline static void multiplyIntCapped(const int multiplier, int *base) {
+ const int temp = *base;
+ if (temp != S_INT_MAX) {
+ // Branch if multiplier == 2 for the optimization
+ if (multiplier == 2) {
+ *base = TWO_31ST_DIV_2 >= temp ? temp << 1 : S_INT_MAX;
+ } else {
+ const int tempRetval = temp * multiplier;
+ *base = tempRetval >= temp ? tempRetval : S_INT_MAX;
+ }
+ }
+}
+
+inline static int powerIntCapped(const int base, const int n) {
+ if (n == 0) return 1;
+ if (base == 2) {
+ return n < 31 ? 1 << n : S_INT_MAX;
+ } else {
+ int ret = base;
+ for (int i = 1; i < n; ++i) multiplyIntCapped(base, &ret);
+ return ret;
+ }
+}
+
+inline static void multiplyRate(const int rate, int *freq) {
+ if (*freq != S_INT_MAX) {
+ if (*freq > 1000000) {
+ *freq /= 100;
+ multiplyIntCapped(rate, freq);
+ } else {
+ multiplyIntCapped(rate, freq);
+ *freq /= 100;
+ }
+ }
+}
+
+//////////////////////
+// RankingAlgorithm //
+//////////////////////
+
+int CorrectionState::RankingAlgorithm::calculateFinalFreq(const int inputIndex, const int depth,
+ const int matchCount, const int freq, const bool sameLength,
+ const CorrectionState* correctionState) {
+ const int skipPos = correctionState->getSkipPos();
+ const int excessivePos = correctionState->getExcessivePos();
+ const int transposedPos = correctionState->getTransposedPos();
+ const int inputLength = correctionState->mInputLength;
+ const int typedLetterMultiplier = correctionState->TYPED_LETTER_MULTIPLIER;
+ const int fullWordMultiplier = correctionState->FULL_WORD_MULTIPLIER;
+ const ProximityInfo *proximityInfo = correctionState->mProximityInfo;
+ const int matchWeight = powerIntCapped(typedLetterMultiplier, matchCount);
+
+ // TODO: Demote by edit distance
+ int finalFreq = freq * matchWeight;
+ if (skipPos >= 0) {
+ if (inputLength >= 2) {
+ const int demotionRate = WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE
+ * (10 * inputLength - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X)
+ / (10 * inputLength
+ - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X + 10);
+ if (DEBUG_DICT_FULL) {
+ LOGI("Demotion rate for missing character is %d.", demotionRate);
+ }
+ multiplyRate(demotionRate, &finalFreq);
+ } else {
+ finalFreq = 0;
+ }
+ }
+ if (transposedPos >= 0) multiplyRate(
+ WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE, &finalFreq);
+ if (excessivePos >= 0) {
+ multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq);
+ if (!proximityInfo->existsAdjacentProximityChars(inputIndex)) {
+ // If an excessive character is not adjacent to the left char or the right char,
+ // we will demote this word.
+ multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE, &finalFreq);
+ }
+ }
+ int lengthFreq = typedLetterMultiplier;
+ multiplyIntCapped(powerIntCapped(typedLetterMultiplier, depth), &lengthFreq);
+ if (lengthFreq == matchWeight) {
+ // Full exact match
+ if (depth > 1) {
+ if (DEBUG_DICT) {
+ LOGI("Found full matched word.");
+ }
+ multiplyRate(FULL_MATCHED_WORDS_PROMOTION_RATE, &finalFreq);
+ }
+ if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0) {
+ finalFreq = capped255MultForFullMatchAccentsOrCapitalizationDifference(finalFreq);
+ }
+ } else if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0 && depth > 0) {
+ // A word with proximity corrections
+ if (DEBUG_DICT) {
+ LOGI("Found one proximity correction.");
+ }
+ multiplyIntCapped(typedLetterMultiplier, &finalFreq);
+ multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq);
+ }
+ if (DEBUG_DICT) {
+ LOGI("calc: %d, %d", depth, sameLength);
+ }
+ if (sameLength) multiplyIntCapped(fullWordMultiplier, &finalFreq);
+ return finalFreq;
+}
+
+int CorrectionState::RankingAlgorithm::calcFreqForSplitTwoWords(
+ const int firstFreq, const int secondFreq, const CorrectionState* correctionState) {
+ const int spaceProximityPos = correctionState->mSpaceProximityPos;
+ const int missingSpacePos = correctionState->mMissingSpacePos;
+ if (DEBUG_DICT) {
+ int inputCount = 0;
+ if (spaceProximityPos >= 0) ++inputCount;
+ if (missingSpacePos >= 0) ++inputCount;
+ assert(inputCount <= 1);
+ }
+ const bool isSpaceProximity = spaceProximityPos >= 0;
+ const int inputLength = correctionState->mInputLength;
+ const int firstWordLength = isSpaceProximity ? spaceProximityPos : missingSpacePos;
+ const int secondWordLength = isSpaceProximity
+ ? (inputLength - spaceProximityPos - 1)
+ : (inputLength - missingSpacePos);
+ const int typedLetterMultiplier = correctionState->TYPED_LETTER_MULTIPLIER;
+
+ if (firstWordLength == 0 || secondWordLength == 0) {
+ return 0;
+ }
+ const int firstDemotionRate = 100 - 100 / (firstWordLength + 1);
+ int tempFirstFreq = firstFreq;
+ multiplyRate(firstDemotionRate, &tempFirstFreq);
+
+ const int secondDemotionRate = 100 - 100 / (secondWordLength + 1);
+ int tempSecondFreq = secondFreq;
+ multiplyRate(secondDemotionRate, &tempSecondFreq);
+
+ const int totalLength = firstWordLength + secondWordLength;
+
+ // Promote pairFreq with multiplying by 2, because the word length is the same as the typed
+ // length.
+ int totalFreq = tempFirstFreq + tempSecondFreq;
+
+ // This is a workaround to try offsetting the not-enough-demotion which will be done in
+ // calcNormalizedScore in Utils.java.
+ // In calcNormalizedScore the score will be demoted by (1 - 1 / length)
+ // but we demoted only (1 - 1 / (length + 1)) so we will additionally adjust freq by
+ // (1 - 1 / length) / (1 - 1 / (length + 1)) = (1 - 1 / (length * length))
+ const int normalizedScoreNotEnoughDemotionAdjustment = 100 - 100 / (totalLength * totalLength);
+ multiplyRate(normalizedScoreNotEnoughDemotionAdjustment, &totalFreq);
+
+ // At this moment, totalFreq is calculated by the following formula:
+ // (firstFreq * (1 - 1 / (firstWordLength + 1)) + secondFreq * (1 - 1 / (secondWordLength + 1)))
+ // * (1 - 1 / totalLength) / (1 - 1 / (totalLength + 1))
+
+ multiplyIntCapped(powerIntCapped(typedLetterMultiplier, totalLength), &totalFreq);
+
+ // This is another workaround to offset the demotion which will be done in
+ // calcNormalizedScore in Utils.java.
+ // In calcNormalizedScore the score will be demoted by (1 - 1 / length) so we have to promote
+ // the same amount because we already have adjusted the synthetic freq of this "missing or
+ // mistyped space" suggestion candidate above in this method.
+ const int normalizedScoreDemotionRateOffset = (100 + 100 / totalLength);
+ multiplyRate(normalizedScoreDemotionRateOffset, &totalFreq);
+
+ if (isSpaceProximity) {
+ // A word pair with one space proximity correction
+ if (DEBUG_DICT) {
+ LOGI("Found a word pair with space proximity correction.");
+ }
+ multiplyIntCapped(typedLetterMultiplier, &totalFreq);
+ multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &totalFreq);
+ }
+
+ multiplyRate(WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE, &totalFreq);
+ return totalFreq;
+}
+
} // namespace latinime
diff --git a/native/src/correction_state.h b/native/src/correction_state.h
index 5b7392590..e03b2a17c 100644
--- a/native/src/correction_state.h
+++ b/native/src/correction_state.h
@@ -26,10 +26,12 @@ namespace latinime {
class ProximityInfo;
class CorrectionState {
+
public:
- CorrectionState();
- void setCorrectionParams(const ProximityInfo *pi, const int inputLength, const int skipPos,
- const int excessivePos, const int transposedPos);
+ CorrectionState(const int typedLetterMultiplier, const int fullWordMultiplier);
+ void initCorrectionState(const ProximityInfo *pi, const int inputLength);
+ void setCorrectionParams(const int skipPos, const int excessivePos, const int transposedPos,
+ const int spaceProximityPos, const int missingSpacePos);
void checkState();
virtual ~CorrectionState();
int getSkipPos() const {
@@ -41,12 +43,36 @@ public:
int getTransposedPos() const {
return mTransposedPos;
}
+ int getSpaceProximityPos() const {
+ return mSpaceProximityPos;
+ }
+ int getMissingSpacePos() const {
+ return mMissingSpacePos;
+ }
+ int getFreqForSplitTwoWords(const int firstFreq, const int secondFreq);
+ int getFinalFreq(const int inputIndex, const int depth, const int matchWeight, const int freq,
+ const bool sameLength);
+
private:
+
+ const int TYPED_LETTER_MULTIPLIER;
+ const int FULL_WORD_MULTIPLIER;
const ProximityInfo *mProximityInfo;
int mInputLength;
int mSkipPos;
int mExcessivePos;
int mTransposedPos;
+ int mSpaceProximityPos;
+ int mMissingSpacePos;
+
+ class RankingAlgorithm {
+ public:
+ static int calculateFinalFreq(const int inputIndex, const int depth,
+ const int matchCount, const int freq, const bool sameLength,
+ const CorrectionState* correctionState);
+ static int calcFreqForSplitTwoWords(const int firstFreq, const int secondFreq,
+ const CorrectionState* correctionState);
+ };
};
} // namespace latinime
#endif // LATINIME_CORRECTION_INFO_H
diff --git a/native/src/dictionary.cpp b/native/src/dictionary.cpp
index 9e32ee80f..a49769bdb 100644
--- a/native/src/dictionary.cpp
+++ b/native/src/dictionary.cpp
@@ -57,12 +57,4 @@ bool Dictionary::isValidWord(unsigned short *word, int length) {
return mUnigramDictionary->isValidWord(word, length);
}
-int Dictionary::getBigramPosition(unsigned short *word, int length) {
- if (IS_LATEST_DICT_VERSION) {
- return mUnigramDictionary->getBigramPosition(DICTIONARY_HEADER_SIZE, word, 0, length);
- } else {
- return mUnigramDictionary->getBigramPosition(0, word, 0, length);
- }
-}
-
} // namespace latinime
diff --git a/native/src/dictionary.h b/native/src/dictionary.h
index 73e03d8fd..d5de0083a 100644
--- a/native/src/dictionary.h
+++ b/native/src/dictionary.h
@@ -64,8 +64,6 @@ public:
const int pos, unsigned short *c, int *childrenPosition,
bool *terminal, int *freq);
static inline unsigned short toBaseLowerCase(unsigned short c);
- // TODO: delete this
- int getBigramPosition(unsigned short *word, int length);
private:
bool hasBigram();
diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp
index f0bb384fb..eb28538f1 100644
--- a/native/src/unigram_dictionary.cpp
+++ b/native/src/unigram_dictionary.cpp
@@ -48,7 +48,7 @@ UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typed
if (DEBUG_DICT) {
LOGI("UnigramDictionary - constructor");
}
- mCorrectionState = new CorrectionState();
+ mCorrectionState = new CorrectionState(typedLetterMultiplier, fullWordMultiplier);
}
UnigramDictionary::~UnigramDictionary() {
@@ -187,6 +187,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
PROF_START(0);
initSuggestions(
proximityInfo, xcoordinates, ycoordinates, codes, codesSize, outWords, frequencies);
+ mCorrectionState->initCorrectionState(mProximityInfo, mInputLength);
if (DEBUG_DICT) assert(codesSize == mInputLength);
const int MAX_DEPTH = min(mInputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH);
@@ -242,7 +243,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
if (DEBUG_DICT) {
LOGI("--- Suggest missing space characters %d", i);
}
- getMissingSpaceWords(mInputLength, i);
+ getMissingSpaceWords(mInputLength, i, mCorrectionState);
}
}
PROF_END(5);
@@ -261,7 +262,7 @@ void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo,
i, x, y, proximityInfo->hasSpaceProximity(x, y));
}
if (proximityInfo->hasSpaceProximity(x, y)) {
- getMistypedSpaceWords(mInputLength, i);
+ getMistypedSpaceWords(mInputLength, i, mCorrectionState);
}
}
}
@@ -355,8 +356,8 @@ void UnigramDictionary::getSuggestionCandidates(const int skipPos,
assert(excessivePos < mInputLength);
assert(missingPos < mInputLength);
}
- mCorrectionState->setCorrectionParams(mProximityInfo, mInputLength, skipPos, excessivePos,
- transposedPos);
+ mCorrectionState->setCorrectionParams(skipPos, excessivePos, transposedPos,
+ -1 /* spaceProximityPos */, -1 /* missingSpacePos */);
int rootPosition = ROOT_POS;
// Get the number of children of root, then increment the position
int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition);
@@ -364,7 +365,7 @@ void UnigramDictionary::getSuggestionCandidates(const int skipPos,
mStackChildCount[0] = childCount;
mStackTraverseAll[0] = (mInputLength <= 0);
- mStackNodeFreq[0] = 1;
+ mStackMatchCount[0] = 0;
mStackInputIndex[0] = 0;
mStackDiffs[0] = 0;
mStackSiblingPos[0] = rootPosition;
@@ -375,7 +376,7 @@ void UnigramDictionary::getSuggestionCandidates(const int skipPos,
if (mStackChildCount[depth] > 0) {
--mStackChildCount[depth];
bool traverseAllNodes = mStackTraverseAll[depth];
- int matchWeight = mStackNodeFreq[depth];
+ int matchCount = mStackMatchCount[depth];
int inputIndex = mStackInputIndex[depth];
int diffs = mStackDiffs[depth];
int siblingPos = mStackSiblingPos[depth];
@@ -384,9 +385,9 @@ void UnigramDictionary::getSuggestionCandidates(const int skipPos,
// depth will never be greater than maxDepth because in that case,
// needsToTraverseChildrenNodes should be false
const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, outputIndex,
- maxDepth, traverseAllNodes, matchWeight, inputIndex, diffs,
+ maxDepth, traverseAllNodes, matchCount, inputIndex, diffs,
nextLetters, nextLettersSize, mCorrectionState, &childCount,
- &firstChildPos, &traverseAllNodes, &matchWeight, &inputIndex, &diffs,
+ &firstChildPos, &traverseAllNodes, &matchCount, &inputIndex, &diffs,
&siblingPos, &outputIndex);
// Update next sibling pos
mStackSiblingPos[depth] = siblingPos;
@@ -395,7 +396,7 @@ void UnigramDictionary::getSuggestionCandidates(const int skipPos,
++depth;
mStackChildCount[depth] = childCount;
mStackTraverseAll[depth] = traverseAllNodes;
- mStackNodeFreq[depth] = matchWeight;
+ mStackMatchCount[depth] = matchCount;
mStackInputIndex[depth] = inputIndex;
mStackDiffs[depth] = diffs;
mStackSiblingPos[depth] = firstChildPos;
@@ -408,11 +409,6 @@ void UnigramDictionary::getSuggestionCandidates(const int skipPos,
}
}
-static const int TWO_31ST_DIV_255 = S_INT_MAX / 255;
-static inline int capped255MultForFullMatchAccentsOrCapitalizationDifference(const int num) {
- return (num < TWO_31ST_DIV_255 ? 255 * num : S_INT_MAX);
-}
-
static const int TWO_31ST_DIV_2 = S_INT_MAX / 2;
inline static void multiplyIntCapped(const int multiplier, int *base) {
const int temp = *base;
@@ -427,153 +423,18 @@ inline static void multiplyIntCapped(const int multiplier, int *base) {
}
}
-inline static int powerIntCapped(const int base, const int n) {
- if (base == 2) {
- return n < 31 ? 1 << n : S_INT_MAX;
- } else {
- int ret = base;
- for (int i = 1; i < n; ++i) multiplyIntCapped(base, &ret);
- return ret;
- }
+void UnigramDictionary::getMissingSpaceWords(
+ const int inputLength, const int missingSpacePos, CorrectionState *correctionState) {
+ correctionState->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
+ -1 /* transposedPos */, -1 /* spaceProximityPos */, missingSpacePos);
+ getSplitTwoWordsSuggestion(inputLength, correctionState);
}
-inline static void multiplyRate(const int rate, int *freq) {
- if (*freq != S_INT_MAX) {
- if (*freq > 1000000) {
- *freq /= 100;
- multiplyIntCapped(rate, freq);
- } else {
- multiplyIntCapped(rate, freq);
- *freq /= 100;
- }
- }
-}
-
-inline static int calcFreqForSplitTwoWords(
- const int typedLetterMultiplier, const int firstWordLength, const int secondWordLength,
- const int firstFreq, const int secondFreq, const bool isSpaceProximity) {
- if (firstWordLength == 0 || secondWordLength == 0) {
- return 0;
- }
- const int firstDemotionRate = 100 - 100 / (firstWordLength + 1);
- int tempFirstFreq = firstFreq;
- multiplyRate(firstDemotionRate, &tempFirstFreq);
-
- const int secondDemotionRate = 100 - 100 / (secondWordLength + 1);
- int tempSecondFreq = secondFreq;
- multiplyRate(secondDemotionRate, &tempSecondFreq);
-
- const int totalLength = firstWordLength + secondWordLength;
-
- // Promote pairFreq with multiplying by 2, because the word length is the same as the typed
- // length.
- int totalFreq = tempFirstFreq + tempSecondFreq;
-
- // This is a workaround to try offsetting the not-enough-demotion which will be done in
- // calcNormalizedScore in Utils.java.
- // In calcNormalizedScore the score will be demoted by (1 - 1 / length)
- // but we demoted only (1 - 1 / (length + 1)) so we will additionally adjust freq by
- // (1 - 1 / length) / (1 - 1 / (length + 1)) = (1 - 1 / (length * length))
- const int normalizedScoreNotEnoughDemotionAdjustment = 100 - 100 / (totalLength * totalLength);
- multiplyRate(normalizedScoreNotEnoughDemotionAdjustment, &totalFreq);
-
- // At this moment, totalFreq is calculated by the following formula:
- // (firstFreq * (1 - 1 / (firstWordLength + 1)) + secondFreq * (1 - 1 / (secondWordLength + 1)))
- // * (1 - 1 / totalLength) / (1 - 1 / (totalLength + 1))
-
- multiplyIntCapped(powerIntCapped(typedLetterMultiplier, totalLength), &totalFreq);
-
- // This is another workaround to offset the demotion which will be done in
- // calcNormalizedScore in Utils.java.
- // In calcNormalizedScore the score will be demoted by (1 - 1 / length) so we have to promote
- // the same amount because we already have adjusted the synthetic freq of this "missing or
- // mistyped space" suggestion candidate above in this method.
- const int normalizedScoreDemotionRateOffset = (100 + 100 / totalLength);
- multiplyRate(normalizedScoreDemotionRateOffset, &totalFreq);
-
- if (isSpaceProximity) {
- // A word pair with one space proximity correction
- if (DEBUG_DICT) {
- LOGI("Found a word pair with space proximity correction.");
- }
- multiplyIntCapped(typedLetterMultiplier, &totalFreq);
- multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &totalFreq);
- }
-
- multiplyRate(WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE, &totalFreq);
- return totalFreq;
-}
-
-bool UnigramDictionary::getMissingSpaceWords(const int inputLength, const int missingSpacePos) {
- return getSplitTwoWordsSuggestion(
- inputLength, 0, missingSpacePos, missingSpacePos, inputLength - missingSpacePos, false);
-}
-
-bool UnigramDictionary::getMistypedSpaceWords(const int inputLength, const int spaceProximityPos) {
- return getSplitTwoWordsSuggestion(
- inputLength, 0, spaceProximityPos, spaceProximityPos + 1,
- inputLength - spaceProximityPos - 1, true);
-}
-
-inline int UnigramDictionary::calculateFinalFreq(const int inputIndex, const int depth,
- const int matchWeight, const int freq, const bool sameLength,
- CorrectionState *correctionState) const {
- const int skipPos = correctionState->getSkipPos();
- const int excessivePos = correctionState->getExcessivePos();
- const int transposedPos = correctionState->getTransposedPos();
-
- // TODO: Demote by edit distance
- int finalFreq = freq * matchWeight;
- if (skipPos >= 0) {
- if (mInputLength >= 2) {
- const int demotionRate = WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE
- * (10 * mInputLength - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X)
- / (10 * mInputLength
- - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X + 10);
- if (DEBUG_DICT_FULL) {
- LOGI("Demotion rate for missing character is %d.", demotionRate);
- }
- multiplyRate(demotionRate, &finalFreq);
- } else {
- finalFreq = 0;
- }
- }
- if (transposedPos >= 0) multiplyRate(
- WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE, &finalFreq);
- if (excessivePos >= 0) {
- multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq);
- if (!mProximityInfo->existsAdjacentProximityChars(inputIndex)) {
- // If an excessive character is not adjacent to the left char or the right char,
- // we will demote this word.
- multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE, &finalFreq);
- }
- }
- int lengthFreq = TYPED_LETTER_MULTIPLIER;
- multiplyIntCapped(powerIntCapped(TYPED_LETTER_MULTIPLIER, depth), &lengthFreq);
- if (lengthFreq == matchWeight) {
- // Full exact match
- if (depth > 1) {
- if (DEBUG_DICT) {
- LOGI("Found full matched word.");
- }
- multiplyRate(FULL_MATCHED_WORDS_PROMOTION_RATE, &finalFreq);
- }
- if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0) {
- finalFreq = capped255MultForFullMatchAccentsOrCapitalizationDifference(finalFreq);
- }
- } else if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0 && depth > 0) {
- // A word with proximity corrections
- if (DEBUG_DICT) {
- LOGI("Found one proximity correction.");
- }
- multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &finalFreq);
- multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq);
- }
- if (DEBUG_DICT) {
- LOGI("calc: %d, %d", depth, sameLength);
- }
- if (sameLength) multiplyIntCapped(FULL_WORD_MULTIPLIER, &finalFreq);
- return finalFreq;
+void UnigramDictionary::getMistypedSpaceWords(
+ const int inputLength, const int spaceProximityPos, CorrectionState *correctionState) {
+ correctionState->setCorrectionParams(-1 /* skipPos */, -1 /* excessivePos */,
+ -1 /* transposedPos */, spaceProximityPos, -1 /* missingSpacePos */);
+ getSplitTwoWordsSuggestion(inputLength, correctionState);
}
inline bool UnigramDictionary::needsToSkipCurrentNode(const unsigned short c,
@@ -586,7 +447,7 @@ inline bool UnigramDictionary::needsToSkipCurrentNode(const unsigned short c,
inline void UnigramDictionary::onTerminal(unsigned short int* word, const int depth,
const uint8_t* const root, const uint8_t flags, const int pos,
- const int inputIndex, const int matchWeight, const int freq, const bool sameLength,
+ const int inputIndex, const int matchCount, const int freq, const bool sameLength,
int* nextLetters, const int nextLettersSize, CorrectionState *correctionState) {
const int skipPos = correctionState->getSkipPos();
@@ -594,8 +455,8 @@ inline void UnigramDictionary::onTerminal(unsigned short int* word, const int de
if (isSameAsTyped) return;
if (depth >= MIN_SUGGEST_DEPTH) {
- const int finalFreq = calculateFinalFreq(inputIndex, depth, matchWeight,
- freq, sameLength, correctionState);
+ const int finalFreq = correctionState->getFinalFreq(inputIndex, depth, matchCount,
+ freq, sameLength);
if (!isSameAsTyped)
addWord(word, depth + 1, finalFreq);
}
@@ -605,13 +466,29 @@ inline void UnigramDictionary::onTerminal(unsigned short int* word, const int de
}
}
-bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength,
- const int firstWordStartPos, const int firstWordLength, const int secondWordStartPos,
- const int secondWordLength, const bool isSpaceProximity) {
- if (inputLength >= MAX_WORD_LENGTH) return false;
+void UnigramDictionary::getSplitTwoWordsSuggestion(
+ const int inputLength, CorrectionState* correctionState) {
+ const int spaceProximityPos = correctionState->getSpaceProximityPos();
+ const int missingSpacePos = correctionState->getMissingSpacePos();
+ if (DEBUG_DICT) {
+ int inputCount = 0;
+ if (spaceProximityPos >= 0) ++inputCount;
+ if (missingSpacePos >= 0) ++inputCount;
+ assert(inputCount <= 1);
+ }
+ const bool isSpaceProximity = spaceProximityPos >= 0;
+ const int firstWordStartPos = 0;
+ const int secondWordStartPos = isSpaceProximity ? (spaceProximityPos + 1) : missingSpacePos;
+ const int firstWordLength = isSpaceProximity ? spaceProximityPos : missingSpacePos;
+ const int secondWordLength = isSpaceProximity
+ ? (inputLength - spaceProximityPos - 1)
+ : (inputLength - missingSpacePos);
+
+ if (inputLength >= MAX_WORD_LENGTH) return;
if (0 >= firstWordLength || 0 >= secondWordLength || firstWordStartPos >= secondWordStartPos
|| firstWordStartPos < 0 || secondWordStartPos + secondWordLength > inputLength)
- return false;
+ return;
+
const int newWordLength = firstWordLength + secondWordLength + 1;
// Allocating variable length array on stack
unsigned short word[newWordLength];
@@ -619,7 +496,7 @@ bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength,
if (DEBUG_DICT) {
LOGI("First freq: %d", firstFreq);
}
- if (firstFreq <= 0) return false;
+ if (firstFreq <= 0) return;
for (int i = 0; i < firstWordLength; ++i) {
word[i] = mWord[i];
@@ -629,21 +506,19 @@ bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength,
if (DEBUG_DICT) {
LOGI("Second freq: %d", secondFreq);
}
- if (secondFreq <= 0) return false;
+ if (secondFreq <= 0) return;
word[firstWordLength] = SPACE;
for (int i = (firstWordLength + 1); i < newWordLength; ++i) {
word[i] = mWord[i - firstWordLength - 1];
}
- int pairFreq = calcFreqForSplitTwoWords(TYPED_LETTER_MULTIPLIER, firstWordLength,
- secondWordLength, firstFreq, secondFreq, isSpaceProximity);
+ const int pairFreq = mCorrectionState->getFreqForSplitTwoWords(firstFreq, secondFreq);
if (DEBUG_DICT) {
- LOGI("Split two words: %d, %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength,
- TYPED_LETTER_MULTIPLIER);
+ LOGI("Split two words: %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength);
}
addWord(word, newWordLength, pairFreq);
- return true;
+ return;
}
// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous
@@ -803,7 +678,7 @@ int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offs
// the current node in nextSiblingPosition. Thus, the caller must keep count of the nodes at any
// given level, as output into newCount when traversing this level's parent.
inline bool UnigramDictionary::processCurrentNode(const int initialPos, const int initialDepth,
- const int maxDepth, const bool initialTraverseAllNodes, int matchWeight, int inputIndex,
+ const int maxDepth, const bool initialTraverseAllNodes, int matchCount, int inputIndex,
const int initialDiffs, int *nextLetters, const int nextLettersSize,
CorrectionState *correctionState, int *newCount, int *newChildrenPosition,
bool *newTraverseAllNodes, int *newMatchRate, int *newInputIndex, int *newDiffs,
@@ -868,7 +743,7 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos, const in
// The frequency should be here, because we come here only if this is actually
// a terminal node, and we are on its last char.
const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
- onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight,
+ onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchCount,
freq, false, nextLetters, nextLettersSize, mCorrectionState);
}
if (!hasChildren) {
@@ -913,13 +788,13 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos, const in
// If inputIndex is greater than mInputLength, that means there is no
// proximity chars. So, we don't need to check proximity.
if (ProximityInfo::SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR == matchedProximityCharId) {
- multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &matchWeight);
+ ++matchCount;
}
const bool isSameAsUserTypedLength = mInputLength == inputIndex + 1
|| (excessivePos == mInputLength - 1 && inputIndex == mInputLength - 2);
if (isSameAsUserTypedLength && isTerminal) {
const int freq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos);
- onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight,
+ onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchCount,
freq, true, nextLetters, nextLettersSize, mCorrectionState);
}
// This character matched the typed character (enough to traverse the node at least)
@@ -975,7 +850,7 @@ inline bool UnigramDictionary::processCurrentNode(const int initialPos, const in
// All the output values that are purely computation by this function are held in local
// variables. Output them to the caller.
*newTraverseAllNodes = traverseAllNodes;
- *newMatchRate = matchWeight;
+ *newMatchRate = matchCount;
*newDiffs = diffs;
*newInputIndex = inputIndex;
*newOutputIndex = depth;
diff --git a/native/src/unigram_dictionary.h b/native/src/unigram_dictionary.h
index 41e381860..f18ed6841 100644
--- a/native/src/unigram_dictionary.h
+++ b/native/src/unigram_dictionary.h
@@ -74,6 +74,7 @@ public:
virtual ~UnigramDictionary();
private:
+
void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates,
const int *ycoordinates, const int *codes, const int codesSize,
unsigned short *outWords, int *frequencies);
@@ -89,13 +90,11 @@ private:
const int transposedPos, int *nextLetters, const int nextLettersSize,
const int maxDepth);
bool addWord(unsigned short *word, int length, int frequency);
- bool getSplitTwoWordsSuggestion(const int inputLength,
- const int firstWordStartPos, const int firstWordLength,
- const int secondWordStartPos, const int secondWordLength, const bool isSpaceProximity);
- bool getMissingSpaceWords(const int inputLength, const int missingSpacePos);
- bool getMistypedSpaceWords(const int inputLength, const int spaceProximityPos);
- int calculateFinalFreq(const int inputIndex, const int depth, const int snr,
- const int freq, const bool sameLength, CorrectionState *correctionState) const;
+ void getSplitTwoWordsSuggestion(const int inputLength, CorrectionState *correctionState);
+ void getMissingSpaceWords(
+ const int inputLength, const int missingSpacePos, CorrectionState *correctionState);
+ void getMistypedSpaceWords(
+ const int inputLength, const int spaceProximityPos, CorrectionState *correctionState);
void onTerminal(unsigned short int* word, const int depth,
const uint8_t* const root, const uint8_t flags, const int pos,
const int inputIndex, const int matchWeight, const int freq, const bool sameLength,
@@ -145,7 +144,7 @@ private:
int mStackChildCount[MAX_WORD_LENGTH_INTERNAL];
bool mStackTraverseAll[MAX_WORD_LENGTH_INTERNAL];
- int mStackNodeFreq[MAX_WORD_LENGTH_INTERNAL];
+ int mStackMatchCount[MAX_WORD_LENGTH_INTERNAL];
int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL];
int mStackDiffs[MAX_WORD_LENGTH_INTERNAL];
int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL];
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-06-02 11:23:28 +0000