aboutsummaryrefslogtreecommitdiffstats
path: root/native
diff options
context:
space:
mode:
Diffstat (limited to 'native')
-rw-r--r--native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp29
-rw-r--r--native/src/correction.cpp82
-rw-r--r--native/src/correction.h24
3 files changed, 115 insertions, 20 deletions
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index 42d0e3207..71a893ca7 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -18,6 +18,7 @@
#define LOG_TAG "LatinIME: jni: BinaryDictionary"
#include "binary_format.h"
+#include "correction.h"
#include "com_android_inputmethod_latin_BinaryDictionary.h"
#include "dictionary.h"
#include "jni.h"
@@ -188,6 +189,29 @@ static jboolean latinime_BinaryDictionary_isValidWord(JNIEnv *env, jobject objec
return result;
}
+static jdouble latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jobject object,
+ jcharArray before, jint beforeLength, jcharArray after, jint afterLength, jint score) {
+ jchar *beforeChars = env->GetCharArrayElements(before, 0);
+ jchar *afterChars = env->GetCharArrayElements(after, 0);
+ jdouble result = Correction::RankingAlgorithm::calcNormalizedScore(
+ (unsigned short*)beforeChars, beforeLength, (unsigned short*)afterChars, afterLength,
+ score);
+ env->ReleaseCharArrayElements(before, beforeChars, JNI_ABORT);
+ env->ReleaseCharArrayElements(after, afterChars, JNI_ABORT);
+ return result;
+}
+
+static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jobject object,
+ jcharArray before, jint beforeLength, jcharArray after, jint afterLength) {
+ jchar *beforeChars = env->GetCharArrayElements(before, 0);
+ jchar *afterChars = env->GetCharArrayElements(after, 0);
+ jint result = Correction::RankingAlgorithm::editDistance(
+ (unsigned short*)beforeChars, beforeLength, (unsigned short*)afterChars, afterLength);
+ env->ReleaseCharArrayElements(before, beforeChars, JNI_ABORT);
+ env->ReleaseCharArrayElements(after, afterChars, JNI_ABORT);
+ return result;
+}
+
static void latinime_BinaryDictionary_close(JNIEnv *env, jobject object, jlong dict) {
Dictionary *dictionary = (Dictionary*)dict;
if (!dictionary) return;
@@ -222,7 +246,10 @@ static JNINativeMethod sMethods[] = {
{"closeNative", "(J)V", (void*)latinime_BinaryDictionary_close},
{"getSuggestionsNative", "(JJ[I[I[III[C[I)I", (void*)latinime_BinaryDictionary_getSuggestions},
{"isValidWordNative", "(J[CI)Z", (void*)latinime_BinaryDictionary_isValidWord},
- {"getBigramsNative", "(J[CI[II[C[IIII)I", (void*)latinime_BinaryDictionary_getBigrams}
+ {"getBigramsNative", "(J[CI[II[C[IIII)I", (void*)latinime_BinaryDictionary_getBigrams},
+ {"calcNormalizedScoreNative", "([CI[CII)D",
+ (void*)latinime_BinaryDictionary_calcNormalizedScore},
+ {"editDistanceNative", "([CI[CI)I", (void*)latinime_BinaryDictionary_editDistance}
};
int register_BinaryDictionary(JNIEnv *env) {
diff --git a/native/src/correction.cpp b/native/src/correction.cpp
index 503fb7c53..dc31bfae7 100644
--- a/native/src/correction.cpp
+++ b/native/src/correction.cpp
@@ -16,6 +16,7 @@
#include <assert.h>
#include <ctype.h>
+#include <math.h>
#include <stdio.h>
#include <string.h>
@@ -933,14 +934,14 @@ int Correction::RankingAlgorithm::calcFreqForSplitTwoWords(
return totalFreq;
}
-#if 0 /* no longer used. keep just for reference */
-inline static int editDistance(
- int* editDistanceTable, const unsigned short* input,
- const int inputLength, const unsigned short* output, const int outputLength) {
+/* Damerau-Levenshtein distance */
+inline static int editDistanceInternal(
+ int* editDistanceTable, const unsigned short* before,
+ const int beforeLength, const unsigned short* after, const int afterLength) {
// dp[li][lo] dp[a][b] = dp[ a * lo + b]
int* dp = editDistanceTable;
- const int li = inputLength + 1;
- const int lo = outputLength + 1;
+ const int li = beforeLength + 1;
+ const int lo = afterLength + 1;
for (int i = 0; i < li; ++i) {
dp[lo * i] = i;
}
@@ -950,13 +951,13 @@ inline static int editDistance(
for (int i = 0; i < li - 1; ++i) {
for (int j = 0; j < lo - 1; ++j) {
- const uint32_t ci = toBaseLowerCase(input[i]);
- const uint32_t co = toBaseLowerCase(output[j]);
+ const uint32_t ci = toBaseLowerCase(before[i]);
+ const uint32_t co = toBaseLowerCase(after[j]);
const uint16_t cost = (ci == co) ? 0 : 1;
dp[(i + 1) * lo + (j + 1)] = min(dp[i * lo + (j + 1)] + 1,
min(dp[(i + 1) * lo + j] + 1, dp[i * lo + j] + cost));
- if (i > 0 && j > 0 && ci == toBaseLowerCase(output[j - 1])
- && co == toBaseLowerCase(input[i - 1])) {
+ if (i > 0 && j > 0 && ci == toBaseLowerCase(after[j - 1])
+ && co == toBaseLowerCase(before[i - 1])) {
dp[(i + 1) * lo + (j + 1)] = min(
dp[(i + 1) * lo + (j + 1)], dp[(i - 1) * lo + (j - 1)] + cost);
}
@@ -964,7 +965,7 @@ inline static int editDistance(
}
if (DEBUG_EDIT_DISTANCE) {
- LOGI("IN = %d, OUT = %d", inputLength, outputLength);
+ LOGI("IN = %d, OUT = %d", beforeLength, afterLength);
for (int i = 0; i < li; ++i) {
for (int j = 0; j < lo; ++j) {
LOGI("EDIT[%d][%d], %d", i, j, dp[i * lo + j]);
@@ -973,6 +974,63 @@ inline static int editDistance(
}
return dp[li * lo - 1];
}
-#endif
+
+int Correction::RankingAlgorithm::editDistance(const unsigned short* before,
+ const int beforeLength, const unsigned short* after, const int afterLength) {
+ int table[(beforeLength + 1) * (afterLength + 1)];
+ return editDistanceInternal(table, before, beforeLength, after, afterLength);
+}
+
+
+// In dictionary.cpp, getSuggestion() method,
+// suggestion scores are computed using the below formula.
+// original score
+// := pow(mTypedLetterMultiplier (this is defined 2),
+// (the number of matched characters between typed word and suggested word))
+// * (individual word's score which defined in the unigram dictionary,
+// and this score is defined in range [0, 255].)
+// Then, the following processing is applied.
+// - If the dictionary word is matched up to the point of the user entry
+// (full match up to min(before.length(), after.length())
+// => Then multiply by FULL_MATCHED_WORDS_PROMOTION_RATE (this is defined 1.2)
+// - If the word is a true full match except for differences in accents or
+// capitalization, then treat it as if the score was 255.
+// - If before.length() == after.length()
+// => multiply by mFullWordMultiplier (this is defined 2))
+// So, maximum original score is pow(2, min(before.length(), after.length())) * 255 * 2 * 1.2
+// For historical reasons we ignore the 1.2 modifier (because the measure for a good
+// autocorrection threshold was done at a time when it didn't exist). This doesn't change
+// the result.
+// So, we can normalize original score by dividing pow(2, min(b.l(),a.l())) * 255 * 2.
+
+/* static */
+double Correction::RankingAlgorithm::calcNormalizedScore(const unsigned short* before,
+ const int beforeLength, const unsigned short* after, const int afterLength,
+ const int score) {
+ if (0 == beforeLength || 0 == afterLength) {
+ return 0;
+ }
+ const int distance = editDistance(before, beforeLength, after, afterLength);
+ int spaceCount = 0;
+ for (int i = 0; i < afterLength; ++i) {
+ if (after[i] == CODE_SPACE) {
+ ++spaceCount;
+ }
+ }
+
+ if (spaceCount == afterLength) {
+ return 0;
+ }
+
+ const double maxScore = score >= S_INT_MAX ? S_INT_MAX : MAX_INITIAL_SCORE
+ * pow((double)TYPED_LETTER_MULTIPLIER,
+ (double)min(beforeLength, afterLength - spaceCount)) * FULL_WORD_MULTIPLIER;
+
+ // add a weight based on edit distance.
+ // distance <= max(afterLength, beforeLength) == afterLength,
+ // so, 0 <= distance / afterLength <= 1
+ const double weight = 1.0 - (double) distance / afterLength;
+ return (score / maxScore) * weight;
+}
} // namespace latinime
diff --git a/native/src/correction.h b/native/src/correction.h
index 9ba472955..4012e7e82 100644
--- a/native/src/correction.h
+++ b/native/src/correction.h
@@ -95,6 +95,23 @@ class Correction {
return mCorrectionStates[index].mParentIndex;
}
+ class RankingAlgorithm {
+ public:
+ static int calculateFinalFreq(const int inputIndex, const int depth,
+ const int freq, int *editDistanceTable, const Correction* correction);
+ static int calcFreqForSplitTwoWords(const int firstFreq, const int secondFreq,
+ const Correction* correction, const unsigned short *word);
+ static double calcNormalizedScore(const unsigned short* before, const int beforeLength,
+ const unsigned short* after, const int afterLength, const int score);
+ static int editDistance(const unsigned short* before,
+ const int beforeLength, const unsigned short* after, const int afterLength);
+ private:
+ static const int CODE_SPACE = ' ';
+ static const int MAX_INITIAL_SCORE = 255;
+ static const int TYPED_LETTER_MULTIPLIER = 2;
+ static const int FULL_WORD_MULTIPLIER = 2;
+ };
+
private:
inline void incrementInputIndex();
inline void incrementOutputIndex();
@@ -153,13 +170,6 @@ class Correction {
bool mTransposing;
bool mSkipping;
- class RankingAlgorithm {
- public:
- static int calculateFinalFreq(const int inputIndex, const int depth,
- const int freq, int *editDistanceTable, const Correction* correction);
- static int calcFreqForSplitTwoWords(const int firstFreq, const int secondFreq,
- const Correction* correction, const unsigned short *word);
- };
};
} // namespace latinime
#endif // LATINIME_CORRECTION_H
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-05-31 04:32:40 +0000