aboutsummaryrefslogtreecommitdiffstats
path: root/native
diff options
context:
space:
mode:
Diffstat (limited to 'native')
-rw-r--r--native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp43
-rw-r--r--native/jni/src/suggest/core/dictionary/property/word_property.cpp3
-rw-r--r--native/jni/src/utils/char_utils.h4
-rw-r--r--native/jni/src/utils/jni_data_utils.h24
4 files changed, 56 insertions, 18 deletions
diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
index c2cd2addd..2654a4a0a 100644
--- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
+++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp
@@ -301,7 +301,7 @@ static jint latinime_BinaryDictionary_getBigramProbability(JNIEnv *env, jclass c
// If token is 0, this method newly starts iterating the dictionary. This method returns 0 when
// the dictionary does not have a next word.
static jint latinime_BinaryDictionary_getNextWord(JNIEnv *env, jclass clazz,
- jlong dict, jint token, jintArray outCodePoints) {
+ jlong dict, jint token, jintArray outCodePoints, jbooleanArray outIsBeginningOfSentence) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return 0;
const jsize codePointBufSize = env->GetArrayLength(outCodePoints);
@@ -317,19 +317,39 @@ static jint latinime_BinaryDictionary_getNextWord(JNIEnv *env, jclass clazz,
JniDataUtils::outputCodePoints(env, outCodePoints, 0 /* start */,
MAX_WORD_LENGTH /* maxLength */, wordCodePoints, wordCodePointCount,
false /* needsNullTermination */);
+ bool isBeginningOfSentence = false;
+ if (wordCodePointCount > 0 && wordCodePoints[0] == CODE_POINT_BEGINNING_OF_SENTENCE) {
+ isBeginningOfSentence = true;
+ }
+ JniDataUtils::putBooleanToArray(env, outIsBeginningOfSentence, 0 /* index */,
+ isBeginningOfSentence);
return nextToken;
}
static void latinime_BinaryDictionary_getWordProperty(JNIEnv *env, jclass clazz,
- jlong dict, jintArray word, jintArray outCodePoints, jbooleanArray outFlags,
- jintArray outProbabilityInfo, jobject outBigramTargets, jobject outBigramProbabilityInfo,
- jobject outShortcutTargets, jobject outShortcutProbabilities) {
+ jlong dict, jintArray word, jboolean isBeginningOfSentence, jintArray outCodePoints,
+ jbooleanArray outFlags, jintArray outProbabilityInfo, jobject outBigramTargets,
+ jobject outBigramProbabilityInfo, jobject outShortcutTargets,
+ jobject outShortcutProbabilities) {
Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
if (!dictionary) return;
const jsize wordLength = env->GetArrayLength(word);
- int wordCodePoints[wordLength];
+ if (wordLength > MAX_WORD_LENGTH) {
+ AKLOGE("Invalid wordLength: %d", wordLength);
+ return;
+ }
+ int wordCodePoints[MAX_WORD_LENGTH];
env->GetIntArrayRegion(word, 0, wordLength, wordCodePoints);
- const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, wordLength);
+ int codePointCount = wordLength;
+ if (isBeginningOfSentence) {
+ codePointCount = CharUtils::attachBeginningOfSentenceMarker(
+ wordCodePoints, wordLength, MAX_WORD_LENGTH);
+ if (codePointCount < 0) {
+ AKLOGE("Cannot attach Beginning-of-Sentence marker.");
+ return;
+ }
+ }
+ const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints, codePointCount);
wordProperty.outputProperties(env, outCodePoints, outFlags, outProbabilityInfo,
outBigramTargets, outBigramProbabilityInfo, outShortcutTargets,
outShortcutProbabilities);
@@ -554,7 +574,6 @@ static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, j
return false;
}
- // TODO: Migrate historical information.
int wordCodePoints[MAX_WORD_LENGTH];
int wordCodePointCount = 0;
int token = 0;
@@ -563,6 +582,10 @@ static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, j
token = dictionary->getNextWordAndNextToken(token, wordCodePoints, &wordCodePointCount);
const WordProperty wordProperty = dictionary->getWordProperty(wordCodePoints,
wordCodePointCount);
+ if (wordCodePoints[0] == CODE_POINT_BEGINNING_OF_SENTENCE) {
+ // Skip beginning-of-sentence unigram.
+ continue;
+ }
if (dictionaryStructureWithBufferPolicy->needsToRunGC(true /* mindsBlockByGC */)) {
dictionaryStructureWithBufferPolicy = runGCAndGetNewStructurePolicy(
std::move(dictionaryStructureWithBufferPolicy), dictFilePathChars);
@@ -592,7 +615,7 @@ static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, j
}
}
const PrevWordsInfo prevWordsInfo(wordCodePoints, wordCodePointCount,
- false /* isBeginningOfSentence */);
+ wordProperty.getUnigramProperty()->representsBeginningOfSentence());
for (const BigramProperty &bigramProperty : *wordProperty.getBigramProperties()) {
if (!dictionaryStructureWithBufferPolicy->addNgramEntry(&prevWordsInfo,
&bigramProperty)) {
@@ -669,13 +692,13 @@ static const JNINativeMethod sMethods[] = {
},
{
const_cast<char *>("getWordPropertyNative"),
- const_cast<char *>("(J[I[I[Z[ILjava/util/ArrayList;Ljava/util/ArrayList;"
+ const_cast<char *>("(J[IZ[I[Z[ILjava/util/ArrayList;Ljava/util/ArrayList;"
"Ljava/util/ArrayList;Ljava/util/ArrayList;)V"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getWordProperty)
},
{
const_cast<char *>("getNextWordNative"),
- const_cast<char *>("(JI[I)I"),
+ const_cast<char *>("(JI[I[Z)I"),
reinterpret_cast<void *>(latinime_BinaryDictionary_getNextWord)
},
{
diff --git a/native/jni/src/suggest/core/dictionary/property/word_property.cpp b/native/jni/src/suggest/core/dictionary/property/word_property.cpp
index 6f5f808f8..5bdd5606b 100644
--- a/native/jni/src/suggest/core/dictionary/property/word_property.cpp
+++ b/native/jni/src/suggest/core/dictionary/property/word_property.cpp
@@ -28,7 +28,8 @@ void WordProperty::outputProperties(JNIEnv *const env, jintArray outCodePoints,
MAX_WORD_LENGTH /* maxLength */, mCodePoints.data(), mCodePoints.size(),
false /* needsNullTermination */);
jboolean flags[] = {mUnigramProperty.isNotAWord(), mUnigramProperty.isBlacklisted(),
- !mBigrams.empty(), mUnigramProperty.hasShortcuts()};
+ !mBigrams.empty(), mUnigramProperty.hasShortcuts(),
+ mUnigramProperty.representsBeginningOfSentence()};
env->SetBooleanArrayRegion(outFlags, 0 /* start */, NELEMS(flags), flags);
int probabilityInfo[] = {mUnigramProperty.getProbability(), mUnigramProperty.getTimestamp(),
mUnigramProperty.getLevel(), mUnigramProperty.getCount()};
diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h
index f28ed5682..63786502b 100644
--- a/native/jni/src/utils/char_utils.h
+++ b/native/jni/src/utils/char_utils.h
@@ -98,6 +98,10 @@ class CharUtils {
// Beginning-of-Sentence.
static AK_FORCE_INLINE int attachBeginningOfSentenceMarker(int *const codePoints,
const int codePointCount, const int maxCodePoint) {
+ if (codePointCount > 0 && codePoints[0] == CODE_POINT_BEGINNING_OF_SENTENCE) {
+ // Marker has already been attached.
+ return codePointCount;
+ }
if (codePointCount >= maxCodePoint) {
// the code points cannot be marked as a Beginning-of-Sentence.
return 0;
diff --git a/native/jni/src/utils/jni_data_utils.h b/native/jni/src/utils/jni_data_utils.h
index 67a66fdfe..3514aeeb0 100644
--- a/native/jni/src/utils/jni_data_utils.h
+++ b/native/jni/src/utils/jni_data_utils.h
@@ -69,18 +69,23 @@ class JniDataUtils {
static void outputCodePoints(JNIEnv *env, jintArray intArrayToOutputCodePoints, const int start,
const int maxLength, const int *const codePoints, const int codePointCount,
const bool needsNullTermination) {
- const int outputCodePointCount = std::min(maxLength, codePointCount);
- int outputCodePonts[outputCodePointCount];
- for (int i = 0; i < outputCodePointCount; ++i) {
+ const int codePointBufSize = std::min(maxLength, codePointCount);
+ int outputCodePonts[codePointBufSize];
+ int outputCodePointCount = 0;
+ for (int i = 0; i < codePointBufSize; ++i) {
const int codePoint = codePoints[i];
+ int codePointToOutput = codePoint;
if (!CharUtils::isInUnicodeSpace(codePoint)) {
- outputCodePonts[i] = CODE_POINT_REPLACEMENT_CHARACTER;
+ if (codePoint == CODE_POINT_BEGINNING_OF_SENTENCE) {
+ // Just skip Beginning-of-Sentence marker.
+ continue;
+ }
+ codePointToOutput = CODE_POINT_REPLACEMENT_CHARACTER;
} else if (codePoint >= 0x01 && codePoint <= 0x1F) {
// Control code.
- outputCodePonts[i] = CODE_POINT_REPLACEMENT_CHARACTER;
- } else {
- outputCodePonts[i] = codePoint;
+ codePointToOutput = CODE_POINT_REPLACEMENT_CHARACTER;
}
+ outputCodePonts[outputCodePointCount++] = codePointToOutput;
}
env->SetIntArrayRegion(intArrayToOutputCodePoints, start, outputCodePointCount,
outputCodePonts);
@@ -90,6 +95,11 @@ class JniDataUtils {
}
}
+ static void putBooleanToArray(JNIEnv *env, jbooleanArray array, const int index,
+ const jboolean value) {
+ env->SetBooleanArrayRegion(array, index, 1 /* len */, &value);
+ }
+
static void putIntToArray(JNIEnv *env, jintArray array, const int index, const int value) {
env->SetIntArrayRegion(array, index, 1 /* len */, &value);
}
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-06-01 17:41:32 +0000