aboutsummaryrefslogtreecommitdiffstats
path: root/native/jni/src
diff options
context:
space:
mode:
Diffstat (limited to 'native/jni/src')
-rw-r--r--native/jni/src/suggest/core/dicnode/dic_node_utils.cpp5
-rw-r--r--native/jni/src/suggest/core/dictionary/property/word_property.cpp48
-rw-r--r--native/jni/src/suggest/core/dictionary/property/word_property.h6
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp4
-rw-r--r--native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp12
-rw-r--r--native/jni/src/utils/jni_data_utils.h3
6 files changed, 53 insertions, 25 deletions
diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
index 7d2898b7a..ea438922f 100644
--- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
+++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp
@@ -74,8 +74,9 @@ namespace latinime {
}
const WordAttributes wordAttributes = dictionaryStructurePolicy->getWordAttributesInContext(
dicNode->getPrevWordIds(), dicNode->getWordId(), multiBigramMap);
- if (dicNode->hasMultipleWords()
- && (wordAttributes.isBlacklisted() || wordAttributes.isNotAWord())) {
+ if (wordAttributes.getProbability() == NOT_A_PROBABILITY
+ || (dicNode->hasMultipleWords()
+ && (wordAttributes.isBlacklisted() || wordAttributes.isNotAWord()))) {
return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
}
// TODO: This equation to calculate the improbability looks unreasonable. Investigate this.
diff --git a/native/jni/src/suggest/core/dictionary/property/word_property.cpp b/native/jni/src/suggest/core/dictionary/property/word_property.cpp
index a707f1ba2..019f0880f 100644
--- a/native/jni/src/suggest/core/dictionary/property/word_property.cpp
+++ b/native/jni/src/suggest/core/dictionary/property/word_property.cpp
@@ -22,8 +22,9 @@
namespace latinime {
void WordProperty::outputProperties(JNIEnv *const env, jintArray outCodePoints,
- jbooleanArray outFlags, jintArray outProbabilityInfo, jobject outBigramTargets,
- jobject outBigramProbabilities, jobject outShortcutTargets,
+ jbooleanArray outFlags, jintArray outProbabilityInfo,
+ jobject outNgramPrevWordsArray, jobject outNgramPrevWordIsBeginningOfSentenceArray,
+ jobject outNgramTargets, jobject outNgramProbabilities, jobject outShortcutTargets,
jobject outShortcutProbabilities) const {
JniDataUtils::outputCodePoints(env, outCodePoints, 0 /* start */,
MAX_WORD_LENGTH /* maxLength */, mCodePoints.data(), mCodePoints.size(),
@@ -43,16 +44,39 @@ void WordProperty::outputProperties(JNIEnv *const env, jintArray outCodePoints,
jclass arrayListClass = env->FindClass("java/util/ArrayList");
jmethodID addMethodId = env->GetMethodID(arrayListClass, "add", "(Ljava/lang/Object;)Z");
- // Output bigrams.
- // TODO: Support n-gram
+ // Output ngrams.
+ jclass intArrayClass = env->FindClass("[I");
for (const auto &ngramProperty : mNgrams) {
- const std::vector<int> *const word1CodePoints = ngramProperty.getTargetCodePoints();
- jintArray bigramWord1CodePointArray = env->NewIntArray(word1CodePoints->size());
- JniDataUtils::outputCodePoints(env, bigramWord1CodePointArray, 0 /* start */,
- word1CodePoints->size(), word1CodePoints->data(), word1CodePoints->size(),
- false /* needsNullTermination */);
- env->CallBooleanMethod(outBigramTargets, addMethodId, bigramWord1CodePointArray);
- env->DeleteLocalRef(bigramWord1CodePointArray);
+ const NgramContext *const ngramContext = ngramProperty.getNgramContext();
+ jobjectArray prevWordWordCodePointsArray = env->NewObjectArray(
+ ngramContext->getPrevWordCount(), intArrayClass, nullptr);
+ jbooleanArray prevWordIsBeginningOfSentenceArray =
+ env->NewBooleanArray(ngramContext->getPrevWordCount());
+ for (size_t i = 0; i < ngramContext->getPrevWordCount(); ++i) {
+ const CodePointArrayView codePoints = ngramContext->getNthPrevWordCodePoints(i + 1);
+ jintArray prevWordCodePoints = env->NewIntArray(codePoints.size());
+ JniDataUtils::outputCodePoints(env, prevWordCodePoints, 0 /* start */,
+ codePoints.size(), codePoints.data(), codePoints.size(),
+ false /* needsNullTermination */);
+ env->SetObjectArrayElement(prevWordWordCodePointsArray, i, prevWordCodePoints);
+ env->DeleteLocalRef(prevWordCodePoints);
+ JniDataUtils::putBooleanToArray(env, prevWordIsBeginningOfSentenceArray, i,
+ ngramContext->isNthPrevWordBeginningOfSentence(i + 1));
+ }
+ env->CallBooleanMethod(outNgramPrevWordsArray, addMethodId, prevWordWordCodePointsArray);
+ env->CallBooleanMethod(outNgramPrevWordIsBeginningOfSentenceArray, addMethodId,
+ prevWordIsBeginningOfSentenceArray);
+ env->DeleteLocalRef(prevWordWordCodePointsArray);
+ env->DeleteLocalRef(prevWordIsBeginningOfSentenceArray);
+
+ const std::vector<int> *const targetWordCodePoints = ngramProperty.getTargetCodePoints();
+ jintArray targetWordCodePointArray = env->NewIntArray(targetWordCodePoints->size());
+ JniDataUtils::outputCodePoints(env, targetWordCodePointArray, 0 /* start */,
+ targetWordCodePoints->size(), targetWordCodePoints->data(),
+ targetWordCodePoints->size(), false /* needsNullTermination */);
+ env->CallBooleanMethod(outNgramTargets, addMethodId, targetWordCodePointArray);
+ env->DeleteLocalRef(targetWordCodePointArray);
+
const HistoricalInfo &ngramHistoricalInfo = ngramProperty.getHistoricalInfo();
int bigramProbabilityInfo[] = {ngramProperty.getProbability(),
ngramHistoricalInfo.getTimestamp(), ngramHistoricalInfo.getLevel(),
@@ -60,7 +84,7 @@ void WordProperty::outputProperties(JNIEnv *const env, jintArray outCodePoints,
jintArray bigramProbabilityInfoArray = env->NewIntArray(NELEMS(bigramProbabilityInfo));
env->SetIntArrayRegion(bigramProbabilityInfoArray, 0 /* start */,
NELEMS(bigramProbabilityInfo), bigramProbabilityInfo);
- env->CallBooleanMethod(outBigramProbabilities, addMethodId, bigramProbabilityInfoArray);
+ env->CallBooleanMethod(outNgramProbabilities, addMethodId, bigramProbabilityInfoArray);
env->DeleteLocalRef(bigramProbabilityInfoArray);
}
diff --git a/native/jni/src/suggest/core/dictionary/property/word_property.h b/native/jni/src/suggest/core/dictionary/property/word_property.h
index 01b8987b5..b5314faaa 100644
--- a/native/jni/src/suggest/core/dictionary/property/word_property.h
+++ b/native/jni/src/suggest/core/dictionary/property/word_property.h
@@ -39,8 +39,10 @@ class WordProperty {
mNgrams(*ngrams) {}
void outputProperties(JNIEnv *const env, jintArray outCodePoints, jbooleanArray outFlags,
- jintArray outProbabilityInfo, jobject outBigramTargets, jobject outBigramProbabilities,
- jobject outShortcutTargets, jobject outShortcutProbabilities) const;
+ jintArray outProbabilityInfo, jobject outNgramPrevWordsArray,
+ jobject outNgramPrevWordIsBeginningOfSentenceArray, jobject outNgramTargets,
+ jobject outNgramProbabilities, jobject outShortcutTargets,
+ jobject outShortcutProbabilities) const;
const UnigramProperty *getUnigramProperty() const {
return &mUnigramProperty;
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
index b96290437..509bd683b 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp
@@ -90,8 +90,8 @@ const WordAttributes LanguageModelDictContent::getWordAttributes(const WordIdArr
// TODO: Some flags in unigramProbabilityEntry should be overwritten by flags in
// probabilityEntry.
const ProbabilityEntry unigramProbabilityEntry = getProbabilityEntry(wordId);
- return WordAttributes(probability, unigramProbabilityEntry.isNotAWord(),
- unigramProbabilityEntry.isBlacklisted(),
+ return WordAttributes(probability, unigramProbabilityEntry.isBlacklisted(),
+ unigramProbabilityEntry.isNotAWord(),
unigramProbabilityEntry.isPossiblyOffensive());
}
// Cannot find the word.
diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
index 193326d82..249d822b2 100644
--- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
+++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp
@@ -488,9 +488,6 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(
AKLOGE("getWordProperty is called for invalid word.");
return WordProperty();
}
- const int ptNodePos =
- mBuffers->getTerminalPositionLookupTable()->getTerminalPtNodePosition(wordId);
- const PtNodeParams ptNodeParams = mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos);
const LanguageModelDictContent *const languageModelDictContent =
mBuffers->getLanguageModelDictContent();
// Fetch ngram information.
@@ -541,12 +538,13 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(
shortcutProbability);
}
}
- const ProbabilityEntry probabilityEntry = languageModelDictContent->getProbabilityEntry(
- ptNodeParams.getTerminalId());
+ const WordAttributes wordAttributes = languageModelDictContent->getWordAttributes(
+ WordIdArrayView(), wordId, mHeaderPolicy);
+ const ProbabilityEntry probabilityEntry = languageModelDictContent->getProbabilityEntry(wordId);
const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo();
const UnigramProperty unigramProperty(probabilityEntry.representsBeginningOfSentence(),
- probabilityEntry.isNotAWord(), probabilityEntry.isBlacklisted(),
- probabilityEntry.isPossiblyOffensive(), probabilityEntry.getProbability(),
+ wordAttributes.isNotAWord(), wordAttributes.isBlacklisted(),
+ wordAttributes.isPossiblyOffensive(), wordAttributes.getProbability(),
*historicalInfo, std::move(shortcuts));
return WordProperty(wordCodePoints.toVector(), &unigramProperty, &ngrams);
}
diff --git a/native/jni/src/utils/jni_data_utils.h b/native/jni/src/utils/jni_data_utils.h
index 25cc41742..a259e1cd0 100644
--- a/native/jni/src/utils/jni_data_utils.h
+++ b/native/jni/src/utils/jni_data_utils.h
@@ -50,6 +50,7 @@ class JniDataUtils {
const jsize keyUtf8Length = env->GetStringUTFLength(keyString);
char keyChars[keyUtf8Length + 1];
env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars);
+ env->DeleteLocalRef(keyString);
keyChars[keyUtf8Length] = '\0';
DictionaryHeaderStructurePolicy::AttributeMap::key_type key;
HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key);
@@ -59,6 +60,7 @@ class JniDataUtils {
const jsize valueUtf8Length = env->GetStringUTFLength(valueString);
char valueChars[valueUtf8Length + 1];
env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars);
+ env->DeleteLocalRef(valueString);
valueChars[valueUtf8Length] = '\0';
DictionaryHeaderStructurePolicy::AttributeMap::mapped_type value;
HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value);
@@ -113,6 +115,7 @@ class JniDataUtils {
continue;
}
env->GetIntArrayRegion(prevWord, 0, prevWordLength, prevWordCodePoints[i]);
+ env->DeleteLocalRef(prevWord);
prevWordCodePointCount[i] = prevWordLength;
jboolean isBeginningOfSentenceBoolean = JNI_FALSE;
env->GetBooleanArrayRegion(isBeginningOfSentenceArray, i, 1 /* len */,
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-05-05 09:56:47 +0000