aboutsummaryrefslogtreecommitdiffstats
path: root/native
diff options
context:
space:
mode:
Diffstat (limited to 'native')
-rw-r--r--native/jni/Android.mk2
-rw-r--r--native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h16
-rw-r--r--native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp (renamed from native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.cpp)31
-rw-r--r--native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h (renamed from native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.h)66
-rw-r--r--native/jni/src/suggest/core/layout/proximity_info_state.h15
-rw-r--r--native/jni/src/suggest/policyimpl/typing/scoring_params.cpp1
-rw-r--r--native/jni/src/suggest/policyimpl/typing/scoring_params.h1
-rw-r--r--native/jni/src/suggest/policyimpl/typing/typing_weighting.h22
8 files changed, 74 insertions, 80 deletions
diff --git a/native/jni/Android.mk b/native/jni/Android.mk
index fb60139d3..d5df6b62e 100644
--- a/native/jni/Android.mk
+++ b/native/jni/Android.mk
@@ -53,10 +53,10 @@ LATIN_IME_CORE_SRC_FILES := \
dic_nodes_cache.cpp) \
$(addprefix suggest/core/dictionary/, \
bigram_dictionary.cpp \
- binary_dictionary_bigrams_reading_utils.cpp \
binary_dictionary_format_utils.cpp \
binary_dictionary_header.cpp \
binary_dictionary_header_reading_utils.cpp \
+ binary_dictionary_terminal_attributes_reading_utils.cpp \
bloom_filter.cpp \
byte_array_utils.cpp \
dictionary.cpp \
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h
index 0856840b2..f2b48e960 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_iterator.h
@@ -18,8 +18,8 @@
#define LATINIME_BINARY_DICTIONARY_BIGRAMS_ITERATOR_H
#include "defines.h"
-#include "suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
+#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
namespace latinime {
@@ -35,15 +35,17 @@ class BinaryDictionaryBigramsIterator {
}
AK_FORCE_INLINE void next() {
- mBigramFlags = BinaryDictionaryBigramsReadingUtils::getFlagsAndForwardPointer(
+ mBigramFlags = BinaryDictionaryTerminalAttributesReadingUtils::getFlagsAndForwardPointer(
mBinaryDictionaryInfo, &mPos);
- mBigramPos = BinaryDictionaryBigramsReadingUtils::getBigramAddressAndForwardPointer(
- mBinaryDictionaryInfo, mBigramFlags, &mPos);
- mHasNext = BinaryDictionaryBigramsReadingUtils::hasNext(mBigramFlags);
+ mBigramPos =
+ BinaryDictionaryTerminalAttributesReadingUtils::getBigramAddressAndForwardPointer(
+ mBinaryDictionaryInfo, mBigramFlags, &mPos);
+ mHasNext = BinaryDictionaryTerminalAttributesReadingUtils::hasNext(mBigramFlags);
}
AK_FORCE_INLINE int getProbability() const {
- return BinaryDictionaryBigramsReadingUtils::getBigramProbability(mBigramFlags);
+ return BinaryDictionaryTerminalAttributesReadingUtils::getProbabilityFromFlags(
+ mBigramFlags);
}
AK_FORCE_INLINE int getBigramPos() const {
@@ -59,7 +61,7 @@ class BinaryDictionaryBigramsIterator {
const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
int mPos;
- BinaryDictionaryBigramsReadingUtils::BigramFlags mBigramFlags;
+ BinaryDictionaryTerminalAttributesReadingUtils::BigramFlags mBigramFlags;
int mBigramPos;
bool mHasNext;
};
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp
index 78a54b141..0a7509c8b 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.cpp
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.cpp
@@ -14,33 +14,28 @@
* limitations under the License.
*/
-#include "suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.h"
+#include "suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/byte_array_utils.h"
namespace latinime {
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
+typedef BinaryDictionaryTerminalAttributesReadingUtils TaUtils;
+
+const TaUtils::TerminalAttributeFlags TaUtils::MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30;
+const TaUtils::TerminalAttributeFlags TaUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10;
+const TaUtils::TerminalAttributeFlags TaUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20;
+const TaUtils::TerminalAttributeFlags TaUtils::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30;
+const TaUtils::TerminalAttributeFlags TaUtils::FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40;
// Flag for presence of more attributes
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
+const TaUtils::TerminalAttributeFlags TaUtils::FLAG_ATTRIBUTE_HAS_NEXT = 0x80;
// Mask for attribute probability, stored on 4 bits inside the flags byte.
-const BinaryDictionaryBigramsReadingUtils::BigramFlags
- BinaryDictionaryBigramsReadingUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
-const int BinaryDictionaryBigramsReadingUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
+const TaUtils::TerminalAttributeFlags TaUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F;
+const int TaUtils::ATTRIBUTE_ADDRESS_SHIFT = 4;
-/* static */ int BinaryDictionaryBigramsReadingUtils::getBigramAddressAndForwardPointer(
- const BinaryDictionaryInfo *const binaryDictionaryInfo, const BigramFlags flags,
+/* static */ int TaUtils::getBigramAddressAndForwardPointer(
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, const TerminalAttributeFlags flags,
int *const pos) {
int offset = 0;
const int origin = *pos;
diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h
index e71f2a17a..f38fd5aaa 100644
--- a/native/jni/src/suggest/core/dictionary/binary_dictionary_bigrams_reading_utils.h
+++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_terminal_attributes_reading_utils.h
@@ -14,8 +14,8 @@
* limitations under the License.
*/
-#ifndef LATINIME_BINARY_DICTIONARY_BIGRAM_READING_UTILS_H
-#define LATINIME_BINARY_DICTIONARY_BIGRAM_READING_UTILS_H
+#ifndef LATINIME_BINARY_DICTIONARY_TERMINAL_ATTRIBUTES_READING_UTILS_H
+#define LATINIME_BINARY_DICTIONARY_TERMINAL_ATTRIBUTES_READING_UTILS_H
#include <stdint.h>
@@ -25,55 +25,57 @@
namespace latinime {
-class BinaryDictionaryBigramsReadingUtils {
+class BinaryDictionaryTerminalAttributesReadingUtils {
public:
- typedef uint8_t BigramFlags;
+ typedef uint8_t TerminalAttributeFlags;
+ typedef TerminalAttributeFlags BigramFlags;
- static AK_FORCE_INLINE void skipExistingBigrams(
- const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
- BigramFlags flags = getFlagsAndForwardPointer(binaryDictionaryInfo, pos);
- while (hasNext(flags)) {
- *pos += attributeAddressSize(flags);
- flags = getFlagsAndForwardPointer(binaryDictionaryInfo, pos);
- }
- *pos += attributeAddressSize(flags);
- }
-
- static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer(
+ static AK_FORCE_INLINE TerminalAttributeFlags getFlagsAndForwardPointer(
const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
return ByteArrayUtils::readUint8andAdvancePosition(
binaryDictionaryInfo->getDictRoot(), pos);
}
- static AK_FORCE_INLINE int getBigramProbability(const BigramFlags flags) {
+ static AK_FORCE_INLINE int getProbabilityFromFlags(const TerminalAttributeFlags flags) {
return flags & MASK_ATTRIBUTE_PROBABILITY;
}
- static AK_FORCE_INLINE bool isOffsetNegative(const BigramFlags flags) {
- return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0;
+ static AK_FORCE_INLINE bool hasNext(const TerminalAttributeFlags flags) {
+ return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0;
}
- static AK_FORCE_INLINE bool hasNext(const BigramFlags flags) {
- return (flags & FLAG_ATTRIBUTE_HAS_NEXT) != 0;
+ // Bigrams reading methods
+ static AK_FORCE_INLINE void skipExistingBigrams(
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, int *const pos) {
+ BigramFlags flags = getFlagsAndForwardPointer(binaryDictionaryInfo, pos);
+ while (hasNext(flags)) {
+ *pos += attributeAddressSize(flags);
+ flags = getFlagsAndForwardPointer(binaryDictionaryInfo, pos);
+ }
+ *pos += attributeAddressSize(flags);
}
static int getBigramAddressAndForwardPointer(
- const BinaryDictionaryInfo *const binaryDictionaryInfo,
- const BigramFlags flags, int *const pos);
+ const BinaryDictionaryInfo *const binaryDictionaryInfo, const BigramFlags flags,
+ int *const pos);
private:
- DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryBigramsReadingUtils);
+ DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryTerminalAttributesReadingUtils);
- static const BigramFlags MASK_ATTRIBUTE_ADDRESS_TYPE;
- static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
- static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
- static const BigramFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
- static const BigramFlags FLAG_ATTRIBUTE_OFFSET_NEGATIVE;
- static const BigramFlags FLAG_ATTRIBUTE_HAS_NEXT;
- static const BigramFlags MASK_ATTRIBUTE_PROBABILITY;
+ static const TerminalAttributeFlags MASK_ATTRIBUTE_ADDRESS_TYPE;
+ static const TerminalAttributeFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE;
+ static const TerminalAttributeFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES;
+ static const TerminalAttributeFlags FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES;
+ static const TerminalAttributeFlags FLAG_ATTRIBUTE_OFFSET_NEGATIVE;
+ static const TerminalAttributeFlags FLAG_ATTRIBUTE_HAS_NEXT;
+ static const TerminalAttributeFlags MASK_ATTRIBUTE_PROBABILITY;
static const int ATTRIBUTE_ADDRESS_SHIFT;
- static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) {
+ static AK_FORCE_INLINE bool isOffsetNegative(const TerminalAttributeFlags flags) {
+ return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0;
+ }
+
+ static AK_FORCE_INLINE int attributeAddressSize(const TerminalAttributeFlags flags) {
return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT;
/* Note: this is a value-dependant optimization of what may probably be
more readably written this way:
@@ -87,4 +89,4 @@ class BinaryDictionaryBigramsReadingUtils {
}
};
}
-#endif /* LATINIME_BINARY_DICTIONARY_BIGRAM_READING_UTILS_H */
+#endif /* LATINIME_BINARY_DICTIONARY_TERMINAL_ATTRIBUTES_READING_UTILS_H */
diff --git a/native/jni/src/suggest/core/layout/proximity_info_state.h b/native/jni/src/suggest/core/layout/proximity_info_state.h
index cc6410af1..dbcd54488 100644
--- a/native/jni/src/suggest/core/layout/proximity_info_state.h
+++ b/native/jni/src/suggest/core/layout/proximity_info_state.h
@@ -90,20 +90,7 @@ class ProximityInfoState {
return false;
}
- // TODO: Promote insertion letter correction if that letter is a proximity of the previous
- // letter like follows:
- // // Demotion for a word with excessive character
- // if (excessiveCount > 0) {
- // multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq);
- // if (!lastCharExceeded
- // && !proximityInfoState->existsAdjacentProximityChars(excessivePos)) {
- // // If an excessive character is not adjacent to the left char or the right char,
- // // we will demote this word.
- // multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE,
- // &finalFreq);
- // }
- // }
- inline bool existsAdjacentProximityChars(const int index) const {
+ AK_FORCE_INLINE bool existsAdjacentProximityChars(const int index) const {
if (index < 0 || index >= mSampledInputSize) return false;
const int currentCodePoint = getPrimaryCodePointAt(index);
const int leftIndex = index - 1;
diff --git a/native/jni/src/suggest/policyimpl/typing/scoring_params.cpp b/native/jni/src/suggest/policyimpl/typing/scoring_params.cpp
index f87989286..2659e4a23 100644
--- a/native/jni/src/suggest/policyimpl/typing/scoring_params.cpp
+++ b/native/jni/src/suggest/policyimpl/typing/scoring_params.cpp
@@ -33,6 +33,7 @@ const float ScoringParams::OMISSION_COST_SAME_CHAR = 0.491f;
const float ScoringParams::OMISSION_COST_FIRST_CHAR = 0.582f;
const float ScoringParams::INSERTION_COST = 0.730f;
const float ScoringParams::INSERTION_COST_SAME_CHAR = 0.586f;
+const float ScoringParams::INSERTION_COST_PROXIMITY_CHAR = 0.70f;
const float ScoringParams::INSERTION_COST_FIRST_CHAR = 0.623f;
const float ScoringParams::TRANSPOSITION_COST = 0.516f;
const float ScoringParams::SPACE_SUBSTITUTION_COST = 0.319f;
diff --git a/native/jni/src/suggest/policyimpl/typing/scoring_params.h b/native/jni/src/suggest/policyimpl/typing/scoring_params.h
index 53ac999c1..c39c41779 100644
--- a/native/jni/src/suggest/policyimpl/typing/scoring_params.h
+++ b/native/jni/src/suggest/policyimpl/typing/scoring_params.h
@@ -42,6 +42,7 @@ class ScoringParams {
static const float OMISSION_COST_FIRST_CHAR;
static const float INSERTION_COST;
static const float INSERTION_COST_SAME_CHAR;
+ static const float INSERTION_COST_PROXIMITY_CHAR;
static const float INSERTION_COST_FIRST_CHAR;
static const float TRANSPOSITION_COST;
static const float SPACE_SUBSTITUTION_COST;
diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
index 7ba4af5f9..e098f353e 100644
--- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
+++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h
@@ -122,19 +122,25 @@ class TypingWeighting : public Weighting {
float getInsertionCost(const DicTraverseSession *const traverseSession,
const DicNode *const parentDicNode, const DicNode *const dicNode) const {
- const int16_t parentPointIndex = parentDicNode->getInputIndex(0);
- const int prevCodePoint =
- traverseSession->getProximityInfoState(0)->getPrimaryCodePointAt(parentPointIndex);
-
+ const int16_t insertedPointIndex = parentDicNode->getInputIndex(0);
+ const int prevCodePoint = traverseSession->getProximityInfoState(0)->getPrimaryCodePointAt(
+ insertedPointIndex);
const int currentCodePoint = dicNode->getNodeCodePoint();
const bool sameCodePoint = prevCodePoint == currentCodePoint;
+ const bool existsAdjacentProximityChars = traverseSession->getProximityInfoState(0)
+ ->existsAdjacentProximityChars(insertedPointIndex);
const float dist = traverseSession->getProximityInfoState(0)->getPointToKeyLength(
- parentPointIndex + 1, currentCodePoint);
+ insertedPointIndex + 1, dicNode->getNodeCodePoint());
const float weightedDistance = dist * ScoringParams::DISTANCE_WEIGHT_LENGTH;
const bool singleChar = dicNode->getNodeCodePointCount() == 1;
- const float cost = (singleChar ? ScoringParams::INSERTION_COST_FIRST_CHAR : 0.0f)
- + (sameCodePoint ? ScoringParams::INSERTION_COST_SAME_CHAR
- : ScoringParams::INSERTION_COST);
+ float cost = (singleChar ? ScoringParams::INSERTION_COST_FIRST_CHAR : 0.0f);
+ if (sameCodePoint) {
+ cost += ScoringParams::INSERTION_COST_SAME_CHAR;
+ } else if (existsAdjacentProximityChars) {
+ cost += ScoringParams::INSERTION_COST_PROXIMITY_CHAR;
+ } else {
+ cost += ScoringParams::INSERTION_COST;
+ }
return cost + weightedDistance;
}
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-08-07 03:29:06 +0000