diff options
38 files changed, 232 insertions, 104 deletions
diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml index 5c62003d3..769a1d986 100644 --- a/java/res/values/attrs.xml +++ b/java/res/values/attrs.xml @@ -74,6 +74,7 @@ <!-- Size of the text for spacebar language label, in the proportion of key height. --> <attr name="languageOnSpacebarTextRatio" format="fraction" /> <attr name="languageOnSpacebarTextColor" format="color" /> + <attr name="languageOnSpacebarTextShadowRadius" format="float" /> <attr name="languageOnSpacebarTextShadowColor" format="color" /> <!-- Background image for the spacebar. --> <attr name="spacebarBackground" format="reference" /> diff --git a/java/res/values/colors.xml b/java/res/values/colors.xml index dabb4d65e..baa0887bc 100644 --- a/java/res/values/colors.xml +++ b/java/res/values/colors.xml @@ -26,7 +26,6 @@ <color name="suggested_word_color_ics">#B233B5E5</color> <color name="highlight_translucent_color_ics">#9933B5E5</color> <color name="key_text_color_holo">@android:color/white</color> - <color name="key_text_shadow_color_holo">@android:color/transparent</color> <color name="key_text_inactivated_color_holo">#66E0E4E5</color> <color name="key_hint_letter_color_holo">#80000000</color> <color name="key_hint_label_color_holo">#A0FFFFFF</color> diff --git a/java/res/values/strings-talkback-descriptions.xml b/java/res/values/strings-talkback-descriptions.xml index 4ffca10c8..80406d02f 100644 --- a/java/res/values/strings-talkback-descriptions.xml +++ b/java/res/values/strings-talkback-descriptions.xml @@ -35,10 +35,14 @@ <string name="spoken_description_unknown">Key code %d</string> <!-- Spoken description for the "Shift" keyboard key when "Shift" is off. --> <string name="spoken_description_shift">Shift</string> + <!-- Spoken description for the "Shift" keyboard key in symbols mode. --> + <string name="spoken_description_symbols_shift">More symbols</string> <!-- Spoken description for the "Shift" keyboard key when "Shift" is on. --> - <string name="spoken_description_shift_shifted">Shift on (tap to disable)</string> + <string name="spoken_description_shift_shifted">Shift</string> + <!-- Spoken description for the "Shift" keyboard key in 2nd symbols (a.k.a. symbols shift) mode. --> + <string name="spoken_description_symbols_shift_shifted">Symbols</string> <!-- Spoken description for the "Shift" keyboard key when "Caps lock" is on. --> - <string name="spoken_description_caps_lock">Caps lock on (tap to disable)</string> + <string name="spoken_description_caps_lock">Shift</string> <!-- Spoken description for the "Delete" keyboard key. --> <string name="spoken_description_delete">Delete</string> <!-- Spoken description for the "To Symbol" keyboard key. --> @@ -76,8 +80,8 @@ <string name="spoken_description_shiftmode_locked">Caps lock enabled</string> <!-- Spoken feedback after changing to the symbols keyboard. --> <string name="spoken_description_mode_symbol">Symbols mode</string> - <!-- Spoken feedback after changing to the symbols shift keyboard. --> - <string name="spoken_description_mode_symbol_shift">Symbols shift mode</string> + <!-- Spoken feedback after changing to the 2nd symbols (a.k.a. symbols shift) keyboard. --> + <string name="spoken_description_mode_symbol_shift">More symbols mode</string> <!-- Spoken feedback after changing to the alphanumeric keyboard. --> <string name="spoken_description_mode_alpha">Letters mode</string> <!-- Spoken feedback after changing to the phone dialer keyboard. --> diff --git a/java/res/values/themes-ics.xml b/java/res/values/themes-ics.xml index 616dbd82c..d7943eeaf 100644 --- a/java/res/values/themes-ics.xml +++ b/java/res/values/themes-ics.xml @@ -56,8 +56,8 @@ <item name="keyShiftedLetterHintInactivatedColor">@color/key_shifted_letter_hint_inactivated_color_holo</item> <item name="keyShiftedLetterHintActivatedColor">@color/key_shifted_letter_hint_activated_color_holo</item> <item name="keyPreviewTextColor">@color/key_text_color_holo</item> - <item name="keyTextShadowColor">@color/key_text_shadow_color_holo</item> - <item name="keyTextShadowRadius">0.0</item> + <!-- A negative value to disable key text shadow layer. --> + <item name="keyTextShadowRadius">-1.0</item> </style> <style name="MainKeyboardView.ICS" @@ -71,6 +71,7 @@ <item name="autoCorrectionSpacebarLedEnabled">false</item> <item name="autoCorrectionSpacebarLedIcon">@drawable/sym_keyboard_space_led_holo</item> <item name="languageOnSpacebarTextColor">@color/spacebar_text_color_holo</item> + <item name="languageOnSpacebarTextShadowRadius">1.0</item> <item name="languageOnSpacebarTextShadowColor">@color/spacebar_text_shadow_color_holo</item> <item name="spacebarBackground">@drawable/btn_keyboard_spacebar_ics</item> </style> diff --git a/java/res/values/themes-klp.xml b/java/res/values/themes-klp.xml index 9bb3d79a7..13500fef2 100644 --- a/java/res/values/themes-klp.xml +++ b/java/res/values/themes-klp.xml @@ -56,8 +56,8 @@ <item name="keyShiftedLetterHintInactivatedColor">@color/key_shifted_letter_hint_inactivated_color_holo</item> <item name="keyShiftedLetterHintActivatedColor">@color/key_shifted_letter_hint_activated_color_holo</item> <item name="keyPreviewTextColor">@color/key_text_color_holo</item> - <item name="keyTextShadowColor">@color/key_text_shadow_color_holo</item> - <item name="keyTextShadowRadius">0.0</item> + <!-- A negative value to disable key text shadow layer. --> + <item name="keyTextShadowRadius">-1.0</item> </style> <style name="MainKeyboardView.KLP" @@ -71,6 +71,7 @@ <item name="autoCorrectionSpacebarLedEnabled">false</item> <item name="autoCorrectionSpacebarLedIcon">@drawable/sym_keyboard_space_led_holo</item> <item name="languageOnSpacebarTextColor">@color/spacebar_text_color_holo</item> + <item name="languageOnSpacebarTextShadowRadius">1.0</item> <item name="languageOnSpacebarTextShadowColor">@color/spacebar_text_shadow_color_holo</item> <item name="spacebarBackground">@drawable/btn_keyboard_spacebar_klp</item> </style> diff --git a/java/res/values/themes-lmp.xml b/java/res/values/themes-lmp.xml index 773da196d..41c4d09d5 100644 --- a/java/res/values/themes-lmp.xml +++ b/java/res/values/themes-lmp.xml @@ -56,8 +56,8 @@ <item name="keyShiftedLetterHintInactivatedColor">@color/key_shifted_letter_hint_inactivated_color_holo</item> <item name="keyShiftedLetterHintActivatedColor">@color/key_shifted_letter_hint_activated_color_holo</item> <item name="keyPreviewTextColor">@color/key_text_color_holo</item> - <item name="keyTextShadowColor">@color/key_text_shadow_color_holo</item> - <item name="keyTextShadowRadius">0.0</item> + <!-- A negative value to disable key text shadow layer. --> + <item name="keyTextShadowRadius">-1.0</item> </style> <style name="MainKeyboardView.LMP" @@ -71,6 +71,7 @@ <item name="autoCorrectionSpacebarLedEnabled">false</item> <item name="autoCorrectionSpacebarLedIcon">@drawable/sym_keyboard_space_led_holo</item> <item name="languageOnSpacebarTextColor">@color/spacebar_text_color_holo</item> + <item name="languageOnSpacebarTextShadowRadius">1.0</item> <item name="languageOnSpacebarTextShadowColor">@color/spacebar_text_shadow_color_holo</item> <item name="spacebarBackground">@drawable/btn_keyboard_spacebar_lmp</item> </style> diff --git a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java index 2e6649bf2..0499a3456 100644 --- a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java +++ b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java @@ -189,9 +189,14 @@ public final class KeyCodeDescriptionMapper { break; case KeyboardId.ELEMENT_ALPHABET_AUTOMATIC_SHIFTED: case KeyboardId.ELEMENT_ALPHABET_MANUAL_SHIFTED: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_shift_shifted; break; + case KeyboardId.ELEMENT_SYMBOLS: + resId = R.string.spoken_description_symbols_shift; + break; + case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: + resId = R.string.spoken_description_symbols_shift_shifted; + break; default: resId = R.string.spoken_description_shift; } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java index 8ca00b005..4450a4474 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java @@ -82,6 +82,7 @@ public class KeyboardView extends View { private final float mVerticalCorrection; private final Drawable mKeyBackground; private final Rect mKeyBackgroundPadding = new Rect(); + private static final float KET_TEXT_SHADOW_RADIUS_DISABLED = -1.0f; // HORIZONTAL ELLIPSIS "...", character for popup hint. private static final String POPUP_HINT_CHAR = "\u2026"; @@ -133,7 +134,7 @@ public class KeyboardView extends View { mKeyShiftedLetterHintPadding = keyboardViewAttr.getDimension( R.styleable.KeyboardView_keyShiftedLetterHintPadding, 0.0f); mKeyTextShadowRadius = keyboardViewAttr.getFloat( - R.styleable.KeyboardView_keyTextShadowRadius, 0.0f); + R.styleable.KeyboardView_keyTextShadowRadius, KET_TEXT_SHADOW_RADIUS_DISABLED); mVerticalCorrection = keyboardViewAttr.getDimension( R.styleable.KeyboardView_verticalCorrection, 0.0f); keyboardViewAttr.recycle(); @@ -414,18 +415,23 @@ public class KeyboardView extends View { } } - paint.setColor(key.selectTextColor(params)); if (key.isEnabled()) { - // Set a drop shadow for the text - paint.setShadowLayer(mKeyTextShadowRadius, 0.0f, 0.0f, params.mTextShadowColor); + paint.setColor(key.selectTextColor(params)); + // Set a drop shadow for the text if the shadow radius is positive value. + if (mKeyTextShadowRadius > 0.0f) { + paint.setShadowLayer(mKeyTextShadowRadius, 0.0f, 0.0f, params.mTextShadowColor); + } else { + paint.clearShadowLayer(); + } } else { // Make label invisible paint.setColor(Color.TRANSPARENT); + paint.clearShadowLayer(); } blendAlpha(paint, params.mAnimAlpha); canvas.drawText(label, 0, label.length(), positionX, baseline, paint); // Turn off drop shadow and reset x-scale. - paint.setShadowLayer(0.0f, 0.0f, 0.0f, Color.TRANSPARENT); + paint.clearShadowLayer(); paint.setTextScaleX(1.0f); if (icon != null) { diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java index 8f79a9128..1a8e4b7e9 100644 --- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java @@ -74,6 +74,7 @@ import java.util.WeakHashMap; * @attr ref R.styleable#MainKeyboardView_autoCorrectionSpacebarLedIcon * @attr ref R.styleable#MainKeyboardView_languageOnSpacebarTextRatio * @attr ref R.styleable#MainKeyboardView_languageOnSpacebarTextColor + * @attr ref R.styleable#MainKeyboardView_languageOnSpacebarTextShadowRadius * @attr ref R.styleable#MainKeyboardView_languageOnSpacebarTextShadowColor * @attr ref R.styleable#MainKeyboardView_spacebarBackground * @attr ref R.styleable#MainKeyboardView_languageOnSpacebarFinalAlpha @@ -129,7 +130,9 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack private final float mLanguageOnSpacebarTextRatio; private float mLanguageOnSpacebarTextSize; private final int mLanguageOnSpacebarTextColor; + private final float mLanguageOnSpacebarTextShadowRadius; private final int mLanguageOnSpacebarTextShadowColor; + private static final float LANGUAGE_ON_SPACEBAR_TEXT_SHADOW_RADIUS_DISABLED = -1.0f; // The minimum x-scale to fit the language name on spacebar. private static final float MINIMUM_XSCALE_OF_LANGUAGE_NAME = 0.8f; // Stuff to draw auto correction LED on spacebar. @@ -231,6 +234,9 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack R.styleable.MainKeyboardView_languageOnSpacebarTextRatio, 1, 1, 1.0f); mLanguageOnSpacebarTextColor = mainKeyboardViewAttr.getColor( R.styleable.MainKeyboardView_languageOnSpacebarTextColor, 0); + mLanguageOnSpacebarTextShadowRadius = mainKeyboardViewAttr.getFloat( + R.styleable.MainKeyboardView_languageOnSpacebarTextShadowRadius, + LANGUAGE_ON_SPACEBAR_TEXT_SHADOW_RADIUS_DISABLED); mLanguageOnSpacebarTextShadowColor = mainKeyboardViewAttr.getColor( R.styleable.MainKeyboardView_languageOnSpacebarTextShadowColor, 0); mLanguageOnSpacebarFinalAlpha = mainKeyboardViewAttr.getInt( @@ -948,12 +954,17 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack final float descent = paint.descent(); final float textHeight = -paint.ascent() + descent; final float baseline = height / 2 + textHeight / 2; - paint.setColor(mLanguageOnSpacebarTextShadowColor); - paint.setAlpha(mLanguageOnSpacebarAnimAlpha); - canvas.drawText(language, width / 2, baseline - descent - 1, paint); + if (mLanguageOnSpacebarTextShadowRadius > 0.0f) { + paint.setShadowLayer(mLanguageOnSpacebarTextShadowRadius, 0, 0, + mLanguageOnSpacebarTextShadowColor); + } else { + paint.clearShadowLayer(); + } paint.setColor(mLanguageOnSpacebarTextColor); paint.setAlpha(mLanguageOnSpacebarAnimAlpha); canvas.drawText(language, width / 2, baseline - descent, paint); + paint.clearShadowLayer(); + paint.setTextScaleX(1.0f); } // Draw the spacebar icon at the bottom diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 94a1e3658..999508e92 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -414,8 +414,8 @@ public final class BinaryDictionary extends Dictionary { public WordProperty mWordProperty; public int mNextToken; - public GetNextWordPropertyResult(final WordProperty wordPreperty, final int nextToken) { - mWordProperty = wordPreperty; + public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { + mWordProperty = wordProperty; mNextToken = nextToken; } } diff --git a/native/jni/NativeFileList.mk b/native/jni/NativeFileList.mk index 34c190718..6ccfec911 100644 --- a/native/jni/NativeFileList.mk +++ b/native/jni/NativeFileList.mk @@ -46,23 +46,22 @@ LATIN_IME_CORE_SRC_FILES := \ $(addprefix suggest/policyimpl/dictionary/, \ header/header_policy.cpp \ header/header_read_write_utils.cpp \ - shortcut/shortcut_list_reading_utils.cpp \ structure/dictionary_structure_with_buffer_policy_factory.cpp) \ - $(addprefix suggest/policyimpl/dictionary/bigram/, \ - bigram_list_read_write_utils.cpp \ - ver4_bigram_list_policy.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/pt_common/, \ + bigram/bigram_list_read_write_utils.cpp \ dynamic_pt_gc_event_listeners.cpp \ dynamic_pt_reading_helper.cpp \ dynamic_pt_reading_utils.cpp \ dynamic_pt_updating_helper.cpp \ dynamic_pt_writing_utils.cpp \ - patricia_trie_reading_utils.cpp) \ + patricia_trie_reading_utils.cpp \ + shortcut/shortcut_list_reading_utils.cpp ) \ $(addprefix suggest/policyimpl/dictionary/structure/v2/, \ patricia_trie_policy.cpp \ ver2_patricia_trie_node_reader.cpp \ ver2_pt_node_array_reader.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v4/, \ + bigram/ver4_bigram_list_policy.cpp \ ver4_dict_buffers.cpp \ ver4_dict_constants.cpp \ ver4_patricia_trie_node_reader.cpp \ diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index a3d8ec158..a55b2da96 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -335,7 +335,7 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, if (!shortcutTargetCodePoints.empty()) { shortcuts.emplace_back(&shortcutTargetCodePoints, shortcutProbability); } - // Use 1 for count to indicate the word has inputed. + // Use 1 for count to indicate the word has inputted. const UnigramProperty unigramProperty(isNotAWord, isBlacklisted, probability, timestamp, 0 /* level */, 1 /* count */, &shortcuts); dictionary->addUnigramWord(codePoints, codePointCount, &unigramProperty); @@ -353,8 +353,12 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, jsize word1Length = env->GetArrayLength(word1); int word1CodePoints[word1Length]; env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); - dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, - word1Length, probability, timestamp); + const std::vector<int> bigramTargetCodePoints( + word1CodePoints, word1CodePoints + word1Length); + // Use 1 for count to indicate the bigram has inputted. + const BigramProperty bigramProperty(&bigramTargetCodePoints, probability, + timestamp, 0 /* level */, 1 /* count */); + dictionary->addBigramWords(word0CodePoints, word0Length, &bigramProperty); } static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict, @@ -437,14 +441,18 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j env->GetIntField(languageModelParam, shortcutProbabilityFieldId); shortcuts.emplace_back(&shortcutTargetCodePoints, shortcutProbability); } - // Use 1 for count to indicate the word has inputed. + // Use 1 for count to indicate the word has inputted. const UnigramProperty unigramProperty(isNotAWord, isBlacklisted, unigramProbability, timestamp, 0 /* level */, 1 /* count */, &shortcuts); dictionary->addUnigramWord(word1CodePoints, word1Length, &unigramProperty); if (word0) { jint bigramProbability = env->GetIntField(languageModelParam, bigramProbabilityFieldId); - dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length, - bigramProbability, timestamp); + const std::vector<int> bigramTargetCodePoints( + word1CodePoints, word1CodePoints + word1Length); + // Use 1 for count to indicate the bigram has inputted. + const BigramProperty bigramProperty(&bigramTargetCodePoints, bigramProbability, + timestamp, 0 /* level */, 1 /* count */); + dictionary->addBigramWords(word0CodePoints, word0Length, &bigramProperty); } if (dictionary->needsToRunGC(true /* mindsBlockByGC */)) { return i + 1; @@ -558,11 +566,9 @@ static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, j return false; } } - for (const BigramProperty &bigarmProperty : *wordProperty.getBigramProperties()) { - const std::vector<int> *targetCodePoints = bigarmProperty.getTargetCodePoints(); + for (const BigramProperty &bigramProperty : *wordProperty.getBigramProperties()) { if (!dictionaryStructureWithBufferPolicy->addBigramWords(wordCodePoints, wordLength, - targetCodePoints->data(), targetCodePoints->size(), - bigarmProperty.getProbability(), bigarmProperty.getTimestamp())) { + &bigramProperty)) { LogUtils::logToJava(env, "Cannot add bigram to the new dict."); return false; } diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index e288413a3..fdc893653 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -88,11 +88,10 @@ void Dictionary::addUnigramWord(const int *const word, const int length, mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, unigramProperty); } -void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1, const int probability, const int timestamp) { +void Dictionary::addBigramWords(const int *const word0, const int length0, + const BigramProperty *const bigramProperty) { TimeKeeper::setCurrentTime(); - mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1, - probability, timestamp); + mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, bigramProperty); } void Dictionary::removeBigramWords(const int *const word0, const int length0, diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index b6149b338..f0a7e5b6a 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -76,8 +76,8 @@ class Dictionary { void addUnigramWord(const int *const codePoints, const int codePointCount, const UnigramProperty *const unigramProperty); - void addBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1, const int probability, const int timestamp); + void addBigramWords(const int *const word0, const int length0, + const BigramProperty *const bigramProperty); void removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1); diff --git a/native/jni/src/suggest/core/layout/proximity_info.cpp b/native/jni/src/suggest/core/layout/proximity_info.cpp index c40a2bdca..4c75a188e 100644 --- a/native/jni/src/suggest/core/layout/proximity_info.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info.cpp @@ -226,7 +226,7 @@ int ProximityInfo::getKeyCenterXOfKeyIdG( // When the referencePointY is NOT_A_COORDINATE, this method calculates the return value without // using the line segment. int ProximityInfo::getKeyCenterYOfKeyIdG( - const int keyId, const int referencePointY, const bool isGeometric) const { + const int keyId, const int referencePointY, const bool isGeometric) const { // TODO: Remove "isGeometric" and have separate "proximity_info"s for gesture and typing. if (keyId < 0) { return 0; diff --git a/native/jni/src/suggest/core/layout/proximity_info_state_utils.h b/native/jni/src/suggest/core/layout/proximity_info_state_utils.h index 71e83a80c..211a79737 100644 --- a/native/jni/src/suggest/core/layout/proximity_info_state_utils.h +++ b/native/jni/src/suggest/core/layout/proximity_info_state_utils.h @@ -56,7 +56,7 @@ class ProximityInfoStateUtils { const std::vector<int> *const sampledLengthCache, const std::vector<int> *const sampledInputIndice, std::vector<float> *sampledSpeedRates, std::vector<float> *sampledDirections); - static void refreshBeelineSpeedRates(const int mostCommonKeyWidth, const float averageSpeed, + static void refreshBeelineSpeedRates(const int mostCommonKeyWidth, const float averageSpeed, const int inputSize, const int *const xCoordinates, const int *const yCoordinates, const int *times, const int sampledInputSize, const std::vector<int> *const sampledInputXs, diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index 807f9b8dd..ce5a49f83 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -73,8 +73,8 @@ class DictionaryStructureWithBufferPolicy { const UnigramProperty *const unigramProperty) = 0; // Returns whether the update was success or not. - virtual bool addBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1, const int probability, const int timestamp) = 0; + virtual bool addBigramWords(const int *const word0, const int length0, + const BigramProperty *const bigramProperty) = 0; // Returns whether the update was success or not. virtual bool removeBigramWords(const int *const word0, const int length0, diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.cpp index 7d0d09631..08b4e0b5e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h index 15f924a6a..15f924a6a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp index a527f03bd..9e575858a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.cpp @@ -85,13 +85,13 @@ bool DynamicPtUpdatingHelper::addUnigramWord( } bool DynamicPtUpdatingHelper::addBigramWords(const int word0Pos, const int word1Pos, - const int probability, const int timestamp, bool *const outAddedNewBigram) { + const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) { const PtNodeParams sourcePtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word0Pos)); const PtNodeParams targetPtNodeParams( mPtNodeReader->fetchNodeInfoInBufferFromPtNodePos(word1Pos)); - return mPtNodeWriter->addNewBigramEntry(&sourcePtNodeParams, &targetPtNodeParams, probability, - timestamp, outAddedNewBigram); + return mPtNodeWriter->addNewBigramEntry(&sourcePtNodeParams, &targetPtNodeParams, + bigramProperty, outAddedNewBigram); } // Remove a bigram relation from word0Pos to word1Pos. diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h index 44914fe4c..f10d15a9b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h @@ -22,6 +22,7 @@ namespace latinime { +class BigramProperty; class BufferWithExtendableBuffer; class DynamicPtReadingHelper; class PtNodeReader; @@ -42,8 +43,8 @@ class DynamicPtUpdatingHelper { const UnigramProperty *const unigramProperty, bool *const outAddedNewUnigram); // Add a bigram relation from word0Pos to word1Pos. - bool addBigramWords(const int word0Pos, const int word1Pos, const int probability, - const int timestamp, bool *const outAddedNewBigram); + bool addBigramWords(const int word0Pos, const int word1Pos, + const BigramProperty *const bigramProperty, bool *const outAddedNewBigram); // Remove a bigram relation from word0Pos to word1Pos. bool removeBigramWords(const int word0Pos, const int word1Pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h index cbca3fe35..a8029f73f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_writer.h @@ -24,6 +24,7 @@ namespace latinime { +class BigramProperty; class UnigramProperty; // Interface class used to write PtNode information. @@ -70,7 +71,7 @@ class PtNodeWriter { const UnigramProperty *const unigramProperty, int *const ptNodeWritingPos) = 0; virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams, - const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp, + const PtNodeParams *const targetPtNodeParam, const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) = 0; virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.cpp index 847dcdee5..91c76941c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" @@ -44,7 +44,7 @@ const int ShortcutListReadingUtils::WHITELIST_SHORTCUT_PROBABILITY = 15; } /* static */ int ShortcutListReadingUtils::readShortcutTarget( - const uint8_t *const dictRoot, const int maxLength, int *const outWord, int *const pos) { + const uint8_t *const dictRoot, const int maxLength, int *const outWord, int *const pos) { return ByteArrayUtils::readStringAndAdvancePosition(dictRoot, maxLength, outWord, pos); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h index d065bf7fd..d065bf7fd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/bigram/bigram_list_policy.h index a898e2afc..00bb502dc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/bigram/bigram_list_policy.h @@ -21,7 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" -#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h index 85f46603e..54d1e0f6d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h @@ -22,9 +22,9 @@ #include "defines.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" -#include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/v2/bigram/bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/v2/shortcut/shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v2/ver2_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v2/ver2_pt_node_array_reader.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" @@ -88,8 +88,8 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { return false; } - bool addBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1, const int probability, const int timestamp) { + bool addBigramWords(const int *const word0, const int length0, + const BigramProperty *const bigramProperty) { // This method should not be called for non-updatable dictionary. AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/shortcut/shortcut_list_policy.h index 6d2b4778c..8e16ccc05 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v2/shortcut/shortcut_list_policy.h @@ -21,7 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp index 1645039d3..7a52fd180 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.cpp @@ -14,10 +14,11 @@ * limitations under the License. */ -#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h" -#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" +#include "suggest/core/dictionary/property/bigram_property.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" @@ -49,13 +50,12 @@ void Ver4BigramListPolicy::getNextBigram(int *const outBigramPos, int *const out } bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTargetTerminalId, - const int newProbability, const int timestamp, bool *const outAddedNewEntry) { + const BigramProperty *const bigramProperty, bool *const outAddedNewEntry) { // 1. The word has no bigrams yet. // 2. The word has bigrams, and there is the target in the list. // 3. The word has bigrams, and there is an invalid entry that can be reclaimed. // 4. The word has bigrams. We have to append new bigram entry to the list. // 5. Same as 4, but the list is the last entry of the content file. - if (outAddedNewEntry) { *outAddedNewEntry = false; } @@ -69,7 +69,7 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY, newTargetTerminalId); const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(&newBigramEntry, - newProbability, timestamp); + bigramProperty); // Write an entry. const int writingPos = mBigramDictContent->getBigramListHeadPos(terminalId); if (!mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, writingPos)) { @@ -102,7 +102,7 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY, newTargetTerminalId); const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom( - &newBigramEntry, newProbability, timestamp); + &newBigramEntry, bigramProperty); if (!mBigramDictContent->writeBigramEntryAtTail(&bigramEntryToWrite)) { return false; } @@ -128,7 +128,7 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget const BigramEntry updatedBigramEntry = originalBigramEntry.updateTargetTerminalIdAndGetEntry(newTargetTerminalId); const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom( - &updatedBigramEntry, newProbability, timestamp); + &updatedBigramEntry, bigramProperty); return mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, entryPosToUpdate); } @@ -253,19 +253,19 @@ int Ver4BigramListPolicy::getEntryPosToUpdate(const int targetTerminalIdToFind, } const BigramEntry Ver4BigramListPolicy::createUpdatedBigramEntryFrom( - const BigramEntry *const originalBigramEntry, const int newProbability, - const int timestamp) const { + const BigramEntry *const originalBigramEntry, + const BigramProperty *const bigramProperty) const { // TODO: Consolidate historical info and probability. if (mHeaderPolicy->hasHistoricalInfoOfWords()) { - // Use 1 for count to indicate the bigram has inputed. - const HistoricalInfo historicalInfoForUpdate(timestamp, 0 /* level */, 1 /* count */); + const HistoricalInfo historicalInfoForUpdate(bigramProperty->getTimestamp(), + bigramProperty->getLevel(), bigramProperty->getCount()); const HistoricalInfo updatedHistoricalInfo = ForgettingCurveUtils::createUpdatedHistoricalInfo( - originalBigramEntry->getHistoricalInfo(), newProbability, + originalBigramEntry->getHistoricalInfo(), bigramProperty->getProbability(), &historicalInfoForUpdate, mHeaderPolicy); return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo); } else { - return originalBigramEntry->updateProbabilityAndGetEntry(newProbability); + return originalBigramEntry->updateProbabilityAndGetEntry(bigramProperty->getProbability()); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h index c1f33359b..1613941c4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h @@ -24,6 +24,7 @@ namespace latinime { class BigramDictContent; +class BigramProperty; class HeaderPolicy; class TerminalPositionLookupTable; @@ -43,8 +44,8 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy { // Do nothing because we don't need to skip bigram lists in ver4 dictionaries. } - bool addNewEntry(const int terminalId, const int newTargetTerminalId, const int newProbability, - const int timestamp, bool *const outAddedNewEntry); + bool addNewEntry(const int terminalId, const int newTargetTerminalId, + const BigramProperty *const bigramProperty, bool *const outAddedNewEntry); bool removeEntry(const int terminalId, const int targetTerminalId); @@ -60,7 +61,7 @@ class Ver4BigramListPolicy : public DictionaryBigramsStructurePolicy { int *const outTailEntryPos) const; const BigramEntry createUpdatedBigramEntryFrom(const BigramEntry *const originalBigramEntry, - const int newProbability, const int timestamp) const; + const BigramProperty *const bigramProperty) const; bool updateHasNextFlag(const bool hasNext, const int bigramEntryPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h index fe984615c..790273541 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h @@ -19,7 +19,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/pt_common/shortcut/shortcut_list_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/shortcut_dict_content.h" #include "suggest/policyimpl/dictionary/structure/v4/content/terminal_position_lookup_table.h" diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp index 67420a252..0a435e91c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.cpp @@ -95,4 +95,4 @@ const PtNodeParams Ver4PatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProce } } -} +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index cc3a24a22..f89d3d7a0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -17,13 +17,13 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" #include "suggest/core/dictionary/property/unigram_property.h" -#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_writing_utils.h" #include "suggest/policyimpl/dictionary/structure/pt_common/patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" +#include "suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -76,7 +76,7 @@ bool Ver4PatriciaTrieNodeWriter::markPtNodeAsMoved( PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const PatriciaTrieReadingUtils::NodeFlags updatedFlags = DynamicPtReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, - false /* isDeleted */, false /* willBecomeNonTerminal */); + false /* isDeleted */, false /* willBecomeNonTerminal */); int writingPos = toBeUpdatedPtNodeParams->getHeadPos(); // Update flags. if (!DynamicPtWritingUtils::writeFlagsAndAdvancePosition(mTrieBuffer, updatedFlags, @@ -223,11 +223,10 @@ bool Ver4PatriciaTrieNodeWriter::writeNewTerminalPtNodeAndAdvancePosition( } bool Ver4PatriciaTrieNodeWriter::addNewBigramEntry( - const PtNodeParams *const sourcePtNodeParams, - const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp, - bool *const outAddedNewBigram) { + const PtNodeParams *const sourcePtNodeParams, const PtNodeParams *const targetPtNodeParam, + const BigramProperty *const bigramProperty, bool *const outAddedNewBigram) { if (!mBigramPolicy->addNewEntry(sourcePtNodeParams->getTerminalId(), - targetPtNodeParam->getTerminalId(), probability, timestamp, outAddedNewBigram)) { + targetPtNodeParam->getTerminalId(), bigramProperty, outAddedNewBigram)) { AKLOGE("Cannot add new bigram entry. terminalId: %d, targetTerminalId: %d", sourcePtNodeParams->getTerminalId(), targetPtNodeParam->getTerminalId()); return false; @@ -416,4 +415,4 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeFlags(const int ptNodePos, return true; } -} +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h index f20d3a241..e90bc44c0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h @@ -76,7 +76,7 @@ class Ver4PatriciaTrieNodeWriter : public PtNodeWriter { const UnigramProperty *const unigramProperty, int *const ptNodeWritingPos); virtual bool addNewBigramEntry(const PtNodeParams *const sourcePtNodeParams, - const PtNodeParams *const targetPtNodeParam, const int probability, const int timestamp, + const PtNodeParams *const targetPtNodeParam, const BigramProperty *const bigramProperty, bool *const outAddedNewBigram); virtual bool removeBigramEntry(const PtNodeParams *const sourcePtNodeParams, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 9999e0692..2fb3decee 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -209,8 +209,7 @@ bool Ver4PatriciaTriePolicy::addUnigramWord(const int *const word, const int len } bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int length0, - const int *const word1, const int length1, const int probability, - const int timestamp) { + const BigramProperty *const bigramProperty) { if (!mBuffers->isUpdatable()) { AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); return false; @@ -220,9 +219,10 @@ bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int le mDictBuffer->getTailPosition()); return false; } - if (length0 > MAX_WORD_LENGTH || length1 > MAX_WORD_LENGTH) { + if (length0 > MAX_WORD_LENGTH + || bigramProperty->getTargetCodePoints()->size() > MAX_WORD_LENGTH) { AKLOGE("Either src word or target word is too long to insert the bigram to the dictionary. " - "length0: %d, length1: %d", length0, length1); + "length0: %d, length1: %d", length0, bigramProperty->getTargetCodePoints()->size()); return false; } const int word0Pos = getTerminalPtNodePositionOfWord(word0, length0, @@ -230,14 +230,14 @@ bool Ver4PatriciaTriePolicy::addBigramWords(const int *const word0, const int le if (word0Pos == NOT_A_DICT_POS) { return false; } - const int word1Pos = getTerminalPtNodePositionOfWord(word1, length1, - false /* forceLowerCaseSearch */); + const int word1Pos = getTerminalPtNodePositionOfWord( + bigramProperty->getTargetCodePoints()->data(), + bigramProperty->getTargetCodePoints()->size(), false /* forceLowerCaseSearch */); if (word1Pos == NOT_A_DICT_POS) { return false; } bool addedNewBigram = false; - if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, probability, timestamp, - &addedNewBigram)) { + if (mUpdatingHelper.addBigramWords(word0Pos, word1Pos, bigramProperty, &addedNewBigram)) { if (addedNewBigram) { mBigramCount++; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h index 8f981def5..b78576484 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h @@ -21,10 +21,10 @@ #include "defines.h" #include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" -#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/pt_common/dynamic_pt_updating_helper.h" +#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" @@ -93,8 +93,8 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool addUnigramWord(const int *const word, const int length, const UnigramProperty *const unigramProperty); - bool addBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1, const int probability, const int timestamp); + bool addBigramWords(const int *const word0, const int length0, + const BigramProperty *const bigramProperty); bool removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 12298d967..f31c50253 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -19,9 +19,9 @@ #include <cstring> #include <queue> -#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/v4/bigram/ver4_bigram_list_policy.h" +#include "suggest/policyimpl/dictionary/structure/v4/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index aed24c56e..35d9a4e18 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -580,7 +580,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - // TODO: Add tests for bigrams when the implementation gets ready. addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); assertTrue(binaryDictionary.isValidWord("aaa")); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); @@ -590,6 +589,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidBigram("aaa", "abc")); + addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidBigram("aaa", "bbb")); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); assertTrue(binaryDictionary.migrateTo(toFormatVersion)); @@ -600,6 +604,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); assertTrue(binaryDictionary.isValidWord("bbb")); + assertTrue(binaryDictionary.isValidBigram("aaa", "abc")); + assertFalse(binaryDictionary.isValidBigram("aaa", "bbb")); + addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + assertTrue(binaryDictionary.isValidBigram("aaa", "bbb")); binaryDictionary.close(); dictFile.delete(); } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 9ceafa705..770e76e5f 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -1245,6 +1245,12 @@ public class BinaryDictionaryTests extends AndroidTestCase { addUnigramWord(binaryDictionary, "bbb", unigramProbability); final int bigramProbability = 10; addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability); + final int shortcutProbability = 10; + binaryDictionary.addUnigramWord("ccc", unigramProbability, "xxx", shortcutProbability, + false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); + binaryDictionary.addUnigramWord("ddd", unigramProbability, null /* shortcutTarget */, + Dictionary.NOT_A_PROBABILITY, true /* isNotAWord */, + true /* isBlacklisted */, 0 /* timestamp */); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); assertTrue(binaryDictionary.isValidBigram("aaa", "bbb")); @@ -1256,5 +1262,84 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); // TODO: Add tests for bigram frequency when the implementation gets ready. assertTrue(binaryDictionary.isValidBigram("aaa", "bbb")); + WordProperty wordProperty = binaryDictionary.getWordProperty("ccc"); + assertEquals(1, wordProperty.mShortcutTargets.size()); + assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord); + wordProperty = binaryDictionary.getWordProperty("ddd"); + assertTrue(wordProperty.mIsBlacklistEntry); + assertTrue(wordProperty.mIsNotAWord); + } + + public void testLargeDictMigration() { + testLargeDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, FormatSpec.VERSION4); + } + + private void testLargeDictMigration(final int fromFormatVersion, final int toFormatVersion) { + final int UNIGRAM_COUNT = 3000; + final int BIGRAM_COUNT = 3000; + final int codePointSetSize = 50; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<Pair<String, String>> bigrams = new ArrayList<Pair<String,String>>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = + new HashMap<Pair<String, String>, Integer>(); + + for (int i = 0; i < UNIGRAM_COUNT; i++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + final int unigramProbability = random.nextInt(0xFF); + addUnigramWord(binaryDictionary, word, unigramProbability); + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + words.add(word); + unigramProbabilities.put(word, unigramProbability); + } + + for (int i = 0; i < BIGRAM_COUNT; i++) { + final int word0Index = random.nextInt(words.size()); + final int word1Index = random.nextInt(words.size()); + if (word0Index == word1Index) { + continue; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + binaryDictionary.addBigramWords(word0, word1, bigramProbability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); + if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { + binaryDictionary.flushWithGC(); + } + final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + bigrams.add(bigram); + bigramProbabilities.put(bigram, bigramProbability); + } + assertTrue(binaryDictionary.migrateTo(toFormatVersion)); + + for (final String word : words) { + assertEquals((int)unigramProbabilities.get(word), binaryDictionary.getFrequency(word)); + } + assertEquals(unigramProbabilities.size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + + for (final Pair<String, String> bigram : bigrams) { + // TODO: Add tests for bigram frequency when the implementation gets ready. + assertTrue(binaryDictionary.isValidBigram(bigram.first, bigram.second)); + } + assertEquals(bigramProbabilities.size(), Integer.parseInt( + binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); } } |