diff options
38 files changed, 367 insertions, 114 deletions
diff --git a/java/res/xml-sw600dp/keys_comma_period.xml b/java/res/xml-sw600dp/keys_comma_period.xml new file mode 100644 index 000000000..7604e033d --- /dev/null +++ b/java/res/xml-sw600dp/keys_comma_period.xml @@ -0,0 +1,103 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <switch> + <case + latin:mode="email|url" + > + <Key + latin:keyLabel="," + latin:keyHintLabel="-" + latin:moreKeys="-" + latin:backgroundType="functional" + latin:keyStyle="hasShiftedLetterHintStyle" /> + <Key + latin:keyLabel="." + latin:keyHintLabel="_" + latin:moreKeys="_" + latin:backgroundType="functional" + latin:keyStyle="hasShiftedLetterHintStyle" /> + </case> + <case + latin:languageCode="ar" + > + <Key + latin:keyLabel="!text/keylabel_for_apostrophe" + latin:keyHintLabel="!text/keyhintlabel_for_apostrophe" + latin:moreKeys="!text/more_keys_for_apostrophe" + latin:backgroundType="functional" + latin:keyStyle="hasShiftedLetterHintStyle" /> + <Key + latin:keyLabel="." + latin:keyHintLabel="!text/keyhintlabel_for_arabic_diacritics" + latin:keyLabelFlags="hasPopupHint" + latin:moreKeys="!text/more_keys_for_arabic_diacritics" + latin:backgroundType="functional" + latin:keyStyle="hasShiftedLetterHintStyle" /> + </case> + <case + latin:languageCode="fa" + > + <Key + latin:keyLabel="!text/keylabel_for_apostrophe" + latin:keyHintLabel="!text/keyhintlabel_for_apostrophe" + latin:keyLabelFlags="hasPopupHint" + latin:moreKeys="!text/more_keys_for_apostrophe" + latin:backgroundType="functional" + latin:keyStyle="hasShiftedLetterHintStyle" /> + <Key + latin:keyLabel="." + latin:keyHintLabel="!text/keyhintlabel_for_arabic_diacritics" + latin:keyLabelFlags="hasPopupHint" + latin:moreKeys="!text/more_keys_for_arabic_diacritics" + latin:backgroundType="functional" + latin:keyStyle="hasShiftedLetterHintStyle" /> + </case> + <case + latin:languageCode="hy" + > + <!-- U+055D: "՝" ARMENIAN COMMA --> + <Key + latin:keyLabel="՝" + latin:backgroundType="functional" /> + <!-- U+0589: "։" ARMENIAN FULL STOP --> + <Key + latin:keyLabel="։" + latin:keyLabelFlags="hasPopupHint" + latin:backgroundType="functional" + latin:moreKeys="!text/more_keys_for_punctuation" /> + </case> + <default> + <Key + latin:keyLabel="!text/keylabel_for_tablet_comma" + latin:keyHintLabel="!text/keyhintlabel_for_tablet_comma" + latin:backgroundType="functional" + latin:moreKeys="!text/more_keys_for_tablet_comma" /> + <Key + latin:keyLabel="." + latin:keyHintLabel="!text/keyhintlabel_for_period" + latin:backgroundType="functional" + latin:moreKeys="!text/more_keys_for_period" /> + </default> + </switch> +</merge> diff --git a/java/res/xml-sw600dp/keys_exclamation_question.xml b/java/res/xml-sw600dp/keys_exclamation_question.xml index 983ef3897..cd38282ee 100644 --- a/java/res/xml-sw600dp/keys_exclamation_question.xml +++ b/java/res/xml-sw600dp/keys_exclamation_question.xml @@ -22,7 +22,7 @@ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" > <Key - latin:keyLabel="\?" /> - <Key latin:keyLabel="!" /> + <Key + latin:keyLabel="\?" /> </merge> diff --git a/java/res/xml/keys_comma_period.xml b/java/res/xml/keys_comma_period.xml index 02b46c23a..1b51e45ed 100644 --- a/java/res/xml/keys_comma_period.xml +++ b/java/res/xml/keys_comma_period.xml @@ -23,22 +23,6 @@ > <switch> <case - latin:mode="email|url" - > - <Key - latin:keyLabel="." - latin:keyHintLabel="_" - latin:moreKeys="_" - latin:backgroundType="functional" - latin:keyStyle="hasShiftedLetterHintStyle" /> - <Key - latin:keyLabel="," - latin:keyHintLabel="-" - latin:moreKeys="-" - latin:backgroundType="functional" - latin:keyStyle="hasShiftedLetterHintStyle" /> - </case> - <case latin:languageCode="ar" > <Key @@ -76,28 +60,28 @@ <case latin:languageCode="hy" > + <!-- U+055D: "՝" ARMENIAN COMMA --> + <Key + latin:keyLabel="՝" + latin:backgroundType="functional" /> <!-- U+0589: "։" ARMENIAN FULL STOP --> <Key latin:keyLabel="։" latin:keyLabelFlags="hasPopupHint" latin:backgroundType="functional" latin:moreKeys="!text/more_keys_for_punctuation" /> - <!-- U+055D: "՝" ARMENIAN COMMA --> - <Key - latin:keyLabel="՝" - latin:backgroundType="functional" /> </case> <default> <Key - latin:keyLabel="." - latin:keyHintLabel="!text/keyhintlabel_for_tablet_period" - latin:backgroundType="functional" - latin:moreKeys="!text/more_keys_for_tablet_period" /> - <Key latin:keyLabel="!text/keylabel_for_tablet_comma" latin:keyHintLabel="!text/keyhintlabel_for_tablet_comma" latin:backgroundType="functional" latin:moreKeys="!text/more_keys_for_tablet_comma" /> + <Key + latin:keyLabel="." + latin:keyHintLabel="!text/keyhintlabel_for_period" + latin:backgroundType="functional" + latin:moreKeys="!text/more_keys_for_period" /> </default> </switch> </merge> diff --git a/java/res/xml/rowkeys_symbols3.xml b/java/res/xml/rowkeys_symbols3.xml index 9f5e620e6..074078cb6 100644 --- a/java/res/xml/rowkeys_symbols3.xml +++ b/java/res/xml/rowkeys_symbols3.xml @@ -54,9 +54,9 @@ latin:keyLabel="!text/keylabel_for_symbols_semicolon" latin:moreKeys="!text/more_keys_for_symbols_semicolon" /> <Key - latin:keyLabel="!text/keylabel_for_symbols_question" - latin:moreKeys="!text/more_keys_for_symbols_question" /> - <Key latin:keyLabel="!" latin:moreKeys="!text/more_keys_for_symbols_exclamation" /> + <Key + latin:keyLabel="!text/keylabel_for_symbols_question" + latin:moreKeys="!text/more_keys_for_symbols_question" /> </merge> diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java index ee4ac950c..52f190e77 100644 --- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java +++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java @@ -823,14 +823,16 @@ public final class PointerTracker implements PointerTrackerQueue.Element { final int size = sAggregratedPointers.getPointerSize(); if (size > sLastRecognitionPointSize && stroke.hasRecognitionTimePast(eventTime, sLastRecognitionTime)) { - sLastRecognitionPointSize = size; - sLastRecognitionTime = eventTime; if (DEBUG_LISTENER) { Log.d(TAG, String.format("[%d] onUpdateBatchInput: batchPoints=%d", mPointerId, size)); } mTimerProxy.startUpdateBatchInputTimer(this); mListener.onUpdateBatchInput(sAggregratedPointers); + // The listener may change the size of the pointers (when auto-committing + // for example), so we need to get the size from the pointers again. + sLastRecognitionPointSize = sAggregratedPointers.getPointerSize(); + sLastRecognitionTime = eventTime; } } } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index 67553fb75..684cf632b 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -209,8 +209,8 @@ public final class KeyboardTextsSet { /* 104 */ "keylabel_for_tablet_comma", /* 105 */ "keyhintlabel_for_tablet_comma", /* 106 */ "more_keys_for_tablet_comma", - /* 107 */ "keyhintlabel_for_tablet_period", - /* 108 */ "more_keys_for_tablet_period", + /* 107 */ "keyhintlabel_for_period", + /* 108 */ "more_keys_for_period", /* 109 */ "keylabel_for_apostrophe", /* 110 */ "keyhintlabel_for_apostrophe", /* 111 */ "more_keys_for_apostrophe", diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index d9bad7e57..541e69788 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -44,9 +44,9 @@ public final class BinaryDictionary extends Dictionary { private static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; // Must be equal to MAX_RESULTS in native/jni/src/defines.h private static final int MAX_RESULTS = 18; - // Required space count for auto commit. - // TODO: Remove this heuristic. - private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3; + // The cutoff returned by native for auto-commit confidence. + // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h + private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; @UsedForTesting public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; @@ -343,18 +343,7 @@ public final class BinaryDictionary extends Dictionary { @Override public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { - // TODO: actually use the confidence rather than use this completely broken heuristic - final String word = candidate.mWord; - final int length = word.length(); - int remainingSpaces = SPACE_COUNT_FOR_AUTO_COMMIT; - for (int i = 0; i < length; ++i) { - // This is okay because no low-surrogate and no high-surrogate can ever match the - // space character, so we don't need to take care of iterating on code points. - if (Constants.CODE_SPACE == word.charAt(i)) { - if (0 >= --remainingSpaces) return true; - } - } - return false; + return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; } @Override diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index a282f595c..e90137674 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -288,6 +288,8 @@ public final class BinaryDictIOUtils { return BinaryDictEncoderUtils.getByteSize(value); } + // TODO: Remove this method. + @Deprecated static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) { final int flags = dictBuffer.readUnsignedByte(); BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 3796a466c..e251f7df7 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -391,4 +391,6 @@ public abstract class DictDecoder { return readLength; } } + + public abstract void skipPtNode(final FormatOptions formatOptions); } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java index 848277cd4..75d1058ad 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -231,4 +231,40 @@ public class Ver3DictDecoder extends DictDecoder { public boolean hasNextPtNodeArray() { return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; } + + @Override + public void skipPtNode(final FormatOptions formatOptions) { + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + PtNodeReader.readParentAddress(mDictBuffer, formatOptions); + BinaryDictIOUtils.skipString(mDictBuffer, + (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readFrequency(mDictBuffer); + if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { + final int shortcutsSize = mDictBuffer.readUnsignedShort(); + mDictBuffer.position(mDictBuffer.position() + shortcutsSize + - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = mDictBuffer.readUnsignedByte(); + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + mDictBuffer.readUnsignedByte(); + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + mDictBuffer.readUnsignedShort(); + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + mDictBuffer.readUnsignedInt24(); + break; + } + if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; + } + if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode."); + } + } + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 0aa431966..fa19e2677 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -293,4 +293,14 @@ public class Ver4DictDecoder extends DictDecoder { public boolean hasNextPtNodeArray() { return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; } + + @Override + public void skipPtNode(final FormatOptions formatOptions) { + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + PtNodeReader.readParentAddress(mDictBuffer, formatOptions); + BinaryDictIOUtils.skipString(mDictBuffer, + (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer); + PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions); + } } diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 21c15ea67..38159b0f3 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -344,8 +344,7 @@ static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, static const int GET_PROPERTY_RESULT_LENGTH = 100; char resultChars[GET_PROPERTY_RESULT_LENGTH]; resultChars[0] = '\0'; - dictionary->getDictionaryStructurePolicy()->getProperty(queryChars, resultChars, - GET_PROPERTY_RESULT_LENGTH); + dictionary->getProperty(queryChars, resultChars, GET_PROPERTY_RESULT_LENGTH); return env->NewStringUTF(resultChars); } diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h index c920f64b4..742e388e4 100644 --- a/native/jni/src/defines.h +++ b/native/jni/src/defines.h @@ -298,9 +298,19 @@ static inline void prof_out(void) { #define NOT_AN_INDEX (-1) #define NOT_A_PROBABILITY (-1) #define NOT_A_DICT_POS (S_INT_MIN) + // A special value to mean the first word confidence makes no sense in this case, // e.g. this is not a multi-word suggestion. -#define NOT_A_FIRST_WORD_CONFIDENCE (S_INT_MIN) +#define NOT_A_FIRST_WORD_CONFIDENCE (S_INT_MAX) +// How high the confidence needs to be for us to auto-commit. Arbitrary. +// This needs to be the same as CONFIDENCE_FOR_AUTO_COMMIT in BinaryDictionary.java +#define CONFIDENCE_FOR_AUTO_COMMIT (1000000) +// 80% of the full confidence +#define DISTANCE_WEIGHT_FOR_AUTO_COMMIT (80 * CONFIDENCE_FOR_AUTO_COMMIT / 100) +// 100% of the full confidence +#define LENGTH_WEIGHT_FOR_AUTO_COMMIT (CONFIDENCE_FOR_AUTO_COMMIT) +// 80% of the full confidence +#define SPACE_COUNT_WEIGHT_FOR_AUTO_COMMIT (80 * CONFIDENCE_FOR_AUTO_COMMIT / 100) #define KEYCODE_SPACE ' ' #define KEYCODE_SINGLE_QUOTE '\'' diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h index 9099e8285..49cfdecac 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node.h +++ b/native/jni/src/suggest/core/dicnode/dic_node.h @@ -271,7 +271,7 @@ class DicNode { return isTerminalNodes && currentNodeDepth > 0 && currentNodeDepth == terminalNodeDepth; } - bool shouldBeFilterdBySafetyNetForBigram() const { + bool shouldBeFilteredBySafetyNetForBigram() const { const uint16_t currentDepth = getNodeCodePointCount(); const int prevWordLen = mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength() - mDicNodeState.mDicNodeStatePrevWord.getPrevWordStart() - 1; @@ -321,6 +321,16 @@ class DicNode { DUMP_WORD_AND_SCORE("OUTPUT"); } + // "Total" in this context (and other methods in this class) means the whole suggestion. When + // this represents a multi-word suggestion, the referenced PtNode (in mDicNodeState) is only + // the one that corresponds to the last word of the suggestion, and all the previous words + // are concatenated together in mPrevWord - which contains a space at the end. + int getTotalNodeSpaceCount() const { + if (isFirstWord()) return 0; + return CharUtils::getSpaceCount(mDicNodeState.mDicNodeStatePrevWord.mPrevWord, + mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength()); + } + int getSecondWordFirstInputIndex(const ProximityInfoState *const pInfoState) const { const int inputIndex = mDicNodeState.mDicNodeStatePrevWord.getSecondWordFirstInputIndex(); if (inputIndex == NOT_AN_INDEX) { diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index 5969b31cc..59ead1894 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -129,7 +129,7 @@ bool Dictionary::needsToRunGC(const bool mindsBlockByGC) { } void Dictionary::getProperty(const char *const query, char *const outResult, - const int maxResultLength) const { + const int maxResultLength) { return mDictionaryStructureWithBufferPolicy->getProperty(query, outResult, maxResultLength); } diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 43d3b964d..0195d5bf0 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -84,7 +84,7 @@ class Dictionary { bool needsToRunGC(const bool mindsBlockByGC); void getProperty(const char *const query, char *const outResult, - const int maxResultLength) const; + const int maxResultLength); const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const { return mDictionaryStructureWithBufferPolicy; diff --git a/native/jni/src/suggest/core/layout/proximity_info_params.cpp b/native/jni/src/suggest/core/layout/proximity_info_params.cpp index 0e887f700..49df10301 100644 --- a/native/jni/src/suggest/core/layout/proximity_info_params.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info_params.cpp @@ -69,13 +69,13 @@ const float ProximityInfoParams::STRAIGHT_ANGLE_THRESHOLD = M_PI_F * 15.0f / 180 const float ProximityInfoParams::SKIP_CORNER_PROBABILITY = 0.4f; const float ProximityInfoParams::SPEED_MARGIN = 0.1f; const float ProximityInfoParams::CENTER_VALUE_OF_NORMALIZED_DISTRIBUTION = 0.0f; -// TODO: The variance is critical for accuracy; thus, adjusting these parameter by machine +// TODO: The variance is critical for accuracy; thus, adjusting these parameters by machine // learning or something would be efficient. -const float ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DIVIATION = 0.3f; -const float ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDERD_DIVIATION = 0.25f; -const float ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DIVIATION = 0.5f; -const float ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDERD_DIVIATION = 0.15f; -const float ProximityInfoParams::MIN_STANDERD_DIVIATION = 0.37f; +const float ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DEVIATION = 0.3f; +const float ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDARD_DEVIATION = 0.25f; +const float ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DEVIATION = 0.5f; +const float ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDARD_DEVIATION = 0.15f; +const float ProximityInfoParams::MIN_STANDARD_DEVIATION = 0.37f; const float ProximityInfoParams::PREV_DISTANCE_WEIGHT = 0.5f; const float ProximityInfoParams::NEXT_DISTANCE_WEIGHT = 0.6f; diff --git a/native/jni/src/suggest/core/layout/proximity_info_params.h b/native/jni/src/suggest/core/layout/proximity_info_params.h index 4e47f7308..ae1f82c22 100644 --- a/native/jni/src/suggest/core/layout/proximity_info_params.h +++ b/native/jni/src/suggest/core/layout/proximity_info_params.h @@ -73,11 +73,11 @@ class ProximityInfoParams { static const float SKIP_CORNER_PROBABILITY; static const float SPEED_MARGIN; static const float CENTER_VALUE_OF_NORMALIZED_DISTRIBUTION; - static const float SPEEDxANGLE_WEIGHT_FOR_STANDARD_DIVIATION; - static const float MAX_SPEEDxANGLE_RATE_FOR_STANDERD_DIVIATION; - static const float SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DIVIATION; - static const float MAX_SPEEDxNEAREST_RATE_FOR_STANDERD_DIVIATION; - static const float MIN_STANDERD_DIVIATION; + static const float SPEEDxANGLE_WEIGHT_FOR_STANDARD_DEVIATION; + static const float MAX_SPEEDxANGLE_RATE_FOR_STANDARD_DEVIATION; + static const float SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DEVIATION; + static const float MAX_SPEEDxNEAREST_RATE_FOR_STANDARD_DEVIATION; + static const float MIN_STANDARD_DEVIATION; static const float PREV_DISTANCE_WEIGHT; static const float NEXT_DISTANCE_WEIGHT; diff --git a/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp b/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp index 904671f7f..e1b35340b 100644 --- a/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp @@ -708,13 +708,13 @@ namespace latinime { const float inputCharProbability = 1.0f - skipProbability; const float speedxAngleRate = min(speedRate * currentAngle / M_PI_F - * ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DIVIATION, - ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDERD_DIVIATION); + * ProximityInfoParams::SPEEDxANGLE_WEIGHT_FOR_STANDARD_DEVIATION, + ProximityInfoParams::MAX_SPEEDxANGLE_RATE_FOR_STANDARD_DEVIATION); const float speedxNearestKeyDistanceRate = min(speedRate * nearestKeyDistance - * ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DIVIATION, - ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDERD_DIVIATION); + * ProximityInfoParams::SPEEDxNEAREST_WEIGHT_FOR_STANDARD_DEVIATION, + ProximityInfoParams::MAX_SPEEDxNEAREST_RATE_FOR_STANDARD_DEVIATION); const float sigma = speedxAngleRate + speedxNearestKeyDistanceRate - + ProximityInfoParams::MIN_STANDERD_DIVIATION; + + ProximityInfoParams::MIN_STANDARD_DEVIATION; ProximityInfoUtils::NormalDistribution distribution(ProximityInfoParams::CENTER_VALUE_OF_NORMALIZED_DISTRIBUTION, sigma); diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index c7ffef0d5..41f82049f 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -80,8 +80,10 @@ class DictionaryStructureWithBufferPolicy { virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0; + // Currently, this method is used only for testing. You may want to consider creating new + // dedicated method instead of this if you want to use this in the production. virtual void getProperty(const char *const query, char *const outResult, - const int maxResultLength) const = 0; + const int maxResultLength) = 0; protected: DictionaryStructureWithBufferPolicy() {} diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index 51cfba17a..73ccebc88 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -166,7 +166,11 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen // TODO: have partial commit work even with multiple pointers. const bool outputSecondWordFirstLetterInputIndex = traverseSession->isOnlyOnePointerUsed(0 /* pointerId */); - outputAutoCommitFirstWordConfidence[0] = computeFirstWordConfidence(); + if (terminalSize > 0) { + // If we have no suggestions, don't write this + outputAutoCommitFirstWordConfidence[0] = + computeFirstWordConfidence(&terminals[0]); + } // Output suggestion results here for (int terminalIndex = 0; terminalIndex < terminalSize && outputWordIndex < MAX_RESULTS; @@ -255,9 +259,55 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen return outputWordIndex; } -int Suggest::computeFirstWordConfidence() const { - // TODO: implement this. - return NOT_A_FIRST_WORD_CONFIDENCE; +int Suggest::computeFirstWordConfidence(const DicNode *const terminalDicNode) const { + // Get the number of spaces in the first suggestion + const int spaceCount = terminalDicNode->getTotalNodeSpaceCount(); + // Get the number of characters in the first suggestion + const int length = terminalDicNode->getTotalNodeCodePointCount(); + // Get the distance for the first word of the suggestion + const float distance = terminalDicNode->getNormalizedCompoundDistanceAfterFirstWord(); + + // Arbitrarily, we give a score whose useful values range from 0 to 1,000,000. + // 1,000,000 will be the cutoff to auto-commit. It's fine if the number is under 0 or + // above 1,000,000 : under 0 just means it's very bad to commit, and above 1,000,000 means + // we are very confident. + // Expected space count is 1 ~ 5 + static const int MIN_EXPECTED_SPACE_COUNT = 1; + static const int MAX_EXPECTED_SPACE_COUNT = 5; + // Expected length is about 4 ~ 30 + static const int MIN_EXPECTED_LENGTH = 4; + static const int MAX_EXPECTED_LENGTH = 30; + // Expected distance is about 0.2 ~ 2.0, but consider 0.0 ~ 2.0 + static const float MIN_EXPECTED_DISTANCE = 0.0; + static const float MAX_EXPECTED_DISTANCE = 2.0; + // This is not strict: it's where most stuff will be falling, but it's still fine if it's + // outside these values. We want to output a value that reflects all of these. Each factor + // contributes a bit. + + // We need at least a space. + if (spaceCount < 1) return NOT_A_FIRST_WORD_CONFIDENCE; + + // The smaller the edit distance, the higher the contribution. MIN_EXPECTED_DISTANCE means 0 + // contribution, while MAX_EXPECTED_DISTANCE means full contribution according to the + // weight of the distance. Clamp to avoid overflows. + const float clampedDistance = distance < MIN_EXPECTED_DISTANCE ? MIN_EXPECTED_DISTANCE + : distance > MAX_EXPECTED_DISTANCE ? MAX_EXPECTED_DISTANCE : distance; + const int distanceContribution = DISTANCE_WEIGHT_FOR_AUTO_COMMIT + * (MAX_EXPECTED_DISTANCE - clampedDistance) + / (MAX_EXPECTED_DISTANCE - MIN_EXPECTED_DISTANCE); + // The larger the suggestion length, the larger the contribution. MIN_EXPECTED_LENGTH is no + // contribution, MAX_EXPECTED_LENGTH is full contribution according to the weight of the + // length. Length is guaranteed to be between 1 and 48, so we don't need to clamp. + const int lengthContribution = LENGTH_WEIGHT_FOR_AUTO_COMMIT + * (length - MIN_EXPECTED_LENGTH) / (MAX_EXPECTED_LENGTH - MIN_EXPECTED_LENGTH); + // The more spaces, the larger the contribution. MIN_EXPECTED_SPACE_COUNT space is no + // contribution, MAX_EXPECTED_SPACE_COUNT spaces is full contribution according to the + // weight of the space count. + const int spaceContribution = SPACE_COUNT_WEIGHT_FOR_AUTO_COMMIT + * (spaceCount - MIN_EXPECTED_SPACE_COUNT) + / (MAX_EXPECTED_SPACE_COUNT - MIN_EXPECTED_SPACE_COUNT); + + return distanceContribution + lengthContribution + spaceContribution; } /** @@ -395,7 +445,7 @@ void Suggest::processTerminalDicNode( if (!dicNode->isTerminalWordNode()) { return; } - if (dicNode->shouldBeFilterdBySafetyNetForBigram()) { + if (dicNode->shouldBeFilteredBySafetyNetForBigram()) { return; } // Create a non-cached node here. diff --git a/native/jni/src/suggest/core/suggest.h b/native/jni/src/suggest/core/suggest.h index 0e8bd1195..b20343d29 100644 --- a/native/jni/src/suggest/core/suggest.h +++ b/native/jni/src/suggest/core/suggest.h @@ -58,7 +58,7 @@ class Suggest : public SuggestInterface { int outputSuggestions(DicTraverseSession *traverseSession, int *frequencies, int *outputCodePoints, int *outputIndicesToPartialCommit, int *outputTypes, int *outputAutoCommitFirstWordConfidence) const; - int computeFirstWordConfidence() const; + int computeFirstWordConfidence(const DicNode *const terminalDicNode) const; void initializeSearch(DicTraverseSession *traverseSession, int commitPoint) const; void expandCurrentDicNodes(DicTraverseSession *traverseSession) const; void processTerminalDicNode(DicTraverseSession *traverseSession, DicNode *dicNode) const; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index 0d8c92768..a5a42ead1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -33,12 +33,16 @@ namespace latinime { +// Note that these are corresponding definitions in Java side in BinaryDictionaryTests and +// BinaryDictionaryDecayingTests. const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; +const char *const DynamicPatriciaTriePolicy::SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY = + "SET_NEEDS_TO_DECAY_FOR_TESTING"; const int DynamicPatriciaTriePolicy::MAX_DICT_EXTENDED_REGION_SIZE = 1024 * 1024; const int DynamicPatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS = DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - 1024; -const int DynamicPatriciaTriePolicy::MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING = 2 * 60 * 60; +const int DynamicPatriciaTriePolicy::DECAY_INTERVAL_FOR_DECAYING_DICTS = 2 * 60 * 60; void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, DicNodeVector *const childDicNodes) const { @@ -301,7 +305,7 @@ void DynamicPatriciaTriePolicy::flush(const char *const filePath) { return; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); + &mBigramListPolicy, &mShortcutListPolicy, false /* needsToDecay */); writingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount); } @@ -310,9 +314,15 @@ void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) { AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); return; } + const bool runGCwithDecay = needsToDecay(); + DynamicBigramListPolicy bigramListPolicyForGC(&mBufferWithExtendableBuffer, + &mShortcutListPolicy, runGCwithDecay); DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); + &bigramListPolicyForGC, &mShortcutListPolicy, runGCwithDecay); writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy); + if (runGCwithDecay) { + mNeedsToDecayForTesting = false; + } } bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { @@ -340,8 +350,7 @@ bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { } else if (mBigramCount >= DecayingUtils::MAX_BIGRAM_COUNT) { // Bigram count exceeds the limit. return true; - } else if (mindsBlockByGC && mHeaderPolicy.getLastUpdatedTime() - + MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING < time(0)) { + } else if (mindsBlockByGC && needsToDecay()) { // Time to update probabilities for decaying. return true; } @@ -350,12 +359,19 @@ bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { } void DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult, - const int maxResultLength) const { + const int maxResultLength) { if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) { snprintf(outResult, maxResultLength, "%d", mUnigramCount); } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) { snprintf(outResult, maxResultLength, "%d", mBigramCount); + } else if (strncmp(query, SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY, maxResultLength) == 0) { + mNeedsToDecayForTesting = true; } } +bool DynamicPatriciaTriePolicy::needsToDecay() const { + return mHeaderPolicy.isDecayingDict() && (mNeedsToDecayForTesting + || mHeaderPolicy.getLastDecayedTime() + DECAY_INTERVAL_FOR_DECAYING_DICTS < time(0)); +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index d3150c6fc..c3bbe9977 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -40,7 +40,7 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()), mUnigramCount(mHeaderPolicy.getUnigramCount()), - mBigramCount(mHeaderPolicy.getBigramCount()) {} + mBigramCount(mHeaderPolicy.getBigramCount()), mNeedsToDecayForTesting(false) {} ~DynamicPatriciaTriePolicy() { delete mBuffer; @@ -95,16 +95,17 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool needsToRunGC(const bool mindsBlockByGC) const; void getProperty(const char *const query, char *const outResult, - const int maxResultLength) const; + const int maxResultLength); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); - static const char*const UNIGRAM_COUNT_QUERY; - static const char*const BIGRAM_COUNT_QUERY; + static const char *const UNIGRAM_COUNT_QUERY; + static const char *const BIGRAM_COUNT_QUERY; + static const char *const SET_NEEDS_TO_DECAY_FOR_TESTING_QUERY; static const int MAX_DICT_EXTENDED_REGION_SIZE; static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS; - static const int MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING; + static const int DECAY_INTERVAL_FOR_DECAYING_DICTS; const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; @@ -113,6 +114,9 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { DynamicBigramListPolicy mBigramListPolicy; int mUnigramCount; int mBigramCount; + int mNeedsToDecayForTesting; + + bool needsToDecay() const; }; } // namespace latinime #endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index 28124d251..db6d48fdb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -153,7 +153,7 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam const int extendedRegionSize = headerPolicy->getExtendedRegionSize() + mBuffer->getUsedAdditionalBufferSize(); if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */, - unigramCount, bigramCount, extendedRegionSize)) { + false /* updatesLastDecayedTime */, unigramCount, bigramCount, extendedRegionSize)) { return; } DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, mBuffer); @@ -170,7 +170,7 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNod } BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */, - unigramCount, bigramCount, 0 /* extendedRegionSize */)) { + mNeedsToDecay, unigramCount, bigramCount, 0 /* extendedRegionSize */)) { return; } DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, &newDictBuffer); @@ -488,12 +488,12 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, DynamicPatriciaTrieGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( - this, mBuffer, mIsDecayingDict); + this, mBuffer, mNeedsToDecay); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( &traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) { return false; } - if (mIsDecayingDict && traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted + if (mNeedsToDecay && traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted .getValidUnigramCount() > DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC) { // TODO: Remove more unigrams. } @@ -506,7 +506,7 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, return false; } - if (mIsDecayingDict && traversePolicyToUpdateBigramProbability.getValidBigramEntryCount() + if (mNeedsToDecay && traversePolicyToUpdateBigramProbability.getValidBigramEntryCount() > DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC) { // TODO: Remove more bigrams. } @@ -525,7 +525,7 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, // Create policy instance for the GCed dictionary. DynamicShortcutListPolicy newDictShortcutPolicy(bufferToWrite); DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy, - mIsDecayingDict); + mNeedsToDecay); // Create reading helper for the GCed dictionary. DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictBigramPolicy, &newDictShortcutPolicy); @@ -544,7 +544,7 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, int DynamicPatriciaTrieWritingHelper::getUpdatedProbability(const int originalProbability, const int newProbability) { - if (mIsDecayingDict) { + if (mNeedsToDecay) { return DecayingUtils::getUpdatedUnigramProbability(originalProbability, newProbability); } else { return newProbability; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index ecee2cdbf..0caf29120 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -51,9 +51,9 @@ class DynamicPatriciaTrieWritingHelper { DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer, DynamicBigramListPolicy *const bigramPolicy, - DynamicShortcutListPolicy *const shortcutPolicy, const bool isDecayingDict) + DynamicShortcutListPolicy *const shortcutPolicy, const bool needsToDecay) : mBuffer(buffer), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy), - mIsDecayingDict(isDecayingDict) {} + mNeedsToDecay(needsToDecay) {} ~DynamicPatriciaTrieWritingHelper() {} @@ -94,7 +94,7 @@ class DynamicPatriciaTrieWritingHelper { BufferWithExtendableBuffer *const mBuffer; DynamicBigramListPolicy *const mBigramPolicy; DynamicShortcutListPolicy *const mShortcutPolicy; - const bool mIsDecayingDict; + const bool mNeedsToDecay; bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos, const int bigramLinkedNodePos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 9ce9994dd..eb072fbaf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -23,6 +23,7 @@ const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WOR // TODO: Change attribute string to "IS_DECAYING_DICT". const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE"; const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date"; +const char *const HeaderPolicy::LAST_DECAYED_TIME_KEY = "LAST_DECAYED_TIME"; const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT"; const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT"; const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE"; @@ -63,8 +64,8 @@ float HeaderPolicy::readMultipleWordCostMultiplier() const { } bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, - const bool updatesLastUpdatedTime, const int unigramCount, const int bigramCount, - const int extendedRegionSize) const { + const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime, + const int unigramCount, const int bigramCount, const int extendedRegionSize) const { int writingPos = 0; if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion, &writingPos)) { @@ -90,6 +91,11 @@ bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferT HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_UPDATED_TIME_KEY, time(0)); } + if (updatesLastDecayedTime) { + // Set current time as a last updated time. + HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_DECAYED_TIME_KEY, + time(0)); + } if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite, &writingPos)) { return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 4261667fa..a9c7805a8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -40,6 +40,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { IS_DECAYING_DICT_KEY, false /* defaultValue */)), mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), + mLastDecayedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + LAST_DECAYED_TIME_KEY, time(0) /* defaultValue */)), mUnigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, UNIGRAM_COUNT_KEY, 0 /* defaultValue */)), mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, @@ -58,6 +60,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { IS_DECAYING_DICT_KEY, false /* defaultValue */)), mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), + mLastDecayedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0) {} ~HeaderPolicy() {} @@ -90,6 +94,10 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mLastUpdatedTime; } + AK_FORCE_INLINE int getLastDecayedTime() const { + return mLastDecayedTime; + } + AK_FORCE_INLINE int getUnigramCount() const { return mUnigramCount; } @@ -106,8 +114,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { int *outValue, int outValueSize) const; bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, - const bool updatesLastUpdatedTime, const int unigramCount, - const int bigramCount, const int extendedRegionSize) const; + const bool updatesLastUpdatedTime, const bool updatesLastDecayedTime, + const int unigramCount, const int bigramCount, const int extendedRegionSize) const; private: DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy); @@ -115,6 +123,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; static const char *const IS_DECAYING_DICT_KEY; static const char *const LAST_UPDATED_TIME_KEY; + static const char *const LAST_DECAYED_TIME_KEY; static const char *const UNIGRAM_COUNT_KEY; static const char *const BIGRAM_COUNT_KEY; static const char *const EXTENDED_REGION_SIZE_KEY; @@ -128,6 +137,7 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const float mMultiWordCostMultiplier; const bool mIsDecayingDict; const int mLastUpdatedTime; + const int mLastDecayedTime; const int mUnigramCount; const int mBigramCount; const int mExtendedRegionSize; diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index 8d88c68e8..0f8662aea 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -114,7 +114,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { } void getProperty(const char *const query, char *const outResult, - const int maxResultLength) const { + const int maxResultLength) { // getProperty is not supported for this class. if (maxResultLength > 0) { outResult[0] = '\0'; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp index f22e94c6a..994826fa8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -44,7 +44,8 @@ const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); HeaderPolicy headerPolicy(FormatUtils::VERSION_3, attributeMap); headerPolicy.writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */, - 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); + true /* updatesLastDecayedTime */, 0 /* unigramCount */, 0 /* bigramCount */, + 0 /* extendedRegionSize */); BufferWithExtendableBuffer bodyBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(&bodyBuffer, 0 /* rootPos */)) { return false; diff --git a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h index 89e53f441..007c19e0a 100644 --- a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h +++ b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h @@ -101,7 +101,7 @@ class TypingTraversal : public Traversal { } const int16_t pointIndex = dicNode->getInputIndex(0); return pointIndex <= inputSize && !dicNode->isTotalInputSizeExceedingLimit() - && !dicNode->shouldBeFilterdBySafetyNetForBigram(); + && !dicNode->shouldBeFilteredBySafetyNetForBigram(); } AK_FORCE_INLINE bool shouldDepthLevelCache( diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h index 2e735a81c..41663c81a 100644 --- a/native/jni/src/utils/char_utils.h +++ b/native/jni/src/utils/char_utils.h @@ -75,6 +75,16 @@ class CharUtils { return c; } + static AK_FORCE_INLINE int getSpaceCount(const int *const codePointBuffer, const int length) { + int spaceCount = 0; + for (int i = 0; i < length; ++i) { + if (codePointBuffer[i] == KEYCODE_SPACE) { + ++spaceCount; + } + } + return spaceCount; + } + static unsigned short latin_tolower(const unsigned short c); private: diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index cf85d97a0..8a439fc22 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -32,6 +32,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; + // Note that these are corresponding definitions in native code in + // latinime::DynamicPatriciaTriePolicy. + private static final String SET_NEEDS_TO_DECAY_FOR_TESTING_KEY = + "SET_NEEDS_TO_DECAY_FOR_TESTING"; + private static final int DUMMY_PROBABILITY = 0; @Override @@ -45,6 +50,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { + binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY); binaryDictionary.flushWithGC(); } @@ -53,6 +59,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { // typed in 32 GCs are removed. final int count = 32; for (int i = 0; i < count; i++) { + binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY); binaryDictionary.flushWithGC(); } } diff --git a/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml index 9728c9963..4cf742441 100644 --- a/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values-ca/donottranslate-more-keys.xml @@ -72,7 +72,7 @@ <string name="more_keys_for_l">l·l,ł</string> <!-- U+00B7: "·" MIDDLE DOT --> <string name="more_keys_for_punctuation">"!fixedColumnOrder!4,·,!,\\,,\?,:,;,\@"</string> - <string name="more_keys_for_tablet_period">\?,·</string> + <string name="more_keys_for_period">\?,·</string> <!-- U+00E7: "ç" LATIN SMALL LETTER C WITH CEDILLA --> <string name="keylabel_for_spanish_row2_10">ç</string> </resources> diff --git a/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml index 849429629..8e6b4ee06 100644 --- a/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values-es/donottranslate-more-keys.xml @@ -75,7 +75,7 @@ <!-- U+00A1: "¡" INVERTED EXCLAMATION MARK --> <string name="more_keys_for_tablet_comma">"!,¡"</string> <!-- U+00BF: "¿" INVERTED QUESTION MARK --> - <string name="more_keys_for_tablet_period">"\?,¿"</string> + <string name="more_keys_for_period">"\?,¿"</string> <string name="keylabel_for_apostrophe">\"</string> <string name="keyhintlabel_for_apostrophe">\'</string> <string name="more_keys_for_apostrophe">\'</string> diff --git a/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml index 5a03c803c..ab4fbda44 100644 --- a/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values-fa/donottranslate-more-keys.xml @@ -81,8 +81,8 @@ <string name="keylabel_for_tablet_comma">"،"</string> <string name="keyhintlabel_for_tablet_comma">"!"</string> <string name="more_keys_for_tablet_comma">"!,\\,"</string> - <string name="keyhintlabel_for_tablet_period">"؟"</string> - <string name="more_keys_for_tablet_period">"؟,\?"</string> + <string name="keyhintlabel_for_period">"؟"</string> + <string name="more_keys_for_period">"؟,\?"</string> <string name="keylabel_for_apostrophe">،</string> <string name="keyhintlabel_for_apostrophe">؟</string> <string name="more_keys_for_apostrophe">"!fixedColumnOrder!4,:,!,؟,؛,-,/,«|»,»|«"</string> diff --git a/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml index feaed4c98..a1633316f 100644 --- a/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values-iw/donottranslate-more-keys.xml @@ -55,6 +55,6 @@ <string name="keylabel_for_currency">₪</string> <string name="keyhintlabel_for_tablet_comma">!</string> <string name="more_keys_for_tablet_comma">!</string> - <string name="keyhintlabel_for_tablet_period">\?</string> - <string name="more_keys_for_tablet_period">\?</string> + <string name="keyhintlabel_for_period">\?</string> + <string name="more_keys_for_period">\?</string> </resources> diff --git a/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml index cc09f7fe5..44aa64cbe 100644 --- a/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml @@ -169,9 +169,9 @@ <string name="keylabel_for_tablet_comma">,</string> <string name="keyhintlabel_for_tablet_comma"></string> <string name="more_keys_for_tablet_comma"></string> - <string name="keyhintlabel_for_tablet_period"></string> + <string name="keyhintlabel_for_period"></string> <!-- U+2026: "…" HORIZONTAL ELLIPSIS --> - <string name="more_keys_for_tablet_period">…</string> + <string name="more_keys_for_period">…</string> <string name="keylabel_for_apostrophe">\'</string> <string name="keyhintlabel_for_apostrophe">\"</string> <string name="more_keys_for_apostrophe">\"</string> |