diff options
Diffstat (limited to 'native/jni/src')
65 files changed, 1166 insertions, 2151 deletions
diff --git a/native/jni/src/char_utils.h b/native/jni/src/char_utils.h deleted file mode 100644 index b429f40b2..000000000 --- a/native/jni/src/char_utils.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_CHAR_UTILS_H -#define LATINIME_CHAR_UTILS_H - -#include <cctype> - -#include "defines.h" - -namespace latinime { - -inline static bool isAsciiUpper(int c) { - // Note: isupper(...) reports false positives for some Cyrillic characters, causing them to - // be incorrectly lower-cased using toAsciiLower(...) rather than latin_tolower(...). - return (c >= 'A' && c <= 'Z'); -} - -inline static int toAsciiLower(int c) { - return c - 'A' + 'a'; -} - -inline static bool isAscii(int c) { - return isascii(c) != 0; -} - -unsigned short latin_tolower(const unsigned short c); - -/** - * Table mapping most combined Latin, Greek, and Cyrillic characters - * to their base characters. If c is in range, BASE_CHARS[c] == c - * if c is not a combined character, or the base character if it - * is combined. - */ -static const int BASE_CHARS_SIZE = 0x0500; -extern const unsigned short BASE_CHARS[BASE_CHARS_SIZE]; - -inline static int toBaseCodePoint(int c) { - if (c < BASE_CHARS_SIZE) { - return static_cast<int>(BASE_CHARS[c]); - } - return c; -} - -AK_FORCE_INLINE static int toLowerCase(const int c) { - if (isAsciiUpper(c)) { - return toAsciiLower(c); - } - if (isAscii(c)) { - return c; - } - return static_cast<int>(latin_tolower(static_cast<unsigned short>(c))); -} - -AK_FORCE_INLINE static int toBaseLowerCase(const int c) { - return toLowerCase(toBaseCodePoint(c)); -} - -inline static bool isIntentionalOmissionCodePoint(const int codePoint) { - // TODO: Do not hardcode here - return codePoint == KEYCODE_SINGLE_QUOTE || codePoint == KEYCODE_HYPHEN_MINUS; -} - -inline static int getCodePointCount(const int arraySize, const int *const codePoints) { - int size = 0; - for (; size < arraySize; ++size) { - if (codePoints[size] == '\0') { - break; - } - } - return size; -} - -} // namespace latinime -#endif // LATINIME_CHAR_UTILS_H diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h index eb59744f6..e0edff584 100644 --- a/native/jni/src/defines.h +++ b/native/jni/src/defines.h @@ -203,14 +203,12 @@ static inline void prof_out(void) { #define DEBUG_DICT true #define DEBUG_DICT_FULL false #define DEBUG_EDIT_DISTANCE false -#define DEBUG_SHOW_FOUND_WORD false #define DEBUG_NODE DEBUG_DICT_FULL #define DEBUG_TRACE DEBUG_DICT_FULL #define DEBUG_PROXIMITY_INFO false #define DEBUG_PROXIMITY_CHARS false #define DEBUG_CORRECTION false #define DEBUG_CORRECTION_FREQ false -#define DEBUG_WORDS_PRIORITY_QUEUE false #define DEBUG_SAMPLING_POINTS false #define DEBUG_POINTS_PROBABILITY false #define DEBUG_DOUBLE_LETTER false @@ -229,14 +227,12 @@ static inline void prof_out(void) { #define DEBUG_DICT false #define DEBUG_DICT_FULL false #define DEBUG_EDIT_DISTANCE false -#define DEBUG_SHOW_FOUND_WORD false #define DEBUG_NODE false #define DEBUG_TRACE false #define DEBUG_PROXIMITY_INFO false #define DEBUG_PROXIMITY_CHARS false #define DEBUG_CORRECTION false #define DEBUG_CORRECTION_FREQ false -#define DEBUG_WORDS_PRIORITY_QUEUE false #define DEBUG_SAMPLING_POINTS false #define DEBUG_POINTS_PROBABILITY false #define DEBUG_DOUBLE_LETTER false @@ -268,11 +264,6 @@ static inline void prof_out(void) { // of the binary dictionary where a {key,value} string pair scheme is used. #define LARGEST_INT_DIGIT_COUNT 11 -// Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap(). -// We measured and compared performance of both, and found mmap() is fairly good in terms of -// loading time, and acceptable even for several initial lookups which involve page faults. -#define USE_MMAP_FOR_DICTIONARY - #define NOT_VALID_WORD (-99) #define NOT_A_CODE_POINT (-1) #define NOT_A_DISTANCE (-1) @@ -289,61 +280,21 @@ static inline void prof_out(void) { #define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true #define SUGGEST_MULTIPLE_WORDS true -#define USE_SUGGEST_INTERFACE_FOR_TYPING true #define SUGGEST_INTERFACE_OUTPUT_SCALE 1000000.0f -// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent. -#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80 -#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12 -#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 58 -#define WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE 50 -#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75 -#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75 -#define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 70 -#define FULL_MATCHED_WORDS_PROMOTION_RATE 120 -#define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90 -#define WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE 70 -#define WORDS_WITH_MATCH_SKIP_PROMOTION_RATE 105 -#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE 148 -#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER 3 -#define CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE 45 -#define INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE 70 -#define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96 -#define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50 -#define TWO_WORDS_CORRECTION_DEMOTION_BASE 80 -#define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1 #define ZERO_DISTANCE_PROMOTION_RATE 110.0f #define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f #define HALF_SCORE_SQUARED_RADIUS 32.0f #define MAX_PROBABILITY 255 #define MAX_BIGRAM_ENCODED_PROBABILITY 15 +#define MULTIPLE_WORDS_DEMOTION_RATE 80 // Assuming locale strings such as en_US, sr-Latn etc. #define MAX_LOCALE_STRING_LENGTH 10 -// Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used -// for better performance. -// Holds up to 1 candidate for each word -#define SUB_QUEUE_MAX_WORDS 1 -#define SUB_QUEUE_MAX_COUNT 10 -#define SUB_QUEUE_MIN_WORD_LENGTH 4 -// TODO: Extend this limitation -#define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 5 -// TODO: Remove this limitation -#define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12 -// TODO: Remove this limitation -#define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 45 -#define MULTIPLE_WORDS_DEMOTION_RATE 80 -#define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6 - -#define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35f -#define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185f /* heuristic... This should be changed if we change the unit of the probability. */ #define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_PROBABILITY * 58 / 100) -#define MAX_DEPTH_MULTIPLIER 3 -#define FIRST_WORD_INDEX 0 - // Max value for length, distance and probability which are used in weighting // TODO: Remove #define MAX_VALUE_FOR_WEIGHTING 10000000 @@ -351,16 +302,17 @@ static inline void prof_out(void) { // The max number of the keys in one keyboard layout #define MAX_KEY_COUNT_IN_A_KEYBOARD 64 -// TODO: Reduce this constant if possible; check the maximum number of digraphs in the same -// word in the dictionary for languages with digraphs, like German and French -#define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5 - -#define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3 - // TODO: Remove #define MAX_POINTER_COUNT 1 #define MAX_POINTER_COUNT_G 2 +// Queue IDs and size for DicNodesCache +#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_ACTIVE 0 +#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_NEXT_ACTIVE 1 +#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_TERMINAL 2 +#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_CACHE_FOR_CONTINUOUS_SUGGESTION 3 +#define DIC_NODES_CACHE_PRIORITY_QUEUES_SIZE 4 + // Size, in bytes, of the bloom filter index for bigrams // 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k, // where k is the number of hash functions, n the number of bigrams, and m the number of diff --git a/native/jni/src/dic_traverse_wrapper.h b/native/jni/src/dic_traverse_wrapper.h deleted file mode 100644 index 1108a45c8..000000000 --- a/native/jni/src/dic_traverse_wrapper.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DIC_TRAVERSE_WRAPPER_H -#define LATINIME_DIC_TRAVERSE_WRAPPER_H - -#include "defines.h" -#include "jni.h" - -namespace latinime { -class Dictionary; -// TODO: Remove -class DicTraverseWrapper { - public: - static void *getDicTraverseSession(JNIEnv *env, jstring locale) { - if (sDicTraverseSessionFactoryMethod) { - return sDicTraverseSessionFactoryMethod(env, locale); - } - return 0; - } - static void initDicTraverseSession(void *traverseSession, const Dictionary *const dictionary, - const int *prevWord, const int prevWordLength) { - if (sDicTraverseSessionInitMethod) { - sDicTraverseSessionInitMethod(traverseSession, dictionary, prevWord, prevWordLength); - } - } - static void releaseDicTraverseSession(void *traverseSession) { - if (sDicTraverseSessionReleaseMethod) { - sDicTraverseSessionReleaseMethod(traverseSession); - } - } - static void setTraverseSessionFactoryMethod(void *(*factoryMethod)(JNIEnv *, jstring)) { - sDicTraverseSessionFactoryMethod = factoryMethod; - } - static void setTraverseSessionInitMethod( - void (*initMethod)(void *, const Dictionary *const, const int *, const int)) { - sDicTraverseSessionInitMethod = initMethod; - } - static void setTraverseSessionReleaseMethod(void (*releaseMethod)(void *)) { - sDicTraverseSessionReleaseMethod = releaseMethod; - } - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DicTraverseWrapper); - static void *(*sDicTraverseSessionFactoryMethod)(JNIEnv *, jstring); - static void (*sDicTraverseSessionInitMethod)( - void *, const Dictionary *const, const int *, const int); - static void (*sDicTraverseSessionReleaseMethod)(void *); -}; -} // namespace latinime -#endif // LATINIME_DIC_TRAVERSE_WRAPPER_H diff --git a/native/jni/src/geometry_utils.h b/native/jni/src/geometry_utils.h deleted file mode 100644 index 4cbb127e8..000000000 --- a/native/jni/src/geometry_utils.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_GEOMETRY_UTILS_H -#define LATINIME_GEOMETRY_UTILS_H - -#include <cmath> - -#include "defines.h" - -#define ROUND_FLOAT_10000(f) ((f) < 1000.0f && (f) > 0.001f) \ - ? (floorf((f) * 10000.0f) / 10000.0f) : (f) - -namespace latinime { - -static inline float SQUARE_FLOAT(const float x) { return x * x; } - -static AK_FORCE_INLINE float getAngle(const int x1, const int y1, const int x2, const int y2) { - const int dx = x1 - x2; - const int dy = y1 - y2; - if (dx == 0 && dy == 0) return 0.0f; - return atan2f(static_cast<float>(dy), static_cast<float>(dx)); -} - -static AK_FORCE_INLINE float getAngleDiff(const float a1, const float a2) { - const float deltaA = fabsf(a1 - a2); - const float diff = ROUND_FLOAT_10000(deltaA); - if (diff > M_PI_F) { - const float normalizedDiff = 2.0f * M_PI_F - diff; - return ROUND_FLOAT_10000(normalizedDiff); - } - return diff; -} - -static AK_FORCE_INLINE int getDistanceInt(const int x1, const int y1, const int x2, - const int y2) { - return static_cast<int>(hypotf(static_cast<float>(x1 - x2), static_cast<float>(y1 - y2))); -} -} // namespace latinime -#endif // LATINIME_GEOMETRY_UTILS_H diff --git a/native/jni/src/correction.cpp b/native/jni/src/obsolete/correction.cpp index 61bf3f619..6b80ed8ea 100644 --- a/native/jni/src/correction.cpp +++ b/native/jni/src/obsolete/correction.cpp @@ -18,18 +18,39 @@ #include <cmath> -#include "char_utils.h" -#include "correction.h" #include "defines.h" -#include "proximity_info_state.h" -#include "suggest_utils.h" +#include "obsolete/correction.h" +#include "suggest/core/layout/proximity_info_state.h" +#include "suggest/core/layout/touch_position_correction_utils.h" #include "suggest/policyimpl/utils/edit_distance.h" #include "suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h" +#include "utils/char_utils.h" namespace latinime { class ProximityInfo; +// private static const member variables +// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent. +const int Correction::WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE = 80; +const int Correction::WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X = 12; +const int Correction::WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE = 58; +const int Correction::WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE = 50; +const int Correction::WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE = 75; +const int Correction::WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE = 75; +const int Correction::WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE = 70; +const int Correction::FULL_MATCHED_WORDS_PROMOTION_RATE = 120; +const int Correction::WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE = 90; +const int Correction::WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE = 70; +const int Correction::WORDS_WITH_MATCH_SKIP_PROMOTION_RATE = 105; +const int Correction::WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE = 148; +const int Correction::WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER = 3; +const int Correction::CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE = 45; +const int Correction::INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE = 70; +const int Correction::FIRST_CHAR_DIFFERENT_DEMOTION_RATE = 96; +const int Correction::TWO_WORDS_CAPITALIZED_DEMOTION_RATE = 50; +const int Correction::TWO_WORDS_CORRECTION_DEMOTION_BASE = 80; + ///////////////////////////// // edit distance funcitons // ///////////////////////////// @@ -528,7 +549,7 @@ inline static int getQuoteCount(const int *word, const int length) { } inline static bool isUpperCase(unsigned short c) { - return isAsciiUpper(toBaseCodePoint(c)); + return CharUtils::isAsciiUpper(CharUtils::toBaseCodePoint(c)); } ////////////////////// @@ -676,8 +697,8 @@ inline static bool isUpperCase(unsigned short c) { if (i < adjustedProximityMatchedCount) { multiplyIntCapped(typedLetterMultiplier, &finalFreq); } - const float factor = - SuggestUtils::getLengthScalingFactor(static_cast<float>(squaredDistance)); + const float factor = TouchPositionCorrectionUtils::getLengthScalingFactor( + static_cast<float>(squaredDistance)); if (factor > 0.0f) { multiplyRate(static_cast<int>(factor * 100.0f), &finalFreq); } else if (squaredDistance == PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO) { @@ -918,11 +939,15 @@ inline static bool isUpperCase(unsigned short c) { // In dictionary.cpp, getSuggestion() method, // When USE_SUGGEST_INTERFACE_FOR_TYPING is true: +// +// // TODO: Revise the following logic thoroughly by referring to the logic +// // marked as "Otherwise" below. // SUGGEST_INTERFACE_OUTPUT_SCALE was multiplied to the original suggestion scores to convert // them to integers. // score = (int)((original score) * SUGGEST_INTERFACE_OUTPUT_SCALE) // Undo the scaling here to recover the original score. // normalizedScore = ((float)score) / SUGGEST_INTERFACE_OUTPUT_SCALE +// // Otherwise: suggestion scores are computed using the below formula. // original score // := powf(mTypedLetterMultiplier (this is defined 2), @@ -965,9 +990,11 @@ inline static bool isUpperCase(unsigned short c) { // so, 0 <= distance / afterLength <= 1 const float weight = 1.0f - static_cast<float>(distance) / static_cast<float>(afterLength); - if (USE_SUGGEST_INTERFACE_FOR_TYPING) { + // TODO: Revise the following logic thoroughly by referring to... + if (true /* USE_SUGGEST_INTERFACE_FOR_TYPING */) { return (static_cast<float>(score) / SUGGEST_INTERFACE_OUTPUT_SCALE) * weight; } + // ...this logic. const float maxScore = score >= S_INT_MAX ? static_cast<float>(S_INT_MAX) : static_cast<float>(MAX_INITIAL_SCORE) * powf(static_cast<float>(TYPED_LETTER_MULTIPLIER), diff --git a/native/jni/src/correction.h b/native/jni/src/obsolete/correction.h index a9e9b48a6..47dcef2d7 100644 --- a/native/jni/src/correction.h +++ b/native/jni/src/obsolete/correction.h @@ -19,9 +19,10 @@ #include <cstring> // for memset() -#include "correction_state.h" #include "defines.h" -#include "proximity_info_state.h" +#include "obsolete/correction_state.h" +#include "suggest/core/layout/proximity_info_state.h" +#include "utils/char_utils.h" namespace latinime { @@ -134,6 +135,27 @@ class Correction { private: DISALLOW_COPY_AND_ASSIGN(Correction); + // The following "rate"s are used as a multiplier before dividing by 100, so they are in + // percent. + static const int WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE; + static const int WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X; + static const int WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE; + static const int WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE; + static const int WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE; + static const int WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE; + static const int WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE; + static const int FULL_MATCHED_WORDS_PROMOTION_RATE; + static const int WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE; + static const int WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE; + static const int WORDS_WITH_MATCH_SKIP_PROMOTION_RATE; + static const int WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE; + static const int WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER; + static const int CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE; + static const int INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE; + static const int FIRST_CHAR_DIFFERENT_DEMOTION_RATE; + static const int TWO_WORDS_CAPITALIZED_DEMOTION_RATE; + static const int TWO_WORDS_CORRECTION_DEMOTION_BASE; + ///////////////////////// // static inline utils // ///////////////////////// @@ -342,13 +364,13 @@ AK_FORCE_INLINE static void calcEditDistanceOneStep(int *editDistanceTable, cons const int *const prevprev = outputLength >= 2 ? editDistanceTable + (outputLength - 2) * (inputSize + 1) : 0; current[0] = outputLength; - const int co = toBaseLowerCase(output[outputLength - 1]); - const int prevCO = outputLength >= 2 ? toBaseLowerCase(output[outputLength - 2]) : 0; + const int co = CharUtils::toBaseLowerCase(output[outputLength - 1]); + const int prevCO = outputLength >= 2 ? CharUtils::toBaseLowerCase(output[outputLength - 2]) : 0; for (int i = 1; i <= inputSize; ++i) { - const int ci = toBaseLowerCase(input[i - 1]); + const int ci = CharUtils::toBaseLowerCase(input[i - 1]); const int cost = (ci == co) ? 0 : 1; current[i] = min(current[i - 1] + 1, min(prev[i] + 1, prev[i - 1] + cost)); - if (i >= 2 && prevprev && ci == prevCO && co == toBaseLowerCase(input[i - 2])) { + if (i >= 2 && prevprev && ci == prevCO && co == CharUtils::toBaseLowerCase(input[i - 2])) { current[i] = min(current[i], prevprev[i - 2] + 1); } } diff --git a/native/jni/src/correction_state.h b/native/jni/src/obsolete/correction_state.h index a63d4aa94..a63d4aa94 100644 --- a/native/jni/src/correction_state.h +++ b/native/jni/src/obsolete/correction_state.h diff --git a/native/jni/src/suggest/core/dicnode/dic_node.cpp b/native/jni/src/suggest/core/dicnode/dic_node.cpp index 8c48c587b..de088c7d0 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_node.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "dic_node.h" +#include "suggest/core/dicnode/dic_node.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h index 4225bb3e5..3f64d07b2 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node.h +++ b/native/jni/src/suggest/core/dicnode/dic_node.h @@ -17,13 +17,13 @@ #ifndef LATINIME_DIC_NODE_H #define LATINIME_DIC_NODE_H -#include "char_utils.h" #include "defines.h" -#include "dic_node_state.h" -#include "dic_node_profiler.h" -#include "dic_node_properties.h" -#include "dic_node_release_listener.h" -#include "digraph_utils.h" +#include "suggest/core/dicnode/dic_node_state.h" +#include "suggest/core/dicnode/dic_node_profiler.h" +#include "suggest/core/dicnode/dic_node_properties.h" +#include "suggest/core/dicnode/dic_node_release_listener.h" +#include "suggest/core/dictionary/digraph_utils.h" +#include "utils/char_utils.h" #if DEBUG_DICT #define LOGI_SHOW_ADD_COST_PROP \ @@ -221,7 +221,7 @@ class DicNode { bool isFirstCharUppercase() const { const int c = getOutputWordBuf()[0]; - return isAsciiUpper(c); + return CharUtils::isAsciiUpper(c); } bool isFirstWord() const { @@ -375,7 +375,7 @@ class DicNode { // Whether the current codepoint can be an intentional omission, in which case the traversal // algorithm will always check for a possible omission here. bool canBeIntentionalOmission() const { - return isIntentionalOmissionCodePoint(getNodeCodePoint()); + return CharUtils::isIntentionalOmissionCodePoint(getNodeCodePoint()); } // Whether the omission is so frequent that it should incur zero cost. diff --git a/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h b/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h index d3f28a8bd..970e3bda4 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h @@ -21,8 +21,8 @@ #include <vector> #include "defines.h" -#include "dic_node.h" -#include "dic_node_release_listener.h" +#include "suggest/core/dicnode/dic_node.h" +#include "suggest/core/dicnode/dic_node_release_listener.h" #define MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY 200 diff --git a/native/jni/src/suggest/core/dicnode/dic_node_properties.h b/native/jni/src/suggest/core/dicnode/dic_node_properties.h index 63a6b1340..d2f87c10b 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_properties.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_properties.h @@ -19,8 +19,8 @@ #include <stdint.h> -#include "binary_format.h" #include "defines.h" +#include "suggest/core/dictionary/binary_format.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_node_state.h b/native/jni/src/suggest/core/dicnode/dic_node_state.h index 239b63c32..d35e7d79f 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_state.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_state.h @@ -18,10 +18,10 @@ #define LATINIME_DIC_NODE_STATE_H #include "defines.h" -#include "dic_node_state_input.h" -#include "dic_node_state_output.h" -#include "dic_node_state_prevword.h" -#include "dic_node_state_scoring.h" +#include "suggest/core/dicnode/dic_node_state_input.h" +#include "suggest/core/dicnode/dic_node_state_output.h" +#include "suggest/core/dicnode/dic_node_state_prevword.h" +#include "suggest/core/dicnode/dic_node_state_scoring.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_node_state_prevword.h b/native/jni/src/suggest/core/dicnode/dic_node_state_prevword.h index e3b892bda..c3968c090 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_state_prevword.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_state_prevword.h @@ -21,7 +21,7 @@ #include <stdint.h> #include "defines.h" -#include "dic_node_utils.h" +#include "suggest/core/dicnode/dic_node_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_node_state_scoring.h b/native/jni/src/suggest/core/dicnode/dic_node_state_scoring.h index dca9d60da..4c884225a 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_state_scoring.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_state_scoring.h @@ -20,7 +20,7 @@ #include <stdint.h> #include "defines.h" -#include "digraph_utils.h" +#include "suggest/core/dictionary/digraph_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp index 5357c3773..3deee1a42 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp @@ -17,13 +17,16 @@ #include <cstring> #include <vector> -#include "binary_format.h" -#include "dic_node.h" -#include "dic_node_utils.h" -#include "dic_node_vector.h" -#include "multi_bigram_map.h" -#include "proximity_info.h" -#include "proximity_info_state.h" +#include "suggest/core/dicnode/dic_node.h" +#include "suggest/core/dicnode/dic_node_utils.h" +#include "suggest/core/dicnode/dic_node_vector.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" +#include "suggest/core/dictionary/binary_format.h" +#include "suggest/core/dictionary/multi_bigram_map.h" +#include "suggest/core/dictionary/probability_utils.h" +#include "suggest/core/layout/proximity_info.h" +#include "suggest/core/layout/proximity_info_state.h" +#include "utils/char_utils.h" namespace latinime { @@ -31,20 +34,23 @@ namespace latinime { // Node initialization utils // /////////////////////////////// -/* static */ void DicNodeUtils::initAsRoot(const int rootPos, const uint8_t *const dicRoot, - const int prevWordNodePos, DicNode *newRootNode) { - int curPos = rootPos; +/* static */ void DicNodeUtils::initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo, + const int prevWordNodePos, DicNode *const newRootNode) { + int curPos = binaryDictionaryInfo->getRootPosition(); const int pos = curPos; - const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos); + const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &curPos); const int childrenPos = curPos; newRootNode->initAsRoot(pos, childrenPos, childrenCount, prevWordNodePos); } -/*static */ void DicNodeUtils::initAsRootWithPreviousWord(const int rootPos, - const uint8_t *const dicRoot, DicNode *prevWordLastNode, DicNode *newRootNode) { - int curPos = rootPos; +/*static */ void DicNodeUtils::initAsRootWithPreviousWord( + const BinaryDictionaryInfo *const binaryDictionaryInfo, + DicNode *const prevWordLastNode, DicNode *const newRootNode) { + int curPos = binaryDictionaryInfo->getRootPosition(); const int pos = curPos; - const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &curPos); + const int childrenCount = BinaryFormat::getGroupCountAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &curPos); const int childrenPos = curPos; newRootNode->initAsRootWithPreviousWord(prevWordLastNode, pos, childrenPos, childrenCount); } @@ -62,24 +68,27 @@ namespace latinime { DicNodeVector *childDicNodes) { // Passing multiple chars node. No need to traverse child const int codePoint = dicNode->getNodeTypedCodePoint(); - const int baseLowerCaseCodePoint = toBaseLowerCase(codePoint); + const int baseLowerCaseCodePoint = CharUtils::toBaseLowerCase(codePoint); const bool isMatch = isMatchedNodeCodePoint(pInfoState, pointIndex, exactOnly, codePoint); - if (isMatch || isIntentionalOmissionCodePoint(baseLowerCaseCodePoint)) { + if (isMatch || CharUtils::isIntentionalOmissionCodePoint(baseLowerCaseCodePoint)) { childDicNodes->pushPassingChild(dicNode); } } /* static */ int DicNodeUtils::createAndGetLeavingChildNode(DicNode *dicNode, int pos, - const uint8_t *const dicRoot, const int terminalDepth, const ProximityInfoState *pInfoState, - const int pointIndex, const bool exactOnly, const std::vector<int> *const codePointsFilter, - const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) { + const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth, + const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, + const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo, + DicNodeVector *childDicNodes) { int nextPos = pos; - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dicRoot, &pos); + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &pos); const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); const bool isTerminal = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags)); const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags); - int codePoint = BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos); + int codePoint = BinaryFormat::getCodePointAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &pos); ASSERT(NOT_A_CODE_POINT != codePoint); const int nodeCodePoint = codePoint; // TODO: optimize this @@ -89,7 +98,8 @@ namespace latinime { do { const int nextCodePoint = hasMultipleChars - ? BinaryFormat::getCodePointAndForwardPointer(dicRoot, &pos) : NOT_A_CODE_POINT; + ? BinaryFormat::getCodePointAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &pos) : NOT_A_CODE_POINT; const bool isLastChar = (NOT_A_CODE_POINT == nextCodePoint); if (!isLastChar) { additionalWordBuf[additionalSubwordLength++] = nextCodePoint; @@ -97,12 +107,14 @@ namespace latinime { codePoint = nextCodePoint; } while (NOT_A_CODE_POINT != codePoint); - const int probability = - isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer(dicRoot, pos) : -1; + const int probability = isTerminal ? BinaryFormat::readProbabilityWithoutMovingPointer( + binaryDictionaryInfo->getDictRoot(), pos) : -1; pos = BinaryFormat::skipProbability(flags, pos); - int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition(dicRoot, flags, pos) : 0; + int childrenPos = hasChildren ? BinaryFormat::readChildrenPosition( + binaryDictionaryInfo->getDictRoot(), flags, pos) : 0; const int attributesPos = BinaryFormat::skipChildrenPosition(flags, pos); - const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(dicRoot, flags, pos); + const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes( + binaryDictionaryInfo->getDictRoot(), flags, pos); if (isDicNodeFilteredOut(nodeCodePoint, pInfo, codePointsFilter)) { return siblingPos; @@ -110,8 +122,8 @@ namespace latinime { if (!isMatchedNodeCodePoint(pInfoState, pointIndex, exactOnly, nodeCodePoint)) { return siblingPos; } - const int childrenCount = hasChildren - ? BinaryFormat::getGroupCountAndForwardPointer(dicRoot, &childrenPos) : 0; + const int childrenCount = hasChildren ? BinaryFormat::getGroupCountAndForwardPointer( + binaryDictionaryInfo->getDictRoot(), &childrenPos) : 0; childDicNodes->pushLeavingChild(dicNode, nextPos, flags, childrenPos, attributesPos, siblingPos, nodeCodePoint, childrenCount, probability, -1 /* bigramProbability */, isTerminal, hasMultipleChars, hasChildren, additionalSubwordLength, additionalWordBuf); @@ -125,13 +137,13 @@ namespace latinime { return false; } if (pInfo && (pInfo->getKeyIndexOf(nodeCodePoint) == NOT_AN_INDEX - || isIntentionalOmissionCodePoint(nodeCodePoint))) { + || CharUtils::isIntentionalOmissionCodePoint(nodeCodePoint))) { // If normalized nodeCodePoint is not on the keyboard or skippable, this child is never // filtered. return false; } - const int lowerCodePoint = toLowerCase(nodeCodePoint); - const int baseLowerCodePoint = toBaseCodePoint(lowerCodePoint); + const int lowerCodePoint = CharUtils::toLowerCase(nodeCodePoint); + const int baseLowerCodePoint = CharUtils::toBaseCodePoint(lowerCodePoint); // TODO: Avoid linear search for (int i = 0; i < filterSize; ++i) { // Checking if a normalized code point is in filter characters when pInfo is not @@ -147,16 +159,18 @@ namespace latinime { } /* static */ void DicNodeUtils::createAndGetAllLeavingChildNodes(DicNode *dicNode, - const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex, - const bool exactOnly, const std::vector<int> *const codePointsFilter, - const ProximityInfo *const pInfo, DicNodeVector *childDicNodes) { + const BinaryDictionaryInfo *const binaryDictionaryInfo, + const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, + const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo, + DicNodeVector *childDicNodes) { const int terminalDepth = dicNode->getLeavingDepth(); const int childCount = dicNode->getChildrenCount(); int nextPos = dicNode->getChildrenPos(); for (int i = 0; i < childCount; i++) { const int filterSize = codePointsFilter ? codePointsFilter->size() : 0; - nextPos = createAndGetLeavingChildNode(dicNode, nextPos, dicRoot, terminalDepth, pInfoState, - pointIndex, exactOnly, codePointsFilter, pInfo, childDicNodes); + nextPos = createAndGetLeavingChildNode(dicNode, nextPos, binaryDictionaryInfo, + terminalDepth, pInfoState, pointIndex, exactOnly, codePointsFilter, pInfo, + childDicNodes); if (!pInfo && filterSize > 0 && childDicNodes->exceeds(filterSize)) { // All code points have been found. break; @@ -164,14 +178,15 @@ namespace latinime { } } -/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot, - DicNodeVector *childDicNodes) { - getProximityChildDicNodes(dicNode, dicRoot, 0, 0, false, childDicNodes); +/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes) { + getProximityChildDicNodes(dicNode, binaryDictionaryInfo, 0, 0, false, childDicNodes); } /* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode, - const uint8_t *const dicRoot, const ProximityInfoState *pInfoState, const int pointIndex, - bool exactOnly, DicNodeVector *childDicNodes) { + const BinaryDictionaryInfo *const binaryDictionaryInfo, + const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly, + DicNodeVector *childDicNodes) { if (dicNode->isTotalInputSizeExceedingLimit()) { return; } @@ -179,9 +194,9 @@ namespace latinime { DicNodeUtils::createAndGetPassingChildNode(dicNode, pInfoState, pointIndex, exactOnly, childDicNodes); } else { - DicNodeUtils::createAndGetAllLeavingChildNodes(dicNode, dicRoot, pInfoState, pointIndex, - exactOnly, 0 /* codePointsFilter */, 0 /* pInfo */, - childDicNodes); + DicNodeUtils::createAndGetAllLeavingChildNodes( + dicNode, binaryDictionaryInfo, pInfoState, pointIndex, exactOnly, + 0 /* codePointsFilter */, 0 /* pInfo */, childDicNodes); } } @@ -191,32 +206,35 @@ namespace latinime { /** * Computes the combined bigram / unigram cost for the given dicNode. */ -/* static */ float DicNodeUtils::getBigramNodeImprobability(const uint8_t *const dicRoot, +/* static */ float DicNodeUtils::getBigramNodeImprobability( + const BinaryDictionaryInfo *const binaryDictionaryInfo, const DicNode *const node, MultiBigramMap *multiBigramMap) { if (node->isImpossibleBigramWord()) { return static_cast<float>(MAX_VALUE_FOR_WEIGHTING); } - const int probability = getBigramNodeProbability(dicRoot, node, multiBigramMap); + const int probability = getBigramNodeProbability(binaryDictionaryInfo, node, multiBigramMap); // TODO: This equation to calculate the improbability looks unreasonable. Investigate this. const float cost = static_cast<float>(MAX_PROBABILITY - probability) / static_cast<float>(MAX_PROBABILITY); return cost; } -/* static */ int DicNodeUtils::getBigramNodeProbability(const uint8_t *const dicRoot, +/* static */ int DicNodeUtils::getBigramNodeProbability( + const BinaryDictionaryInfo *const binaryDictionaryInfo, const DicNode *const node, MultiBigramMap *multiBigramMap) { const int unigramProbability = node->getProbability(); const int wordPos = node->getPos(); const int prevWordPos = node->getPrevWordPos(); if (NOT_VALID_WORD == wordPos || NOT_VALID_WORD == prevWordPos) { // Note: Normally wordPos comes from the dictionary and should never equal NOT_VALID_WORD. - return backoff(unigramProbability); + return ProbabilityUtils::backoff(unigramProbability); } if (multiBigramMap) { return multiBigramMap->getBigramProbability( - dicRoot, prevWordPos, wordPos, unigramProbability); + binaryDictionaryInfo, prevWordPos, wordPos, unigramProbability); } - return BinaryFormat::getBigramProbability(dicRoot, prevWordPos, wordPos, unigramProbability); + return BinaryFormat::getBigramProbability( + binaryDictionaryInfo->getDictRoot(), prevWordPos, wordPos, unigramProbability); } /////////////////////////////////////// diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.h b/native/jni/src/suggest/core/dicnode/dic_node_utils.h index 5bc542d05..e198d6181 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.h @@ -24,6 +24,7 @@ namespace latinime { +class BinaryDictionaryInfo; class DicNode; class DicNodeVector; class ProximityInfo; @@ -34,19 +35,20 @@ class DicNodeUtils { public: static int appendTwoWords(const int *src0, const int16_t length0, const int *src1, const int16_t length1, int *dest); - static void initAsRoot(const int rootPos, const uint8_t *const dicRoot, + static void initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int prevWordNodePos, DicNode *newRootNode); - static void initAsRootWithPreviousWord(const int rootPos, const uint8_t *const dicRoot, + static void initAsRootWithPreviousWord(const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNode *prevWordLastNode, DicNode *newRootNode); static void initByCopy(DicNode *srcNode, DicNode *destNode); - static void getAllChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot, - DicNodeVector *childDicNodes); - static float getBigramNodeImprobability(const uint8_t *const dicRoot, + static void getAllChildDicNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes); + static float getBigramNodeImprobability(const BinaryDictionaryInfo *const binaryDictionaryInfo, const DicNode *const node, MultiBigramMap *const multiBigramMap); static bool isDicNodeFilteredOut(const int nodeCodePoint, const ProximityInfo *const pInfo, const std::vector<int> *const codePointsFilter); // TODO: Move to private - static void getProximityChildDicNodes(DicNode *dicNode, const uint8_t *const dicRoot, + static void getProximityChildDicNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly, DicNodeVector *childDicNodes); @@ -60,16 +62,18 @@ class DicNodeUtils { // Max number of bigrams to look up static const int MAX_BIGRAMS_CONSIDERED_PER_CONTEXT = 500; - static int getBigramNodeProbability(const uint8_t *const dicRoot, const DicNode *const node, - MultiBigramMap *multiBigramMap); + static int getBigramNodeProbability(const BinaryDictionaryInfo *const binaryDictionaryInfo, + const DicNode *const node, MultiBigramMap *multiBigramMap); static void createAndGetPassingChildNode(DicNode *dicNode, const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, DicNodeVector *childDicNodes); - static void createAndGetAllLeavingChildNodes(DicNode *dicNode, const uint8_t *const dicRoot, + static void createAndGetAllLeavingChildNodes(DicNode *dicNode, + const BinaryDictionaryInfo *const binaryDictionaryInfo, const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo, DicNodeVector *childDicNodes); - static int createAndGetLeavingChildNode(DicNode *dicNode, int pos, const uint8_t *const dicRoot, - const int terminalDepth, const ProximityInfoState *pInfoState, const int pointIndex, + static int createAndGetLeavingChildNode(DicNode *dicNode, int pos, + const BinaryDictionaryInfo *const binaryDictionaryInfo, const int terminalDepth, + const ProximityInfoState *pInfoState, const int pointIndex, const bool exactOnly, const std::vector<int> *const codePointsFilter, const ProximityInfo *const pInfo, DicNodeVector *childDicNodes); diff --git a/native/jni/src/suggest/core/dicnode/dic_node_vector.h b/native/jni/src/suggest/core/dicnode/dic_node_vector.h index ca07edaee..e23c411f0 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_vector.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_vector.h @@ -20,7 +20,7 @@ #include <vector> #include "defines.h" -#include "dic_node.h" +#include "suggest/core/dicnode/dic_node.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp index b9a60780b..c3d2a2e74 100644 --- a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp @@ -17,9 +17,9 @@ #include <list> #include "defines.h" -#include "dic_node_priority_queue.h" -#include "dic_node_utils.h" -#include "dic_nodes_cache.h" +#include "suggest/core/dicnode/dic_node_priority_queue.h" +#include "suggest/core/dicnode/dic_node_utils.h" +#include "suggest/core/dicnode/dic_nodes_cache.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h index a62aa422a..7aab0906e 100644 --- a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h +++ b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h @@ -20,13 +20,7 @@ #include <stdint.h> #include "defines.h" -#include "dic_node_priority_queue.h" - -#define INITIAL_QUEUE_ID_ACTIVE 0 -#define INITIAL_QUEUE_ID_NEXT_ACTIVE 1 -#define INITIAL_QUEUE_ID_TERMINAL 2 -#define INITIAL_QUEUE_ID_CACHE_FOR_CONTINUOUS_SUGGESTION 3 -#define PRIORITY_QUEUES_SIZE 4 +#include "suggest/core/dicnode/dic_node_priority_queue.h" namespace latinime { @@ -38,11 +32,12 @@ class DicNode; class DicNodesCache { public: AK_FORCE_INLINE DicNodesCache() - : mActiveDicNodes(&mDicNodePriorityQueues[INITIAL_QUEUE_ID_ACTIVE]), - mNextActiveDicNodes(&mDicNodePriorityQueues[INITIAL_QUEUE_ID_NEXT_ACTIVE]), - mTerminalDicNodes(&mDicNodePriorityQueues[INITIAL_QUEUE_ID_TERMINAL]), - mCachedDicNodesForContinuousSuggestion( - &mDicNodePriorityQueues[INITIAL_QUEUE_ID_CACHE_FOR_CONTINUOUS_SUGGESTION]), + : mActiveDicNodes(&mDicNodePriorityQueues[DIC_NODES_CACHE_INITIAL_QUEUE_ID_ACTIVE]), + mNextActiveDicNodes(&mDicNodePriorityQueues[ + DIC_NODES_CACHE_INITIAL_QUEUE_ID_NEXT_ACTIVE]), + mTerminalDicNodes(&mDicNodePriorityQueues[DIC_NODES_CACHE_INITIAL_QUEUE_ID_TERMINAL]), + mCachedDicNodesForContinuousSuggestion(&mDicNodePriorityQueues[ + DIC_NODES_CACHE_INITIAL_QUEUE_ID_CACHE_FOR_CONTINUOUS_SUGGESTION]), mInputIndex(0), mLastCachedInputIndex(0) { } @@ -147,9 +142,8 @@ class DicNodesCache { mCachedDicNodesForContinuousSuggestion->dump(); } mInputIndex = mLastCachedInputIndex; - mCachedDicNodesForContinuousSuggestion = - moveNodesAndReturnReusableEmptyQueue( - mCachedDicNodesForContinuousSuggestion, &mActiveDicNodes); + mCachedDicNodesForContinuousSuggestion = moveNodesAndReturnReusableEmptyQueue( + mCachedDicNodesForContinuousSuggestion, &mActiveDicNodes); } AK_FORCE_INLINE static DicNodePriorityQueue *moveNodesAndReturnReusableEmptyQueue( @@ -169,7 +163,7 @@ class DicNodesCache { mTerminalDicNodes->clear(); } - DicNodePriorityQueue mDicNodePriorityQueues[PRIORITY_QUEUES_SIZE]; + DicNodePriorityQueue mDicNodePriorityQueues[DIC_NODES_CACHE_PRIORITY_QUEUES_SIZE]; // Active dicNodes currently being expanded. DicNodePriorityQueue *mActiveDicNodes; // Next dicNodes to be expanded. diff --git a/native/jni/src/bigram_dictionary.cpp b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp index 9053e7226..59d1b19b6 100644 --- a/native/jni/src/bigram_dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp @@ -19,15 +19,19 @@ #define LOG_TAG "LatinIME: bigram_dictionary.cpp" #include "bigram_dictionary.h" -#include "binary_format.h" -#include "bloom_filter.h" -#include "char_utils.h" + #include "defines.h" -#include "dictionary.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" +#include "suggest/core/dictionary/binary_format.h" +#include "suggest/core/dictionary/bloom_filter.h" +#include "suggest/core/dictionary/dictionary.h" +#include "suggest/core/dictionary/probability_utils.h" +#include "utils/char_utils.h" namespace latinime { -BigramDictionary::BigramDictionary(const uint8_t *const streamStart) : DICT_ROOT(streamStart) { +BigramDictionary::BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo) + : mBinaryDictionaryInfo(binaryDictionaryInfo) { if (DEBUG_DICT) { AKLOGI("BigramDictionary - constructor"); } @@ -51,7 +55,7 @@ void BigramDictionary::addWordBigram(int *word, int length, int probability, int int insertAt = 0; while (insertAt < MAX_RESULTS) { if (probability > bigramProbability[insertAt] || (bigramProbability[insertAt] == probability - && length < getCodePointCount(MAX_WORD_LENGTH, + && length < CharUtils::getCodePointCount(MAX_WORD_LENGTH, bigramCodePoints + insertAt * MAX_WORD_LENGTH))) { break; } @@ -102,7 +106,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i // TODO: remove unused arguments, and refrain from storing stuff in members of this class // TODO: have "in" arguments before "out" ones, and make out args explicit in the name - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = getBigramListPositionForWord(prevWord, prevWordLength, false /* forceLowerCaseSearch */); // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams @@ -133,7 +137,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i // resulting probability is 8 - although in the practice it's never bigger than 3 or 4 // in very bad cases. This means that sometimes, we'll see some bigrams interverted // here, but it can't get too bad. - const int probability = BinaryFormat::computeProbabilityForBigram( + const int probability = ProbabilityUtils::computeProbabilityForBigram( unigramProbability, bigramProbabilityTemp); addWordBigram(bigramBuffer, length, probability, bigramProbability, bigramCodePoints, outputTypes); @@ -148,7 +152,7 @@ int BigramDictionary::getBigrams(const int *prevWord, int prevWordLength, int *i int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength, const bool forceLowerCaseSearch) const { if (0 >= prevWordLength) return 0; - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength, forceLowerCaseSearch); @@ -169,7 +173,7 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in void BigramDictionary::fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength, std::map<int, int> *map, uint8_t *filter) const { memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE); - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = getBigramListPositionForWord(prevWord, prevWordLength, false /* forceLowerCaseSearch */); if (0 == pos) { @@ -195,9 +199,9 @@ bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodePoints) cons // what user typed. int maxAlt = MAX_ALTERNATIVES; - const int firstBaseLowerCodePoint = toBaseLowerCase(*word); + const int firstBaseLowerCodePoint = CharUtils::toBaseLowerCase(*word); while (maxAlt > 0) { - if (toBaseLowerCase(*inputCodePoints) == firstBaseLowerCodePoint) { + if (CharUtils::toBaseLowerCase(*inputCodePoints) == firstBaseLowerCodePoint) { return true; } inputCodePoints++; @@ -208,7 +212,7 @@ bool BigramDictionary::checkFirstCharacter(int *word, int *inputCodePoints) cons bool BigramDictionary::isValidBigram(const int *word1, int length1, const int *word2, int length2) const { - const uint8_t *const root = DICT_ROOT; + const uint8_t *const root = mBinaryDictionaryInfo->getDictRoot(); int pos = getBigramListPositionForWord(word1, length1, false /* forceLowerCaseSearch */); // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams if (0 == pos) return false; diff --git a/native/jni/src/bigram_dictionary.h b/native/jni/src/suggest/core/dictionary/bigram_dictionary.h index b86e564c3..8b7a253a2 100644 --- a/native/jni/src/bigram_dictionary.h +++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.h @@ -24,9 +24,12 @@ namespace latinime { +class BinaryDictionaryInfo; + class BigramDictionary { public: - BigramDictionary(const uint8_t *const streamStart); + BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo); + int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords, int *frequencies, int *outputTypes) const; void fillBigramAddressToProbabilityMapAndFilter(const int *prevWord, const int prevWordLength, @@ -35,13 +38,14 @@ class BigramDictionary { ~BigramDictionary(); private: DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary); + void addWordBigram(int *word, int length, int probability, int *bigramProbability, int *bigramCodePoints, int *outputTypes) const; bool checkFirstCharacter(int *word, int *inputCodePoints) const; int getBigramListPositionForWord(const int *prevWord, const int prevWordLength, const bool forceLowerCaseSearch) const; - const uint8_t *const DICT_ROOT; + const BinaryDictionaryInfo *const mBinaryDictionaryInfo; // TODO: Re-implement proximity correction for bigram correction static const int MAX_ALTERNATIVES = 1; }; diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp new file mode 100644 index 000000000..50e0211d7 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/core/dictionary/binary_dictionary_format.h" + +namespace latinime { + +/** + * Dictionary size + */ +// Any file smaller than this is not a dictionary. +const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4; + +/** + * Format versions + */ +// Originally, format version 1 had a 16-bit magic number, then the version number `01' +// then options that must be 0. Hence the first 32-bits of the format are always as follow +// and it's okay to consider them a magic number as a whole. +const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; +const int BinaryDictionaryFormat::FORMAT_VERSION_1_HEADER_SIZE = 5; + +// The versions of Latin IME that only handle format version 1 only test for the magic +// number, so we had to change it so that version 2 files would be rejected by older +// implementations. On this occasion, we made the magic number 32 bits long. +const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; +// Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12 +const int BinaryDictionaryFormat::FORMAT_VERSION_2_MINIMUM_SIZE = 12; +const int BinaryDictionaryFormat::VERSION_2_MAGIC_NUMBER_SIZE = 4; +const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_VERSION_SIZE = 2; +const int BinaryDictionaryFormat::VERSION_2_DICTIONARY_FLAG_SIZE = 2; + +/* static */ BinaryDictionaryFormat::FORMAT_VERSION BinaryDictionaryFormat::detectFormatVersion( + const uint8_t *const dict, const int dictSize) { + // The magic number is stored big-endian. + // If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't + // understand this format. + if (dictSize < DICTIONARY_MINIMUM_SIZE) { + return UNKNOWN_VERSION; + } + const uint32_t magicNumber = ByteArrayUtils::readUint32(dict, 0); + switch (magicNumber) { + case FORMAT_VERSION_1_MAGIC_NUMBER: + // Format 1 header is exactly 5 bytes long and looks like: + // Magic number (2 bytes) 0x78 0xB1 + // Version number (1 byte) 0x01 + // Options (2 bytes) must be 0x00 0x00 + return VERSION_1; + case FORMAT_VERSION_2_MAGIC_NUMBER: + // Version 2 dictionaries are at least 12 bytes long. + // If this dictionary has the version 2 magic number but is less than 12 bytes long, + // then it's an unknown format and we need to avoid confidently reading the next bytes. + if (dictSize < FORMAT_VERSION_2_MINIMUM_SIZE) { + return UNKNOWN_VERSION; + } + // Format 2 header is as follows: + // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE + // Version number (2 bytes) 0x00 0x02 + // Options (2 bytes) + // Header size (4 bytes) : integer, big endian + if (ByteArrayUtils::readUint16(dict, 4) == 2) { + return VERSION_2; + } else { + return UNKNOWN_VERSION; + } + default: + return UNKNOWN_VERSION; + } +} + +} // namespace latinime diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h new file mode 100644 index 000000000..3aa1662da --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_format.h @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BINARY_DICTIONARY_FORMAT_H +#define LATINIME_BINARY_DICTIONARY_FORMAT_H + +#include <stdint.h> + +#include "defines.h" +#include "suggest/core/dictionary/byte_array_utils.h" + +namespace latinime { + +/** + * Methods to handle binary dictionary format version. + * + * Currently, we have a file with a similar name, binary_format.h. binary_format.h contains binary + * reading methods and utility methods for various purposes. + * On the other hand, this file deals with only about dictionary format version. + */ +class BinaryDictionaryFormat { + public: + // TODO: Remove obsolete version logic + enum FORMAT_VERSION { + VERSION_1, + VERSION_2, + UNKNOWN_VERSION + }; + + static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize); + + static AK_FORCE_INLINE int getHeaderSize( + const uint8_t *const dict, const FORMAT_VERSION format) { + switch (format) { + case VERSION_1: + return FORMAT_VERSION_1_HEADER_SIZE; + case VERSION_2: + // See the format of the header in the comment in detectFormat() above + return ByteArrayUtils::readUint32(dict, 8); + default: + return S_INT_MAX; + } + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormat); + + static const int DICTIONARY_MINIMUM_SIZE; + static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER; + static const int FORMAT_VERSION_1_HEADER_SIZE; + static const uint32_t FORMAT_VERSION_2_MAGIC_NUMBER; + static const int FORMAT_VERSION_2_MINIMUM_SIZE; + static const int VERSION_2_MAGIC_NUMBER_SIZE; + static const int VERSION_2_DICTIONARY_VERSION_SIZE ; + static const int VERSION_2_DICTIONARY_FLAG_SIZE; +}; +} // namespace latinime +#endif /* LATINIME_BINARY_DICTIONARY_FORMAT_H */ diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h new file mode 100644 index 000000000..8508c6786 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/binary_dictionary_info.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BINARY_DICTIONARY_INFO_H +#define LATINIME_BINARY_DICTIONARY_INFO_H + +#include <stdint.h> + +#include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_format.h" + +namespace latinime { + +class BinaryDictionaryInfo { + public: + BinaryDictionaryInfo(const uint8_t *const dictBuf, const int dictSize) + : mDictBuf(dictBuf), + mFormat(BinaryDictionaryFormat::detectFormatVersion(mDictBuf, dictSize)), + mDictRoot(mDictBuf + BinaryDictionaryFormat::getHeaderSize(mDictBuf, mFormat)) {} + + AK_FORCE_INLINE const uint8_t *getDictBuf() const { + return mDictBuf; + } + + AK_FORCE_INLINE const uint8_t *getDictRoot() const { + return mDictRoot; + } + + AK_FORCE_INLINE BinaryDictionaryFormat::FORMAT_VERSION getFormat() const { + return mFormat; + } + + AK_FORCE_INLINE int getRootPosition() const { + return 0; + } + + private: + DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo); + + const uint8_t *const mDictBuf; + const BinaryDictionaryFormat::FORMAT_VERSION mFormat; + const uint8_t *const mDictRoot; +}; +} +#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */ diff --git a/native/jni/src/binary_format.h b/native/jni/src/suggest/core/dictionary/binary_format.h index 98241532f..c82065f97 100644 --- a/native/jni/src/binary_format.h +++ b/native/jni/src/suggest/core/dictionary/binary_format.h @@ -18,12 +18,12 @@ #define LATINIME_BINARY_FORMAT_H #include <cstdlib> -#include <map> #include <stdint.h> -#include "bloom_filter.h" -#include "char_utils.h" -#include "hash_map_compat.h" +#include "suggest/core/dictionary/bloom_filter.h" +#include "suggest/core/dictionary/probability_utils.h" +#include "utils/char_utils.h" +#include "utils/hash_map_compat.h" namespace latinime { @@ -91,10 +91,6 @@ class BinaryFormat { const int length, const bool forceLowerCaseSearch); static int getWordAtAddress(const uint8_t *const root, const int address, const int maxDepth, int *outWord, int *outUnigramProbability); - static int computeProbabilityForBigram( - const int unigramProbability, const int bigramProbability); - static int getProbability(const int position, const std::map<int, int> *bigramMap, - const uint8_t *bigramFilter, const int unigramProbability); static int getBigramProbabilityFromHashMap(const int position, const hash_map_compat<int, int> *bigramMap, const int unigramProbability); static float getMultiWordCostMultiplier(const uint8_t *const dict, const int dictSize); @@ -271,8 +267,7 @@ AK_FORCE_INLINE int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t * inline float BinaryFormat::getMultiWordCostMultiplier(const uint8_t *const dict, const int dictSize) { - const int headerValue = readHeaderValueInt(dict, dictSize, - "MULTIPLE_WORDS_DEMOTION_RATE"); + const int headerValue = readHeaderValueInt(dict, dictSize, "MULTIPLE_WORDS_DEMOTION_RATE"); if (headerValue == S_INT_MIN) { return 1.0f; } @@ -473,7 +468,8 @@ AK_FORCE_INLINE int BinaryFormat::getTerminalPosition(const uint8_t *const root, // there was no match (or we would have found it). if (wordPos >= length) return NOT_VALID_WORD; int charGroupCount = BinaryFormat::getGroupCountAndForwardPointer(root, &pos); - const int wChar = forceLowerCaseSearch ? toLowerCase(inWord[wordPos]) : inWord[wordPos]; + const int wChar = forceLowerCaseSearch + ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos]; while (true) { // If there are no more character groups in this node, it means we could not // find a matching character for this depth, therefore there is no match. @@ -677,51 +673,18 @@ AK_FORCE_INLINE int BinaryFormat::getWordAtAddress(const uint8_t *const root, co return 0; } -static inline int backoff(const int unigramProbability) { - return unigramProbability; - // For some reason, applying the backoff weight gives bad results in tests. To apply the - // backoff weight, we divide the probability by 2, which in our storing format means - // decreasing the score by 8. - // TODO: figure out what's wrong with this. - // return unigramProbability > 8 ? unigramProbability - 8 : (0 == unigramProbability ? 0 : 8); -} - -inline int BinaryFormat::computeProbabilityForBigram( - const int unigramProbability, const int bigramProbability) { - // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want the - // unigram probability to be the median value of the 17th step from the top. A value of - // 0 for the bigram probability represents the middle of the 16th step from the top, - // while a value of 15 represents the middle of the top step. - // See makedict.BinaryDictInputOutput for details. - const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability) - / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY); - return unigramProbability - + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize); -} - -// This returns a probability in log space. -inline int BinaryFormat::getProbability(const int position, const std::map<int, int> *bigramMap, - const uint8_t *bigramFilter, const int unigramProbability) { - if (!bigramMap || !bigramFilter) return backoff(unigramProbability); - if (!isInFilter(bigramFilter, position)) return backoff(unigramProbability); - const std::map<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position); - if (bigramProbabilityIt != bigramMap->end()) { - const int bigramProbability = bigramProbabilityIt->second; - return computeProbabilityForBigram(unigramProbability, bigramProbability); - } - return backoff(unigramProbability); -} - // This returns a probability in log space. inline int BinaryFormat::getBigramProbabilityFromHashMap(const int position, const hash_map_compat<int, int> *bigramMap, const int unigramProbability) { - if (!bigramMap) return backoff(unigramProbability); + if (!bigramMap) { + return ProbabilityUtils::backoff(unigramProbability); + } const hash_map_compat<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position); if (bigramProbabilityIt != bigramMap->end()) { const int bigramProbability = bigramProbabilityIt->second; - return computeProbabilityForBigram(unigramProbability, bigramProbability); + return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, bigramProbability); } - return backoff(unigramProbability); + return ProbabilityUtils::backoff(unigramProbability); } AK_FORCE_INLINE void BinaryFormat::fillBigramProbabilityToHashMap( @@ -742,7 +705,9 @@ AK_FORCE_INLINE void BinaryFormat::fillBigramProbabilityToHashMap( AK_FORCE_INLINE int BinaryFormat::getBigramProbability(const uint8_t *const root, int position, const int nextPosition, const int unigramProbability) { position = getBigramListPositionForWordPosition(root, position); - if (0 == position) return backoff(unigramProbability); + if (0 == position) { + return ProbabilityUtils::backoff(unigramProbability); + } uint8_t bigramFlags; do { @@ -751,10 +716,11 @@ AK_FORCE_INLINE int BinaryFormat::getBigramProbability(const uint8_t *const root root, bigramFlags, &position); if (bigramPos == nextPosition) { const int bigramProbability = MASK_ATTRIBUTE_PROBABILITY & bigramFlags; - return computeProbabilityForBigram(unigramProbability, bigramProbability); + return ProbabilityUtils::computeProbabilityForBigram( + unigramProbability, bigramProbability); } } while (FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags); - return backoff(unigramProbability); + return ProbabilityUtils::backoff(unigramProbability); } // Returns a pointer to the start of the bigram list. diff --git a/native/jni/src/bloom_filter.h b/native/jni/src/suggest/core/dictionary/bloom_filter.h index bcce1f7ea..bcce1f7ea 100644 --- a/native/jni/src/bloom_filter.h +++ b/native/jni/src/suggest/core/dictionary/bloom_filter.h diff --git a/native/jni/src/dic_traverse_wrapper.cpp b/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp index 88ca9fa0d..68b1d5d15 100644 --- a/native/jni/src/dic_traverse_wrapper.cpp +++ b/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp @@ -1,11 +1,11 @@ /* - * Copyright (C) 2012, The Android Open Source Project + * Copyright (C) 2013 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,13 +14,11 @@ * limitations under the License. */ -#define LOG_TAG "LatinIME: jni: Session" - -#include "dic_traverse_wrapper.h" +#include "suggest/core/dictionary/byte_array_utils.h" namespace latinime { -void *(*DicTraverseWrapper::sDicTraverseSessionFactoryMethod)(JNIEnv *, jstring) = 0; -void (*DicTraverseWrapper::sDicTraverseSessionReleaseMethod)(void *) = 0; -void (*DicTraverseWrapper::sDicTraverseSessionInitMethod)( - void *, const Dictionary *const, const int *, const int) = 0; + +const uint8_t ByteArrayUtils::MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; +const uint8_t ByteArrayUtils::CHARACTER_ARRAY_TERMINATOR = 0x1F; + } // namespace latinime diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.h b/native/jni/src/suggest/core/dictionary/byte_array_utils.h new file mode 100644 index 000000000..832b74725 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/byte_array_utils.h @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BYTE_ARRAY_UTILS_H +#define LATINIME_BYTE_ARRAY_UTILS_H + +#include <stdint.h> + +#include "defines.h" + +namespace latinime { + +/** + * Utility methods for reading byte arrays. + */ +class ByteArrayUtils { + public: + /** + * Integer + * + * Each method read a corresponding size integer in a big endian manner. + */ + static AK_FORCE_INLINE uint32_t readUint32(const uint8_t *const buffer, const int pos) { + return (buffer[pos] << 24) ^ (buffer[pos + 1] << 16) + ^ (buffer[pos + 2] << 8) ^ buffer[pos + 3]; + } + + static AK_FORCE_INLINE uint32_t readUint24(const uint8_t *const buffer, const int pos) { + return (buffer[pos] << 16) ^ (buffer[pos + 1] << 8) ^ buffer[pos + 2]; + } + + static AK_FORCE_INLINE uint16_t readUint16(const uint8_t *const buffer, const int pos) { + return (buffer[pos] << 8) ^ buffer[pos + 1]; + } + + static AK_FORCE_INLINE uint8_t readUint8(const uint8_t *const buffer, const int pos) { + return buffer[pos]; + } + + static AK_FORCE_INLINE uint32_t readUint32andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint32_t value = readUint32(buffer, *pos); + *pos += 4; + return value; + } + + static AK_FORCE_INLINE uint32_t readUint24andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint32_t value = readUint24(buffer, *pos); + *pos += 3; + return value; + } + + static AK_FORCE_INLINE uint16_t readUint16andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint16_t value = readUint16(buffer, *pos); + *pos += 2; + return value; + } + + static AK_FORCE_INLINE uint8_t readUint8andAdvancePosition( + const uint8_t *const buffer, int *const pos) { + return buffer[(*pos)++]; + } + + /** + * Code Point + * + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + */ + static AK_FORCE_INLINE int readCodePoint(const uint8_t *const buffer, const int pos) { + int p = pos; + return readCodePointAndAdvancePosition(buffer, &p); + } + + static AK_FORCE_INLINE int readCodePointAndAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint8_t firstByte = readUint8(buffer, *pos); + if (firstByte < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + if (firstByte == CHARACTER_ARRAY_TERMINATOR) { + *pos += 1; + return NOT_A_CODE_POINT; + } else { + return readUint24andAdvancePosition(buffer, pos); + } + } else { + *pos += 1; + return firstByte; + } + } + + /** + * String (array of code points) + * + * Reads code points until the terminator is found. + */ + // Returns the length of the string. + static int readStringAndAdvancePosition(const uint8_t *const buffer, int *const pos, + int *const outBuffer, const int maxLength) { + int length = 0; + int codePoint = readCodePointAndAdvancePosition(buffer, pos); + while (NOT_A_CODE_POINT != codePoint && length < maxLength) { + outBuffer[length++] = codePoint; + codePoint = readCodePointAndAdvancePosition(buffer, pos); + } + return length; + } + + // Advances the position and returns the length of the string. + static int advancePositionToBehindString( + const uint8_t *const buffer, int *const pos, const int maxLength) { + int length = 0; + int codePoint = readCodePointAndAdvancePosition(buffer, pos); + while (NOT_A_CODE_POINT != codePoint && length < maxLength) { + codePoint = readCodePointAndAdvancePosition(buffer, pos); + } + return length; + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils); + + static const uint8_t MINIMAL_ONE_BYTE_CHARACTER_VALUE; + static const uint8_t CHARACTER_ARRAY_TERMINATOR; +}; +} // namespace latinime +#endif /* LATINIME_BYTE_ARRAY_UTILS_H */ diff --git a/native/jni/src/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index dadb2bab2..2d4ad5df5 100644 --- a/native/jni/src/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -16,50 +16,47 @@ #define LOG_TAG "LatinIME: dictionary.cpp" -#include "dictionary.h" +#include "suggest/core/dictionary/dictionary.h" #include <map> // TODO: remove #include <stdint.h> -#include "bigram_dictionary.h" -#include "binary_format.h" #include "defines.h" -#include "dic_traverse_wrapper.h" +#include "suggest/core/dictionary/bigram_dictionary.h" +#include "suggest/core/dictionary/binary_format.h" +#include "suggest/core/session/dic_traverse_session.h" #include "suggest/core/suggest.h" +#include "suggest/core/suggest_options.h" #include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h" #include "suggest/policyimpl/typing/typing_suggest_policy_factory.h" -#include "unigram_dictionary.h" namespace latinime { Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust) - : mDict(static_cast<unsigned char *>(dict)), - mOffsetDict((static_cast<unsigned char *>(dict)) - + BinaryFormat::getHeaderSize(mDict, dictSize)), - mDictSize(dictSize), mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust), - mUnigramDictionary(new UnigramDictionary(mOffsetDict, - BinaryFormat::getFlags(mDict, dictSize))), - mBigramDictionary(new BigramDictionary(mOffsetDict)), + : mBinaryDicitonaryInfo(static_cast<const uint8_t *>(dict), dictSize), + mDictSize(dictSize), + mDictFlags(BinaryFormat::getFlags(mBinaryDicitonaryInfo.getDictBuf(), dictSize)), + mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust), + mBigramDictionary(new BigramDictionary(&mBinaryDicitonaryInfo)), mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())), mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) { } Dictionary::~Dictionary() { - delete mUnigramDictionary; delete mBigramDictionary; delete mGestureSuggest; delete mTypingSuggest; } -int Dictionary::getSuggestions(ProximityInfo *proximityInfo, void *traverseSession, +int Dictionary::getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession, int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, - int inputSize, int *prevWordCodePoints, int prevWordLength, int commitPoint, bool isGesture, - bool useFullEditDistance, int *outWords, int *frequencies, int *spaceIndices, - int *outputTypes) const { + int inputSize, int *prevWordCodePoints, int prevWordLength, int commitPoint, + const SuggestOptions *const suggestOptions, int *outWords, int *frequencies, + int *spaceIndices, int *outputTypes) const { int result = 0; - if (isGesture) { - DicTraverseWrapper::initDicTraverseSession( - traverseSession, this, prevWordCodePoints, prevWordLength); + if (suggestOptions->isGesture()) { + DicTraverseSession::initSessionInstance( + traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions); result = mGestureSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates, ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, outWords, frequencies, spaceIndices, outputTypes); @@ -68,26 +65,15 @@ int Dictionary::getSuggestions(ProximityInfo *proximityInfo, void *traverseSessi } return result; } else { - if (USE_SUGGEST_INTERFACE_FOR_TYPING) { - DicTraverseWrapper::initDicTraverseSession( - traverseSession, this, prevWordCodePoints, prevWordLength); - result = mTypingSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates, - ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, - outWords, frequencies, spaceIndices, outputTypes); - if (DEBUG_DICT) { - DUMP_RESULT(outWords, frequencies); - } - return result; - } else { - std::map<int, int> bigramMap; - uint8_t bigramFilter[BIGRAM_FILTER_BYTE_SIZE]; - mBigramDictionary->fillBigramAddressToProbabilityMapAndFilter(prevWordCodePoints, - prevWordLength, &bigramMap, bigramFilter); - result = mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates, - inputCodePoints, inputSize, &bigramMap, bigramFilter, useFullEditDistance, - outWords, frequencies, outputTypes); - return result; + DicTraverseSession::initSessionInstance( + traverseSession, this, prevWordCodePoints, prevWordLength, suggestOptions); + result = mTypingSuggest->getSuggestions(proximityInfo, traverseSession, xcoordinates, + ycoordinates, times, pointerIds, inputCodePoints, inputSize, commitPoint, + outWords, frequencies, spaceIndices, outputTypes); + if (DEBUG_DICT) { + DUMP_RESULT(outWords, frequencies); } + return result; } } @@ -99,7 +85,27 @@ int Dictionary::getBigrams(const int *word, int length, int *inputCodePoints, in } int Dictionary::getProbability(const int *word, int length) const { - return mUnigramDictionary->getProbability(word, length); + const uint8_t *const root = mBinaryDicitonaryInfo.getDictRoot(); + int pos = BinaryFormat::getTerminalPosition(root, word, length, + false /* forceLowerCaseSearch */); + if (NOT_VALID_WORD == pos) { + return NOT_A_PROBABILITY; + } + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + if (flags & (BinaryFormat::FLAG_IS_BLACKLISTED | BinaryFormat::FLAG_IS_NOT_A_WORD)) { + // If this is not a word, or if it's a blacklisted entry, it should behave as + // having no probability outside of the suggestion process (where it should be used + // for shortcuts). + return NOT_A_PROBABILITY; + } + const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); + if (hasMultipleChars) { + pos = BinaryFormat::skipOtherCharacters(root, pos); + } else { + BinaryFormat::getCodePointAndForwardPointer(root, &pos); + } + const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(root, pos); + return unigramProbability; } bool Dictionary::isValidBigram(const int *word1, int length1, const int *word2, int length2) const { @@ -107,7 +113,7 @@ bool Dictionary::isValidBigram(const int *word1, int length1, const int *word2, } int Dictionary::getDictFlags() const { - return mUnigramDictionary->getDictFlags(); + return mDictFlags; } } // namespace latinime diff --git a/native/jni/src/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 2ad5b6c0b..1f25080b1 100644 --- a/native/jni/src/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -20,13 +20,15 @@ #include <stdint.h> #include "defines.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" namespace latinime { class BigramDictionary; +class DicTraverseSession; class ProximityInfo; class SuggestInterface; -class UnigramDictionary; +class SuggestOptions; class Dictionary { public: @@ -41,6 +43,10 @@ class Dictionary { static const int KIND_APP_DEFINED = 6; // Suggested by the application static const int KIND_SHORTCUT = 7; // A shortcut static const int KIND_PREDICTION = 8; // A prediction (== a suggestion with no input) + // KIND_RESUMED: A resumed suggestion (comes from a span, currently this type is used only + // in java for re-correction) + static const int KIND_RESUMED = 9; + static const int KIND_OOV_CORRECTION = 10; // Most probable string correction static const int KIND_MASK_FLAGS = 0xFFFFFF00; // Mask to get the flags static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000; @@ -48,22 +54,19 @@ class Dictionary { Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust); - int getSuggestions(ProximityInfo *proximityInfo, void *traverseSession, int *xcoordinates, - int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, int inputSize, - int *prevWordCodePoints, int prevWordLength, int commitPoint, bool isGesture, - bool useFullEditDistance, int *outWords, int *frequencies, int *spaceIndices, - int *outputTypes) const; + int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession, + int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints, + int inputSize, int *prevWordCodePoints, int prevWordLength, int commitPoint, + const SuggestOptions *const suggestOptions, int *outWords, int *frequencies, + int *spaceIndices, int *outputTypes) const; int getBigrams(const int *word, int length, int *inputCodePoints, int inputSize, int *outWords, int *frequencies, int *outputTypes) const; int getProbability(const int *word, int length) const; bool isValidBigram(const int *word1, int length1, const int *word2, int length2) const; - const uint8_t *getDict() const { // required to release dictionary buffer - return mDict; - } - const uint8_t *getOffsetDict() const { - return mOffsetDict; + const BinaryDictionaryInfo *getBinaryDictionaryInfo() const { + return &mBinaryDicitonaryInfo; } int getDictSize() const { return mDictSize; } int getMmapFd() const { return mMmapFd; } @@ -73,16 +76,15 @@ class Dictionary { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); - const uint8_t *mDict; - const uint8_t *mOffsetDict; + const BinaryDictionaryInfo mBinaryDicitonaryInfo; // Used only for the mmap version of dictionary loading, but we use these as dummy variables // also for the malloc version. const int mDictSize; + const int mDictFlags; const int mMmapFd; const int mDictBufAdjust; - const UnigramDictionary *mUnigramDictionary; const BigramDictionary *mBigramDictionary; SuggestInterface *mGestureSuggest; SuggestInterface *mTypingSuggest; diff --git a/native/jni/src/digraph_utils.cpp b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp index 083442669..f53e56ef1 100644 --- a/native/jni/src/digraph_utils.cpp +++ b/native/jni/src/suggest/core/dictionary/digraph_utils.cpp @@ -14,10 +14,11 @@ * limitations under the License. */ -#include "char_utils.h" -#include "binary_format.h" +#include "suggest/core/dictionary/digraph_utils.h" + #include "defines.h" -#include "digraph_utils.h" +#include "suggest/core/dictionary/binary_format.h" +#include "utils/char_utils.h" namespace latinime { @@ -121,7 +122,7 @@ const DigraphUtils::DigraphType DigraphUtils::USED_DIGRAPH_TYPES[] = /* static */ const DigraphUtils::digraph_t *DigraphUtils::getDigraphForDigraphTypeAndCodePoint( const DigraphUtils::DigraphType digraphType, const int compositeGlyphCodePoint) { const DigraphUtils::digraph_t *digraphs = 0; - const int compositeGlyphLowerCodePoint = toLowerCase(compositeGlyphCodePoint); + const int compositeGlyphLowerCodePoint = CharUtils::toLowerCase(compositeGlyphCodePoint); const int digraphsSize = DigraphUtils::getAllDigraphsForDictionaryAndReturnSize(digraphType, &digraphs); for (int i = 0; i < digraphsSize; i++) { diff --git a/native/jni/src/digraph_utils.h b/native/jni/src/suggest/core/dictionary/digraph_utils.h index 94435228e..c1205940c 100644 --- a/native/jni/src/digraph_utils.h +++ b/native/jni/src/suggest/core/dictionary/digraph_utils.h @@ -17,6 +17,8 @@ #ifndef DIGRAPH_UTILS_H #define DIGRAPH_UTILS_H +#include "defines.h" + namespace latinime { class DigraphUtils { diff --git a/native/jni/src/multi_bigram_map.h b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h index 7e1b6301f..ba97e5842 100644 --- a/native/jni/src/multi_bigram_map.h +++ b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h @@ -17,12 +17,10 @@ #ifndef LATINIME_MULTI_BIGRAM_MAP_H #define LATINIME_MULTI_BIGRAM_MAP_H -#include <cstring> -#include <stdint.h> - #include "defines.h" -#include "binary_format.h" -#include "hash_map_compat.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" +#include "suggest/core/dictionary/binary_format.h" +#include "utils/hash_map_compat.h" namespace latinime { @@ -36,20 +34,20 @@ class MultiBigramMap { // Look up the bigram probability for the given word pair from the cached bigram maps. // Also caches the bigrams if there is space remaining and they have not been cached already. - int getBigramProbability(const uint8_t *const dicRoot, const int wordPosition, - const int nextWordPosition, const int unigramProbability) { + int getBigramProbability(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const int wordPosition, const int nextWordPosition, const int unigramProbability) { hash_map_compat<int, BigramMap>::const_iterator mapPosition = mBigramMaps.find(wordPosition); if (mapPosition != mBigramMaps.end()) { return mapPosition->second.getBigramProbability(nextWordPosition, unigramProbability); } if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) { - addBigramsForWordPosition(dicRoot, wordPosition); + addBigramsForWordPosition(binaryDicitonaryInfo, wordPosition); return mBigramMaps[wordPosition].getBigramProbability( nextWordPosition, unigramProbability); } - return BinaryFormat::getBigramProbability( - dicRoot, wordPosition, nextWordPosition, unigramProbability); + return BinaryFormat::getBigramProbability(binaryDicitonaryInfo->getDictRoot(), + wordPosition, nextWordPosition, unigramProbability); } void clear() { @@ -64,8 +62,9 @@ class MultiBigramMap { BigramMap() : mBigramMap(DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP) {} ~BigramMap() {} - void init(const uint8_t *const dicRoot, int position) { - BinaryFormat::fillBigramProbabilityToHashMap(dicRoot, position, &mBigramMap); + void init(const BinaryDictionaryInfo *const binaryDicitonaryInfo, const int position) { + BinaryFormat::fillBigramProbabilityToHashMap( + binaryDicitonaryInfo->getDictRoot(), position, &mBigramMap); } inline int getBigramProbability(const int nextWordPosition, const int unigramProbability) @@ -79,8 +78,9 @@ class MultiBigramMap { hash_map_compat<int, int> mBigramMap; }; - void addBigramsForWordPosition(const uint8_t *const dicRoot, const int position) { - mBigramMaps[position].init(dicRoot, position); + void addBigramsForWordPosition(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const int position) { + mBigramMaps[position].init(binaryDicitonaryInfo, position); } hash_map_compat<int, BigramMap> mBigramMaps; diff --git a/native/jni/src/suggest/core/dictionary/probability_utils.h b/native/jni/src/suggest/core/dictionary/probability_utils.h new file mode 100644 index 000000000..14d2f8436 --- /dev/null +++ b/native/jni/src/suggest/core/dictionary/probability_utils.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_PROBABILITY_UTILS_H +#define LATINIME_PROBABILITY_UTILS_H + +#include <map> +#include <stdint.h> + +#include "defines.h" + +namespace latinime { + +class ProbabilityUtils { + public: + static AK_FORCE_INLINE int backoff(const int unigramProbability) { + return unigramProbability; + // For some reason, applying the backoff weight gives bad results in tests. To apply the + // backoff weight, we divide the probability by 2, which in our storing format means + // decreasing the score by 8. + // TODO: figure out what's wrong with this. + // return unigramProbability > 8 ? + // unigramProbability - 8 : (0 == unigramProbability ? 0 : 8); + } + + static AK_FORCE_INLINE int computeProbabilityForBigram( + const int unigramProbability, const int bigramProbability) { + // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want + // the unigram probability to be the median value of the 17th step from the top. A value of + // 0 for the bigram probability represents the middle of the 16th step from the top, + // while a value of 15 represents the middle of the top step. + // See makedict.BinaryDictInputOutput for details. + const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability) + / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY); + return unigramProbability + + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize); + } + + // This returns a probability in log space. + static AK_FORCE_INLINE int getProbability(const int position, + const std::map<int, int> *const bigramMap, + const uint8_t *bigramFilter, const int unigramProbability) { + if (!bigramMap || !bigramFilter) { + return backoff(unigramProbability); + } + if (!isInFilter(bigramFilter, position)){ + return backoff(unigramProbability); + } + const std::map<int, int>::const_iterator bigramProbabilityIt = bigramMap->find(position); + if (bigramProbabilityIt != bigramMap->end()) { + const int bigramProbability = bigramProbabilityIt->second; + return computeProbabilityForBigram(unigramProbability, bigramProbability); + } + return backoff(unigramProbability); + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils); +}; +} +#endif /* LATINIME_PROBABILITY_UTILS_H */ diff --git a/native/jni/src/suggest/core/dictionary/shortcut_utils.h b/native/jni/src/suggest/core/dictionary/shortcut_utils.h index c411408ec..601ac5f5a 100644 --- a/native/jni/src/suggest/core/dictionary/shortcut_utils.h +++ b/native/jni/src/suggest/core/dictionary/shortcut_utils.h @@ -19,7 +19,7 @@ #include "defines.h" #include "suggest/core/dicnode/dic_node_utils.h" -#include "terminal_attributes.h" +#include "suggest/core/dictionary/terminal_attributes.h" namespace latinime { diff --git a/native/jni/src/terminal_attributes.h b/native/jni/src/suggest/core/dictionary/terminal_attributes.h index 92ef71c2c..bbd9af090 100644 --- a/native/jni/src/terminal_attributes.h +++ b/native/jni/src/suggest/core/dictionary/terminal_attributes.h @@ -18,7 +18,9 @@ #define LATINIME_TERMINAL_ATTRIBUTES_H #include <stdint.h> -#include "binary_format.h" + +#include "suggest/core/dictionary/binary_dictionary_info.h" +#include "suggest/core/dictionary/binary_format.h" namespace latinime { @@ -31,8 +33,9 @@ class TerminalAttributes { public: class ShortcutIterator { public: - ShortcutIterator(const uint8_t *dict, const int pos, const uint8_t flags) - : mDict(dict), mPos(pos), + ShortcutIterator(const BinaryDictionaryInfo *const binaryDictionaryInfo, const int pos, + const uint8_t flags) + : mBinaryDicitionaryInfo(binaryDictionaryInfo), mPos(pos), mHasNextShortcutTarget(0 != (flags & BinaryFormat::FLAG_HAS_SHORTCUT_TARGETS)) { } @@ -43,11 +46,13 @@ class TerminalAttributes { // Gets the shortcut target itself as an int string. For parameters and return value // see BinaryFormat::getWordAtAddress. inline int getNextShortcutTarget(const int maxDepth, int *outWord, int *outFreq) { - const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(mDict, &mPos); + const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer( + mBinaryDicitionaryInfo->getDictRoot(), &mPos); mHasNextShortcutTarget = 0 != (shortcutFlags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT); unsigned int i; for (i = 0; i < MAX_WORD_LENGTH; ++i) { - const int codePoint = BinaryFormat::getCodePointAndForwardPointer(mDict, &mPos); + const int codePoint = BinaryFormat::getCodePointAndForwardPointer( + mBinaryDicitionaryInfo->getDictRoot(), &mPos); if (NOT_A_CODE_POINT == codePoint) break; outWord[i] = codePoint; } @@ -56,19 +61,21 @@ class TerminalAttributes { } private: - const uint8_t *const mDict; + const BinaryDictionaryInfo *const mBinaryDicitionaryInfo; int mPos; bool mHasNextShortcutTarget; }; - TerminalAttributes(const uint8_t *const dict, const uint8_t flags, const int pos) - : mDict(dict), mFlags(flags), mStartPos(pos) { + TerminalAttributes(const BinaryDictionaryInfo *const binaryDicitonaryInfo, + const uint8_t flags, const int pos) + : mBinaryDicitionaryInfo(binaryDicitonaryInfo), mFlags(flags), mStartPos(pos) { } inline ShortcutIterator getShortcutIterator() const { // The size of the shortcuts is stored here so that the whole shortcut chunk can be // skipped quickly, so we ignore it. - return ShortcutIterator(mDict, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags); + return ShortcutIterator( + mBinaryDicitionaryInfo, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags); } bool isBlacklistedOrNotAWord() const { @@ -77,7 +84,7 @@ class TerminalAttributes { private: DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes); - const uint8_t *const mDict; + const BinaryDictionaryInfo *const mBinaryDicitionaryInfo; const uint8_t mFlags; const int mStartPos; }; diff --git a/native/jni/src/additional_proximity_chars.cpp b/native/jni/src/suggest/core/layout/additional_proximity_chars.cpp index 661c50e91..34b8b37b0 100644 --- a/native/jni/src/additional_proximity_chars.cpp +++ b/native/jni/src/suggest/core/layout/additional_proximity_chars.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "additional_proximity_chars.h" +#include "suggest/core/layout/additional_proximity_chars.h" namespace latinime { // TODO: Stop using hardcoded additional proximity characters. diff --git a/native/jni/src/additional_proximity_chars.h b/native/jni/src/suggest/core/layout/additional_proximity_chars.h index a88fd6cea..a88fd6cea 100644 --- a/native/jni/src/additional_proximity_chars.h +++ b/native/jni/src/suggest/core/layout/additional_proximity_chars.h diff --git a/native/jni/src/suggest/core/layout/geometry_utils.h b/native/jni/src/suggest/core/layout/geometry_utils.h new file mode 100644 index 000000000..b667df68f --- /dev/null +++ b/native/jni/src/suggest/core/layout/geometry_utils.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_GEOMETRY_UTILS_H +#define LATINIME_GEOMETRY_UTILS_H + +#include <cmath> + +#include "defines.h" + +#define ROUND_FLOAT_10000(f) ((f) < 1000.0f && (f) > 0.001f) \ + ? (floorf((f) * 10000.0f) / 10000.0f) : (f) + +namespace latinime { + +class GeometryUtils { + public: + static inline float SQUARE_FLOAT(const float x) { return x * x; } + + static AK_FORCE_INLINE float getAngle(const int x1, const int y1, const int x2, const int y2) { + const int dx = x1 - x2; + const int dy = y1 - y2; + if (dx == 0 && dy == 0) return 0.0f; + return atan2f(static_cast<float>(dy), static_cast<float>(dx)); + } + + static AK_FORCE_INLINE float getAngleDiff(const float a1, const float a2) { + const float deltaA = fabsf(a1 - a2); + const float diff = ROUND_FLOAT_10000(deltaA); + if (diff > M_PI_F) { + const float normalizedDiff = 2.0f * M_PI_F - diff; + return ROUND_FLOAT_10000(normalizedDiff); + } + return diff; + } + + static AK_FORCE_INLINE int getDistanceInt(const int x1, const int y1, const int x2, + const int y2) { + return static_cast<int>(hypotf(static_cast<float>(x1 - x2), static_cast<float>(y1 - y2))); + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(GeometryUtils); +}; +} // namespace latinime +#endif // LATINIME_GEOMETRY_UTILS_H diff --git a/native/jni/src/proximity_info.cpp b/native/jni/src/suggest/core/layout/proximity_info.cpp index 88d670d61..80355c148 100644 --- a/native/jni/src/proximity_info.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info.cpp @@ -14,18 +14,19 @@ * limitations under the License. */ +#define LOG_TAG "LatinIME: proximity_info.cpp" + +#include "suggest/core/layout/proximity_info.h" + #include <cstring> #include <cmath> -#define LOG_TAG "LatinIME: proximity_info.cpp" - -#include "additional_proximity_chars.h" -#include "char_utils.h" #include "defines.h" -#include "geometry_utils.h" #include "jni.h" -#include "proximity_info.h" -#include "proximity_info_params.h" +#include "suggest/core/layout/additional_proximity_chars.h" +#include "suggest/core/layout/geometry_utils.h" +#include "suggest/core/layout/proximity_info_params.h" +#include "utils/char_utils.h" namespace latinime { @@ -58,7 +59,7 @@ ProximityInfo::ProximityInfo(JNIEnv *env, const jstring localeJStr, MOST_COMMON_KEY_WIDTH_SQUARE(mostCommonKeyWidth * mostCommonKeyWidth), MOST_COMMON_KEY_HEIGHT(mostCommonKeyHeight), NORMALIZED_SQUARED_MOST_COMMON_KEY_HYPOTENUSE(1.0f + - SQUARE_FLOAT(static_cast<float>(mostCommonKeyHeight) / + GeometryUtils::SQUARE_FLOAT(static_cast<float>(mostCommonKeyHeight) / static_cast<float>(mostCommonKeyWidth))), CELL_WIDTH((keyboardWidth + gridWidth - 1) / gridWidth), CELL_HEIGHT((keyboardHeight + gridHeight - 1) / gridHeight), @@ -150,7 +151,7 @@ float ProximityInfo::getNormalizedSquaredDistanceFromCenterFloatG( const float touchY = static_cast<float>(y); const float keyWidth = static_cast<float>(getMostCommonKeyWidth()); return ProximityInfoUtils::getSquaredDistanceFloat(centerX, centerY, touchX, touchY) - / SQUARE_FLOAT(keyWidth); + / GeometryUtils::SQUARE_FLOAT(keyWidth); } int ProximityInfo::getCodePointOf(const int keyIndex) const { @@ -164,7 +165,7 @@ void ProximityInfo::initializeG() { // TODO: Optimize for (int i = 0; i < KEY_COUNT; ++i) { const int code = mKeyCodePoints[i]; - const int lowerCode = toLowerCase(code); + const int lowerCode = CharUtils::toLowerCase(code); mCenterXsG[i] = mKeyXCoordinates[i] + mKeyWidths[i] / 2; mCenterYsG[i] = mKeyYCoordinates[i] + mKeyHeights[i] / 2; mCodeToKeyMap[lowerCode] = i; @@ -173,7 +174,7 @@ void ProximityInfo::initializeG() { for (int i = 0; i < KEY_COUNT; i++) { mKeyKeyDistancesG[i][i] = 0; for (int j = i + 1; j < KEY_COUNT; j++) { - mKeyKeyDistancesG[i][j] = getDistanceInt( + mKeyKeyDistancesG[i][j] = GeometryUtils::getDistanceInt( mCenterXsG[i], mCenterYsG[i], mCenterXsG[j], mCenterYsG[j]); mKeyKeyDistancesG[j][i] = mKeyKeyDistancesG[i][j]; } diff --git a/native/jni/src/proximity_info.h b/native/jni/src/suggest/core/layout/proximity_info.h index deb9ae0de..6ca2fdd7b 100644 --- a/native/jni/src/proximity_info.h +++ b/native/jni/src/suggest/core/layout/proximity_info.h @@ -18,9 +18,9 @@ #define LATINIME_PROXIMITY_INFO_H #include "defines.h" -#include "hash_map_compat.h" #include "jni.h" -#include "proximity_info_utils.h" +#include "suggest/core/layout/proximity_info_utils.h" +#include "utils/hash_map_compat.h" namespace latinime { diff --git a/native/jni/src/proximity_info_params.cpp b/native/jni/src/suggest/core/layout/proximity_info_params.cpp index 2675d9e70..0e887f700 100644 --- a/native/jni/src/proximity_info_params.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info_params.cpp @@ -15,7 +15,7 @@ */ #include "defines.h" -#include "proximity_info_params.h" +#include "suggest/core/layout/proximity_info_params.h" namespace latinime { const float ProximityInfoParams::NOT_A_DISTANCE_FLOAT = -1.0f; diff --git a/native/jni/src/proximity_info_params.h b/native/jni/src/suggest/core/layout/proximity_info_params.h index 4e47f7308..4e47f7308 100644 --- a/native/jni/src/proximity_info_params.h +++ b/native/jni/src/suggest/core/layout/proximity_info_params.h diff --git a/native/jni/src/proximity_info_state.cpp b/native/jni/src/suggest/core/layout/proximity_info_state.cpp index cc5b736bd..4e53992d4 100644 --- a/native/jni/src/proximity_info_state.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info_state.cpp @@ -14,17 +14,19 @@ * limitations under the License. */ +#define LOG_TAG "LatinIME: proximity_info_state.cpp" + +#include "suggest/core/layout/proximity_info_state.h" + #include <cstring> // for memset() and memcpy() #include <sstream> // for debug prints #include <vector> -#define LOG_TAG "LatinIME: proximity_info_state.cpp" - #include "defines.h" -#include "geometry_utils.h" -#include "proximity_info.h" -#include "proximity_info_state.h" -#include "proximity_info_state_utils.h" +#include "suggest/core/layout/geometry_utils.h" +#include "suggest/core/layout/proximity_info.h" +#include "suggest/core/layout/proximity_info_state_utils.h" +#include "utils/char_utils.h" namespace latinime { @@ -174,7 +176,7 @@ float ProximityInfoState::getPointToKeyLength( const int index = inputIndex * mProximityInfo->getKeyCount() + keyId; return min(mSampledNormalizedSquaredLengthCache[index], mMaxPointToKeyLength); } - if (isIntentionalOmissionCodePoint(codePoint)) { + if (CharUtils::isIntentionalOmissionCodePoint(codePoint)) { return 0.0f; } // If the char is not a key on the keyboard then return the max length. @@ -202,7 +204,7 @@ ProximityType ProximityInfoState::getProximityType(const int index, const int co const bool checkProximityChars, int *proximityIndex) const { const int *currentCodePoints = getProximityCodePointsAt(index); const int firstCodePoint = currentCodePoints[0]; - const int baseLowerC = toBaseLowerCase(codePoint); + const int baseLowerC = CharUtils::toBaseLowerCase(codePoint); // The first char in the array is what user typed. If it matches right away, that means the // user typed that same char for this pos. @@ -214,7 +216,7 @@ ProximityType ProximityInfoState::getProximityType(const int index, const int co // If the non-accented, lowercased version of that first character matches c, then we have a // non-accented version of the accented character the user typed. Treat it as a close char. - if (toBaseLowerCase(firstCodePoint) == baseLowerC) { + if (CharUtils::toBaseLowerCase(firstCodePoint) == baseLowerC) { return PROXIMITY_CHAR; } @@ -256,8 +258,8 @@ ProximityType ProximityInfoState::getProximityTypeG(const int index, const int c if (!isUsed()) { return UNRELATED_CHAR; } - const int lowerCodePoint = toLowerCase(codePoint); - const int baseLowerCodePoint = toBaseCodePoint(lowerCodePoint); + const int lowerCodePoint = CharUtils::toLowerCase(codePoint); + const int baseLowerCodePoint = CharUtils::toBaseCodePoint(lowerCodePoint); for (int i = 0; i < static_cast<int>(mSampledSearchKeyVectors[index].size()); ++i) { if (mSampledSearchKeyVectors[index][i] == lowerCodePoint || mSampledSearchKeyVectors[index][i] == baseLowerCodePoint) { diff --git a/native/jni/src/proximity_info_state.h b/native/jni/src/suggest/core/layout/proximity_info_state.h index bbe8af240..0079ab5b8 100644 --- a/native/jni/src/proximity_info_state.h +++ b/native/jni/src/suggest/core/layout/proximity_info_state.h @@ -20,11 +20,10 @@ #include <cstring> // for memset() #include <vector> -#include "char_utils.h" #include "defines.h" -#include "hash_map_compat.h" -#include "proximity_info_params.h" -#include "proximity_info_state_utils.h" +#include "suggest/core/layout/proximity_info_params.h" +#include "suggest/core/layout/proximity_info_state_utils.h" +#include "utils/hash_map_compat.h" namespace latinime { diff --git a/native/jni/src/proximity_info_state_utils.cpp b/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp index 359673cd8..6f88833a2 100644 --- a/native/jni/src/proximity_info_state_utils.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info_state_utils.cpp @@ -14,16 +14,17 @@ * limitations under the License. */ +#include "suggest/core/layout/proximity_info_state_utils.h" + #include <cmath> #include <cstring> // for memset() #include <sstream> // for debug prints #include <vector> #include "defines.h" -#include "geometry_utils.h" -#include "proximity_info.h" -#include "proximity_info_params.h" -#include "proximity_info_state_utils.h" +#include "suggest/core/layout/geometry_utils.h" +#include "suggest/core/layout/proximity_info.h" +#include "suggest/core/layout/proximity_info_params.h" namespace latinime { @@ -103,12 +104,12 @@ namespace latinime { const int time = times ? times[i] : -1; if (i > 1) { - const float prevAngle = getAngle( + const float prevAngle = GeometryUtils::getAngle( inputXCoordinates[i - 2], inputYCoordinates[i - 2], inputXCoordinates[i - 1], inputYCoordinates[i - 1]); - const float currentAngle = getAngle( + const float currentAngle = GeometryUtils::getAngle( inputXCoordinates[i - 1], inputYCoordinates[i - 1], x, y); - sumAngle += getAngleDiff(prevAngle, currentAngle); + sumAngle += GeometryUtils::getAngleDiff(prevAngle, currentAngle); } if (pushTouchPoint(proximityInfo, maxPointToKeyLength, i, c, x, y, time, @@ -157,7 +158,8 @@ namespace latinime { const float sweetSpotCenterY = proximityInfo->getSweetSpotCenterYAt(keyIndex); const float inputX = static_cast<float>((*sampledInputXs)[inputIndex]); const float inputY = static_cast<float>((*sampledInputYs)[inputIndex]); - return SQUARE_FLOAT(inputX - sweetSpotCenterX) + SQUARE_FLOAT(inputY - sweetSpotCenterY); + return GeometryUtils::SQUARE_FLOAT(inputX - sweetSpotCenterX) + + GeometryUtils::SQUARE_FLOAT(inputY - sweetSpotCenterY); } /* static */ float ProximityInfoStateUtils::calculateNormalizedSquaredDistance( @@ -174,7 +176,8 @@ namespace latinime { } const float squaredDistance = calculateSquaredDistanceFromSweetSpotCenter(proximityInfo, sampledInputXs, sampledInputYs, keyIndex, inputIndex); - const float squaredRadius = SQUARE_FLOAT(proximityInfo->getSweetSpotRadiiAt(keyIndex)); + const float squaredRadius = GeometryUtils::SQUARE_FLOAT( + proximityInfo->getSweetSpotRadiiAt(keyIndex)); return squaredDistance / squaredRadius; } @@ -285,7 +288,7 @@ namespace latinime { if (i < sampledInputSize - 1 && j >= (*sampledInputIndice)[i + 1]) { break; } - length += getDistanceInt(xCoordinates[j], yCoordinates[j], + length += GeometryUtils::getDistanceInt(xCoordinates[j], yCoordinates[j], xCoordinates[j + 1], yCoordinates[j + 1]); duration += times[j + 1] - times[j]; } @@ -296,7 +299,7 @@ namespace latinime { break; } // TODO: use mSampledLengthCache instead? - length += getDistanceInt(xCoordinates[j], yCoordinates[j], + length += GeometryUtils::getDistanceInt(xCoordinates[j], yCoordinates[j], xCoordinates[j + 1], yCoordinates[j + 1]); duration += times[j + 1] - times[j]; } @@ -349,7 +352,7 @@ namespace latinime { const int y1 = (*sampledInputYs)[index0]; const int x2 = (*sampledInputXs)[index1]; const int y2 = (*sampledInputYs)[index1]; - return getAngle(x1, y1, x2, y2); + return GeometryUtils::getAngle(x1, y1, x2, y2); } // Calculating point to key distance for all near keys and returning the distance between @@ -411,9 +414,9 @@ namespace latinime { } const int baseSampleRate = mostCommonKeyWidth; - const int distPrev = getDistanceInt(sampledInputXs->back(), sampledInputYs->back(), - (*sampledInputXs)[size - 2], (*sampledInputYs)[size - 2]) - * ProximityInfoParams::DISTANCE_BASE_SCALE; + const int distPrev = GeometryUtils::getDistanceInt(sampledInputXs->back(), + sampledInputYs->back(), (*sampledInputXs)[size - 2], + (*sampledInputYs)[size - 2]) * ProximityInfoParams::DISTANCE_BASE_SCALE; float score = 0.0f; // Location @@ -425,10 +428,11 @@ namespace latinime { score += ProximityInfoParams::LOCALMIN_DISTANCE_AND_NEAR_TO_KEY_SCORE; } // Angle - const float angle1 = getAngle(x, y, sampledInputXs->back(), sampledInputYs->back()); - const float angle2 = getAngle(sampledInputXs->back(), sampledInputYs->back(), + const float angle1 = GeometryUtils::getAngle(x, y, sampledInputXs->back(), + sampledInputYs->back()); + const float angle2 = GeometryUtils::getAngle(sampledInputXs->back(), sampledInputYs->back(), (*sampledInputXs)[size - 2], (*sampledInputYs)[size - 2]); - const float angleDiff = getAngleDiff(angle1, angle2); + const float angleDiff = GeometryUtils::getAngleDiff(angle1, angle2); // Save corner if (distPrev > baseSampleRate * ProximityInfoParams::CORNER_CHECK_DISTANCE_THRESHOLD_SCALE @@ -472,13 +476,13 @@ namespace latinime { } // Check if the last point should be skipped. if (isLastPoint && size > 0) { - if (getDistanceInt(x, y, sampledInputXs->back(), sampledInputYs->back()) + if (GeometryUtils::getDistanceInt(x, y, sampledInputXs->back(), sampledInputYs->back()) * ProximityInfoParams::LAST_POINT_SKIP_DISTANCE_SCALE < mostCommonKeyWidth) { // This point is not used because it's too close to the previous point. if (DEBUG_GEO_FULL) { AKLOGI("p0: size = %zd, x = %d, y = %d, lx = %d, ly = %d, dist = %d, " "width = %d", size, x, y, sampledInputXs->back(), - sampledInputYs->back(), getDistanceInt( + sampledInputYs->back(), GeometryUtils::getDistanceInt( x, y, sampledInputXs->back(), sampledInputYs->back()), mostCommonKeyWidth / ProximityInfoParams::LAST_POINT_SKIP_DISTANCE_SCALE); @@ -499,7 +503,7 @@ namespace latinime { // Pushing point information. if (size > 0) { sampledLengthCache->push_back( - sampledLengthCache->back() + getDistanceInt( + sampledLengthCache->back() + GeometryUtils::getDistanceInt( x, y, sampledInputXs->back(), sampledInputYs->back())); } else { sampledLengthCache->push_back(0); @@ -540,7 +544,8 @@ namespace latinime { while (start > 0 && tempBeelineDistance < lookupRadius) { tempTime += times[start] - times[start - 1]; --start; - tempBeelineDistance = getDistanceInt(x0, y0, xCoordinates[start], yCoordinates[start]); + tempBeelineDistance = GeometryUtils::getDistanceInt(x0, y0, xCoordinates[start], + yCoordinates[start]); } // Exclusive unless this is an edge point if (start > 0 && start < actualInputIndex) { @@ -553,7 +558,8 @@ namespace latinime { while (end < (inputSize - 1) && tempBeelineDistance < lookupRadius) { tempTime += times[end + 1] - times[end]; ++end; - tempBeelineDistance = getDistanceInt(x0, y0, xCoordinates[end], yCoordinates[end]); + tempBeelineDistance = GeometryUtils::getDistanceInt(x0, y0, xCoordinates[end], + yCoordinates[end]); } // Exclusive unless this is an edge point if (end > actualInputIndex && end < (inputSize - 1)) { @@ -571,7 +577,7 @@ namespace latinime { const int y2 = yCoordinates[start]; const int x3 = xCoordinates[end]; const int y3 = yCoordinates[end]; - const int beelineDistance = getDistanceInt(x2, y2, x3, y3); + const int beelineDistance = GeometryUtils::getDistanceInt(x2, y2, x3, y3); int adjustedStartTime = times[start]; if (start == 0 && actualInputIndex == 0 && inputSize > 1) { adjustedStartTime += ProximityInfoParams::FIRST_POINT_TIME_OFFSET_MILLIS; @@ -613,7 +619,7 @@ namespace latinime { } const float previousDirection = getDirection(sampledInputXs, sampledInputYs, index - 1, index); const float nextDirection = getDirection(sampledInputXs, sampledInputYs, index, index + 1); - const float directionDiff = getAngleDiff(previousDirection, nextDirection); + const float directionDiff = GeometryUtils::getAngleDiff(previousDirection, nextDirection); return directionDiff; } @@ -636,7 +642,7 @@ namespace latinime { } const float previousDirection = getDirection(sampledInputXs, sampledInputYs, index0, index1); const float nextDirection = getDirection(sampledInputXs, sampledInputYs, index1, index2); - return getAngleDiff(previousDirection, nextDirection); + return GeometryUtils::getAngleDiff(previousDirection, nextDirection); } // This function basically converts from a length to an edit distance. Accordingly, it's obviously diff --git a/native/jni/src/proximity_info_state_utils.h b/native/jni/src/suggest/core/layout/proximity_info_state_utils.h index 1837c7ab6..66fe07926 100644 --- a/native/jni/src/proximity_info_state_utils.h +++ b/native/jni/src/suggest/core/layout/proximity_info_state_utils.h @@ -21,7 +21,7 @@ #include <vector> #include "defines.h" -#include "hash_map_compat.h" +#include "utils/hash_map_compat.h" namespace latinime { class ProximityInfo; diff --git a/native/jni/src/proximity_info_utils.h b/native/jni/src/suggest/core/layout/proximity_info_utils.h index 71c97e325..54f7539d1 100644 --- a/native/jni/src/proximity_info_utils.h +++ b/native/jni/src/suggest/core/layout/proximity_info_utils.h @@ -19,11 +19,11 @@ #include <cmath> -#include "additional_proximity_chars.h" -#include "char_utils.h" #include "defines.h" -#include "geometry_utils.h" -#include "hash_map_compat.h" +#include "suggest/core/layout/additional_proximity_chars.h" +#include "suggest/core/layout/geometry_utils.h" +#include "utils/char_utils.h" +#include "utils/hash_map_compat.h" namespace latinime { class ProximityInfoUtils { @@ -37,7 +37,7 @@ class ProximityInfoUtils { if (c == NOT_A_CODE_POINT) { return NOT_AN_INDEX; } - const int lowerCode = toLowerCase(c); + const int lowerCode = CharUtils::toLowerCase(c); hash_map_compat<int, int>::const_iterator mapPos = codeToKeyMap->find(lowerCode); if (mapPos != codeToKeyMap->end()) { return mapPos->second; @@ -87,7 +87,7 @@ class ProximityInfoUtils { static inline float getSquaredDistanceFloat(const float x1, const float y1, const float x2, const float y2) { - return SQUARE_FLOAT(x1 - x2) + SQUARE_FLOAT(y1 - y2); + return GeometryUtils::SQUARE_FLOAT(x1 - x2) + GeometryUtils::SQUARE_FLOAT(y1 - y2); } static inline float pointToLineSegSquaredDistanceFloat(const float x, const float y, @@ -98,7 +98,8 @@ class ProximityInfoUtils { const float ray2y = y2 - y1; const float dotProduct = ray1x * ray2x + ray1y * ray2y; - const float lineLengthSqr = SQUARE_FLOAT(ray2x) + SQUARE_FLOAT(ray2y); + const float lineLengthSqr = GeometryUtils::SQUARE_FLOAT(ray2x) + + GeometryUtils::SQUARE_FLOAT(ray2y); const float projectionLengthSqr = dotProduct / lineLengthSqr; float projectionX; @@ -121,12 +122,14 @@ class ProximityInfoUtils { public: NormalDistribution(const float u, const float sigma) : mU(u), mSigma(sigma), - mPreComputedNonExpPart(1.0f / sqrtf(2.0f * M_PI_F * SQUARE_FLOAT(sigma))), - mPreComputedExponentPart(-1.0f / (2.0f * SQUARE_FLOAT(sigma))) {} + mPreComputedNonExpPart(1.0f / sqrtf(2.0f * M_PI_F + * GeometryUtils::SQUARE_FLOAT(sigma))), + mPreComputedExponentPart(-1.0f / (2.0f * GeometryUtils::SQUARE_FLOAT(sigma))) {} float getProbabilityDensity(const float x) const { const float shiftedX = x - mU; - return mPreComputedNonExpPart * expf(mPreComputedExponentPart * SQUARE_FLOAT(shiftedX)); + return mPreComputedNonExpPart + * expf(mPreComputedExponentPart * GeometryUtils::SQUARE_FLOAT(shiftedX)); } private: diff --git a/native/jni/src/suggest_utils.h b/native/jni/src/suggest/core/layout/touch_position_correction_utils.h index e053dd662..429dcae0d 100644 --- a/native/jni/src/suggest_utils.h +++ b/native/jni/src/suggest/core/layout/touch_position_correction_utils.h @@ -14,14 +14,14 @@ * limitations under the License. */ -#ifndef LATINIME_SUGGEST_UTILS_H -#define LATINIME_SUGGEST_UTILS_H +#ifndef LATINIME_TOUCH_POSITION_CORRECTION_UTILS_H +#define LATINIME_TOUCH_POSITION_CORRECTION_UTILS_H #include "defines.h" -#include "proximity_info_params.h" +#include "suggest/core/layout/proximity_info_params.h" namespace latinime { -class SuggestUtils { +class TouchPositionCorrectionUtils { public: // TODO: (OLD) Remove static float getLengthScalingFactor(const float normalizedSquaredDistance) { @@ -82,7 +82,7 @@ class SuggestUtils { } } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(SuggestUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(TouchPositionCorrectionUtils); }; } // namespace latinime -#endif // LATINIME_SUGGEST_UTILS_H +#endif // LATINIME_TOUCH_POSITION_CORRECTION_UTILS_H diff --git a/native/jni/src/suggest/core/policy/weighting.cpp b/native/jni/src/suggest/core/policy/weighting.cpp index d01531f07..0c57ca001 100644 --- a/native/jni/src/suggest/core/policy/weighting.cpp +++ b/native/jni/src/suggest/core/policy/weighting.cpp @@ -16,7 +16,6 @@ #include "suggest/core/policy/weighting.h" -#include "char_utils.h" #include "defines.h" #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_profiler.h" @@ -143,7 +142,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n case CT_TERMINAL: { const float languageImprobability = DicNodeUtils::getBigramNodeImprobability( - traverseSession->getOffsetDict(), dicNode, multiBigramMap); + traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap); return weighting->getTerminalLanguageCost(traverseSession, dicNode, languageImprobability); } case CT_NEW_WORD_SPACE_SUBSTITUTION: diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.cpp b/native/jni/src/suggest/core/session/dic_traverse_session.cpp index 6408f0163..c398caefa 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.cpp +++ b/native/jni/src/suggest/core/session/dic_traverse_session.cpp @@ -16,67 +16,35 @@ #include "suggest/core/session/dic_traverse_session.h" -#include "binary_format.h" #include "defines.h" -#include "dictionary.h" -#include "dic_traverse_wrapper.h" #include "jni.h" #include "suggest/core/dicnode/dic_node_utils.h" +#include "suggest/core/dictionary/binary_dictionary_info.h" +#include "suggest/core/dictionary/binary_format.h" +#include "suggest/core/dictionary/dictionary.h" namespace latinime { -const int DicTraverseSession::CACHE_START_INPUT_LENGTH_THRESHOLD = 20; - -// A factory method for DicTraverseSession -static void *getSessionInstance(JNIEnv *env, jstring localeStr) { - return new DicTraverseSession(env, localeStr); -} - -// TODO: Pass "DicTraverseSession *traverseSession" when the source code structure settles down. -static void initSessionInstance(void *traverseSession, const Dictionary *const dictionary, - const int *prevWord, const int prevWordLength) { - if (traverseSession) { - DicTraverseSession *tSession = static_cast<DicTraverseSession *>(traverseSession); - tSession->init(dictionary, prevWord, prevWordLength); - } -} - -// TODO: Pass "DicTraverseSession *traverseSession" when the source code structure settles down. -static void releaseSessionInstance(void *traverseSession) { - delete static_cast<DicTraverseSession *>(traverseSession); -} - -// An ad-hoc internal class to register the factory method defined above -class TraverseSessionFactoryRegisterer { - public: - TraverseSessionFactoryRegisterer() { - DicTraverseWrapper::setTraverseSessionFactoryMethod(getSessionInstance); - DicTraverseWrapper::setTraverseSessionInitMethod(initSessionInstance); - DicTraverseWrapper::setTraverseSessionReleaseMethod(releaseSessionInstance); - } - private: - DISALLOW_COPY_AND_ASSIGN(TraverseSessionFactoryRegisterer); -}; - -// To invoke the TraverseSessionFactoryRegisterer constructor in the global constructor. -static TraverseSessionFactoryRegisterer traverseSessionFactoryRegisterer; - void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord, - int prevWordLength) { + int prevWordLength, const SuggestOptions *const suggestOptions) { mDictionary = dictionary; - mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier(mDictionary->getDict(), + mMultiWordCostMultiplier = BinaryFormat::getMultiWordCostMultiplier( + mDictionary->getBinaryDictionaryInfo()->getDictBuf(), mDictionary->getDictSize()); + mSuggestOptions = suggestOptions; if (!prevWord) { mPrevWordPos = NOT_VALID_WORD; return; } // TODO: merge following similar calls to getTerminalPosition into one case-insensitive call. - mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord, + mPrevWordPos = BinaryFormat::getTerminalPosition( + dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord, prevWordLength, false /* forceLowerCaseSearch */); if (mPrevWordPos == NOT_VALID_WORD) { // Check bigrams for lower-cased previous word if original was not found. Useful for // auto-capitalized words like "The [current_word]". - mPrevWordPos = BinaryFormat::getTerminalPosition(dictionary->getOffsetDict(), prevWord, + mPrevWordPos = BinaryFormat::getTerminalPosition( + dictionary->getBinaryDictionaryInfo()->getDictRoot(), prevWord, prevWordLength, true /* forceLowerCaseSearch */); } } @@ -91,8 +59,8 @@ void DicTraverseSession::setupForGetSuggestions(const ProximityInfo *pInfo, maxSpatialDistance, maxPointerCount); } -const uint8_t *DicTraverseSession::getOffsetDict() const { - return mDictionary->getOffsetDict(); +const BinaryDictionaryInfo *DicTraverseSession::getBinaryDictionaryInfo() const { + return mDictionary->getBinaryDictionaryInfo(); } int DicTraverseSession::getDictFlags() const { diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h index d88be5b88..630b3b59b 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.h +++ b/native/jni/src/suggest/core/session/dic_traverse_session.h @@ -22,20 +22,41 @@ #include "defines.h" #include "jni.h" -#include "multi_bigram_map.h" -#include "proximity_info_state.h" #include "suggest/core/dicnode/dic_nodes_cache.h" +#include "suggest/core/dictionary/multi_bigram_map.h" +#include "suggest/core/layout/proximity_info_state.h" namespace latinime { +class BinaryDictionaryInfo; class Dictionary; class ProximityInfo; +class SuggestOptions; class DicTraverseSession { public: + + // A factory method for DicTraverseSession + static AK_FORCE_INLINE void *getSessionInstance(JNIEnv *env, jstring localeStr) { + return new DicTraverseSession(env, localeStr); + } + + static AK_FORCE_INLINE void initSessionInstance(DicTraverseSession *traverseSession, + const Dictionary *const dictionary, const int *prevWord, const int prevWordLength, + const SuggestOptions *const suggestOptions) { + if (traverseSession) { + DicTraverseSession *tSession = static_cast<DicTraverseSession *>(traverseSession); + tSession->init(dictionary, prevWord, prevWordLength, suggestOptions); + } + } + + static AK_FORCE_INLINE void releaseSessionInstance(DicTraverseSession *traverseSession) { + delete traverseSession; + } + AK_FORCE_INLINE DicTraverseSession(JNIEnv *env, jstring localeStr) : mPrevWordPos(NOT_VALID_WORD), mProximityInfo(0), - mDictionary(0), mDicNodesCache(), mMultiBigramMap(), + mDictionary(0), mSuggestOptions(0), mDicNodesCache(), mMultiBigramMap(), mInputSize(0), mPartiallyCommited(false), mMaxPointerCount(1), mMultiWordCostMultiplier(1.0f) { // NOTE: mProximityInfoStates is an array of instances. @@ -45,7 +66,8 @@ class DicTraverseSession { // Non virtual inline destructor -- never inherit this class AK_FORCE_INLINE ~DicTraverseSession() {} - void init(const Dictionary *dictionary, const int *prevWord, int prevWordLength); + void init(const Dictionary *dictionary, const int *prevWord, int prevWordLength, + const SuggestOptions *const suggestOptions); // TODO: Remove and merge into init void setupForGetSuggestions(const ProximityInfo *pInfo, const int *inputCodePoints, const int inputSize, const int *const inputXs, const int *const inputYs, @@ -54,13 +76,14 @@ class DicTraverseSession { void resetCache(const int nextActiveCacheSize, const int maxWords); // TODO: Remove - const uint8_t *getOffsetDict() const; + const BinaryDictionaryInfo *getBinaryDictionaryInfo() const; int getDictFlags() const; //-------------------- // getters and setters //-------------------- const ProximityInfo *getProximityInfo() const { return mProximityInfo; } + const SuggestOptions *getSuggestOptions() const { return mSuggestOptions; } int getPrevWordPos() const { return mPrevWordPos; } // TODO: REMOVE void setPrevWordPos(int pos) { mPrevWordPos = pos; } @@ -167,6 +190,7 @@ class DicTraverseSession { int mPrevWordPos; const ProximityInfo *mProximityInfo; const Dictionary *mDictionary; + const SuggestOptions *mSuggestOptions; DicNodesCache mDicNodesCache; // Temporary cache for bigram frequencies diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index a18794850..1f108e400 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -16,19 +16,18 @@ #include "suggest/core/suggest.h" -#include "char_utils.h" -#include "dictionary.h" -#include "digraph_utils.h" -#include "proximity_info.h" #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_priority_queue.h" #include "suggest/core/dicnode/dic_node_vector.h" +#include "suggest/core/dictionary/dictionary.h" +#include "suggest/core/dictionary/digraph_utils.h" #include "suggest/core/dictionary/shortcut_utils.h" +#include "suggest/core/dictionary/terminal_attributes.h" +#include "suggest/core/layout/proximity_info.h" #include "suggest/core/policy/scoring.h" #include "suggest/core/policy/traversal.h" #include "suggest/core/policy/weighting.h" #include "suggest/core/session/dic_traverse_session.h" -#include "terminal_attributes.h" namespace latinime { @@ -106,8 +105,8 @@ void Suggest::initializeSearch(DicTraverseSession *traverseSession, int commitPo traverseSession->resetCache(TRAVERSAL->getMaxCacheSize(), MAX_RESULTS); // Create a new dic node here DicNode rootNode; - DicNodeUtils::initAsRoot(traverseSession->getDicRootPos(), - traverseSession->getOffsetDict(), traverseSession->getPrevWordPos(), &rootNode); + DicNodeUtils::initAsRoot(traverseSession->getBinaryDictionaryInfo(), + traverseSession->getPrevWordPos(), &rootNode); traverseSession->getDicTraverseCache()->copyPushActive(&rootNode); } } @@ -159,7 +158,7 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen terminalIndex, doubleLetterTerminalIndex, doubleLetterLevel); const float compoundDistance = terminalDicNode->getCompoundDistance(languageWeight) + doubleLetterCost; - const TerminalAttributes terminalAttributes(traverseSession->getOffsetDict(), + const TerminalAttributes terminalAttributes(traverseSession->getBinaryDictionaryInfo(), terminalDicNode->getFlags(), terminalDicNode->getAttributesPos()); const bool isPossiblyOffensiveWord = terminalDicNode->getProbability() <= 0; const bool isExactMatch = terminalDicNode->isExactMatch(); @@ -285,7 +284,7 @@ void Suggest::expandCurrentDicNodes(DicTraverseSession *traverseSession) const { } DicNodeUtils::getAllChildDicNodes( - &dicNode, traverseSession->getOffsetDict(), &childDicNodes); + &dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes); const int childDicNodesSize = childDicNodes.getSizeAndLock(); for (int i = 0; i < childDicNodesSize; ++i) { @@ -432,7 +431,8 @@ void Suggest::processDicNodeAsDigraph(DicTraverseSession *traverseSession, void Suggest::processDicNodeAsOmission( DicTraverseSession *traverseSession, DicNode *dicNode) const { DicNodeVector childDicNodes; - DicNodeUtils::getAllChildDicNodes(dicNode, traverseSession->getOffsetDict(), &childDicNodes); + DicNodeUtils::getAllChildDicNodes( + dicNode, traverseSession->getBinaryDictionaryInfo(), &childDicNodes); const int size = childDicNodes.getSizeAndLock(); for (int i = 0; i < size; i++) { @@ -457,7 +457,7 @@ void Suggest::processDicNodeAsInsertion(DicTraverseSession *traverseSession, DicNode *dicNode) const { const int16_t pointIndex = dicNode->getInputIndex(0); DicNodeVector childDicNodes; - DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(), + DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(), traverseSession->getProximityInfoState(0), pointIndex + 1, true, &childDicNodes); const int size = childDicNodes.getSizeAndLock(); for (int i = 0; i < size; i++) { @@ -475,14 +475,14 @@ void Suggest::processDicNodeAsTransposition(DicTraverseSession *traverseSession, DicNode *dicNode) const { const int16_t pointIndex = dicNode->getInputIndex(0); DicNodeVector childDicNodes1; - DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getOffsetDict(), + DicNodeUtils::getProximityChildDicNodes(dicNode, traverseSession->getBinaryDictionaryInfo(), traverseSession->getProximityInfoState(0), pointIndex + 1, false, &childDicNodes1); const int childSize1 = childDicNodes1.getSizeAndLock(); for (int i = 0; i < childSize1; i++) { if (childDicNodes1[i]->hasChildren()) { DicNodeVector childDicNodes2; DicNodeUtils::getProximityChildDicNodes( - childDicNodes1[i], traverseSession->getOffsetDict(), + childDicNodes1[i], traverseSession->getBinaryDictionaryInfo(), traverseSession->getProximityInfoState(0), pointIndex, false, &childDicNodes2); const int childSize2 = childDicNodes2.getSizeAndLock(); for (int j = 0; j < childSize2; j++) { @@ -522,8 +522,8 @@ void Suggest::createNextWordDicNode(DicTraverseSession *traverseSession, DicNode // Create a non-cached node here. DicNode newDicNode; - DicNodeUtils::initAsRootWithPreviousWord(traverseSession->getDicRootPos(), - traverseSession->getOffsetDict(), dicNode, &newDicNode); + DicNodeUtils::initAsRootWithPreviousWord( + traverseSession->getBinaryDictionaryInfo(), dicNode, &newDicNode); const CorrectionType correctionType = spaceSubstitution ? CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION; Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode, diff --git a/native/jni/src/suggest/core/suggest_options.h b/native/jni/src/suggest/core/suggest_options.h new file mode 100644 index 000000000..1b21aafcf --- /dev/null +++ b/native/jni/src/suggest/core/suggest_options.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_SUGGEST_OPTIONS_H +#define LATINIME_SUGGEST_OPTIONS_H + +#include "defines.h" + +namespace latinime { + +class SuggestOptions{ + public: + SuggestOptions(const int *const options, const int length) + : mOptions(options), mLength(length) {} + + AK_FORCE_INLINE bool isGesture() const { + return getBoolOption(IS_GESTURE); + } + + AK_FORCE_INLINE bool useFullEditDistance() const { + return getBoolOption(USE_FULL_EDIT_DISTANCE); + } + + AK_FORCE_INLINE bool getAdditionalFeaturesBoolOption(const int key) const { + return getBoolOption(key + ADDITIONAL_FEATURES_OPTIONS); + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(SuggestOptions); + + // Need to update com.android.inputmethod.latin.NativeSuggestOptions when you add, remove or + // reorder options. + static const int IS_GESTURE = 0; + static const int USE_FULL_EDIT_DISTANCE = 1; + // Additional features options are stored after the other options and used as setting values of + // experimental features. + static const int ADDITIONAL_FEATURES_OPTIONS = 2; + + const int *const mOptions; + const int mLength; + + AK_FORCE_INLINE bool isValidKey(const int key) const { + return 0 <= key && key < mLength; + } + + AK_FORCE_INLINE bool getBoolOption(const int key) const { + if (isValidKey(key)) { + return mOptions[key] != 0; + } + return false; + } + + AK_FORCE_INLINE int getIntOption(const int key) const { + if (isValidKey(key)) { + return mOptions[key]; + } + return 0; + } +}; +} // namespace latinime +#endif // LATINIME_SUGGEST_OPTIONS_H diff --git a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h index 12110d54f..e21b318e6 100644 --- a/native/jni/src/suggest/policyimpl/typing/typing_traversal.h +++ b/native/jni/src/suggest/policyimpl/typing/typing_traversal.h @@ -19,14 +19,14 @@ #include <stdint.h> -#include "char_utils.h" #include "defines.h" -#include "proximity_info_state.h" #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_vector.h" +#include "suggest/core/layout/proximity_info_state.h" #include "suggest/core/policy/traversal.h" #include "suggest/core/session/dic_traverse_session.h" #include "suggest/policyimpl/typing/scoring_params.h" +#include "utils/char_utils.h" namespace latinime { class TypingTraversal : public Traversal { @@ -64,9 +64,9 @@ class TypingTraversal : public Traversal { } const int point0Index = dicNode->getInputIndex(0); const int currentBaseLowerCodePoint = - toBaseLowerCase(childDicNode->getNodeCodePoint()); + CharUtils::toBaseLowerCase(childDicNode->getNodeCodePoint()); const int typedBaseLowerCodePoint = - toBaseLowerCase(traverseSession->getProximityInfoState(0) + CharUtils::toBaseLowerCase(traverseSession->getProximityInfoState(0) ->getPrimaryCodePointAt(point0Index)); return (currentBaseLowerCodePoint != typedBaseLowerCodePoint); } @@ -172,7 +172,7 @@ class TypingTraversal : public Traversal { } const int c = dicNode->getOutputWordBuf()[0]; const bool shortCappedWord = dicNode->getDepth() - < ScoringParams::THRESHOLD_SHORT_WORD_LENGTH && isAsciiUpper(c); + < ScoringParams::THRESHOLD_SHORT_WORD_LENGTH && CharUtils::isAsciiUpper(c); return !shortCappedWord || probability >= ScoringParams::THRESHOLD_NEXT_WORD_PROBABILITY_FOR_CAPPED; } diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h index 3938c0ec5..17fa11082 100644 --- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h +++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h @@ -18,11 +18,12 @@ #define LATINIME_TYPING_WEIGHTING_H #include "defines.h" -#include "suggest_utils.h" #include "suggest/core/dicnode/dic_node_utils.h" +#include "suggest/core/layout/touch_position_correction_utils.h" #include "suggest/core/policy/weighting.h" #include "suggest/core/session/dic_traverse_session.h" #include "suggest/policyimpl/typing/scoring_params.h" +#include "utils/char_utils.h" namespace latinime { @@ -74,7 +75,7 @@ class TypingWeighting : public Weighting { // the keyboard (like accented letters) const float normalizedSquaredLength = traverseSession->getProximityInfoState(0) ->getPointToKeyLength(pointIndex, dicNode->getNodeCodePoint()); - const float normalizedDistance = SuggestUtils::getSweetSpotFactor( + const float normalizedDistance = TouchPositionCorrectionUtils::getSweetSpotFactor( traverseSession->isTouchPositionCorrectionEnabled(), normalizedSquaredLength); const float weightedDistance = ScoringParams::DISTANCE_WEIGHT_LENGTH * normalizedDistance; @@ -98,9 +99,9 @@ class TypingWeighting : public Weighting { bool isProximityDicNode(const DicTraverseSession *const traverseSession, const DicNode *const dicNode) const { const int pointIndex = dicNode->getInputIndex(0); - const int primaryCodePoint = toBaseLowerCase( + const int primaryCodePoint = CharUtils::toBaseLowerCase( traverseSession->getProximityInfoState(0)->getPrimaryCodePointAt(pointIndex)); - const int dicNodeChar = toBaseLowerCase(dicNode->getNodeCodePoint()); + const int dicNodeChar = CharUtils::toBaseLowerCase(dicNode->getNodeCodePoint()); return primaryCodePoint != dicNodeChar; } @@ -145,7 +146,7 @@ class TypingWeighting : public Weighting { float getNewWordBigramCost(const DicTraverseSession *const traverseSession, const DicNode *const dicNode, MultiBigramMap *const multiBigramMap) const { - return DicNodeUtils::getBigramNodeImprobability(traverseSession->getOffsetDict(), + return DicNodeUtils::getBigramNodeImprobability(traverseSession->getBinaryDictionaryInfo(), dicNode, multiBigramMap) * ScoringParams::DISTANCE_WEIGHT_LANGUAGE; } diff --git a/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h b/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h index ec1457455..81614bc9c 100644 --- a/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h +++ b/native/jni/src/suggest/policyimpl/utils/damerau_levenshtein_edit_distance_policy.h @@ -17,8 +17,8 @@ #ifndef LATINIME_DAEMARU_LEVENSHTEIN_EDIT_DISTANCE_POLICY_H #define LATINIME_DAEMARU_LEVENSHTEIN_EDIT_DISTANCE_POLICY_H -#include "char_utils.h" #include "suggest/policyimpl/utils/edit_distance_policy.h" +#include "utils/char_utils.h" namespace latinime { @@ -31,8 +31,8 @@ class DamerauLevenshteinEditDistancePolicy : public EditDistancePolicy { ~DamerauLevenshteinEditDistancePolicy() {} AK_FORCE_INLINE float getSubstitutionCost(const int index0, const int index1) const { - const int c0 = toBaseLowerCase(mString0[index0]); - const int c1 = toBaseLowerCase(mString1[index1]); + const int c0 = CharUtils::toBaseLowerCase(mString0[index0]); + const int c1 = CharUtils::toBaseLowerCase(mString1[index1]); return (c0 == c1) ? 0.0f : 1.0f; } @@ -45,10 +45,10 @@ class DamerauLevenshteinEditDistancePolicy : public EditDistancePolicy { } AK_FORCE_INLINE bool allowTransposition(const int index0, const int index1) const { - const int c0 = toBaseLowerCase(mString0[index0]); - const int c1 = toBaseLowerCase(mString1[index1]); - if (index0 > 0 && index1 > 0 && c0 == toBaseLowerCase(mString1[index1 - 1]) - && c1 == toBaseLowerCase(mString0[index0 - 1])) { + const int c0 = CharUtils::toBaseLowerCase(mString0[index0]); + const int c1 = CharUtils::toBaseLowerCase(mString1[index1]); + if (index0 > 0 && index1 > 0 && c0 == CharUtils::toBaseLowerCase(mString1[index1 - 1]) + && c1 == CharUtils::toBaseLowerCase(mString0[index0 - 1])) { return true; } return false; diff --git a/native/jni/src/unigram_dictionary.cpp b/native/jni/src/unigram_dictionary.cpp deleted file mode 100644 index a672294b5..000000000 --- a/native/jni/src/unigram_dictionary.cpp +++ /dev/null @@ -1,988 +0,0 @@ -/* - * Copyright (C) 2010, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <cstring> - -#define LOG_TAG "LatinIME: unigram_dictionary.cpp" - -#include "binary_format.h" -#include "char_utils.h" -#include "defines.h" -#include "dictionary.h" -#include "digraph_utils.h" -#include "proximity_info.h" -#include "terminal_attributes.h" -#include "unigram_dictionary.h" -#include "words_priority_queue.h" -#include "words_priority_queue_pool.h" - -namespace latinime { - -// TODO: check the header -UnigramDictionary::UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags) - : DICT_ROOT(streamStart), ROOT_POS(0), - MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), DICT_FLAGS(dictFlags) { - if (DEBUG_DICT) { - AKLOGI("UnigramDictionary - constructor"); - } -} - -UnigramDictionary::~UnigramDictionary() { -} - -// TODO: This needs to take a const int* and not tinker with its contents -static void addWord(int *word, int length, int probability, WordsPriorityQueue *queue, int type) { - queue->push(probability, word, length, type); -} - -// Return the replacement code point for a digraph, or 0 if none. -int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int inputSize, - const DigraphUtils::digraph_t *const digraphs, const unsigned int digraphsSize) const { - - // There can't be a digraph if we don't have at least 2 characters to examine - if (i + 2 > inputSize) return false; - - // Search for the first char of some digraph - int lastDigraphIndex = -1; - const int thisChar = codes[i]; - for (lastDigraphIndex = digraphsSize - 1; lastDigraphIndex >= 0; --lastDigraphIndex) { - if (thisChar == digraphs[lastDigraphIndex].first) break; - } - // No match: return early - if (lastDigraphIndex < 0) return 0; - - // It's an interesting digraph if the second char matches too. - if (digraphs[lastDigraphIndex].second == codes[i + 1]) { - return digraphs[lastDigraphIndex].compositeGlyph; - } else { - return 0; - } -} - -// Mostly the same arguments as the non-recursive version, except: -// codes is the original value. It points to the start of the work buffer, and gets passed as is. -// inputSize is the size of the user input (thus, it is the size of codesSrc). -// codesDest is the current point in the work buffer. -// codesSrc is the current point in the user-input, original, content-unmodified buffer. -// codesRemain is the remaining size in codesSrc. -void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, - const int *xcoordinates, const int *ycoordinates, const int *codesBuffer, - int *xCoordinatesBuffer, int *yCoordinatesBuffer, - const int codesBufferSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, const int *codesSrc, - const int codesRemain, const int currentDepth, int *codesDest, Correction *correction, - WordsPriorityQueuePool *queuePool, - const DigraphUtils::digraph_t *const digraphs, const unsigned int digraphsSize) const { - ASSERT(sizeof(codesDest[0]) == sizeof(codesSrc[0])); - ASSERT(sizeof(xCoordinatesBuffer[0]) == sizeof(xcoordinates[0])); - ASSERT(sizeof(yCoordinatesBuffer[0]) == sizeof(ycoordinates[0])); - - const int startIndex = static_cast<int>(codesDest - codesBuffer); - if (currentDepth < MAX_DIGRAPH_SEARCH_DEPTH) { - for (int i = 0; i < codesRemain; ++i) { - xCoordinatesBuffer[startIndex + i] = xcoordinates[codesBufferSize - codesRemain + i]; - yCoordinatesBuffer[startIndex + i] = ycoordinates[codesBufferSize - codesRemain + i]; - const int replacementCodePoint = - getDigraphReplacement(codesSrc, i, codesRemain, digraphs, digraphsSize); - if (0 != replacementCodePoint) { - // Found a digraph. We will try both spellings. eg. the word is "pruefen" - - // Copy the word up to the first char of the digraph, including proximity chars, - // and overwrite the primary code with the replacement code point. Then, continue - // processing on the remaining part of the word, skipping the second char of the - // digraph. - // In our example, copy "pru", replace "u" with the version with the diaeresis and - // continue running on "fen". - // Make i the index of the second char of the digraph for simplicity. Forgetting - // to do that results in an infinite recursion so take care! - ++i; - memcpy(codesDest, codesSrc, i * sizeof(codesDest[0])); - codesDest[i - 1] = replacementCodePoint; - getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, - codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize, - bigramMap, bigramFilter, useFullEditDistance, codesSrc + i + 1, - codesRemain - i - 1, currentDepth + 1, codesDest + i, correction, - queuePool, digraphs, digraphsSize); - - // Copy the second char of the digraph in place, then continue processing on - // the remaining part of the word. - // In our example, after "pru" in the buffer copy the "e", and continue on "fen" - memcpy(codesDest + i, codesSrc + i, sizeof(codesDest[0])); - getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, - codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize, - bigramMap, bigramFilter, useFullEditDistance, codesSrc + i, codesRemain - i, - currentDepth + 1, codesDest + i, correction, queuePool, digraphs, - digraphsSize); - return; - } - } - } - - // If we come here, we hit the end of the word: let's check it against the dictionary. - // In our example, we'll come here once for "prufen" and then once for "pruefen". - // If the word contains several digraphs, we'll come it for the product of them. - // eg. if the word is "ueberpruefen" we'll test, in order, against - // "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen". - const unsigned int remainingBytes = sizeof(codesDest[0]) * codesRemain; - if (0 != remainingBytes) { - memcpy(codesDest, codesSrc, remainingBytes); - memcpy(&xCoordinatesBuffer[startIndex], &xcoordinates[codesBufferSize - codesRemain], - sizeof(xCoordinatesBuffer[0]) * codesRemain); - memcpy(&yCoordinatesBuffer[startIndex], &ycoordinates[codesBufferSize - codesRemain], - sizeof(yCoordinatesBuffer[0]) * codesRemain); - } - - getWordSuggestions(proximityInfo, xCoordinatesBuffer, yCoordinatesBuffer, codesBuffer, - startIndex + codesRemain, bigramMap, bigramFilter, useFullEditDistance, correction, - queuePool); -} - -// bigramMap contains the association <bigram address> -> <bigram probability> -// bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter -// in bigram_dictionary.cpp -int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *inputCodePoints, const int inputSize, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, int *outWords, int *frequencies, int *outputTypes) const { - WordsPriorityQueuePool queuePool(MAX_RESULTS, SUB_QUEUE_MAX_WORDS); - queuePool.clearAll(); - Correction masterCorrection; - masterCorrection.resetCorrection(); - const DigraphUtils::digraph_t *digraphs = 0; - const int digraphsSize = - DigraphUtils::getAllDigraphsForDictionaryAndReturnSize(DICT_FLAGS, &digraphs); - if (digraphsSize > 0) - { // Incrementally tune the word and try all possibilities - int codesBuffer[sizeof(*inputCodePoints) * inputSize]; - int xCoordinatesBuffer[inputSize]; - int yCoordinatesBuffer[inputSize]; - getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, - xCoordinatesBuffer, yCoordinatesBuffer, inputSize, bigramMap, bigramFilter, - useFullEditDistance, inputCodePoints, inputSize, 0, codesBuffer, &masterCorrection, - &queuePool, digraphs, digraphsSize); - } else { // Normal processing - getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, inputSize, - bigramMap, bigramFilter, useFullEditDistance, &masterCorrection, &queuePool); - } - - PROF_START(20); - if (DEBUG_DICT) { - float ns = queuePool.getMasterQueue()->getHighestNormalizedScore( - masterCorrection.getPrimaryInputWord(), inputSize, 0, 0, 0); - ns += 0; - AKLOGI("Max normalized score = %f", ns); - } - const int suggestedWordsCount = - queuePool.getMasterQueue()->outputSuggestions(masterCorrection.getPrimaryInputWord(), - inputSize, frequencies, outWords, outputTypes); - - if (DEBUG_DICT) { - float ns = queuePool.getMasterQueue()->getHighestNormalizedScore( - masterCorrection.getPrimaryInputWord(), inputSize, 0, 0, 0); - ns += 0; - AKLOGI("Returning %d words", suggestedWordsCount); - /// Print the returned words - for (int j = 0; j < suggestedWordsCount; ++j) { - int *w = outWords + j * MAX_WORD_LENGTH; - char s[MAX_WORD_LENGTH]; - for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i]; - (void)s; // To suppress compiler warning - AKLOGI("%s %i", s, frequencies[j]); - } - } - PROF_END(20); - PROF_CLOSE; - return suggestedWordsCount; -} - -void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *inputCodePoints, const int inputSize, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, Correction *correction, WordsPriorityQueuePool *queuePool) - const { - PROF_OPEN; - PROF_START(0); - PROF_END(0); - - PROF_START(1); - getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, bigramMap, - bigramFilter, useFullEditDistance, inputSize, correction, queuePool); - PROF_END(1); - - PROF_START(2); - // Note: This line is intentionally left blank - PROF_END(2); - - PROF_START(3); - // Note: This line is intentionally left blank - PROF_END(3); - - PROF_START(4); - bool hasAutoCorrectionCandidate = false; - WordsPriorityQueue *masterQueue = queuePool->getMasterQueue(); - if (masterQueue->size() > 0) { - float nsForMaster = masterQueue->getHighestNormalizedScore( - correction->getPrimaryInputWord(), inputSize, 0, 0, 0); - hasAutoCorrectionCandidate = (nsForMaster > START_TWO_WORDS_CORRECTION_THRESHOLD); - } - PROF_END(4); - - PROF_START(5); - // Multiple word suggestions - if (SUGGEST_MULTIPLE_WORDS - && inputSize >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) { - getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, inputCodePoints, - useFullEditDistance, inputSize, correction, queuePool, - hasAutoCorrectionCandidate); - } - PROF_END(5); - - PROF_START(6); - // Note: This line is intentionally left blank - PROF_END(6); - - if (DEBUG_DICT) { - queuePool->dumpSubQueue1TopSuggestions(); - for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { - WordsPriorityQueue *queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i); - if (queue->size() > 0) { - WordsPriorityQueue::SuggestedWord *sw = queue->top(); - const int score = sw->mScore; - const int *word = sw->mWord; - const int wordLength = sw->mWordLength; - float ns = Correction::RankingAlgorithm::calcNormalizedScore( - correction->getPrimaryInputWord(), i, word, wordLength, score); - ns += 0; - AKLOGI("--- TOP SUB WORDS for %d --- %d %f [%d]", i, score, ns, - (ns > TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD)); - DUMP_WORD(correction->getPrimaryInputWord(), i); - DUMP_WORD(word, wordLength); - } - } - } -} - -void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates, - const int *yCoordinates, const int *codes, const int inputSize, - Correction *correction) const { - if (DEBUG_DICT) { - AKLOGI("initSuggest"); - DUMP_WORD(codes, inputSize); - } - correction->initInputParams(proximityInfo, codes, inputSize, xCoordinates, yCoordinates); - const int maxDepth = min(inputSize * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH); - correction->initCorrection(proximityInfo, inputSize, maxDepth); -} - -void UnigramDictionary::getOneWordSuggestions(ProximityInfo *proximityInfo, - const int *xcoordinates, const int *ycoordinates, const int *codes, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, const int inputSize, - Correction *correction, WordsPriorityQueuePool *queuePool) const { - initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputSize, correction); - getSuggestionCandidates(useFullEditDistance, inputSize, bigramMap, bigramFilter, correction, - queuePool, true /* doAutoCompletion */, DEFAULT_MAX_ERRORS, FIRST_WORD_INDEX); -} - -void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, - const int inputSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - Correction *correction, WordsPriorityQueuePool *queuePool, - const bool doAutoCompletion, const int maxErrors, const int currentWordIndex) const { - uint8_t totalTraverseCount = correction->pushAndGetTotalTraverseCount(); - if (DEBUG_DICT) { - AKLOGI("Traverse count %d", totalTraverseCount); - } - if (totalTraverseCount > MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT) { - if (DEBUG_DICT) { - AKLOGI("Abort traversing %d", totalTraverseCount); - } - return; - } - // TODO: Remove setCorrectionParams - correction->setCorrectionParams(0, 0, 0, - -1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance, - doAutoCompletion, maxErrors); - int rootPosition = ROOT_POS; - // Get the number of children of root, then increment the position - int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition); - int outputIndex = 0; - - correction->initCorrectionState(rootPosition, childCount, (inputSize <= 0)); - - // Depth first search - while (outputIndex >= 0) { - if (correction->initProcessState(outputIndex)) { - int siblingPos = correction->getTreeSiblingPos(outputIndex); - int firstChildPos; - - const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, - bigramMap, bigramFilter, correction, &childCount, &firstChildPos, &siblingPos, - queuePool, currentWordIndex); - // Update next sibling pos - correction->setTreeSiblingPos(outputIndex, siblingPos); - - if (needsToTraverseChildrenNodes) { - // Goes to child node - outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos); - } - } else { - // Goes to parent sibling node - outputIndex = correction->getTreeParentIndex(outputIndex); - } - } -} - -void UnigramDictionary::onTerminal(const int probability, - const TerminalAttributes &terminalAttributes, Correction *correction, - WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, - const int currentWordIndex) const { - const int inputIndex = correction->getInputIndex(); - const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT; - - int wordLength; - int *wordPointer; - - if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) { - WordsPriorityQueue *masterQueue = queuePool->getMasterQueue(); - const int finalProbability = - correction->getFinalProbability(probability, &wordPointer, &wordLength); - - if (0 != finalProbability && !terminalAttributes.isBlacklistedOrNotAWord()) { - // If the probability is 0, we don't want to add this word. However we still - // want to add its shortcuts (including a possible whitelist entry) if any. - // Furthermore, if this is not a word (shortcut only for example) or a blacklisted - // entry then we never want to suggest this. - addWord(wordPointer, wordLength, finalProbability, masterQueue, - Dictionary::KIND_CORRECTION); - } - - const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0; - // Please note that the shortcut candidates will be added to the master queue only. - TerminalAttributes::ShortcutIterator iterator = terminalAttributes.getShortcutIterator(); - while (iterator.hasNextShortcutTarget()) { - // TODO: addWord only supports weak ordering, meaning we have no means - // to control the order of the shortcuts relative to one another or to the word. - // We need to either modulate the probability of each shortcut according - // to its own shortcut probability or to make the queue - // so that the insert order is protected inside the queue for words - // with the same score. For the moment we use -1 to make sure the shortcut will - // never be in front of the word. - int shortcutTarget[MAX_WORD_LENGTH]; - int shortcutFrequency; - const int shortcutTargetStringLength = iterator.getNextShortcutTarget( - MAX_WORD_LENGTH, shortcutTarget, &shortcutFrequency); - int shortcutScore; - int kind; - if (shortcutFrequency == BinaryFormat::WHITELIST_SHORTCUT_PROBABILITY - && correction->sameAsTyped()) { - shortcutScore = S_INT_MAX; - kind = Dictionary::KIND_WHITELIST; - } else { - shortcutScore = shortcutProbability; - kind = Dictionary::KIND_CORRECTION; - } - addWord(shortcutTarget, shortcutTargetStringLength, shortcutScore, - masterQueue, kind); - } - } - - // We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH - // or more length. - if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) { - WordsPriorityQueue *subQueue; - subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex); - if (!subQueue) { - return; - } - const int finalProbability = correction->getFinalProbabilityForSubQueue( - probability, &wordPointer, &wordLength, inputIndex); - addWord(wordPointer, wordLength, finalProbability, subQueue, Dictionary::KIND_CORRECTION); - } -} - -int UnigramDictionary::getSubStringSuggestion( - ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, - const int *codes, const bool useFullEditDistance, Correction *correction, - WordsPriorityQueuePool *queuePool, const int inputSize, - const bool hasAutoCorrectionCandidate, const int currentWordIndex, - const int inputWordStartPos, const int inputWordLength, - const int outputWordStartPos, const bool isSpaceProximity, int *freqArray, - int *wordLengthArray, int *outputWord, int *outputWordLength) const { - if (inputWordLength > MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH) { - return FLAG_MULTIPLE_SUGGEST_ABORT; - } - - ///////////////////////////////////////////// - // safety net for multiple word suggestion // - // TODO: Remove this safety net // - ///////////////////////////////////////////// - int smallWordCount = 0; - int singleLetterWordCount = 0; - if (inputWordLength == 1) { - ++singleLetterWordCount; - } - if (inputWordLength <= 2) { - // small word == single letter or 2-letter word - ++smallWordCount; - } - for (int i = 0; i < currentWordIndex; ++i) { - const int length = wordLengthArray[i]; - if (length == 1) { - ++singleLetterWordCount; - // Safety net to avoid suggesting sequential single letter words - if (i < (currentWordIndex - 1)) { - if (wordLengthArray[i + 1] == 1) { - return FLAG_MULTIPLE_SUGGEST_ABORT; - } - } else if (inputWordLength == 1) { - return FLAG_MULTIPLE_SUGGEST_ABORT; - } - } - if (length <= 2) { - ++smallWordCount; - } - // Safety net to avoid suggesting multiple words with many (4 or more, for now) small words - if (singleLetterWordCount >= 3 || smallWordCount >= 4) { - return FLAG_MULTIPLE_SUGGEST_ABORT; - } - } - ////////////////////////////////////////////// - // TODO: Remove the safety net above // - ////////////////////////////////////////////// - - int *tempOutputWord = 0; - int nextWordLength = 0; - // TODO: Optimize init suggestion - initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, - inputSize, correction); - - int word[MAX_WORD_LENGTH]; - int freq = getMostProbableWordLike( - inputWordStartPos, inputWordLength, correction, word); - if (freq > 0) { - nextWordLength = inputWordLength; - tempOutputWord = word; - } else if (!hasAutoCorrectionCandidate) { - if (inputWordStartPos > 0) { - const int offset = inputWordStartPos; - initSuggestions(proximityInfo, &xcoordinates[offset], &ycoordinates[offset], - codes + offset, inputWordLength, correction); - queuePool->clearSubQueue(currentWordIndex); - // TODO: pass the bigram list for substring suggestion - getSuggestionCandidates(useFullEditDistance, inputWordLength, - 0 /* bigramMap */, 0 /* bigramFilter */, correction, queuePool, - false /* doAutoCompletion */, MAX_ERRORS_FOR_TWO_WORDS, currentWordIndex); - if (DEBUG_DICT) { - if (currentWordIndex < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) { - AKLOGI("Dump word candidates(%d) %d", currentWordIndex, inputWordLength); - for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { - queuePool->getSubQueue(currentWordIndex, i)->dumpTopWord(); - } - } - } - } - WordsPriorityQueue *queue = queuePool->getSubQueue(currentWordIndex, inputWordLength); - // TODO: Return the correct value depending on doAutoCompletion - if (!queue || queue->size() <= 0) { - return FLAG_MULTIPLE_SUGGEST_ABORT; - } - int score = 0; - const float ns = queue->getHighestNormalizedScore( - correction->getPrimaryInputWord(), inputWordLength, - &tempOutputWord, &score, &nextWordLength); - if (DEBUG_DICT) { - AKLOGI("NS(%d) = %f, Score = %d", currentWordIndex, ns, score); - } - // Two words correction won't be done if the score of the first word doesn't exceed the - // threshold. - if (ns < TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD - || nextWordLength < SUB_QUEUE_MIN_WORD_LENGTH) { - return FLAG_MULTIPLE_SUGGEST_SKIP; - } - freq = score >> (nextWordLength + TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER); - } - if (DEBUG_DICT) { - AKLOGI("Freq(%d): %d, length: %d, input length: %d, input start: %d (%d)", - currentWordIndex, freq, nextWordLength, inputWordLength, inputWordStartPos, - (currentWordIndex > 0) ? wordLengthArray[0] : 0); - } - if (freq <= 0 || nextWordLength <= 0 - || MAX_WORD_LENGTH <= (outputWordStartPos + nextWordLength)) { - return FLAG_MULTIPLE_SUGGEST_SKIP; - } - for (int i = 0; i < nextWordLength; ++i) { - outputWord[outputWordStartPos + i] = tempOutputWord[i]; - } - - // Put output values - freqArray[currentWordIndex] = freq; - // TODO: put output length instead of input length - wordLengthArray[currentWordIndex] = inputWordLength; - const int tempOutputWordLength = outputWordStartPos + nextWordLength; - if (outputWordLength) { - *outputWordLength = tempOutputWordLength; - } - - if ((inputWordStartPos + inputWordLength) < inputSize) { - if (outputWordStartPos + nextWordLength >= MAX_WORD_LENGTH) { - return FLAG_MULTIPLE_SUGGEST_SKIP; - } - outputWord[tempOutputWordLength] = KEYCODE_SPACE; - if (outputWordLength) { - ++*outputWordLength; - } - } else if (currentWordIndex >= 1) { - // TODO: Handle 3 or more words - const int pairFreq = correction->getFreqForSplitMultipleWords( - freqArray, wordLengthArray, currentWordIndex + 1, isSpaceProximity, outputWord); - if (DEBUG_DICT) { - DUMP_WORD(outputWord, tempOutputWordLength); - for (int i = 0; i < currentWordIndex + 1; ++i) { - AKLOGI("Split %d,%d words: freq = %d, length = %d", i, currentWordIndex + 1, - freqArray[i], wordLengthArray[i]); - } - AKLOGI("Split two words: freq = %d, length = %d, %d, isSpace ? %d", pairFreq, - inputSize, tempOutputWordLength, isSpaceProximity); - } - addWord(outputWord, tempOutputWordLength, pairFreq, queuePool->getMasterQueue(), - Dictionary::KIND_CORRECTION); - } - return FLAG_MULTIPLE_SUGGEST_CONTINUE; -} - -void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, - const int *xcoordinates, const int *ycoordinates, const int *codes, - const bool useFullEditDistance, const int inputSize, Correction *correction, - WordsPriorityQueuePool *queuePool, const bool hasAutoCorrectionCandidate, - const int startInputPos, const int startWordIndex, const int outputWordLength, - int *freqArray, int *wordLengthArray, int *outputWord) const { - if (startWordIndex >= (MULTIPLE_WORDS_SUGGESTION_MAX_WORDS - 1)) { - // Return if the last word index - return; - } - if (startWordIndex >= 1 - && (hasAutoCorrectionCandidate - || inputSize < MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION)) { - // Do not suggest 3+ words if already has auto correction candidate - return; - } - for (int i = startInputPos + 1; i < inputSize; ++i) { - if (DEBUG_CORRECTION_FREQ) { - AKLOGI("Multi words(%d), start in %d sep %d start out %d", - startWordIndex, startInputPos, i, outputWordLength); - DUMP_WORD(outputWord, outputWordLength); - } - int tempOutputWordLength = 0; - // Current word - int inputWordStartPos = startInputPos; - int inputWordLength = i - startInputPos; - const int suggestionFlag = getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, - codes, useFullEditDistance, correction, queuePool, inputSize, - hasAutoCorrectionCandidate, startWordIndex, inputWordStartPos, inputWordLength, - outputWordLength, true /* not used */, freqArray, wordLengthArray, outputWord, - &tempOutputWordLength); - if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_ABORT) { - // TODO: break here - continue; - } else if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_SKIP) { - continue; - } - - if (DEBUG_CORRECTION_FREQ) { - AKLOGI("Do missing space correction"); - } - // Next word - // Missing space - inputWordStartPos = i; - inputWordLength = inputSize - i; - if (getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate, - startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength, - false /* missing space */, freqArray, wordLengthArray, outputWord, 0) - != FLAG_MULTIPLE_SUGGEST_CONTINUE) { - getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, inputSize, correction, queuePool, - hasAutoCorrectionCandidate, inputWordStartPos, startWordIndex + 1, - tempOutputWordLength, freqArray, wordLengthArray, outputWord); - } - - // Mistyped space - ++inputWordStartPos; - --inputWordLength; - - if (inputWordLength <= 0) { - continue; - } - - const int x = xcoordinates[inputWordStartPos - 1]; - const int y = ycoordinates[inputWordStartPos - 1]; - if (!proximityInfo->hasSpaceProximity(x, y)) { - continue; - } - - if (DEBUG_CORRECTION_FREQ) { - AKLOGI("Do mistyped space correction"); - } - getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, correction, queuePool, inputSize, hasAutoCorrectionCandidate, - startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength, - true /* mistyped space */, freqArray, wordLengthArray, outputWord, 0); - } -} - -void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, - const int *xcoordinates, const int *ycoordinates, const int *codes, - const bool useFullEditDistance, const int inputSize, - Correction *correction, WordsPriorityQueuePool *queuePool, - const bool hasAutoCorrectionCandidate) const { - if (inputSize >= MAX_WORD_LENGTH) return; - if (DEBUG_DICT) { - AKLOGI("--- Suggest multiple words"); - } - - // Allocating fixed length array on stack - int outputWord[MAX_WORD_LENGTH]; - int freqArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS]; - int wordLengthArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS]; - const int outputWordLength = 0; - const int startInputPos = 0; - const int startWordIndex = 0; - getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes, - useFullEditDistance, inputSize, correction, queuePool, hasAutoCorrectionCandidate, - startInputPos, startWordIndex, outputWordLength, freqArray, wordLengthArray, - outputWord); -} - -// Wrapper for getMostProbableWordLikeInner, which matches it to the previous -// interface. -int UnigramDictionary::getMostProbableWordLike(const int startInputIndex, const int inputSize, - Correction *correction, int *word) const { - int inWord[inputSize]; - for (int i = 0; i < inputSize; ++i) { - inWord[i] = correction->getPrimaryCodePointAt(startInputIndex + i); - } - return getMostProbableWordLikeInner(inWord, inputSize, word); -} - -// This function will take the position of a character array within a CharGroup, -// and check it actually like-matches the word in inWord starting at startInputIndex, -// that is, it matches it with case and accents squashed. -// The function returns true if there was a full match, false otherwise. -// The function will copy on-the-fly the characters in the CharGroup to outNewWord. -// It will also place the end position of the array in outPos; in outInputIndex, -// it will place the index of the first char AFTER the match if there was a match, -// and the initial position if there was not. It makes sense because if there was -// a match we want to continue searching, but if there was not, we want to go to -// the next CharGroup. -// In and out parameters may point to the same location. This function takes care -// not to use any input parameters after it wrote into its outputs. -static inline bool testCharGroupForContinuedLikeness(const uint8_t flags, - const uint8_t *const root, const int startPos, const int *const inWord, - const int startInputIndex, const int inputSize, int *outNewWord, int *outInputIndex, - int *outPos) { - const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); - int pos = startPos; - int codePoint = BinaryFormat::getCodePointAndForwardPointer(root, &pos); - int baseChar = toBaseLowerCase(codePoint); - const int wChar = toBaseLowerCase(inWord[startInputIndex]); - - if (baseChar != wChar) { - *outPos = hasMultipleChars ? BinaryFormat::skipOtherCharacters(root, pos) : pos; - *outInputIndex = startInputIndex; - return false; - } - int inputIndex = startInputIndex; - outNewWord[inputIndex] = codePoint; - if (hasMultipleChars) { - codePoint = BinaryFormat::getCodePointAndForwardPointer(root, &pos); - while (NOT_A_CODE_POINT != codePoint) { - baseChar = toBaseLowerCase(codePoint); - if (inputIndex + 1 >= inputSize || toBaseLowerCase(inWord[++inputIndex]) != baseChar) { - *outPos = BinaryFormat::skipOtherCharacters(root, pos); - *outInputIndex = startInputIndex; - return false; - } - outNewWord[inputIndex] = codePoint; - codePoint = BinaryFormat::getCodePointAndForwardPointer(root, &pos); - } - } - *outInputIndex = inputIndex + 1; - *outPos = pos; - return true; -} - -// This function is invoked when a word like the word searched for is found. -// It will compare the probability to the max probability, and if greater, will -// copy the word into the output buffer. In output value maxFreq, it will -// write the new maximum probability if it changed. -static inline void onTerminalWordLike(const int freq, int *newWord, const int length, int *outWord, - int *maxFreq) { - if (freq > *maxFreq) { - for (int q = 0; q < length; ++q) { - outWord[q] = newWord[q]; - } - outWord[length] = 0; - *maxFreq = freq; - } -} - -// Will find the highest probability of the words like the one passed as an argument, -// that is, everything that only differs by case/accents. -int UnigramDictionary::getMostProbableWordLikeInner(const int *const inWord, const int inputSize, - int *outWord) const { - int newWord[MAX_WORD_LENGTH]; - int depth = 0; - int maxFreq = -1; - const uint8_t *const root = DICT_ROOT; - int stackChildCount[MAX_WORD_LENGTH]; - int stackInputIndex[MAX_WORD_LENGTH]; - int stackSiblingPos[MAX_WORD_LENGTH]; - - int startPos = 0; - stackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos); - stackInputIndex[0] = 0; - stackSiblingPos[0] = startPos; - while (depth >= 0) { - const int charGroupCount = stackChildCount[depth]; - int pos = stackSiblingPos[depth]; - for (int charGroupIndex = charGroupCount - 1; charGroupIndex >= 0; --charGroupIndex) { - int inputIndex = stackInputIndex[depth]; - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); - // Test whether all chars in this group match with the word we are searching for. If so, - // we want to traverse its children (or if the inputSize match, evaluate its - // probability). Note that this function will output the position regardless, but will - // only write into inputIndex if there is a match. - const bool isAlike = testCharGroupForContinuedLikeness(flags, root, pos, inWord, - inputIndex, inputSize, newWord, &inputIndex, &pos); - if (isAlike && (!(BinaryFormat::FLAG_IS_NOT_A_WORD & flags)) - && (BinaryFormat::FLAG_IS_TERMINAL & flags) && (inputIndex == inputSize)) { - const int probability = - BinaryFormat::readProbabilityWithoutMovingPointer(root, pos); - onTerminalWordLike(probability, newWord, inputIndex, outWord, &maxFreq); - } - pos = BinaryFormat::skipProbability(flags, pos); - const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos); - const int childrenNodePos = BinaryFormat::readChildrenPosition(root, flags, pos); - // If we had a match and the word has children, we want to traverse them. We don't have - // to traverse words longer than the one we are searching for, since they will not match - // anyway, so don't traverse unless inputIndex < inputSize. - if (isAlike && (-1 != childrenNodePos) && (inputIndex < inputSize)) { - // Save position for this depth, to get back to this once children are done - stackChildCount[depth] = charGroupIndex; - stackSiblingPos[depth] = siblingPos; - // Prepare stack values for next depth - ++depth; - int childrenPos = childrenNodePos; - stackChildCount[depth] = - BinaryFormat::getGroupCountAndForwardPointer(root, &childrenPos); - stackSiblingPos[depth] = childrenPos; - stackInputIndex[depth] = inputIndex; - pos = childrenPos; - // Go to the next depth level. - ++depth; - break; - } else { - // No match, or no children, or word too long to ever match: go the next sibling. - pos = siblingPos; - } - } - --depth; - } - return maxFreq; -} - -int UnigramDictionary::getProbability(const int *const inWord, const int length) const { - const uint8_t *const root = DICT_ROOT; - int pos = BinaryFormat::getTerminalPosition(root, inWord, length, - false /* forceLowerCaseSearch */); - if (NOT_VALID_WORD == pos) { - return NOT_A_PROBABILITY; - } - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); - if (flags & (BinaryFormat::FLAG_IS_BLACKLISTED | BinaryFormat::FLAG_IS_NOT_A_WORD)) { - // If this is not a word, or if it's a blacklisted entry, it should behave as - // having no probability outside of the suggestion process (where it should be used - // for shortcuts). - return NOT_A_PROBABILITY; - } - const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); - if (hasMultipleChars) { - pos = BinaryFormat::skipOtherCharacters(root, pos); - } else { - BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos); - } - const int unigramProbability = BinaryFormat::readProbabilityWithoutMovingPointer(root, pos); - return unigramProbability; -} - -// TODO: remove this function. -int UnigramDictionary::getBigramPosition(int pos, int *word, int offset, int length) const { - return -1; -} - -// ProcessCurrentNode returns a boolean telling whether to traverse children nodes or not. -// If the return value is false, then the caller should read in the output "nextSiblingPosition" -// to find out the address of the next sibling node and pass it to a new call of processCurrentNode. -// It is worthy to note that when false is returned, the output values other than -// nextSiblingPosition are undefined. -// If the return value is true, then the caller must proceed to traverse the children of this -// node. processCurrentNode will output the information about the children: their count in -// newCount, their position in newChildrenPosition, the traverseAllNodes flag in -// newTraverseAllNodes, the match weight into newMatchRate, the input index into newInputIndex, the -// diffs into newDiffs, the sibling position in nextSiblingPosition, and the output index into -// newOutputIndex. Please also note the following caveat: processCurrentNode does not know when -// there aren't any more nodes at this level, it merely returns the address of the first byte after -// the current node in nextSiblingPosition. Thus, the caller must keep count of the nodes at any -// given level, as output into newCount when traversing this level's parent. -bool UnigramDictionary::processCurrentNode(const int initialPos, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, Correction *correction, - int *newCount, int *newChildrenPosition, int *nextSiblingPosition, - WordsPriorityQueuePool *queuePool, const int currentWordIndex) const { - if (DEBUG_DICT) { - correction->checkState(); - } - int pos = initialPos; - - // Flags contain the following information: - // - Address type (MASK_GROUP_ADDRESS_TYPE) on two bits: - // - FLAG_GROUP_ADDRESS_TYPE_{ONE,TWO,THREE}_BYTES means there are children and their address - // is on the specified number of bytes. - // - FLAG_GROUP_ADDRESS_TYPE_NOADDRESS means there are no children, and therefore no address. - // - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not. - // - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children) - // - FLAG_HAS_BIGRAMS: whether this node has bigrams or not - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos); - const bool hasMultipleChars = (0 != (BinaryFormat::FLAG_HAS_MULTIPLE_CHARS & flags)); - const bool isTerminalNode = (0 != (BinaryFormat::FLAG_IS_TERMINAL & flags)); - - bool needsToInvokeOnTerminal = false; - - // This gets only ONE character from the stream. Next there will be: - // if FLAG_HAS_MULTIPLE CHARS: the other characters of the same node - // else if FLAG_IS_TERMINAL: the probability - // else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address - // Note that you can't have a node that both is not a terminal and has no children. - int c = BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos); - ASSERT(NOT_A_CODE_POINT != c); - - // We are going to loop through each character and make it look like it's a different - // node each time. To do that, we will process characters in this node in order until - // we find the character terminator. This is signalled by getCodePoint* returning - // NOT_A_CODE_POINT. - // As a special case, if there is only one character in this node, we must not read the - // next bytes so we will simulate the NOT_A_CODE_POINT return by testing the flags. - // This way, each loop run will look like a "virtual node". - do { - // We prefetch the next char. If 'c' is the last char of this node, we will have - // NOT_A_CODE_POINT in the next char. From this we can decide whether this virtual node - // should behave as a terminal or not and whether we have children. - const int nextc = hasMultipleChars - ? BinaryFormat::getCodePointAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CODE_POINT; - const bool isLastChar = (NOT_A_CODE_POINT == nextc); - // If there are more chars in this nodes, then this virtual node is not a terminal. - // If we are on the last char, this virtual node is a terminal if this node is. - const bool isTerminal = isLastChar && isTerminalNode; - - Correction::CorrectionType stateType = correction->processCharAndCalcState( - c, isTerminal); - if (stateType == Correction::TRAVERSE_ALL_ON_TERMINAL - || stateType == Correction::ON_TERMINAL) { - needsToInvokeOnTerminal = true; - } else if (stateType == Correction::UNRELATED || correction->needsToPrune()) { - // We found that this is an unrelated character, so we should give up traversing - // this node and its children entirely. - // However we may not be on the last virtual node yet so we skip the remaining - // characters in this node, the probability if it's there, read the next sibling - // position to output it, then return false. - // We don't have to output other values because we return false, as in - // "don't traverse children". - if (!isLastChar) { - pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos); - } - pos = BinaryFormat::skipProbability(flags, pos); - *nextSiblingPosition = - BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); - return false; - } - - // Prepare for the next character. Promote the prefetched char to current char - the loop - // will take care of prefetching the next. If we finally found our last char, nextc will - // contain NOT_A_CODE_POINT. - c = nextc; - } while (NOT_A_CODE_POINT != c); - - if (isTerminalNode) { - // The probability should be here, because we come here only if this is actually - // a terminal node, and we are on its last char. - const int unigramProbability = - BinaryFormat::readProbabilityWithoutMovingPointer(DICT_ROOT, pos); - const int childrenAddressPos = BinaryFormat::skipProbability(flags, pos); - const int attributesPos = BinaryFormat::skipChildrenPosition(flags, childrenAddressPos); - TerminalAttributes terminalAttributes(DICT_ROOT, flags, attributesPos); - // bigramMap contains the bigram frequencies indexed by addresses for fast lookup. - // bigramFilter is a bloom filter of said frequencies for even faster rejection. - const int probability = BinaryFormat::getProbability(initialPos, bigramMap, bigramFilter, - unigramProbability); - onTerminal(probability, terminalAttributes, correction, queuePool, needsToInvokeOnTerminal, - currentWordIndex); - - // If there are more chars in this node, then this virtual node has children. - // If we are on the last char, this virtual node has children if this node has. - const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags); - - // This character matched the typed character (enough to traverse the node at least) - // so we just evaluated it. Now we should evaluate this virtual node's children - that - // is, if it has any. If it has no children, we're done here - so we skip the end of - // the node, output the siblings position, and return false "don't traverse children". - // Note that !hasChildren implies isLastChar, so we know we don't have to skip any - // remaining char in this group for there can't be any. - if (!hasChildren) { - pos = BinaryFormat::skipProbability(flags, pos); - *nextSiblingPosition = - BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); - return false; - } - - // Optimization: Prune out words that are too long compared to how much was typed. - if (correction->needsToPrune()) { - pos = BinaryFormat::skipProbability(flags, pos); - *nextSiblingPosition = - BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); - if (DEBUG_DICT_FULL) { - AKLOGI("Traversing was pruned."); - } - return false; - } - } - - // Now we finished processing this node, and we want to traverse children. If there are no - // children, we can't come here. - ASSERT(BinaryFormat::hasChildrenInFlags(flags)); - - // If this node was a terminal it still has the probability under the pointer (it may have been - // read, but not skipped - see readProbabilityWithoutMovingPointer). - // Next come the children position, then possibly attributes (attributes are bigrams only for - // now, maybe something related to shortcuts in the future). - // Once this is read, we still need to output the number of nodes in the immediate children of - // this node, so we read and output it before returning true, as in "please traverse children". - pos = BinaryFormat::skipProbability(flags, pos); - int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos); - *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); - *newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos); - *newChildrenPosition = childrenPos; - return true; -} -} // namespace latinime diff --git a/native/jni/src/unigram_dictionary.h b/native/jni/src/unigram_dictionary.h deleted file mode 100644 index a64a539bd..000000000 --- a/native/jni/src/unigram_dictionary.h +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_UNIGRAM_DICTIONARY_H -#define LATINIME_UNIGRAM_DICTIONARY_H - -#include <map> -#include <stdint.h> -#include "defines.h" -#include "digraph_utils.h" - -namespace latinime { - -class Correction; -class ProximityInfo; -class TerminalAttributes; -class WordsPriorityQueuePool; - -class UnigramDictionary { - public: - // Error tolerances - static const int DEFAULT_MAX_ERRORS = 2; - static const int MAX_ERRORS_FOR_TWO_WORDS = 1; - - static const int FLAG_MULTIPLE_SUGGEST_ABORT = 0; - static const int FLAG_MULTIPLE_SUGGEST_SKIP = 1; - static const int FLAG_MULTIPLE_SUGGEST_CONTINUE = 2; - UnigramDictionary(const uint8_t *const streamStart, const unsigned int dictFlags); - int getProbability(const int *const inWord, const int length) const; - int getBigramPosition(int pos, int *word, int offset, int length) const; - int getSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *inputCodePoints, const int inputSize, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, int *outWords, int *frequencies, - int *outputTypes) const; - int getDictFlags() const { return DICT_FLAGS; } - virtual ~UnigramDictionary(); - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(UnigramDictionary); - void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *inputCodePoints, const int inputSize, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - const bool useFullEditDistance, Correction *correction, - WordsPriorityQueuePool *queuePool) const; - int getDigraphReplacement(const int *codes, const int i, const int inputSize, - const DigraphUtils::digraph_t *const digraphs, const unsigned int digraphsSize) const; - void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codesBuffer, int *xCoordinatesBuffer, - int *yCoordinatesBuffer, const int codesBufferSize, const std::map<int, int> *bigramMap, - const uint8_t *bigramFilter, const bool useFullEditDistance, const int *codesSrc, - const int codesRemain, const int currentDepth, int *codesDest, Correction *correction, - WordsPriorityQueuePool *queuePool, const DigraphUtils::digraph_t *const digraphs, - const unsigned int digraphsSize) const; - void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const int inputSize, - Correction *correction) const; - void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const std::map<int, int> *bigramMap, - const uint8_t *bigramFilter, const bool useFullEditDistance, const int inputSize, - Correction *correction, WordsPriorityQueuePool *queuePool) const; - void getSuggestionCandidates( - const bool useFullEditDistance, const int inputSize, - const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, - Correction *correction, WordsPriorityQueuePool *queuePool, const bool doAutoCompletion, - const int maxErrors, const int currentWordIndex) const; - void getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const bool useFullEditDistance, - const int inputSize, Correction *correction, WordsPriorityQueuePool *queuePool, - const bool hasAutoCorrectionCandidate) const; - void onTerminal(const int freq, const TerminalAttributes &terminalAttributes, - Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, - const int currentWordIndex) const; - // Process a node by considering proximity, missing and excessive character - bool processCurrentNode(const int initialPos, const std::map<int, int> *bigramMap, - const uint8_t *bigramFilter, Correction *correction, int *newCount, - int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool, - const int currentWordIndex) const; - int getMostProbableWordLike(const int startInputIndex, const int inputSize, - Correction *correction, int *word) const; - int getMostProbableWordLikeInner(const int *const inWord, const int inputSize, - int *outWord) const; - int getSubStringSuggestion(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const bool useFullEditDistance, - Correction *correction, WordsPriorityQueuePool *queuePool, const int inputSize, - const bool hasAutoCorrectionCandidate, const int currentWordIndex, - const int inputWordStartPos, const int inputWordLength, const int outputWordStartPos, - const bool isSpaceProximity, int *freqArray, int *wordLengthArray, int *outputWord, - int *outputWordLength) const; - void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const bool useFullEditDistance, - const int inputSize, Correction *correction, WordsPriorityQueuePool *queuePool, - const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex, - const int outputWordLength, int *freqArray, int *wordLengthArray, - int *outputWord) const; - - const uint8_t *const DICT_ROOT; - const int ROOT_POS; - const int MAX_DIGRAPH_SEARCH_DEPTH; - const int DICT_FLAGS; -}; -} // namespace latinime -#endif // LATINIME_UNIGRAM_DICTIONARY_H diff --git a/native/jni/src/char_utils.cpp b/native/jni/src/utils/char_utils.cpp index e219beb62..0e7039610 100644 --- a/native/jni/src/char_utils.cpp +++ b/native/jni/src/utils/char_utils.cpp @@ -14,9 +14,10 @@ * limitations under the License. */ +#include "utils/char_utils.h" + #include <cstdlib> -#include "char_utils.h" #include "defines.h" namespace latinime { @@ -36,8 +37,7 @@ struct LatinCapitalSmallPair { * $ apt-get install libicu-dev * * 3. Build the following code - * (You need this file, char_utils.h, and defines.h) - * $ g++ -o char_utils -DUPDATING_CHAR_UTILS char_utils.cpp -licuuc + * $ g++ -o char_utils -I.. -DUPDATING_CHAR_UTILS char_utils.cpp -licuuc */ #ifdef UPDATING_CHAR_UTILS #include <stdio.h> @@ -47,7 +47,7 @@ extern "C" int main() { for (unsigned short c = 0; c < 0xFFFF; c++) { if (c <= 0x7F) continue; const unsigned short icu4cLowerC = u_tolower(c); - const unsigned short myLowerC = latin_tolower(c); + const unsigned short myLowerC = CharUtils::latin_tolower(c); if (c != icu4cLowerC) { #ifdef CONFIRMING_CHAR_UTILS if (icu4cLowerC != myLowerC) { @@ -70,7 +70,7 @@ extern "C" int main() { * * 5. Update the SORTED_CHAR_MAP[] array below with the output above. * Then, rebuild with -DCONFIRMING_CHAR_UTILS and confirm the program exits successfully. - * $ g++ -o char_utils -DUPDATING_CHAR_UTILS -DCONFIRMING_CHAR_UTILS char_utils.cpp -licuuc + * $ g++ -o char_utils -I.. -DUPDATING_CHAR_UTILS -DCONFIRMING_CHAR_UTILS char_utils.cpp -licuuc * $ ./char_utils * $ */ @@ -1054,7 +1054,7 @@ static int compare_pair_capital(const void *a, const void *b) { - static_cast<int>((static_cast<const struct LatinCapitalSmallPair *>(b))->capital); } -unsigned short latin_tolower(const unsigned short c) { +/* static */ unsigned short CharUtils::latin_tolower(const unsigned short c) { struct LatinCapitalSmallPair *p = static_cast<struct LatinCapitalSmallPair *>(bsearch(&c, SORTED_CHAR_MAP, NELEMS(SORTED_CHAR_MAP), sizeof(SORTED_CHAR_MAP[0]), compare_pair_capital)); @@ -1063,7 +1063,7 @@ unsigned short latin_tolower(const unsigned short c) { /* * Table mapping most combined Latin, Greek, and Cyrillic characters - * to their base characters. If c is in range, BASE_CHARS[c] == c + * to their base characters. If c is in range, CharUtils::BASE_CHARS[c] == c * if c is not a combined character, or the base character if it * is combined. * @@ -1074,7 +1074,7 @@ unsigned short latin_tolower(const unsigned short c) { * for ($j = $i; $j < $i + 8; $j++) { \ * printf("0x%04X, ", $base[$j] ? $base[$j] : $j)}; print "\n"; }' */ -const unsigned short BASE_CHARS[BASE_CHARS_SIZE] = { +/* static */ const unsigned short CharUtils::BASE_CHARS[CharUtils::BASE_CHARS_SIZE] = { /* U+0000 */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, /* U+0008 */ 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, /* U+0010 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, diff --git a/native/jni/src/utils/char_utils.h b/native/jni/src/utils/char_utils.h new file mode 100644 index 000000000..2e735a81c --- /dev/null +++ b/native/jni/src/utils/char_utils.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_CHAR_UTILS_H +#define LATINIME_CHAR_UTILS_H + +#include <cctype> + +#include "defines.h" + +namespace latinime { + +class CharUtils { + public: + static AK_FORCE_INLINE bool isAsciiUpper(int c) { + // Note: isupper(...) reports false positives for some Cyrillic characters, causing them to + // be incorrectly lower-cased using toAsciiLower(...) rather than latin_tolower(...). + return (c >= 'A' && c <= 'Z'); + } + + static AK_FORCE_INLINE int toAsciiLower(int c) { + return c - 'A' + 'a'; + } + + static AK_FORCE_INLINE bool isAscii(int c) { + return isascii(c) != 0; + } + + static AK_FORCE_INLINE int toLowerCase(const int c) { + if (isAsciiUpper(c)) { + return toAsciiLower(c); + } + if (isAscii(c)) { + return c; + } + return static_cast<int>(latin_tolower(static_cast<unsigned short>(c))); + } + + static AK_FORCE_INLINE int toBaseLowerCase(const int c) { + return toLowerCase(toBaseCodePoint(c)); + } + + static AK_FORCE_INLINE bool isIntentionalOmissionCodePoint(const int codePoint) { + // TODO: Do not hardcode here + return codePoint == KEYCODE_SINGLE_QUOTE || codePoint == KEYCODE_HYPHEN_MINUS; + } + + static AK_FORCE_INLINE int getCodePointCount(const int arraySize, const int *const codePoints) { + int size = 0; + for (; size < arraySize; ++size) { + if (codePoints[size] == '\0') { + break; + } + } + return size; + } + + static AK_FORCE_INLINE int toBaseCodePoint(int c) { + if (c < BASE_CHARS_SIZE) { + return static_cast<int>(BASE_CHARS[c]); + } + return c; + } + + static unsigned short latin_tolower(const unsigned short c); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(CharUtils); + + /** + * Table mapping most combined Latin, Greek, and Cyrillic characters + * to their base characters. If c is in range, BASE_CHARS[c] == c + * if c is not a combined character, or the base character if it + * is combined. + */ + static const int BASE_CHARS_SIZE = 0x0500; + static const unsigned short BASE_CHARS[BASE_CHARS_SIZE]; +}; +} // namespace latinime +#endif // LATINIME_CHAR_UTILS_H diff --git a/native/jni/src/hash_map_compat.h b/native/jni/src/utils/hash_map_compat.h index a1e982bc4..a1e982bc4 100644 --- a/native/jni/src/hash_map_compat.h +++ b/native/jni/src/utils/hash_map_compat.h diff --git a/native/jni/src/words_priority_queue.cpp b/native/jni/src/words_priority_queue.cpp deleted file mode 100644 index 7e18d0f87..000000000 --- a/native/jni/src/words_priority_queue.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2012, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "words_priority_queue.h" - -namespace latinime { - -int WordsPriorityQueue::outputSuggestions(const int *before, const int beforeLength, - int *frequencies, int *outputCodePoints, int* outputTypes) { - mHighestSuggestedWord = 0; - const int size = min(MAX_WORDS, static_cast<int>(mSuggestions.size())); - SuggestedWord *swBuffer[size]; - int index = size - 1; - while (!mSuggestions.empty() && index >= 0) { - SuggestedWord *sw = mSuggestions.top(); - if (DEBUG_WORDS_PRIORITY_QUEUE) { - AKLOGI("dump word. %d", sw->mScore); - DUMP_WORD(sw->mWord, sw->mWordLength); - } - swBuffer[index] = sw; - mSuggestions.pop(); - --index; - } - if (size >= 2) { - SuggestedWord *nsMaxSw = 0; - int maxIndex = 0; - float maxNs = 0; - for (int i = 0; i < size; ++i) { - SuggestedWord *tempSw = swBuffer[i]; - if (!tempSw) { - continue; - } - const float tempNs = getNormalizedScore(tempSw, before, beforeLength, 0, 0, 0); - if (tempNs >= maxNs) { - maxNs = tempNs; - maxIndex = i; - nsMaxSw = tempSw; - } - } - if (maxIndex > 0 && nsMaxSw) { - memmove(&swBuffer[1], &swBuffer[0], maxIndex * sizeof(swBuffer[0])); - swBuffer[0] = nsMaxSw; - } - } - for (int i = 0; i < size; ++i) { - SuggestedWord *sw = swBuffer[i]; - if (!sw) { - AKLOGE("SuggestedWord is null %d", i); - continue; - } - const int wordLength = sw->mWordLength; - int *targetAddress = outputCodePoints + i * MAX_WORD_LENGTH; - frequencies[i] = sw->mScore; - outputTypes[i] = sw->mType; - memcpy(targetAddress, sw->mWord, wordLength * sizeof(targetAddress[0])); - if (wordLength < MAX_WORD_LENGTH) { - targetAddress[wordLength] = 0; - } - sw->mUsed = false; - } - return size; -} -} // namespace latinime diff --git a/native/jni/src/words_priority_queue.h b/native/jni/src/words_priority_queue.h deleted file mode 100644 index 54e8007a2..000000000 --- a/native/jni/src/words_priority_queue.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_WORDS_PRIORITY_QUEUE_H -#define LATINIME_WORDS_PRIORITY_QUEUE_H - -#include <cstring> // for memcpy() -#include <queue> - -#include "correction.h" -#include "defines.h" - -namespace latinime { - -class WordsPriorityQueue { - public: - struct SuggestedWord { - int mScore; - int mWord[MAX_WORD_LENGTH]; - int mWordLength; - bool mUsed; - int mType; - - void setParams(int score, int *word, int wordLength, int type) { - mScore = score; - mWordLength = wordLength; - memcpy(mWord, word, sizeof(mWord[0]) * wordLength); - mUsed = true; - mType = type; - } - }; - - WordsPriorityQueue(int maxWords) - : mSuggestions(), MAX_WORDS(maxWords), - mSuggestedWords(new SuggestedWord[MAX_WORD_LENGTH]), mHighestSuggestedWord(0) { - for (int i = 0; i < MAX_WORD_LENGTH; ++i) { - mSuggestedWords[i].mUsed = false; - } - } - - // Non virtual inline destructor -- never inherit this class - AK_FORCE_INLINE ~WordsPriorityQueue() { - delete[] mSuggestedWords; - } - - void push(int score, int *word, int wordLength, int type) { - SuggestedWord *sw = 0; - if (size() >= MAX_WORDS) { - sw = mSuggestions.top(); - const int minScore = sw->mScore; - if (minScore >= score) { - return; - } - sw->mUsed = false; - mSuggestions.pop(); - } - if (sw == 0) { - sw = getFreeSuggestedWord(score, word, wordLength, type); - } else { - sw->setParams(score, word, wordLength, type); - } - if (sw == 0) { - AKLOGE("SuggestedWord is accidentally null."); - return; - } - if (DEBUG_WORDS_PRIORITY_QUEUE) { - AKLOGI("Push word. %d, %d", score, wordLength); - DUMP_WORD(word, wordLength); - } - mSuggestions.push(sw); - if (!mHighestSuggestedWord || mHighestSuggestedWord->mScore < sw->mScore) { - mHighestSuggestedWord = sw; - } - } - - SuggestedWord *top() const { - if (mSuggestions.empty()) return 0; - SuggestedWord *sw = mSuggestions.top(); - return sw; - } - - int size() const { - return static_cast<int>(mSuggestions.size()); - } - - AK_FORCE_INLINE void clear() { - mHighestSuggestedWord = 0; - while (!mSuggestions.empty()) { - SuggestedWord *sw = mSuggestions.top(); - if (DEBUG_WORDS_PRIORITY_QUEUE) { - AKLOGI("Clear word. %d", sw->mScore); - DUMP_WORD(sw->mWord, sw->mWordLength); - } - sw->mUsed = false; - mSuggestions.pop(); - } - } - - AK_FORCE_INLINE void dumpTopWord() const { - if (size() <= 0) { - return; - } - DUMP_WORD(mHighestSuggestedWord->mWord, mHighestSuggestedWord->mWordLength); - } - - AK_FORCE_INLINE float getHighestNormalizedScore(const int *before, const int beforeLength, - int **outWord, int *outScore, int *outLength) const { - if (!mHighestSuggestedWord) { - return 0.0f; - } - return getNormalizedScore(mHighestSuggestedWord, before, beforeLength, outWord, outScore, - outLength); - } - - int outputSuggestions(const int *before, const int beforeLength, int *frequencies, - int *outputCodePoints, int* outputTypes); - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(WordsPriorityQueue); - struct wordComparator { - bool operator ()(SuggestedWord * left, SuggestedWord * right) { - return left->mScore > right->mScore; - } - }; - - SuggestedWord *getFreeSuggestedWord(int score, int *word, int wordLength, int type) const { - for (int i = 0; i < MAX_WORD_LENGTH; ++i) { - if (!mSuggestedWords[i].mUsed) { - mSuggestedWords[i].setParams(score, word, wordLength, type); - return &mSuggestedWords[i]; - } - } - return 0; - } - - static float getNormalizedScore(SuggestedWord *sw, const int *before, const int beforeLength, - int **outWord, int *outScore, int *outLength) { - const int score = sw->mScore; - int *word = sw->mWord; - const int wordLength = sw->mWordLength; - if (outScore) { - *outScore = score; - } - if (outWord) { - *outWord = word; - } - if (outLength) { - *outLength = wordLength; - } - return Correction::RankingAlgorithm::calcNormalizedScore(before, beforeLength, word, - wordLength, score); - } - - typedef std::priority_queue<SuggestedWord *, std::vector<SuggestedWord *>, - wordComparator> Suggestions; - Suggestions mSuggestions; - const int MAX_WORDS; - SuggestedWord *mSuggestedWords; - SuggestedWord *mHighestSuggestedWord; -}; -} // namespace latinime -#endif // LATINIME_WORDS_PRIORITY_QUEUE_H diff --git a/native/jni/src/words_priority_queue_pool.h b/native/jni/src/words_priority_queue_pool.h deleted file mode 100644 index 2cd210a05..000000000 --- a/native/jni/src/words_priority_queue_pool.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_WORDS_PRIORITY_QUEUE_POOL_H -#define LATINIME_WORDS_PRIORITY_QUEUE_POOL_H - -#include "defines.h" -#include "words_priority_queue.h" - -namespace latinime { - -class WordsPriorityQueuePool { - public: - WordsPriorityQueuePool(int mainQueueMaxWords, int subQueueMaxWords) - // Note: using placement new() requires the caller to call the destructor explicitly. - : mMasterQueue(new(mMasterQueueBuf) WordsPriorityQueue(mainQueueMaxWords)) { - for (int i = 0, subQueueBufOffset = 0; - i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT; - ++i, subQueueBufOffset += static_cast<int>(sizeof(WordsPriorityQueue))) { - mSubQueues[i] = new(mSubQueueBuf + subQueueBufOffset) - WordsPriorityQueue(subQueueMaxWords); - } - } - - // Non virtual inline destructor -- never inherit this class - ~WordsPriorityQueuePool() { - // Note: these explicit calls to the destructor match the calls to placement new() above. - if (mMasterQueue) mMasterQueue->~WordsPriorityQueue(); - for (int i = 0; i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT; ++i) { - if (mSubQueues[i]) mSubQueues[i]->~WordsPriorityQueue(); - } - } - - WordsPriorityQueue *getMasterQueue() const { - return mMasterQueue; - } - - WordsPriorityQueue *getSubQueue(const int wordIndex, const int inputWordLength) const { - if (wordIndex >= MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) { - return 0; - } - if (inputWordLength < 0 || inputWordLength >= SUB_QUEUE_MAX_COUNT) { - if (DEBUG_WORDS_PRIORITY_QUEUE) { - ASSERT(false); - } - return 0; - } - return mSubQueues[wordIndex * SUB_QUEUE_MAX_COUNT + inputWordLength]; - } - - inline void clearAll() { - mMasterQueue->clear(); - for (int i = 0; i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS; ++i) { - clearSubQueue(i); - } - } - - AK_FORCE_INLINE void clearSubQueue(const int wordIndex) { - for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { - WordsPriorityQueue *queue = getSubQueue(wordIndex, i); - if (queue) { - queue->clear(); - } - } - } - - void dumpSubQueue1TopSuggestions() const { - AKLOGI("DUMP SUBQUEUE1 TOP SUGGESTIONS"); - for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { - getSubQueue(0, i)->dumpTopWord(); - } - } - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(WordsPriorityQueuePool); - char mMasterQueueBuf[sizeof(WordsPriorityQueue)]; - char mSubQueueBuf[SUB_QUEUE_MAX_COUNT * MULTIPLE_WORDS_SUGGESTION_MAX_WORDS - * sizeof(WordsPriorityQueue)]; - WordsPriorityQueue *mMasterQueue; - WordsPriorityQueue *mSubQueues[SUB_QUEUE_MAX_COUNT * MULTIPLE_WORDS_SUGGESTION_MAX_WORDS]; -}; -} // namespace latinime -#endif // LATINIME_WORDS_PRIORITY_QUEUE_POOL_H |