diff options
Diffstat (limited to 'native')
41 files changed, 5646 insertions, 2185 deletions
diff --git a/native/Android.mk b/native/Android.mk index 54b24e42e..5053e7d64 100644 --- a/native/Android.mk +++ b/native/Android.mk @@ -1,50 +1 @@ -LOCAL_PATH := $(call my-dir) -include $(CLEAR_VARS) - -LOCAL_C_INCLUDES += $(LOCAL_PATH)/src $(JNI_H_INCLUDE) - -LOCAL_CFLAGS += -Werror -Wall - -# To suppress compiler warnings for unused variables/functions used for debug features etc. -LOCAL_CFLAGS += -Wno-unused-parameter -Wno-unused-function - -LOCAL_SRC_FILES := \ - jni/com_android_inputmethod_keyboard_ProximityInfo.cpp \ - jni/com_android_inputmethod_latin_BinaryDictionary.cpp \ - jni/jni_common.cpp \ - src/bigram_dictionary.cpp \ - src/char_utils.cpp \ - src/dictionary.cpp \ - src/proximity_info.cpp \ - src/unigram_dictionary.cpp - -#FLAG_DBG := true - -TARGETING_UNBUNDLED_FROYO := true - -ifeq ($(TARGET_ARCH), x86) - TARGETING_UNBUNDLED_FROYO := false -endif - -ifeq ($(FLAG_DBG), true) - TARGETING_UNBUNDLED_FROYO := false -endif - -ifeq ($(TARGETING_UNBUNDLED_FROYO), true) - LOCAL_NDK_VERSION := 4 - LOCAL_SDK_VERSION := 8 -endif - -LOCAL_PRELINK_MODULE := false - -LOCAL_MODULE := libjni_latinime2 - -LOCAL_MODULE_TAGS := optional - -ifeq ($(FLAG_DBG), true) - $(warning Making debug version of native library) - LOCAL_CFLAGS += -DFLAG_DBG - LOCAL_SHARED_LIBRARIES := libcutils libutils -endif - -include $(BUILD_SHARED_LIBRARY) +include $(call all-subdir-makefiles) diff --git a/native/jni/Android.mk b/native/jni/Android.mk new file mode 100644 index 000000000..3bb7b58f1 --- /dev/null +++ b/native/jni/Android.mk @@ -0,0 +1,112 @@ +# Copyright (C) 2011 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCAL_PATH := $(call my-dir) + +############ some local flags +# If you change any of those flags, you need to rebuild both libjni_latinime_static +# and the shared library. +#FLAG_DBG := true +FLAG_DO_PROFILE ?= false + +###################################### +include $(CLEAR_VARS) + +LATIN_IME_SRC_DIR := src + +LOCAL_C_INCLUDES += $(LOCAL_PATH)/$(LATIN_IME_SRC_DIR) + +LOCAL_CFLAGS += -Werror -Wall + +# To suppress compiler warnings for unused variables/functions used for debug features etc. +LOCAL_CFLAGS += -Wno-unused-parameter -Wno-unused-function + +LATIN_IME_JNI_SRC_FILES := \ + com_android_inputmethod_keyboard_ProximityInfo.cpp \ + com_android_inputmethod_latin_BinaryDictionary.cpp \ + com_android_inputmethod_latin_NativeUtils.cpp \ + jni_common.cpp + +LATIN_IME_CORE_SRC_FILES := \ + additional_proximity_chars.cpp \ + basechars.cpp \ + bigram_dictionary.cpp \ + char_utils.cpp \ + correction.cpp \ + dictionary.cpp \ + proximity_info.cpp \ + proximity_info_state.cpp \ + unigram_dictionary.cpp + +LOCAL_SRC_FILES := \ + $(LATIN_IME_JNI_SRC_FILES) \ + $(addprefix $(LATIN_IME_SRC_DIR)/,$(LATIN_IME_CORE_SRC_FILES)) + +ifeq ($(FLAG_DO_PROFILE), true) + $(warning Making profiling version of native library) + LOCAL_CFLAGS += -DFLAG_DO_PROFILE +else # FLAG_DO_PROFILE +ifeq ($(FLAG_DBG), true) + $(warning Making debug version of native library) + LOCAL_CFLAGS += -DFLAG_DBG +endif # FLAG_DBG +endif # FLAG_DO_PROFILE + +LOCAL_MODULE := libjni_latinime_static +LOCAL_MODULE_TAGS := optional + +ifdef HISTORICAL_NDK_VERSIONS_ROOT # In the platform build system +include external/stlport/libstlport.mk +else # In the NDK build system +LOCAL_C_INCLUDES += external/stlport/stlport bionic +endif + +include $(BUILD_STATIC_LIBRARY) + +###################################### +include $(CLEAR_VARS) + +# All code in LOCAL_WHOLE_STATIC_LIBRARIES will be built into this shared library. +LOCAL_WHOLE_STATIC_LIBRARIES := libjni_latinime_static + +ifdef HISTORICAL_NDK_VERSIONS_ROOT # In the platform build system +LOCAL_SHARED_LIBRARIES := libstlport +else # In the NDK build system +LOCAL_SHARED_LIBRARIES := libstlport_static +endif + +ifeq ($(FLAG_DO_PROFILE), true) + $(warning Making profiling version of native library) + LOCAL_SHARED_LIBRARIES += libcutils libutils +else # FLAG_DO_PROFILE +ifeq ($(FLAG_DBG), true) + $(warning Making debug version of native library) + LOCAL_SHARED_LIBRARIES += libcutils libutils +endif # FLAG_DBG +endif # FLAG_DO_PROFILE + +LOCAL_MODULE := libjni_latinime +LOCAL_MODULE_TAGS := optional + +ifdef HISTORICAL_NDK_VERSIONS_ROOT # In the platform build system +include external/stlport/libstlport.mk +endif + +include $(BUILD_SHARED_LIBRARY) + +#################### Clean up the tmp vars +LATIN_IME_CORE_SRC_FILES := +LATIN_IME_JNI_SRC_FILES := +LATIN_IME_SRC_DIR := +TARGETING_UNBUNDLED_FROYO := diff --git a/native/jni/Application.mk b/native/jni/Application.mk new file mode 100644 index 000000000..caf3b2622 --- /dev/null +++ b/native/jni/Application.mk @@ -0,0 +1 @@ +APP_STL := stlport_static diff --git a/native/jni/com_android_inputmethod_keyboard_ProximityInfo.cpp b/native/jni/com_android_inputmethod_keyboard_ProximityInfo.cpp index f3e2a7e60..9eb437c06 100644 --- a/native/jni/com_android_inputmethod_keyboard_ProximityInfo.cpp +++ b/native/jni/com_android_inputmethod_keyboard_ProximityInfo.cpp @@ -25,32 +25,63 @@ #include <assert.h> #include <errno.h> #include <stdio.h> +#include <string> namespace latinime { -static jint latinime_Keyboard_setProximityInfo(JNIEnv *env, jobject object, - jint maxProximityCharsSize, jint displayWidth, jint displayHeight, jint gridWidth, - jint gridHeight, jintArray proximityCharsArray) { - jint* proximityChars = env->GetIntArrayElements(proximityCharsArray, NULL); - ProximityInfo *proximityInfo = new ProximityInfo(maxProximityCharsSize, displayWidth, - displayHeight, gridWidth, gridHeight, (const uint32_t *)proximityChars); +static jlong latinime_Keyboard_setProximityInfo(JNIEnv *env, jobject object, + jstring localejStr, jint maxProximityCharsSize, jint displayWidth, jint displayHeight, + jint gridWidth, jint gridHeight, jint mostCommonkeyWidth, jintArray proximityCharsArray, + jint keyCount, jintArray keyXCoordinateArray, jintArray keyYCoordinateArray, + jintArray keyWidthArray, jintArray keyHeightArray, jintArray keyCharCodeArray, + jfloatArray sweetSpotCenterXArray, jfloatArray sweetSpotCenterYArray, + jfloatArray sweetSpotRadiusArray) { + const char *localeStrPtr = env->GetStringUTFChars(localejStr, 0); + const std::string localeStr(localeStrPtr); + jint *proximityChars = env->GetIntArrayElements(proximityCharsArray, 0); + jint *keyXCoordinates = safeGetIntArrayElements(env, keyXCoordinateArray); + jint *keyYCoordinates = safeGetIntArrayElements(env, keyYCoordinateArray); + jint *keyWidths = safeGetIntArrayElements(env, keyWidthArray); + jint *keyHeights = safeGetIntArrayElements(env, keyHeightArray); + jint *keyCharCodes = safeGetIntArrayElements(env, keyCharCodeArray); + jfloat *sweetSpotCenterXs = safeGetFloatArrayElements(env, sweetSpotCenterXArray); + jfloat *sweetSpotCenterYs = safeGetFloatArrayElements(env, sweetSpotCenterYArray); + jfloat *sweetSpotRadii = safeGetFloatArrayElements(env, sweetSpotRadiusArray); + ProximityInfo *proximityInfo = new ProximityInfo( + localeStr, maxProximityCharsSize, displayWidth, + displayHeight, gridWidth, gridHeight, mostCommonkeyWidth, + (const int32_t*)proximityChars, + keyCount, (const int32_t*)keyXCoordinates, (const int32_t*)keyYCoordinates, + (const int32_t*)keyWidths, (const int32_t*)keyHeights, (const int32_t*)keyCharCodes, + (const float*)sweetSpotCenterXs, (const float*)sweetSpotCenterYs, + (const float*)sweetSpotRadii); + safeReleaseFloatArrayElements(env, sweetSpotRadiusArray, sweetSpotRadii); + safeReleaseFloatArrayElements(env, sweetSpotCenterYArray, sweetSpotCenterYs); + safeReleaseFloatArrayElements(env, sweetSpotCenterXArray, sweetSpotCenterXs); + safeReleaseIntArrayElements(env, keyCharCodeArray, keyCharCodes); + safeReleaseIntArrayElements(env, keyHeightArray, keyHeights); + safeReleaseIntArrayElements(env, keyWidthArray, keyWidths); + safeReleaseIntArrayElements(env, keyYCoordinateArray, keyYCoordinates); + safeReleaseIntArrayElements(env, keyXCoordinateArray, keyXCoordinates); env->ReleaseIntArrayElements(proximityCharsArray, proximityChars, 0); - return (jint)proximityInfo; + env->ReleaseStringUTFChars(localejStr, localeStrPtr); + return (jlong)proximityInfo; } -static void latinime_Keyboard_release(JNIEnv *env, jobject object, jint proximityInfo) { +static void latinime_Keyboard_release(JNIEnv *env, jobject object, jlong proximityInfo) { ProximityInfo *pi = (ProximityInfo*)proximityInfo; if (!pi) return; delete pi; } static JNINativeMethod sKeyboardMethods[] = { - {"setProximityInfoNative", "(IIIII[I)I", (void*)latinime_Keyboard_setProximityInfo}, - {"releaseProximityInfoNative", "(I)V", (void*)latinime_Keyboard_release} + {"setProximityInfoNative", "(Ljava/lang/String;IIIIII[II[I[I[I[I[I[F[F[F)J", + (void*)latinime_Keyboard_setProximityInfo}, + {"releaseProximityInfoNative", "(J)V", (void*)latinime_Keyboard_release} }; int register_ProximityInfo(JNIEnv *env) { - const char* const kClassPathName = "com/android/inputmethod/keyboard/ProximityInfo"; + const char *const kClassPathName = "com/android/inputmethod/keyboard/ProximityInfo"; return registerNativeMethods(env, kClassPathName, sKeyboardMethods, sizeof(sKeyboardMethods) / sizeof(sKeyboardMethods[0])); } diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index ce874d8d4..d10dc962e 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -17,7 +17,10 @@ #define LOG_TAG "LatinIME: jni: BinaryDictionary" +#include "binary_format.h" +#include "correction.h" #include "com_android_inputmethod_latin_BinaryDictionary.h" +#include "defines.h" #include "dictionary.h" #include "jni.h" #include "jni_common.h" @@ -32,175 +35,230 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> +#include <unistd.h> #else // USE_MMAP_FOR_DICTIONARY #include <stdlib.h> #endif // USE_MMAP_FOR_DICTIONARY namespace latinime { -static jint latinime_BinaryDictionary_open(JNIEnv *env, jobject object, +void releaseDictBuf(void* dictBuf, const size_t length, int fd); + +static jlong latinime_BinaryDictionary_open(JNIEnv *env, jobject object, jstring sourceDir, jlong dictOffset, jlong dictSize, - jint typedLetterMultiplier, jint fullWordMultiplier, jint maxWordLength, jint maxWords, - jint maxAlternatives) { + jint typedLetterMultiplier, jint fullWordMultiplier, jint maxWordLength, jint maxWords) { PROF_OPEN; PROF_START(66); - const char *sourceDirChars = env->GetStringUTFChars(sourceDir, NULL); - if (sourceDirChars == NULL) { - LOGE("DICT: Can't get sourceDir string"); + const char *sourceDirChars = env->GetStringUTFChars(sourceDir, 0); + if (sourceDirChars == 0) { + AKLOGE("DICT: Can't get sourceDir string"); return 0; } int fd = 0; - void *dictBuf = NULL; + void *dictBuf = 0; int adjust = 0; #ifdef USE_MMAP_FOR_DICTIONARY /* mmap version */ fd = open(sourceDirChars, O_RDONLY); if (fd < 0) { - LOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno); + AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno); return 0; } int pagesize = getpagesize(); adjust = dictOffset % pagesize; int adjDictOffset = dictOffset - adjust; int adjDictSize = dictSize + adjust; - dictBuf = mmap(NULL, sizeof(char) * adjDictSize, PROT_READ, MAP_PRIVATE, fd, adjDictOffset); + dictBuf = mmap(0, sizeof(char) * adjDictSize, PROT_READ, MAP_PRIVATE, fd, adjDictOffset); if (dictBuf == MAP_FAILED) { - LOGE("DICT: Can't mmap dictionary. errno=%d", errno); + AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno); return 0; } dictBuf = (void *)((char *)dictBuf + adjust); #else // USE_MMAP_FOR_DICTIONARY /* malloc version */ - FILE *file = NULL; + FILE *file = 0; file = fopen(sourceDirChars, "rb"); - if (file == NULL) { - LOGE("DICT: Can't fopen sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno); + if (file == 0) { + AKLOGE("DICT: Can't fopen sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno); return 0; } dictBuf = malloc(sizeof(char) * dictSize); if (!dictBuf) { - LOGE("DICT: Can't allocate memory region for dictionary. errno=%d", errno); + AKLOGE("DICT: Can't allocate memory region for dictionary. errno=%d", errno); return 0; } int ret = fseek(file, (long)dictOffset, SEEK_SET); if (ret != 0) { - LOGE("DICT: Failure in fseek. ret=%d errno=%d", ret, errno); + AKLOGE("DICT: Failure in fseek. ret=%d errno=%d", ret, errno); return 0; } ret = fread(dictBuf, sizeof(char) * dictSize, 1, file); if (ret != 1) { - LOGE("DICT: Failure in fread. ret=%d errno=%d", ret, errno); + AKLOGE("DICT: Failure in fread. ret=%d errno=%d", ret, errno); return 0; } ret = fclose(file); if (ret != 0) { - LOGE("DICT: Failure in fclose. ret=%d errno=%d", ret, errno); + AKLOGE("DICT: Failure in fclose. ret=%d errno=%d", ret, errno); return 0; } #endif // USE_MMAP_FOR_DICTIONARY env->ReleaseStringUTFChars(sourceDir, sourceDirChars); if (!dictBuf) { - LOGE("DICT: dictBuf is null"); + AKLOGE("DICT: dictBuf is null"); return 0; } - Dictionary *dictionary = new Dictionary(dictBuf, dictSize, fd, adjust, typedLetterMultiplier, - fullWordMultiplier, maxWordLength, maxWords, maxAlternatives); + Dictionary *dictionary = 0; + if (BinaryFormat::UNKNOWN_FORMAT == BinaryFormat::detectFormat((uint8_t*)dictBuf)) { + AKLOGE("DICT: dictionary format is unknown, bad magic number"); +#ifdef USE_MMAP_FOR_DICTIONARY + releaseDictBuf(((char*)dictBuf) - adjust, adjDictSize, fd); +#else // USE_MMAP_FOR_DICTIONARY + releaseDictBuf(dictBuf, 0, 0); +#endif // USE_MMAP_FOR_DICTIONARY + } else { + dictionary = new Dictionary(dictBuf, dictSize, fd, adjust, typedLetterMultiplier, + fullWordMultiplier, maxWordLength, maxWords); + } PROF_END(66); PROF_CLOSE; - return (jint)dictionary; + return (jlong)dictionary; } -static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object, jint dict, - jint proximityInfo, jintArray xCoordinatesArray, jintArray yCoordinatesArray, - jintArray inputArray, jint arraySize, jint flags, - jcharArray outputArray, jintArray frequencyArray) { +static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jobject object, jlong dict, + jlong proximityInfo, jintArray xCoordinatesArray, jintArray yCoordinatesArray, + jintArray inputArray, jint arraySize, jintArray prevWordForBigrams, + jboolean useFullEditDistance, jcharArray outputArray, jintArray frequencyArray) { Dictionary *dictionary = (Dictionary*)dict; if (!dictionary) return 0; ProximityInfo *pInfo = (ProximityInfo*)proximityInfo; - - int *xCoordinates = env->GetIntArrayElements(xCoordinatesArray, NULL); - int *yCoordinates = env->GetIntArrayElements(yCoordinatesArray, NULL); - - int *frequencies = env->GetIntArrayElements(frequencyArray, NULL); - int *inputCodes = env->GetIntArrayElements(inputArray, NULL); - jchar *outputChars = env->GetCharArrayElements(outputArray, NULL); - + int *xCoordinates = env->GetIntArrayElements(xCoordinatesArray, 0); + int *yCoordinates = env->GetIntArrayElements(yCoordinatesArray, 0); + int *frequencies = env->GetIntArrayElements(frequencyArray, 0); + int *inputCodes = env->GetIntArrayElements(inputArray, 0); + jchar *outputChars = env->GetCharArrayElements(outputArray, 0); + jint *prevWordChars = prevWordForBigrams + ? env->GetIntArrayElements(prevWordForBigrams, 0) : 0; + jsize prevWordLength = prevWordChars ? env->GetArrayLength(prevWordForBigrams) : 0; int count = dictionary->getSuggestions(pInfo, xCoordinates, yCoordinates, inputCodes, - arraySize, flags, (unsigned short*) outputChars, frequencies); - - env->ReleaseIntArrayElements(frequencyArray, frequencies, 0); + arraySize, prevWordChars, prevWordLength, useFullEditDistance, + (unsigned short*) outputChars, frequencies); + if (prevWordChars) { + env->ReleaseIntArrayElements(prevWordForBigrams, prevWordChars, JNI_ABORT); + } + env->ReleaseCharArrayElements(outputArray, outputChars, 0); env->ReleaseIntArrayElements(inputArray, inputCodes, JNI_ABORT); - env->ReleaseIntArrayElements(xCoordinatesArray, xCoordinates, 0); + env->ReleaseIntArrayElements(frequencyArray, frequencies, 0); env->ReleaseIntArrayElements(yCoordinatesArray, yCoordinates, 0); - env->ReleaseCharArrayElements(outputArray, outputChars, 0); - + env->ReleaseIntArrayElements(xCoordinatesArray, xCoordinates, 0); return count; } -static int latinime_BinaryDictionary_getBigrams(JNIEnv *env, jobject object, jint dict, - jcharArray prevWordArray, jint prevWordLength, jintArray inputArray, jint inputArraySize, - jcharArray outputArray, jintArray frequencyArray, jint maxWordLength, jint maxBigrams, - jint maxAlternatives) { +static int latinime_BinaryDictionary_getBigrams(JNIEnv *env, jobject object, jlong dict, + jintArray prevWordArray, jint prevWordLength, jintArray inputArray, jint inputArraySize, + jcharArray outputArray, jintArray frequencyArray, jint maxWordLength, jint maxBigrams) { Dictionary *dictionary = (Dictionary*)dict; if (!dictionary) return 0; - - jchar *prevWord = env->GetCharArrayElements(prevWordArray, NULL); - int *inputCodes = env->GetIntArrayElements(inputArray, NULL); - jchar *outputChars = env->GetCharArrayElements(outputArray, NULL); - int *frequencies = env->GetIntArrayElements(frequencyArray, NULL); - - int count = dictionary->getBigrams((unsigned short*) prevWord, prevWordLength, inputCodes, - inputArraySize, (unsigned short*) outputChars, frequencies, maxWordLength, maxBigrams, - maxAlternatives); - - env->ReleaseCharArrayElements(prevWordArray, prevWord, JNI_ABORT); - env->ReleaseIntArrayElements(inputArray, inputCodes, JNI_ABORT); - env->ReleaseCharArrayElements(outputArray, outputChars, 0); + jint *prevWord = env->GetIntArrayElements(prevWordArray, 0); + int *inputCodes = env->GetIntArrayElements(inputArray, 0); + jchar *outputChars = env->GetCharArrayElements(outputArray, 0); + int *frequencies = env->GetIntArrayElements(frequencyArray, 0); + int count = dictionary->getBigrams(prevWord, prevWordLength, inputCodes, + inputArraySize, (unsigned short*) outputChars, frequencies, maxWordLength, maxBigrams); env->ReleaseIntArrayElements(frequencyArray, frequencies, 0); - + env->ReleaseCharArrayElements(outputArray, outputChars, 0); + env->ReleaseIntArrayElements(inputArray, inputCodes, JNI_ABORT); + env->ReleaseIntArrayElements(prevWordArray, prevWord, JNI_ABORT); return count; } -static jboolean latinime_BinaryDictionary_isValidWord(JNIEnv *env, jobject object, jint dict, - jcharArray wordArray, jint wordLength) { +static jint latinime_BinaryDictionary_getFrequency(JNIEnv *env, jobject object, jlong dict, + jintArray wordArray, jint wordLength) { + Dictionary *dictionary = (Dictionary*)dict; + if (!dictionary) return (jboolean) false; + jint *word = env->GetIntArrayElements(wordArray, 0); + jint result = dictionary->getFrequency(word, wordLength); + env->ReleaseIntArrayElements(wordArray, word, JNI_ABORT); + return result; +} + +static jboolean latinime_BinaryDictionary_isValidBigram(JNIEnv *env, jobject object, jlong dict, + jintArray wordArray1, jintArray wordArray2) { Dictionary *dictionary = (Dictionary*)dict; if (!dictionary) return (jboolean) false; + jint *word1 = env->GetIntArrayElements(wordArray1, 0); + jint *word2 = env->GetIntArrayElements(wordArray2, 0); + jsize length1 = word1 ? env->GetArrayLength(wordArray1) : 0; + jsize length2 = word2 ? env->GetArrayLength(wordArray2) : 0; + jboolean result = dictionary->isValidBigram(word1, length1, word2, length2); + env->ReleaseIntArrayElements(wordArray2, word2, JNI_ABORT); + env->ReleaseIntArrayElements(wordArray1, word1, JNI_ABORT); + return result; +} - jchar *word = env->GetCharArrayElements(wordArray, NULL); - jboolean result = dictionary->isValidWord((unsigned short*) word, wordLength); - env->ReleaseCharArrayElements(wordArray, word, JNI_ABORT); +static jfloat latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jobject object, + jcharArray before, jint beforeLength, jcharArray after, jint afterLength, jint score) { + jchar *beforeChars = env->GetCharArrayElements(before, 0); + jchar *afterChars = env->GetCharArrayElements(after, 0); + jfloat result = Correction::RankingAlgorithm::calcNormalizedScore((unsigned short*)beforeChars, + beforeLength, (unsigned short*)afterChars, afterLength, score); + env->ReleaseCharArrayElements(after, afterChars, JNI_ABORT); + env->ReleaseCharArrayElements(before, beforeChars, JNI_ABORT); + return result; +} +static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jobject object, + jcharArray before, jint beforeLength, jcharArray after, jint afterLength) { + jchar *beforeChars = env->GetCharArrayElements(before, 0); + jchar *afterChars = env->GetCharArrayElements(after, 0); + jint result = Correction::RankingAlgorithm::editDistance( + (unsigned short*)beforeChars, beforeLength, (unsigned short*)afterChars, afterLength); + env->ReleaseCharArrayElements(after, afterChars, JNI_ABORT); + env->ReleaseCharArrayElements(before, beforeChars, JNI_ABORT); return result; } -static void latinime_BinaryDictionary_close(JNIEnv *env, jobject object, jint dict) { +static void latinime_BinaryDictionary_close(JNIEnv *env, jobject object, jlong dict) { Dictionary *dictionary = (Dictionary*)dict; if (!dictionary) return; void *dictBuf = dictionary->getDict(); if (!dictBuf) return; #ifdef USE_MMAP_FOR_DICTIONARY - int ret = munmap((void *)((char *)dictBuf - dictionary->getDictBufAdjust()), - dictionary->getDictSize() + dictionary->getDictBufAdjust()); + releaseDictBuf((void *)((char *)dictBuf - dictionary->getDictBufAdjust()), + dictionary->getDictSize() + dictionary->getDictBufAdjust(), dictionary->getMmapFd()); +#else // USE_MMAP_FOR_DICTIONARY + releaseDictBuf(dictBuf, 0, 0); +#endif // USE_MMAP_FOR_DICTIONARY + delete dictionary; +} + +void releaseDictBuf(void* dictBuf, const size_t length, int fd) { +#ifdef USE_MMAP_FOR_DICTIONARY + int ret = munmap(dictBuf, length); if (ret != 0) { - LOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); + AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); } - ret = close(dictionary->getMmapFd()); + ret = close(fd); if (ret != 0) { - LOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); + AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); } #else // USE_MMAP_FOR_DICTIONARY free(dictBuf); #endif // USE_MMAP_FOR_DICTIONARY - delete dictionary; } static JNINativeMethod sMethods[] = { - {"openNative", "(Ljava/lang/String;JJIIIII)I", (void*)latinime_BinaryDictionary_open}, - {"closeNative", "(I)V", (void*)latinime_BinaryDictionary_close}, - {"getSuggestionsNative", "(II[I[I[III[C[I)I", (void*)latinime_BinaryDictionary_getSuggestions}, - {"isValidWordNative", "(I[CI)Z", (void*)latinime_BinaryDictionary_isValidWord}, - {"getBigramsNative", "(I[CI[II[C[IIII)I", (void*)latinime_BinaryDictionary_getBigrams} + {"openNative", "(Ljava/lang/String;JJIIII)J", (void*)latinime_BinaryDictionary_open}, + {"closeNative", "(J)V", (void*)latinime_BinaryDictionary_close}, + {"getSuggestionsNative", "(JJ[I[I[II[IZ[C[I)I", + (void*)latinime_BinaryDictionary_getSuggestions}, + {"getFrequencyNative", "(J[II)I", (void*)latinime_BinaryDictionary_getFrequency}, + {"isValidBigramNative", "(J[I[I)Z", (void*)latinime_BinaryDictionary_isValidBigram}, + {"getBigramsNative", "(J[II[II[C[III)I", (void*)latinime_BinaryDictionary_getBigrams}, + {"calcNormalizedScoreNative", "([CI[CII)F", + (void*)latinime_BinaryDictionary_calcNormalizedScore}, + {"editDistanceNative", "([CI[CI)I", (void*)latinime_BinaryDictionary_editDistance} }; int register_BinaryDictionary(JNIEnv *env) { diff --git a/native/jni/com_android_inputmethod_latin_NativeUtils.cpp b/native/jni/com_android_inputmethod_latin_NativeUtils.cpp new file mode 100644 index 000000000..c1e586a4b --- /dev/null +++ b/native/jni/com_android_inputmethod_latin_NativeUtils.cpp @@ -0,0 +1,40 @@ +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#include "com_android_inputmethod_latin_NativeUtils.h" +#include "jni.h" +#include "jni_common.h" + +#include <math.h> + +namespace latinime { + +static float latinime_NativeUtils_powf(float x, float y) { + return powf(x, y); +} + +static JNINativeMethod sMethods[] = { + {"powf", "(FF)F", (void*)latinime_NativeUtils_powf} +}; + +int register_NativeUtils(JNIEnv *env) { + const char* const kClassPathName = "com/android/inputmethod/latin/NativeUtils"; + return registerNativeMethods(env, kClassPathName, sMethods, + sizeof(sMethods) / sizeof(sMethods[0])); +} + +} // namespace latinime diff --git a/native/jni/com_android_inputmethod_latin_NativeUtils.h b/native/jni/com_android_inputmethod_latin_NativeUtils.h new file mode 100644 index 000000000..13a348a5c --- /dev/null +++ b/native/jni/com_android_inputmethod_latin_NativeUtils.h @@ -0,0 +1,29 @@ +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_NATIVEUTILS_H +#define _COM_ANDROID_INPUTMETHOD_LATIN_NATIVEUTILS_H + +#include "jni.h" + +namespace latinime { + +int register_NativeUtils(JNIEnv *env); + +} + +#endif // _COM_ANDROID_INPUTMETHOD_LATIN_NATIVEUTILS_H diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp index 8643f723f..1314bab27 100644 --- a/native/jni/jni_common.cpp +++ b/native/jni/jni_common.cpp @@ -19,6 +19,8 @@ #include "com_android_inputmethod_keyboard_ProximityInfo.h" #include "com_android_inputmethod_latin_BinaryDictionary.h" +#include "com_android_inputmethod_latin_NativeUtils.h" +#include "defines.h" #include "jni.h" #include "proximity_info.h" @@ -32,22 +34,27 @@ using namespace latinime; * Returns the JNI version on success, -1 on failure. */ jint JNI_OnLoad(JavaVM* vm, void* reserved) { - JNIEnv* env = NULL; + JNIEnv* env = 0; jint result = -1; if (vm->GetEnv((void**) &env, JNI_VERSION_1_4) != JNI_OK) { - LOGE("ERROR: GetEnv failed"); + AKLOGE("ERROR: GetEnv failed"); goto bail; } - assert(env != NULL); + assert(env != 0); if (!register_BinaryDictionary(env)) { - LOGE("ERROR: BinaryDictionary native registration failed"); + AKLOGE("ERROR: BinaryDictionary native registration failed"); goto bail; } if (!register_ProximityInfo(env)) { - LOGE("ERROR: ProximityInfo native registration failed"); + AKLOGE("ERROR: ProximityInfo native registration failed"); + goto bail; + } + + if (!register_NativeUtils(env)) { + AKLOGE("ERROR: NativeUtils native registration failed"); goto bail; } @@ -63,12 +70,12 @@ namespace latinime { int registerNativeMethods(JNIEnv* env, const char* className, JNINativeMethod* methods, int numMethods) { jclass clazz = env->FindClass(className); - if (clazz == NULL) { - LOGE("Native registration unable to find class '%s'", className); + if (clazz == 0) { + AKLOGE("Native registration unable to find class '%s'", className); return JNI_FALSE; } if (env->RegisterNatives(clazz, methods, numMethods) < 0) { - LOGE("RegisterNatives failed for '%s'", className); + AKLOGE("RegisterNatives failed for '%s'", className); env->DeleteLocalRef(clazz); return JNI_FALSE; } diff --git a/native/jni/jni_common.h b/native/jni/jni_common.h index c502fa3a8..6741443ac 100644 --- a/native/jni/jni_common.h +++ b/native/jni/jni_common.h @@ -18,13 +18,43 @@ #ifndef LATINIME_JNI_COMMON_H #define LATINIME_JNI_COMMON_H +#include <stdlib.h> + #include "jni.h" namespace latinime { -int registerNativeMethods(JNIEnv* env, const char* className, JNINativeMethod* methods, +int registerNativeMethods(JNIEnv *env, const char *className, JNINativeMethod *methods, int numMethods); +inline jint *safeGetIntArrayElements(JNIEnv *env, jintArray jArray) { + if (jArray) { + return env->GetIntArrayElements(jArray, 0); + } else { + return 0; + } +} + +inline jfloat *safeGetFloatArrayElements(JNIEnv *env, jfloatArray jArray) { + if (jArray) { + return env->GetFloatArrayElements(jArray, 0); + } else { + return 0; + } +} + +inline void safeReleaseIntArrayElements(JNIEnv *env, jintArray jArray, jint *cArray) { + if (jArray) { + env->ReleaseIntArrayElements(jArray, cArray, 0); + } +} + +inline void safeReleaseFloatArrayElements(JNIEnv *env, jfloatArray jArray, jfloat *cArray) { + if (jArray) { + env->ReleaseFloatArrayElements(jArray, cArray, 0); + } +} + } // namespace latinime #endif // LATINIME_JNI_COMMON_H diff --git a/native/jni/src/additional_proximity_chars.cpp b/native/jni/src/additional_proximity_chars.cpp new file mode 100644 index 000000000..224f020f2 --- /dev/null +++ b/native/jni/src/additional_proximity_chars.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "additional_proximity_chars.h" + +namespace latinime { +const std::string AdditionalProximityChars::LOCALE_EN_US("en"); + +const int32_t AdditionalProximityChars::EN_US_ADDITIONAL_A[EN_US_ADDITIONAL_A_SIZE] = { + 'e', 'i', 'o', 'u' +}; + +const int32_t AdditionalProximityChars::EN_US_ADDITIONAL_E[EN_US_ADDITIONAL_E_SIZE] = { + 'a', 'i', 'o', 'u' +}; + +const int32_t AdditionalProximityChars::EN_US_ADDITIONAL_I[EN_US_ADDITIONAL_I_SIZE] = { + 'a', 'e', 'o', 'u' +}; + +const int32_t AdditionalProximityChars::EN_US_ADDITIONAL_O[EN_US_ADDITIONAL_O_SIZE] = { + 'a', 'e', 'i', 'u' +}; + +const int32_t AdditionalProximityChars::EN_US_ADDITIONAL_U[EN_US_ADDITIONAL_U_SIZE] = { + 'a', 'e', 'i', 'o' +}; +} diff --git a/native/jni/src/additional_proximity_chars.h b/native/jni/src/additional_proximity_chars.h new file mode 100644 index 000000000..82c31f860 --- /dev/null +++ b/native/jni/src/additional_proximity_chars.h @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_ADDITIONAL_PROXIMITY_CHARS_H +#define LATINIME_ADDITIONAL_PROXIMITY_CHARS_H + +#include <stdint.h> +#include <string> + +#include "defines.h" + +namespace latinime { + +class AdditionalProximityChars { + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(AdditionalProximityChars); + static const std::string LOCALE_EN_US; + static const int EN_US_ADDITIONAL_A_SIZE = 4; + static const int32_t EN_US_ADDITIONAL_A[]; + static const int EN_US_ADDITIONAL_E_SIZE = 4; + static const int32_t EN_US_ADDITIONAL_E[]; + static const int EN_US_ADDITIONAL_I_SIZE = 4; + static const int32_t EN_US_ADDITIONAL_I[]; + static const int EN_US_ADDITIONAL_O_SIZE = 4; + static const int32_t EN_US_ADDITIONAL_O[]; + static const int EN_US_ADDITIONAL_U_SIZE = 4; + static const int32_t EN_US_ADDITIONAL_U[]; + + static bool isEnLocale(const std::string *locale_str) { + return locale_str && locale_str->size() >= LOCALE_EN_US.size() + && LOCALE_EN_US.compare(0, LOCALE_EN_US.size(), *locale_str); + } + + public: + static int getAdditionalCharsSize(const std::string* locale_str, const int32_t c) { + if (!isEnLocale(locale_str)) { + return 0; + } + switch(c) { + case 'a': + return EN_US_ADDITIONAL_A_SIZE; + case 'e': + return EN_US_ADDITIONAL_E_SIZE; + case 'i': + return EN_US_ADDITIONAL_I_SIZE; + case 'o': + return EN_US_ADDITIONAL_O_SIZE; + case 'u': + return EN_US_ADDITIONAL_U_SIZE; + default: + return 0; + } + } + + static const int32_t* getAdditionalChars(const std::string *locale_str, const int32_t c) { + if (!isEnLocale(locale_str)) { + return 0; + } + switch(c) { + case 'a': + return EN_US_ADDITIONAL_A; + case 'e': + return EN_US_ADDITIONAL_E; + case 'i': + return EN_US_ADDITIONAL_I; + case 'o': + return EN_US_ADDITIONAL_O; + case 'u': + return EN_US_ADDITIONAL_U; + default: + return 0; + } + } + + static bool hasAdditionalChars(const std::string *locale_str, const int32_t c) { + return getAdditionalCharsSize(locale_str, c) > 0; + } +}; + +} + +#endif // LATINIME_ADDITIONAL_PROXIMITY_CHARS_H diff --git a/native/src/basechars.h b/native/jni/src/basechars.cpp index 5a4406606..31f1e18a8 100644 --- a/native/src/basechars.h +++ b/native/jni/src/basechars.cpp @@ -1,10 +1,30 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "char_utils.h" + +namespace latinime { + /** * Table mapping most combined Latin, Greek, and Cyrillic characters * to their base characters. If c is in range, BASE_CHARS[c] == c * if c is not a combined character, or the base character if it * is combined. */ -static unsigned short BASE_CHARS[] = { +const unsigned short BASE_CHARS[BASE_CHARS_SIZE] = { 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, @@ -170,3 +190,5 @@ static unsigned short BASE_CHARS[] = { // generated with: // cat UnicodeData.txt | perl -e 'while (<>) { @foo = split(/;/); $foo[5] =~ s/<.*> //; $base[hex($foo[0])] = hex($foo[5]);} for ($i = 0; $i < 0x500; $i += 8) { for ($j = $i; $j < $i + 8; $j++) { printf("0x%04x, ", $base[$j] ? $base[$j] : $j)}; print "\n"; }' + +} // namespace latinime diff --git a/native/jni/src/bigram_dictionary.cpp b/native/jni/src/bigram_dictionary.cpp new file mode 100644 index 000000000..144336981 --- /dev/null +++ b/native/jni/src/bigram_dictionary.cpp @@ -0,0 +1,218 @@ +/* +** +** Copyright 2010, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#include <string.h> + +#define LOG_TAG "LatinIME: bigram_dictionary.cpp" + +#include "bigram_dictionary.h" +#include "binary_format.h" +#include "bloom_filter.h" +#include "defines.h" +#include "dictionary.h" + +namespace latinime { + +BigramDictionary::BigramDictionary(const unsigned char *dict, int maxWordLength) + : DICT(dict), MAX_WORD_LENGTH(maxWordLength) { + if (DEBUG_DICT) { + AKLOGI("BigramDictionary - constructor"); + } +} + +BigramDictionary::~BigramDictionary() { +} + +bool BigramDictionary::addWordBigram(unsigned short *word, int length, int frequency, + const int maxBigrams, int *bigramFreq, unsigned short *bigramChars) const { + word[length] = 0; + if (DEBUG_DICT) { +#ifdef FLAG_DBG + char s[length + 1]; + for (int i = 0; i <= length; i++) s[i] = word[i]; + AKLOGI("Bigram: Found word = %s, freq = %d :", s, frequency); +#endif + } + + // Find the right insertion point + int insertAt = 0; + while (insertAt < maxBigrams) { + if (frequency > bigramFreq[insertAt] || (bigramFreq[insertAt] == frequency + && length < Dictionary::wideStrLen(bigramChars + insertAt * MAX_WORD_LENGTH))) { + break; + } + insertAt++; + } + if (DEBUG_DICT) { + AKLOGI("Bigram: InsertAt -> %d maxBigrams: %d", insertAt, maxBigrams); + } + if (insertAt < maxBigrams) { + memmove((char*) bigramFreq + (insertAt + 1) * sizeof(bigramFreq[0]), + (char*) bigramFreq + insertAt * sizeof(bigramFreq[0]), + (maxBigrams - insertAt - 1) * sizeof(bigramFreq[0])); + bigramFreq[insertAt] = frequency; + memmove((char*) bigramChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short), + (char*) bigramChars + (insertAt ) * MAX_WORD_LENGTH * sizeof(short), + (maxBigrams - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH); + unsigned short *dest = bigramChars + (insertAt ) * MAX_WORD_LENGTH; + while (length--) { + *dest++ = *word++; + } + *dest = 0; // NULL terminate + if (DEBUG_DICT) { + AKLOGI("Bigram: Added word at %d", insertAt); + } + return true; + } + return false; +} + +/* Parameters : + * prevWord: the word before, the one for which we need to look up bigrams. + * prevWordLength: its length. + * inputCodes: what user typed, in the same format as for UnigramDictionary::getSuggestions. + * codesSize: the size of the codes array. + * bigramChars: an array for output, at the same format as outwords for getSuggestions. + * bigramFreq: an array to output frequencies. + * maxWordLength: the maximum size of a word. + * maxBigrams: the maximum number of bigrams fitting in the bigramChars array. + * This method returns the number of bigrams this word has, for backward compatibility. + * Note: this is not the number of bigrams output in the array, which is the number of + * bigrams this word has WHOSE first letter also matches the letter the user typed. + * TODO: this may not be a sensible thing to do. It makes sense when the bigrams are + * used to match the first letter of the second word, but once the user has typed more + * and the bigrams are used to boost unigram result scores, it makes little sense to + * reduce their scope to the ones that match the first letter. + */ +int BigramDictionary::getBigrams(const int32_t *prevWord, int prevWordLength, int *inputCodes, + int codesSize, unsigned short *bigramChars, int *bigramFreq, int maxWordLength, + int maxBigrams) const { + // TODO: remove unused arguments, and refrain from storing stuff in members of this class + // TODO: have "in" arguments before "out" ones, and make out args explicit in the name + + const uint8_t* const root = DICT; + int pos = getBigramListPositionForWord(prevWord, prevWordLength); + // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams + if (0 == pos) return 0; + int bigramFlags; + int bigramCount = 0; + do { + bigramFlags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + uint16_t bigramBuffer[MAX_WORD_LENGTH]; + int unigramFreq = 0; + const int bigramPos = BinaryFormat::getAttributeAddressAndForwardPointer(root, bigramFlags, + &pos); + const int length = BinaryFormat::getWordAtAddress(root, bigramPos, MAX_WORD_LENGTH, + bigramBuffer, &unigramFreq); + + // codesSize == 0 means we are trying to find bigram predictions. + if (codesSize < 1 || checkFirstCharacter(bigramBuffer, inputCodes)) { + const int bigramFreqTemp = UnigramDictionary::MASK_ATTRIBUTE_FREQUENCY & bigramFlags; + // Due to space constraints, the frequency for bigrams is approximate - the lower the + // unigram frequency, the worse the precision. The theoritical maximum error in + // resulting frequency is 8 - although in the practice it's never bigger than 3 or 4 + // in very bad cases. This means that sometimes, we'll see some bigrams interverted + // here, but it can't get too bad. + const int frequency = + BinaryFormat::computeFrequencyForBigram(unigramFreq, bigramFreqTemp); + if (addWordBigram( + bigramBuffer, length, frequency, maxBigrams, bigramFreq, bigramChars)) { + ++bigramCount; + } + } + } while (UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags); + return bigramCount; +} + +// Returns a pointer to the start of the bigram list. +// If the word is not found or has no bigrams, this function returns 0. +int BigramDictionary::getBigramListPositionForWord(const int32_t *prevWord, + const int prevWordLength) const { + if (0 >= prevWordLength) return 0; + const uint8_t* const root = DICT; + int pos = BinaryFormat::getTerminalPosition(root, prevWord, prevWordLength); + + if (NOT_VALID_WORD == pos) return 0; + const int flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + if (0 == (flags & UnigramDictionary::FLAG_HAS_BIGRAMS)) return 0; + if (0 == (flags & UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS)) { + BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + } else { + pos = BinaryFormat::skipOtherCharacters(root, pos); + } + pos = BinaryFormat::skipFrequency(flags, pos); + pos = BinaryFormat::skipChildrenPosition(flags, pos); + pos = BinaryFormat::skipShortcuts(root, flags, pos); + return pos; +} + +void BigramDictionary::fillBigramAddressToFrequencyMapAndFilter(const int32_t *prevWord, + const int prevWordLength, std::map<int, int> *map, uint8_t *filter) const { + memset(filter, 0, BIGRAM_FILTER_BYTE_SIZE); + const uint8_t* const root = DICT; + int pos = getBigramListPositionForWord(prevWord, prevWordLength); + if (0 == pos) return; + + int bigramFlags; + do { + bigramFlags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + const int frequency = UnigramDictionary::MASK_ATTRIBUTE_FREQUENCY & bigramFlags; + const int bigramPos = BinaryFormat::getAttributeAddressAndForwardPointer(root, bigramFlags, + &pos); + (*map)[bigramPos] = frequency; + setInFilter(filter, bigramPos); + } while (0 != (UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags)); +} + +bool BigramDictionary::checkFirstCharacter(unsigned short *word, int *inputCodes) const { + // Checks whether this word starts with same character or neighboring characters of + // what user typed. + + int maxAlt = MAX_ALTERNATIVES; + const unsigned short firstBaseChar = toBaseLowerCase(*word); + while (maxAlt > 0) { + if (toBaseLowerCase(*inputCodes) == firstBaseChar) { + return true; + } + inputCodes++; + maxAlt--; + } + return false; +} + +bool BigramDictionary::isValidBigram(const int32_t *word1, int length1, const int32_t *word2, + int length2) const { + const uint8_t* const root = DICT; + int pos = getBigramListPositionForWord(word1, length1); + // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams + if (0 == pos) return false; + int nextWordPos = BinaryFormat::getTerminalPosition(root, word2, length2); + if (NOT_VALID_WORD == nextWordPos) return false; + int bigramFlags; + do { + bigramFlags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + const int bigramPos = BinaryFormat::getAttributeAddressAndForwardPointer(root, bigramFlags, + &pos); + if (bigramPos == nextWordPos) { + return true; + } + } while (UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT & bigramFlags); + return false; +} + +// TODO: Move functions related to bigram to here +} // namespace latinime diff --git a/native/src/bigram_dictionary.h b/native/jni/src/bigram_dictionary.h index c07458a38..1ff1b2ec6 100644 --- a/native/src/bigram_dictionary.h +++ b/native/jni/src/bigram_dictionary.h @@ -17,38 +17,39 @@ #ifndef LATINIME_BIGRAM_DICTIONARY_H #define LATINIME_BIGRAM_DICTIONARY_H +#include <map> +#include <stdint.h> + +#include "defines.h" + namespace latinime { class Dictionary; class BigramDictionary { -public: - BigramDictionary(const unsigned char *dict, int maxWordLength, int maxAlternatives, - const bool isLatestDictVersion, const bool hasBigram, Dictionary *parentDictionary); - int getBigrams(unsigned short *word, int length, int *codes, int codesSize, - unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams, - int maxAlternatives); + public: + BigramDictionary(const unsigned char *dict, int maxWordLength); + int getBigrams(const int32_t *word, int length, int *inputCodes, int codesSize, + unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams) const; + int getBigramListPositionForWord(const int32_t *prevWord, const int prevWordLength) const; + void fillBigramAddressToFrequencyMapAndFilter(const int32_t *prevWord, const int prevWordLength, + std::map<int, int> *map, uint8_t *filter) const; + bool isValidBigram(const int32_t *word1, int length1, const int32_t *word2, int length2) const; ~BigramDictionary(); -private: - bool addWordBigram(unsigned short *word, int length, int frequency); + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(BigramDictionary); + bool addWordBigram(unsigned short *word, int length, int frequency, const int maxBigrams, + int *bigramFreq, unsigned short *bigramChars) const; int getBigramAddress(int *pos, bool advance); int getBigramFreq(int *pos); void searchForTerminalNode(int addressLookingFor, int frequency); bool getFirstBitOfByte(int *pos) { return (DICT[*pos] & 0x80) > 0; } bool getSecondBitOfByte(int *pos) { return (DICT[*pos] & 0x40) > 0; } - bool checkFirstCharacter(unsigned short *word); + bool checkFirstCharacter(unsigned short *word, int *inputCodes) const; const unsigned char *DICT; const int MAX_WORD_LENGTH; - const int MAX_ALTERNATIVES; - const bool IS_LATEST_DICT_VERSION; - const bool HAS_BIGRAM; - - Dictionary *mParentDictionary; - int *mBigramFreq; - int mMaxBigrams; - unsigned short *mBigramChars; - int *mInputCodes; - int mInputLength; + // TODO: Re-implement proximity correction for bigram correction + static const int MAX_ALTERNATIVES = 1; }; } // namespace latinime diff --git a/native/jni/src/binary_format.h b/native/jni/src/binary_format.h new file mode 100644 index 000000000..214ecfa8d --- /dev/null +++ b/native/jni/src/binary_format.h @@ -0,0 +1,562 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_BINARY_FORMAT_H +#define LATINIME_BINARY_FORMAT_H + +#include <limits> +#include "bloom_filter.h" +#include "unigram_dictionary.h" + +namespace latinime { + +class BinaryFormat { + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryFormat); + const static int32_t MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; + const static int32_t CHARACTER_ARRAY_TERMINATOR = 0x1F; + const static int MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE = 2; + + public: + const static int UNKNOWN_FORMAT = -1; + // Originally, format version 1 had a 16-bit magic number, then the version number `01' + // then options that must be 0. Hence the first 32-bits of the format are always as follow + // and it's okay to consider them a magic number as a whole. + const static uint32_t FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; + const static unsigned int FORMAT_VERSION_1_HEADER_SIZE = 5; + // The versions of Latin IME that only handle format version 1 only test for the magic + // number, so we had to change it so that version 2 files would be rejected by older + // implementations. On this occasion, we made the magic number 32 bits long. + const static uint32_t FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; + + const static int CHARACTER_ARRAY_TERMINATOR_SIZE = 1; + const static int SHORTCUT_LIST_SIZE_SIZE = 2; + + static int detectFormat(const uint8_t* const dict); + static unsigned int getHeaderSize(const uint8_t* const dict); + static unsigned int getFlags(const uint8_t* const dict); + static int getGroupCountAndForwardPointer(const uint8_t* const dict, int* pos); + static uint8_t getFlagsAndForwardPointer(const uint8_t* const dict, int* pos); + static int32_t getCharCodeAndForwardPointer(const uint8_t* const dict, int* pos); + static int readFrequencyWithoutMovingPointer(const uint8_t* const dict, const int pos); + static int skipOtherCharacters(const uint8_t* const dict, const int pos); + static int skipChildrenPosition(const uint8_t flags, const int pos); + static int skipFrequency(const uint8_t flags, const int pos); + static int skipShortcuts(const uint8_t* const dict, const uint8_t flags, const int pos); + static int skipBigrams(const uint8_t* const dict, const uint8_t flags, const int pos); + static int skipAllAttributes(const uint8_t* const dict, const uint8_t flags, const int pos); + static int skipChildrenPosAndAttributes(const uint8_t* const dict, const uint8_t flags, + const int pos); + static int readChildrenPosition(const uint8_t* const dict, const uint8_t flags, const int pos); + static bool hasChildrenInFlags(const uint8_t flags); + static int getAttributeAddressAndForwardPointer(const uint8_t* const dict, const uint8_t flags, + int *pos); + static int getTerminalPosition(const uint8_t* const root, const int32_t* const inWord, + const int length); + static int getWordAtAddress(const uint8_t* const root, const int address, const int maxDepth, + uint16_t* outWord, int* outUnigramFrequency); + static int computeFrequencyForBigram(const int unigramFreq, const int bigramFreq); + static int getProbability(const int position, const std::map<int, int> *bigramMap, + const uint8_t *bigramFilter, const int unigramFreq); + + // Flags for special processing + // Those *must* match the flags in makedict (BinaryDictInputOutput#*_PROCESSING_FLAG) or + // something very bad (like, the apocalypse) will happen. Please update both at the same time. + enum { + REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1, + REQUIRES_FRENCH_LIGATURES_PROCESSING = 0x4 + }; + const static unsigned int NO_FLAGS = 0; +}; + +inline int BinaryFormat::detectFormat(const uint8_t* const dict) { + // The magic number is stored big-endian. + const uint32_t magicNumber = (dict[0] << 24) + (dict[1] << 16) + (dict[2] << 8) + dict[3]; + switch (magicNumber) { + case FORMAT_VERSION_1_MAGIC_NUMBER: + // Format 1 header is exactly 5 bytes long and looks like: + // Magic number (2 bytes) 0x78 0xB1 + // Version number (1 byte) 0x01 + // Options (2 bytes) must be 0x00 0x00 + return 1; + case FORMAT_VERSION_2_MAGIC_NUMBER: + // Format 2 header is as follows: + // Magic number (4 bytes) 0x9B 0xC1 0x3A 0xFE + // Version number (2 bytes) 0x00 0x02 + // Options (2 bytes) + // Header size (4 bytes) : integer, big endian + return (dict[4] << 8) + dict[5]; + default: + return UNKNOWN_FORMAT; + } +} + +inline unsigned int BinaryFormat::getFlags(const uint8_t* const dict) { + switch (detectFormat(dict)) { + case 1: + return NO_FLAGS; + default: + return (dict[6] << 8) + dict[7]; + } +} + +inline unsigned int BinaryFormat::getHeaderSize(const uint8_t* const dict) { + switch (detectFormat(dict)) { + case 1: + return FORMAT_VERSION_1_HEADER_SIZE; + case 2: + // See the format of the header in the comment in detectFormat() above + return (dict[8] << 24) + (dict[9] << 16) + (dict[10] << 8) + dict[11]; + default: + return std::numeric_limits<unsigned int>::max(); + } +} + +inline int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t* const dict, int* pos) { + const int msb = dict[(*pos)++]; + if (msb < 0x80) return msb; + return ((msb & 0x7F) << 8) | dict[(*pos)++]; +} + +inline uint8_t BinaryFormat::getFlagsAndForwardPointer(const uint8_t* const dict, int* pos) { + return dict[(*pos)++]; +} + +inline int32_t BinaryFormat::getCharCodeAndForwardPointer(const uint8_t* const dict, int* pos) { + const int origin = *pos; + const int32_t character = dict[origin]; + if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + if (character == CHARACTER_ARRAY_TERMINATOR) { + *pos = origin + 1; + return NOT_A_CHARACTER; + } else { + *pos = origin + 3; + const int32_t char_1 = character << 16; + const int32_t char_2 = char_1 + (dict[origin + 1] << 8); + return char_2 + dict[origin + 2]; + } + } else { + *pos = origin + 1; + return character; + } +} + +inline int BinaryFormat::readFrequencyWithoutMovingPointer(const uint8_t* const dict, + const int pos) { + return dict[pos]; +} + +inline int BinaryFormat::skipOtherCharacters(const uint8_t* const dict, const int pos) { + int currentPos = pos; + int32_t character = dict[currentPos++]; + while (CHARACTER_ARRAY_TERMINATOR != character) { + if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + currentPos += MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE; + } + character = dict[currentPos++]; + } + return currentPos; +} + +static inline int attributeAddressSize(const uint8_t flags) { + static const int ATTRIBUTE_ADDRESS_SHIFT = 4; + return (flags & UnigramDictionary::MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT; + /* Note: this is a value-dependant optimization of what may probably be + more readably written this way: + switch (flags * UnigramDictionary::MASK_ATTRIBUTE_ADDRESS_TYPE) { + case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1; + case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2; + case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3; + default: return 0; + } + */ +} + +static inline int skipExistingBigrams(const uint8_t* const dict, const int pos) { + int currentPos = pos; + uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dict, ¤tPos); + while (flags & UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT) { + currentPos += attributeAddressSize(flags); + flags = BinaryFormat::getFlagsAndForwardPointer(dict, ¤tPos); + } + currentPos += attributeAddressSize(flags); + return currentPos; +} + +static inline int childrenAddressSize(const uint8_t flags) { + static const int CHILDREN_ADDRESS_SHIFT = 6; + return (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags) >> CHILDREN_ADDRESS_SHIFT; + /* See the note in attributeAddressSize. The same applies here */ +} + +static inline int shortcutByteSize(const uint8_t* const dict, const int pos) { + return ((int)(dict[pos] << 8)) + (dict[pos + 1]); +} + +inline int BinaryFormat::skipChildrenPosition(const uint8_t flags, const int pos) { + return pos + childrenAddressSize(flags); +} + +inline int BinaryFormat::skipFrequency(const uint8_t flags, const int pos) { + return UnigramDictionary::FLAG_IS_TERMINAL & flags ? pos + 1 : pos; +} + +inline int BinaryFormat::skipShortcuts(const uint8_t* const dict, const uint8_t flags, + const int pos) { + if (UnigramDictionary::FLAG_HAS_SHORTCUT_TARGETS & flags) { + return pos + shortcutByteSize(dict, pos); + } else { + return pos; + } +} + +inline int BinaryFormat::skipBigrams(const uint8_t* const dict, const uint8_t flags, + const int pos) { + if (UnigramDictionary::FLAG_HAS_BIGRAMS & flags) { + return skipExistingBigrams(dict, pos); + } else { + return pos; + } +} + +inline int BinaryFormat::skipAllAttributes(const uint8_t* const dict, const uint8_t flags, + const int pos) { + // This function skips all attributes: shortcuts and bigrams. + int newPos = pos; + newPos = skipShortcuts(dict, flags, newPos); + newPos = skipBigrams(dict, flags, newPos); + return newPos; +} + +inline int BinaryFormat::skipChildrenPosAndAttributes(const uint8_t* const dict, + const uint8_t flags, const int pos) { + int currentPos = pos; + currentPos = skipChildrenPosition(flags, currentPos); + currentPos = skipAllAttributes(dict, flags, currentPos); + return currentPos; +} + +inline int BinaryFormat::readChildrenPosition(const uint8_t* const dict, const uint8_t flags, + const int pos) { + int offset = 0; + switch (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags) { + case UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + offset = dict[pos]; + break; + case UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + offset = dict[pos] << 8; + offset += dict[pos + 1]; + break; + case UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + offset = dict[pos] << 16; + offset += dict[pos + 1] << 8; + offset += dict[pos + 2]; + break; + default: + // If we come here, it means we asked for the children of a word with + // no children. + return -1; + } + return pos + offset; +} + +inline bool BinaryFormat::hasChildrenInFlags(const uint8_t flags) { + return (UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_NOADDRESS + != (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags)); +} + +inline int BinaryFormat::getAttributeAddressAndForwardPointer(const uint8_t* const dict, + const uint8_t flags, int *pos) { + int offset = 0; + const int origin = *pos; + switch (UnigramDictionary::MASK_ATTRIBUTE_ADDRESS_TYPE & flags) { + case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + offset = dict[origin]; + *pos = origin + 1; + break; + case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + offset = dict[origin] << 8; + offset += dict[origin + 1]; + *pos = origin + 2; + break; + case UnigramDictionary::FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + offset = dict[origin] << 16; + offset += dict[origin + 1] << 8; + offset += dict[origin + 2]; + *pos = origin + 3; + break; + } + if (UnigramDictionary::FLAG_ATTRIBUTE_OFFSET_NEGATIVE & flags) { + return origin - offset; + } else { + return origin + offset; + } +} + +// This function gets the byte position of the last chargroup of the exact matching word in the +// dictionary. If no match is found, it returns NOT_VALID_WORD. +inline int BinaryFormat::getTerminalPosition(const uint8_t* const root, + const int32_t* const inWord, const int length) { + int pos = 0; + int wordPos = 0; + + while (true) { + // If we already traversed the tree further than the word is long, there means + // there was no match (or we would have found it). + if (wordPos > length) return NOT_VALID_WORD; + int charGroupCount = BinaryFormat::getGroupCountAndForwardPointer(root, &pos); + const int32_t wChar = inWord[wordPos]; + while (true) { + // If there are no more character groups in this node, it means we could not + // find a matching character for this depth, therefore there is no match. + if (0 >= charGroupCount) return NOT_VALID_WORD; + const int charGroupPos = pos; + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + int32_t character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + if (character == wChar) { + // This is the correct node. Only one character group may start with the same + // char within a node, so either we found our match in this node, or there is + // no match and we can return NOT_VALID_WORD. So we will check all the characters + // in this character group indeed does match. + if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags) { + character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + while (NOT_A_CHARACTER != character) { + ++wordPos; + // If we shoot the length of the word we search for, or if we find a single + // character that does not match, as explained above, it means the word is + // not in the dictionary (by virtue of this chargroup being the only one to + // match the word on the first character, but not matching the whole word). + if (wordPos > length) return NOT_VALID_WORD; + if (inWord[wordPos] != character) return NOT_VALID_WORD; + character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + } + } + // If we come here we know that so far, we do match. Either we are on a terminal + // and we match the length, in which case we found it, or we traverse children. + // If we don't match the length AND don't have children, then a word in the + // dictionary fully matches a prefix of the searched word but not the full word. + ++wordPos; + if (UnigramDictionary::FLAG_IS_TERMINAL & flags) { + if (wordPos == length) { + return charGroupPos; + } + pos = BinaryFormat::skipFrequency(UnigramDictionary::FLAG_IS_TERMINAL, pos); + } + if (UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_NOADDRESS + == (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags)) { + return NOT_VALID_WORD; + } + // We have children and we are still shorter than the word we are searching for, so + // we need to traverse children. Put the pointer on the children position, and + // break + pos = BinaryFormat::readChildrenPosition(root, flags, pos); + break; + } else { + // This chargroup does not match, so skip the remaining part and go to the next. + if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags) { + pos = BinaryFormat::skipOtherCharacters(root, pos); + } + pos = BinaryFormat::skipFrequency(flags, pos); + pos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos); + } + --charGroupCount; + } + } +} + +// This function searches for a terminal in the dictionary by its address. +// Due to the fact that words are ordered in the dictionary in a strict breadth-first order, +// it is possible to check for this with advantageous complexity. For each node, we search +// for groups with children and compare the children address with the address we look for. +// When we shoot the address we look for, it means the word we look for is in the children +// of the previous group. The only tricky part is the fact that if we arrive at the end of a +// node with the last group's children address still less than what we are searching for, we +// must descend the last group's children (for example, if the word we are searching for starts +// with a z, it's the last group of the root node, so all children addresses will be smaller +// than the address we look for, and we have to descend the z node). +/* Parameters : + * root: the dictionary buffer + * address: the byte position of the last chargroup of the word we are searching for (this is + * what is stored as the "bigram address" in each bigram) + * outword: an array to write the found word, with MAX_WORD_LENGTH size. + * outUnigramFrequency: a pointer to an int to write the frequency into. + * Return value : the length of the word, of 0 if the word was not found. + */ +inline int BinaryFormat::getWordAtAddress(const uint8_t* const root, const int address, + const int maxDepth, uint16_t* outWord, int* outUnigramFrequency) { + int pos = 0; + int wordPos = 0; + + // One iteration of the outer loop iterates through nodes. As stated above, we will only + // traverse nodes that are actually a part of the terminal we are searching, so each time + // we enter this loop we are one depth level further than last time. + // The only reason we count nodes is because we want to reduce the probability of infinite + // looping in case there is a bug. Since we know there is an upper bound to the depth we are + // supposed to traverse, it does not hurt to count iterations. + for (int loopCount = maxDepth; loopCount > 0; --loopCount) { + int lastCandidateGroupPos = 0; + // Let's loop through char groups in this node searching for either the terminal + // or one of its ascendants. + for (int charGroupCount = getGroupCountAndForwardPointer(root, &pos); charGroupCount > 0; + --charGroupCount) { + const int startPos = pos; + const uint8_t flags = getFlagsAndForwardPointer(root, &pos); + const int32_t character = getCharCodeAndForwardPointer(root, &pos); + if (address == startPos) { + // We found the address. Copy the rest of the word in the buffer and return + // the length. + outWord[wordPos] = character; + if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags) { + int32_t nextChar = getCharCodeAndForwardPointer(root, &pos); + // We count chars in order to avoid infinite loops if the file is broken or + // if there is some other bug + int charCount = maxDepth; + while (NOT_A_CHARACTER != nextChar && --charCount > 0) { + outWord[++wordPos] = nextChar; + nextChar = getCharCodeAndForwardPointer(root, &pos); + } + } + *outUnigramFrequency = readFrequencyWithoutMovingPointer(root, pos); + return ++wordPos; + } + // We need to skip past this char group, so skip any remaining chars after the + // first and possibly the frequency. + if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags) { + pos = skipOtherCharacters(root, pos); + } + pos = skipFrequency(flags, pos); + + // The fact that this group has children is very important. Since we already know + // that this group does not match, if it has no children we know it is irrelevant + // to what we are searching for. + const bool hasChildren = (UnigramDictionary::FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != + (UnigramDictionary::MASK_GROUP_ADDRESS_TYPE & flags)); + // We will write in `found' whether we have passed the children address we are + // searching for. For example if we search for "beer", the children of b are less + // than the address we are searching for and the children of c are greater. When we + // come here for c, we realize this is too big, and that we should descend b. + bool found; + if (hasChildren) { + // Here comes the tricky part. First, read the children position. + const int childrenPos = readChildrenPosition(root, flags, pos); + if (childrenPos > address) { + // If the children pos is greater than address, it means the previous chargroup, + // which address is stored in lastCandidateGroupPos, was the right one. + found = true; + } else if (1 >= charGroupCount) { + // However if we are on the LAST group of this node, and we have NOT shot the + // address we should descend THIS node. So we trick the lastCandidateGroupPos + // so that we will descend this node, not the previous one. + lastCandidateGroupPos = startPos; + found = true; + } else { + // Else, we should continue looking. + found = false; + } + } else { + // Even if we don't have children here, we could still be on the last group of this + // node. If this is the case, we should descend the last group that had children, + // and their address is already in lastCandidateGroup. + found = (1 >= charGroupCount); + } + + if (found) { + // Okay, we found the group we should descend. Its address is in + // the lastCandidateGroupPos variable, so we just re-read it. + if (0 != lastCandidateGroupPos) { + const uint8_t lastFlags = + getFlagsAndForwardPointer(root, &lastCandidateGroupPos); + const int32_t lastChar = + getCharCodeAndForwardPointer(root, &lastCandidateGroupPos); + // We copy all the characters in this group to the buffer + outWord[wordPos] = lastChar; + if (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & lastFlags) { + int32_t nextChar = + getCharCodeAndForwardPointer(root, &lastCandidateGroupPos); + int charCount = maxDepth; + while (-1 != nextChar && --charCount > 0) { + outWord[++wordPos] = nextChar; + nextChar = getCharCodeAndForwardPointer(root, &lastCandidateGroupPos); + } + } + ++wordPos; + // Now we only need to branch to the children address. Skip the frequency if + // it's there, read pos, and break to resume the search at pos. + lastCandidateGroupPos = skipFrequency(lastFlags, lastCandidateGroupPos); + pos = readChildrenPosition(root, lastFlags, lastCandidateGroupPos); + break; + } else { + // Here is a little tricky part: we come here if we found out that all children + // addresses in this group are bigger than the address we are searching for. + // Should we conclude the word is not in the dictionary? No! It could still be + // one of the remaining chargroups in this node, so we have to keep looking in + // this node until we find it (or we realize it's not there either, in which + // case it's actually not in the dictionary). Pass the end of this group, ready + // to start the next one. + pos = skipChildrenPosAndAttributes(root, flags, pos); + } + } else { + // If we did not find it, we should record the last children address for the next + // iteration. + if (hasChildren) lastCandidateGroupPos = startPos; + // Now skip the end of this group (children pos and the attributes if any) so that + // our pos is after the end of this char group, at the start of the next one. + pos = skipChildrenPosAndAttributes(root, flags, pos); + } + + } + } + // If we have looked through all the chargroups and found no match, the address is + // not the address of a terminal in this dictionary. + return 0; +} + +static inline int backoff(const int unigramFreq) { + return unigramFreq; + // For some reason, applying the backoff weight gives bad results in tests. To apply the + // backoff weight, we divide the probability by 2, which in our storing format means + // decreasing the score by 8. + // TODO: figure out what's wrong with this. + // return unigramFreq > 8 ? unigramFreq - 8 : (0 == unigramFreq ? 0 : 8); +} + +inline int BinaryFormat::computeFrequencyForBigram(const int unigramFreq, const int bigramFreq) { + // We divide the range [unigramFreq..255] in 16.5 steps - in other words, we want the + // unigram frequency to be the median value of the 17th step from the top. A value of + // 0 for the bigram frequency represents the middle of the 16th step from the top, + // while a value of 15 represents the middle of the top step. + // See makedict.BinaryDictInputOutput for details. + const float stepSize = ((float)MAX_FREQ - unigramFreq) / (1.5f + MAX_BIGRAM_FREQ); + return (int)(unigramFreq + (bigramFreq + 1) * stepSize); +} + +// This returns a probability in log space. +inline int BinaryFormat::getProbability(const int position, const std::map<int, int> *bigramMap, + const uint8_t *bigramFilter, const int unigramFreq) { + if (!bigramMap || !bigramFilter) return backoff(unigramFreq); + if (!isInFilter(bigramFilter, position)) return backoff(unigramFreq); + const std::map<int, int>::const_iterator bigramFreqIt = bigramMap->find(position); + if (bigramFreqIt != bigramMap->end()) { + const int bigramFreq = bigramFreqIt->second; + return computeFrequencyForBigram(unigramFreq, bigramFreq); + } else { + return backoff(unigramFreq); + } +} + +} // namespace latinime + +#endif // LATINIME_BINARY_FORMAT_H diff --git a/native/src/char_utils.h b/native/jni/src/bloom_filter.h index a69a35e7a..7ae6a1fa4 100644 --- a/native/src/char_utils.h +++ b/native/jni/src/bloom_filter.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,25 @@ * limitations under the License. */ -#ifndef LATINIME_CHAR_UTILS_H -#define LATINIME_CHAR_UTILS_H +#ifndef LATINIME_BLOOM_FILTER_H +#define LATINIME_BLOOM_FILTER_H + +#include <stdint.h> + +#include "defines.h" namespace latinime { -unsigned short latin_tolower(unsigned short c); +static inline void setInFilter(uint8_t *filter, const int position) { + const unsigned int bucket = position % BIGRAM_FILTER_MODULO; + filter[bucket >> 3] |= (1 << (bucket & 0x7)); +} + +static inline bool isInFilter(const uint8_t *filter, const int position) { + const unsigned int bucket = position % BIGRAM_FILTER_MODULO; + return filter[bucket >> 3] & (1 << (bucket & 0x7)); +} } // namespace latinime -#endif // LATINIME_CHAR_UTILS_H +#endif // LATINIME_BLOOM_FILTER_H diff --git a/native/src/char_utils.cpp b/native/jni/src/char_utils.cpp index a31a0632c..a31a0632c 100644 --- a/native/src/char_utils.cpp +++ b/native/jni/src/char_utils.cpp diff --git a/native/jni/src/char_utils.h b/native/jni/src/char_utils.h new file mode 100644 index 000000000..607dc5195 --- /dev/null +++ b/native/jni/src/char_utils.h @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_CHAR_UTILS_H +#define LATINIME_CHAR_UTILS_H + +namespace latinime { + +inline static int isAsciiUpper(unsigned short c) { + return c >= 'A' && c <= 'Z'; +} + +inline static unsigned short toAsciiLower(unsigned short c) { + return c - 'A' + 'a'; +} + +inline static int isAscii(unsigned short c) { + return c <= 127; +} + +unsigned short latin_tolower(unsigned short c); + +/** + * Table mapping most combined Latin, Greek, and Cyrillic characters + * to their base characters. If c is in range, BASE_CHARS[c] == c + * if c is not a combined character, or the base character if it + * is combined. + */ + +static const int BASE_CHARS_SIZE = 0x0500; +extern const unsigned short BASE_CHARS[BASE_CHARS_SIZE]; + +inline static unsigned short toBaseChar(unsigned short c) { + if (c < BASE_CHARS_SIZE) { + return BASE_CHARS[c]; + } + return c; +} + +inline static unsigned short toBaseLowerCase(unsigned short c) { + c = toBaseChar(c); + if (isAsciiUpper(c)) { + return toAsciiLower(c); + } else if (isAscii(c)) { + return c; + } + return latin_tolower(c); +} + +} // namespace latinime + +#endif // LATINIME_CHAR_UTILS_H diff --git a/native/jni/src/correction.cpp b/native/jni/src/correction.cpp new file mode 100644 index 000000000..827067b9f --- /dev/null +++ b/native/jni/src/correction.cpp @@ -0,0 +1,1145 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <assert.h> +#include <ctype.h> +#include <math.h> +#include <stdio.h> +#include <string.h> + +#define LOG_TAG "LatinIME: correction.cpp" + +#include "char_utils.h" +#include "correction.h" +#include "defines.h" +#include "dictionary.h" +#include "proximity_info.h" +#include "proximity_info_state.h" + +namespace latinime { + +///////////////////////////// +// edit distance funcitons // +///////////////////////////// + +inline static void initEditDistance(int *editDistanceTable) { + for (int i = 0; i <= MAX_WORD_LENGTH_INTERNAL; ++i) { + editDistanceTable[i] = i; + } +} + +inline static void dumpEditDistance10ForDebug(int *editDistanceTable, + const int editDistanceTableWidth, const int outputLength) { + if (DEBUG_DICT) { + AKLOGI("EditDistanceTable"); + for (int i = 0; i <= 10; ++i) { + int c[11]; + for (int j = 0; j <= 10; ++j) { + if (j < editDistanceTableWidth + 1 && i < outputLength + 1) { + c[j] = (editDistanceTable + i * (editDistanceTableWidth + 1))[j]; + } else { + c[j] = -1; + } + } + AKLOGI("[ %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d ]", + c[0], c[1], c[2], c[3], c[4], c[5], c[6], c[7], c[8], c[9], c[10]); + (void)c; + } + } +} + +inline static void calcEditDistanceOneStep(int *editDistanceTable, const unsigned short *input, + const int inputLength, const unsigned short *output, const int outputLength) { + // TODO: Make sure that editDistance[0 ~ MAX_WORD_LENGTH_INTERNAL] is not touched. + // Let dp[i][j] be editDistanceTable[i * (inputLength + 1) + j]. + // Assuming that dp[0][0] ... dp[outputLength - 1][inputLength] are already calculated, + // and calculate dp[ouputLength][0] ... dp[outputLength][inputLength]. + int *const current = editDistanceTable + outputLength * (inputLength + 1); + const int *const prev = editDistanceTable + (outputLength - 1) * (inputLength + 1); + const int *const prevprev = + outputLength >= 2 ? editDistanceTable + (outputLength - 2) * (inputLength + 1) : 0; + current[0] = outputLength; + const uint32_t co = toBaseLowerCase(output[outputLength - 1]); + const uint32_t prevCO = outputLength >= 2 ? toBaseLowerCase(output[outputLength - 2]) : 0; + for (int i = 1; i <= inputLength; ++i) { + const uint32_t ci = toBaseLowerCase(input[i - 1]); + const uint16_t cost = (ci == co) ? 0 : 1; + current[i] = min(current[i - 1] + 1, min(prev[i] + 1, prev[i - 1] + cost)); + if (i >= 2 && prevprev && ci == prevCO && co == toBaseLowerCase(input[i - 2])) { + current[i] = min(current[i], prevprev[i - 2] + 1); + } + } +} + +inline static int getCurrentEditDistance(int *editDistanceTable, const int editDistanceTableWidth, + const int outputLength, const int inputLength) { + if (DEBUG_EDIT_DISTANCE) { + AKLOGI("getCurrentEditDistance %d, %d", inputLength, outputLength); + } + return editDistanceTable[(editDistanceTableWidth + 1) * (outputLength) + inputLength]; +} + +////////////////////// +// inline functions // +////////////////////// +static const char QUOTE = '\''; + +inline bool Correction::isQuote(const unsigned short c) { + const unsigned short userTypedChar = mProximityInfoState.getPrimaryCharAt(mInputIndex); + return (c == QUOTE && userTypedChar != QUOTE); +} + +//////////////// +// Correction // +//////////////// + +void Correction::resetCorrection() { + mTotalTraverseCount = 0; +} + +void Correction::initCorrection(const ProximityInfo *pi, const int inputLength, + const int maxDepth) { + mProximityInfo = pi; + mInputLength = inputLength; + mMaxDepth = maxDepth; + mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2; + // TODO: This is not supposed to be required. Check what's going wrong with + // editDistance[0 ~ MAX_WORD_LENGTH_INTERNAL] + initEditDistance(mEditDistanceTable); +} + +void Correction::initCorrectionState( + const int rootPos, const int childCount, const bool traverseAll) { + latinime::initCorrectionState(mCorrectionStates, rootPos, childCount, traverseAll); + // TODO: remove + mCorrectionStates[0].mTransposedPos = mTransposedPos; + mCorrectionStates[0].mExcessivePos = mExcessivePos; + mCorrectionStates[0].mSkipPos = mSkipPos; +} + +void Correction::setCorrectionParams(const int skipPos, const int excessivePos, + const int transposedPos, const int spaceProximityPos, const int missingSpacePos, + const bool useFullEditDistance, const bool doAutoCompletion, const int maxErrors) { + // TODO: remove + mTransposedPos = transposedPos; + mExcessivePos = excessivePos; + mSkipPos = skipPos; + // TODO: remove + mCorrectionStates[0].mTransposedPos = transposedPos; + mCorrectionStates[0].mExcessivePos = excessivePos; + mCorrectionStates[0].mSkipPos = skipPos; + + mSpaceProximityPos = spaceProximityPos; + mMissingSpacePos = missingSpacePos; + mUseFullEditDistance = useFullEditDistance; + mDoAutoCompletion = doAutoCompletion; + mMaxErrors = maxErrors; +} + +void Correction::checkState() { + if (DEBUG_DICT) { + int inputCount = 0; + if (mSkipPos >= 0) ++inputCount; + if (mExcessivePos >= 0) ++inputCount; + if (mTransposedPos >= 0) ++inputCount; + // TODO: remove this assert + assert(inputCount <= 1); + } +} + +int Correction::getFreqForSplitMultipleWords(const int *freqArray, const int *wordLengthArray, + const int wordCount, const bool isSpaceProximity, const unsigned short *word) { + return Correction::RankingAlgorithm::calcFreqForSplitMultipleWords(freqArray, wordLengthArray, + wordCount, this, isSpaceProximity, word); +} + +int Correction::getFinalProbability(const int probability, unsigned short **word, int *wordLength) { + return getFinalProbabilityInternal(probability, word, wordLength, mInputLength); +} + +int Correction::getFinalProbabilityForSubQueue(const int probability, unsigned short **word, + int *wordLength, const int inputLength) { + return getFinalProbabilityInternal(probability, word, wordLength, inputLength); +} + +int Correction::getFinalProbabilityInternal(const int probability, unsigned short **word, + int *wordLength, const int inputLength) { + const int outputIndex = mTerminalOutputIndex; + const int inputIndex = mTerminalInputIndex; + *wordLength = outputIndex + 1; + if (outputIndex < MIN_SUGGEST_DEPTH) { + return NOT_A_PROBABILITY; + } + + *word = mWord; + int finalProbability= Correction::RankingAlgorithm::calculateFinalProbability( + inputIndex, outputIndex, probability, mEditDistanceTable, this, inputLength); + return finalProbability; +} + +bool Correction::initProcessState(const int outputIndex) { + if (mCorrectionStates[outputIndex].mChildCount <= 0) { + return false; + } + mOutputIndex = outputIndex; + --(mCorrectionStates[outputIndex].mChildCount); + mInputIndex = mCorrectionStates[outputIndex].mInputIndex; + mNeedsToTraverseAllNodes = mCorrectionStates[outputIndex].mNeedsToTraverseAllNodes; + + mEquivalentCharCount = mCorrectionStates[outputIndex].mEquivalentCharCount; + mProximityCount = mCorrectionStates[outputIndex].mProximityCount; + mTransposedCount = mCorrectionStates[outputIndex].mTransposedCount; + mExcessiveCount = mCorrectionStates[outputIndex].mExcessiveCount; + mSkippedCount = mCorrectionStates[outputIndex].mSkippedCount; + mLastCharExceeded = mCorrectionStates[outputIndex].mLastCharExceeded; + + mTransposedPos = mCorrectionStates[outputIndex].mTransposedPos; + mExcessivePos = mCorrectionStates[outputIndex].mExcessivePos; + mSkipPos = mCorrectionStates[outputIndex].mSkipPos; + + mMatching = false; + mProximityMatching = false; + mAdditionalProximityMatching = false; + mTransposing = false; + mExceeding = false; + mSkipping = false; + + return true; +} + +int Correction::goDownTree( + const int parentIndex, const int childCount, const int firstChildPos) { + mCorrectionStates[mOutputIndex].mParentIndex = parentIndex; + mCorrectionStates[mOutputIndex].mChildCount = childCount; + mCorrectionStates[mOutputIndex].mSiblingPos = firstChildPos; + return mOutputIndex; +} + +// TODO: remove +int Correction::getInputIndex() { + return mInputIndex; +} + +void Correction::incrementInputIndex() { + ++mInputIndex; +} + +void Correction::incrementOutputIndex() { + ++mOutputIndex; + mCorrectionStates[mOutputIndex].mParentIndex = mCorrectionStates[mOutputIndex - 1].mParentIndex; + mCorrectionStates[mOutputIndex].mChildCount = mCorrectionStates[mOutputIndex - 1].mChildCount; + mCorrectionStates[mOutputIndex].mSiblingPos = mCorrectionStates[mOutputIndex - 1].mSiblingPos; + mCorrectionStates[mOutputIndex].mInputIndex = mInputIndex; + mCorrectionStates[mOutputIndex].mNeedsToTraverseAllNodes = mNeedsToTraverseAllNodes; + + mCorrectionStates[mOutputIndex].mEquivalentCharCount = mEquivalentCharCount; + mCorrectionStates[mOutputIndex].mProximityCount = mProximityCount; + mCorrectionStates[mOutputIndex].mTransposedCount = mTransposedCount; + mCorrectionStates[mOutputIndex].mExcessiveCount = mExcessiveCount; + mCorrectionStates[mOutputIndex].mSkippedCount = mSkippedCount; + + mCorrectionStates[mOutputIndex].mSkipPos = mSkipPos; + mCorrectionStates[mOutputIndex].mTransposedPos = mTransposedPos; + mCorrectionStates[mOutputIndex].mExcessivePos = mExcessivePos; + + mCorrectionStates[mOutputIndex].mLastCharExceeded = mLastCharExceeded; + + mCorrectionStates[mOutputIndex].mMatching = mMatching; + mCorrectionStates[mOutputIndex].mProximityMatching = mProximityMatching; + mCorrectionStates[mOutputIndex].mAdditionalProximityMatching = mAdditionalProximityMatching; + mCorrectionStates[mOutputIndex].mTransposing = mTransposing; + mCorrectionStates[mOutputIndex].mExceeding = mExceeding; + mCorrectionStates[mOutputIndex].mSkipping = mSkipping; +} + +void Correction::startToTraverseAllNodes() { + mNeedsToTraverseAllNodes = true; +} + +bool Correction::needsToPrune() const { + // TODO: use edit distance here + return mOutputIndex - 1 >= mMaxDepth || mProximityCount > mMaxEditDistance + // Allow one char longer word for missing character + || (!mDoAutoCompletion && (mOutputIndex > mInputLength)); +} + +void Correction::addCharToCurrentWord(const int32_t c) { + mWord[mOutputIndex] = c; + const unsigned short *primaryInputWord = mProximityInfoState.getPrimaryInputWord(); + calcEditDistanceOneStep(mEditDistanceTable, primaryInputWord, mInputLength, + mWord, mOutputIndex + 1); +} + +Correction::CorrectionType Correction::processSkipChar( + const int32_t c, const bool isTerminal, const bool inputIndexIncremented) { + addCharToCurrentWord(c); + mTerminalInputIndex = mInputIndex - (inputIndexIncremented ? 1 : 0); + mTerminalOutputIndex = mOutputIndex; + if (mNeedsToTraverseAllNodes && isTerminal) { + incrementOutputIndex(); + return TRAVERSE_ALL_ON_TERMINAL; + } else { + incrementOutputIndex(); + return TRAVERSE_ALL_NOT_ON_TERMINAL; + } +} + +Correction::CorrectionType Correction::processUnrelatedCorrectionType() { + // Needs to set mTerminalInputIndex and mTerminalOutputIndex before returning any CorrectionType + mTerminalInputIndex = mInputIndex; + mTerminalOutputIndex = mOutputIndex; + return UNRELATED; +} + +inline bool isEquivalentChar(ProximityType type) { + return type == EQUIVALENT_CHAR; +} + +inline bool isProximityCharOrEquivalentChar(ProximityType type) { + return type == EQUIVALENT_CHAR || type == NEAR_PROXIMITY_CHAR; +} + +Correction::CorrectionType Correction::processCharAndCalcState( + const int32_t c, const bool isTerminal) { + const int correctionCount = (mSkippedCount + mExcessiveCount + mTransposedCount); + if (correctionCount > mMaxErrors) { + return processUnrelatedCorrectionType(); + } + + // TODO: Change the limit if we'll allow two or more corrections + const bool noCorrectionsHappenedSoFar = correctionCount == 0; + const bool canTryCorrection = noCorrectionsHappenedSoFar; + int proximityIndex = 0; + mDistances[mOutputIndex] = NOT_A_DISTANCE; + + // Skip checking this node + if (mNeedsToTraverseAllNodes || isQuote(c)) { + bool incremented = false; + if (mLastCharExceeded && mInputIndex == mInputLength - 1) { + // TODO: Do not check the proximity if EditDistance exceeds the threshold + const ProximityType matchId = mProximityInfoState.getMatchedProximityId( + mInputIndex, c, true, &proximityIndex); + if (isEquivalentChar(matchId)) { + mLastCharExceeded = false; + --mExcessiveCount; + mDistances[mOutputIndex] = + mProximityInfoState.getNormalizedSquaredDistance(mInputIndex, 0); + } else if (matchId == NEAR_PROXIMITY_CHAR) { + mLastCharExceeded = false; + --mExcessiveCount; + ++mProximityCount; + mDistances[mOutputIndex] = mProximityInfoState.getNormalizedSquaredDistance( + mInputIndex, proximityIndex); + } + if (!isQuote(c)) { + incrementInputIndex(); + incremented = true; + } + } + return processSkipChar(c, isTerminal, incremented); + } + + // Check possible corrections. + if (mExcessivePos >= 0) { + if (mExcessiveCount == 0 && mExcessivePos < mOutputIndex) { + mExcessivePos = mOutputIndex; + } + if (mExcessivePos < mInputLength - 1) { + mExceeding = mExcessivePos == mInputIndex && canTryCorrection; + } + } + + if (mSkipPos >= 0) { + if (mSkippedCount == 0 && mSkipPos < mOutputIndex) { + if (DEBUG_DICT) { + assert(mSkipPos == mOutputIndex - 1); + } + mSkipPos = mOutputIndex; + } + mSkipping = mSkipPos == mOutputIndex && canTryCorrection; + } + + if (mTransposedPos >= 0) { + if (mTransposedCount == 0 && mTransposedPos < mOutputIndex) { + mTransposedPos = mOutputIndex; + } + if (mTransposedPos < mInputLength - 1) { + mTransposing = mInputIndex == mTransposedPos && canTryCorrection; + } + } + + bool secondTransposing = false; + if (mTransposedCount % 2 == 1) { + if (isEquivalentChar(mProximityInfoState.getMatchedProximityId( + mInputIndex - 1, c, false))) { + ++mTransposedCount; + secondTransposing = true; + } else if (mCorrectionStates[mOutputIndex].mExceeding) { + --mTransposedCount; + ++mExcessiveCount; + --mExcessivePos; + incrementInputIndex(); + } else { + --mTransposedCount; + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + DUMP_WORD(mWord, mOutputIndex); + AKLOGI("UNRELATED(0): %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + return processUnrelatedCorrectionType(); + } + } + + // TODO: Change the limit if we'll allow two or more proximity chars with corrections + // Work around: When the mMaxErrors is 1, we only allow just one error + // including proximity correction. + const bool checkProximityChars = (mMaxErrors > 1) + ? (noCorrectionsHappenedSoFar || mProximityCount == 0) + : (noCorrectionsHappenedSoFar && mProximityCount == 0); + + ProximityType matchedProximityCharId = secondTransposing + ? EQUIVALENT_CHAR + : mProximityInfoState.getMatchedProximityId( + mInputIndex, c, checkProximityChars, &proximityIndex); + + if (UNRELATED_CHAR == matchedProximityCharId + || ADDITIONAL_PROXIMITY_CHAR == matchedProximityCharId) { + if (canTryCorrection && mOutputIndex > 0 + && mCorrectionStates[mOutputIndex].mProximityMatching + && mCorrectionStates[mOutputIndex].mExceeding + && isEquivalentChar(mProximityInfoState.getMatchedProximityId( + mInputIndex, mWord[mOutputIndex - 1], false))) { + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + AKLOGI("CONVERSION p->e %c", mWord[mOutputIndex - 1]); + } + // Conversion p->e + // Example: + // wearth -> earth + // px -> (E)mmmmm + ++mExcessiveCount; + --mProximityCount; + mExcessivePos = mOutputIndex - 1; + ++mInputIndex; + // Here, we are doing something equivalent to matchedProximityCharId, + // but we already know that "excessive char correction" just happened + // so that we just need to check "mProximityCount == 0". + matchedProximityCharId = mProximityInfoState.getMatchedProximityId( + mInputIndex, c, mProximityCount == 0, &proximityIndex); + } + } + + if (UNRELATED_CHAR == matchedProximityCharId + || ADDITIONAL_PROXIMITY_CHAR == matchedProximityCharId) { + if (ADDITIONAL_PROXIMITY_CHAR == matchedProximityCharId) { + mAdditionalProximityMatching = true; + } + // TODO: Optimize + // As the current char turned out to be an unrelated char, + // we will try other correction-types. Please note that mCorrectionStates[mOutputIndex] + // here refers to the previous state. + if (mInputIndex < mInputLength - 1 && mOutputIndex > 0 && mTransposedCount > 0 + && !mCorrectionStates[mOutputIndex].mTransposing + && mCorrectionStates[mOutputIndex - 1].mTransposing + && isEquivalentChar(mProximityInfoState.getMatchedProximityId( + mInputIndex, mWord[mOutputIndex - 1], false)) + && isEquivalentChar( + mProximityInfoState.getMatchedProximityId(mInputIndex + 1, c, false))) { + // Conversion t->e + // Example: + // occaisional -> occa sional + // mmmmttx -> mmmm(E)mmmmmm + mTransposedCount -= 2; + ++mExcessiveCount; + ++mInputIndex; + } else if (mOutputIndex > 0 && mInputIndex > 0 && mTransposedCount > 0 + && !mCorrectionStates[mOutputIndex].mTransposing + && mCorrectionStates[mOutputIndex - 1].mTransposing + && isEquivalentChar( + mProximityInfoState.getMatchedProximityId(mInputIndex - 1, c, false))) { + // Conversion t->s + // Example: + // chcolate -> chocolate + // mmttx -> mmsmmmmmm + mTransposedCount -= 2; + ++mSkippedCount; + --mInputIndex; + } else if (canTryCorrection && mInputIndex > 0 + && mCorrectionStates[mOutputIndex].mProximityMatching + && mCorrectionStates[mOutputIndex].mSkipping + && isEquivalentChar( + mProximityInfoState.getMatchedProximityId(mInputIndex - 1, c, false))) { + // Conversion p->s + // Note: This logic tries saving cases like contrst --> contrast -- "a" is one of + // proximity chars of "s", but it should rather be handled as a skipped char. + ++mSkippedCount; + --mProximityCount; + return processSkipChar(c, isTerminal, false); + } else if (mInputIndex - 1 < mInputLength + && mSkippedCount > 0 + && mCorrectionStates[mOutputIndex].mSkipping + && mCorrectionStates[mOutputIndex].mAdditionalProximityMatching + && isProximityCharOrEquivalentChar( + mProximityInfoState.getMatchedProximityId(mInputIndex + 1, c, false))) { + // Conversion s->a + incrementInputIndex(); + --mSkippedCount; + mProximityMatching = true; + ++mProximityCount; + mDistances[mOutputIndex] = ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO; + } else if ((mExceeding || mTransposing) && mInputIndex - 1 < mInputLength + && isEquivalentChar( + mProximityInfoState.getMatchedProximityId(mInputIndex + 1, c, false))) { + // 1.2. Excessive or transpose correction + if (mTransposing) { + ++mTransposedCount; + } else { + ++mExcessiveCount; + incrementInputIndex(); + } + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + DUMP_WORD(mWord, mOutputIndex); + if (mTransposing) { + AKLOGI("TRANSPOSE: %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } else { + AKLOGI("EXCEED: %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + } + } else if (mSkipping) { + // 3. Skip correction + ++mSkippedCount; + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + AKLOGI("SKIP: %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + return processSkipChar(c, isTerminal, false); + } else if (ADDITIONAL_PROXIMITY_CHAR == matchedProximityCharId) { + // As a last resort, use additional proximity characters + mProximityMatching = true; + ++mProximityCount; + mDistances[mOutputIndex] = ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO; + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + AKLOGI("ADDITIONALPROX: %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + } else { + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + DUMP_WORD(mWord, mOutputIndex); + AKLOGI("UNRELATED(1): %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + return processUnrelatedCorrectionType(); + } + } else if (secondTransposing) { + // If inputIndex is greater than mInputLength, that means there is no + // proximity chars. So, we don't need to check proximity. + mMatching = true; + } else if (isEquivalentChar(matchedProximityCharId)) { + mMatching = true; + ++mEquivalentCharCount; + mDistances[mOutputIndex] = mProximityInfoState.getNormalizedSquaredDistance(mInputIndex, 0); + } else if (NEAR_PROXIMITY_CHAR == matchedProximityCharId) { + mProximityMatching = true; + ++mProximityCount; + mDistances[mOutputIndex] = + mProximityInfoState.getNormalizedSquaredDistance(mInputIndex, proximityIndex); + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 + || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + AKLOGI("PROX: %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + } + + addCharToCurrentWord(c); + + // 4. Last char excessive correction + mLastCharExceeded = mExcessiveCount == 0 && mSkippedCount == 0 && mTransposedCount == 0 + && mProximityCount == 0 && (mInputIndex == mInputLength - 2); + const bool isSameAsUserTypedLength = (mInputLength == mInputIndex + 1) || mLastCharExceeded; + if (mLastCharExceeded) { + ++mExcessiveCount; + } + + // Start traversing all nodes after the index exceeds the user typed length + if (isSameAsUserTypedLength) { + startToTraverseAllNodes(); + } + + const bool needsToTryOnTerminalForTheLastPossibleExcessiveChar = + mExceeding && mInputIndex == mInputLength - 2; + + // Finally, we are ready to go to the next character, the next "virtual node". + // We should advance the input index. + // We do this in this branch of the 'if traverseAllNodes' because we are still matching + // characters to input; the other branch is not matching them but searching for + // completions, this is why it does not have to do it. + incrementInputIndex(); + // Also, the next char is one "virtual node" depth more than this char. + incrementOutputIndex(); + + if ((needsToTryOnTerminalForTheLastPossibleExcessiveChar + || isSameAsUserTypedLength) && isTerminal) { + mTerminalInputIndex = mInputIndex - 1; + mTerminalOutputIndex = mOutputIndex - 1; + if (DEBUG_CORRECTION + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == mInputLength) + && (MIN_OUTPUT_INDEX_FOR_DEBUG <= 0 || MIN_OUTPUT_INDEX_FOR_DEBUG < mOutputIndex)) { + DUMP_WORD(mWord, mOutputIndex); + AKLOGI("ONTERMINAL(1): %d, %d, %d, %d, %c", mProximityCount, mSkippedCount, + mTransposedCount, mExcessiveCount, c); + } + return ON_TERMINAL; + } else { + mTerminalInputIndex = mInputIndex - 1; + mTerminalOutputIndex = mOutputIndex - 1; + return NOT_ON_TERMINAL; + } +} + +Correction::~Correction() { +} + +inline static int getQuoteCount(const unsigned short* word, const int length) { + int quoteCount = 0; + for (int i = 0; i < length; ++i) { + if(word[i] == '\'') { + ++quoteCount; + } + } + return quoteCount; +} + +inline static bool isUpperCase(unsigned short c) { + return isAsciiUpper(toBaseChar(c)); +} + +////////////////////// +// RankingAlgorithm // +////////////////////// + +/* static */ +int Correction::RankingAlgorithm::calculateFinalProbability(const int inputIndex, + const int outputIndex, const int freq, int* editDistanceTable, const Correction* correction, + const int inputLength) { + const int excessivePos = correction->getExcessivePos(); + const int typedLetterMultiplier = correction->TYPED_LETTER_MULTIPLIER; + const int fullWordMultiplier = correction->FULL_WORD_MULTIPLIER; + const ProximityInfoState *proximityInfoState = &correction->mProximityInfoState; + const int skippedCount = correction->mSkippedCount; + const int transposedCount = correction->mTransposedCount / 2; + const int excessiveCount = correction->mExcessiveCount + correction->mTransposedCount % 2; + const int proximityMatchedCount = correction->mProximityCount; + const bool lastCharExceeded = correction->mLastCharExceeded; + const bool useFullEditDistance = correction->mUseFullEditDistance; + const int outputLength = outputIndex + 1; + if (skippedCount >= inputLength || inputLength == 0) { + return -1; + } + + // TODO: find more robust way + bool sameLength = lastCharExceeded ? (inputLength == inputIndex + 2) + : (inputLength == inputIndex + 1); + + // TODO: use mExcessiveCount + const int matchCount = inputLength - correction->mProximityCount - excessiveCount; + + const unsigned short* word = correction->mWord; + const bool skipped = skippedCount > 0; + + const int quoteDiffCount = max(0, getQuoteCount(word, outputLength) + - getQuoteCount(proximityInfoState->getPrimaryInputWord(), inputLength)); + + // TODO: Calculate edit distance for transposed and excessive + int ed = 0; + if (DEBUG_DICT_FULL) { + dumpEditDistance10ForDebug(editDistanceTable, correction->mInputLength, outputLength); + } + int adjustedProximityMatchedCount = proximityMatchedCount; + + int finalFreq = freq; + + if (DEBUG_CORRECTION_FREQ + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == inputLength)) { + AKLOGI("FinalFreq0: %d", finalFreq); + } + // TODO: Optimize this. + if (transposedCount > 0 || proximityMatchedCount > 0 || skipped || excessiveCount > 0) { + ed = getCurrentEditDistance(editDistanceTable, correction->mInputLength, outputLength, + inputLength) - transposedCount; + + const int matchWeight = powerIntCapped(typedLetterMultiplier, + max(inputLength, outputLength) - ed); + multiplyIntCapped(matchWeight, &finalFreq); + + // TODO: Demote further if there are two or more excessive chars with longer user input? + if (inputLength > outputLength) { + multiplyRate(INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE, &finalFreq); + } + + ed = max(0, ed - quoteDiffCount); + adjustedProximityMatchedCount = min(max(0, ed - (outputLength - inputLength)), + proximityMatchedCount); + if (transposedCount <= 0) { + if (ed == 1 && (inputLength == outputLength - 1 || inputLength == outputLength + 1)) { + // Promote a word with just one skipped or excessive char + if (sameLength) { + multiplyRate(WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE + + WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER * outputLength, + &finalFreq); + } else { + multiplyIntCapped(typedLetterMultiplier, &finalFreq); + } + } else if (ed == 0) { + multiplyIntCapped(typedLetterMultiplier, &finalFreq); + sameLength = true; + } + } + } else { + const int matchWeight = powerIntCapped(typedLetterMultiplier, matchCount); + multiplyIntCapped(matchWeight, &finalFreq); + } + + if (proximityInfoState->getMatchedProximityId(0, word[0], true) == UNRELATED_CHAR) { + multiplyRate(FIRST_CHAR_DIFFERENT_DEMOTION_RATE, &finalFreq); + } + + /////////////////////////////////////////////// + // Promotion and Demotion for each correction + + // Demotion for a word with missing character + if (skipped) { + const int demotionRate = WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE + * (10 * inputLength - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X) + / (10 * inputLength + - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X + 10); + if (DEBUG_DICT_FULL) { + AKLOGI("Demotion rate for missing character is %d.", demotionRate); + } + multiplyRate(demotionRate, &finalFreq); + } + + // Demotion for a word with transposed character + if (transposedCount > 0) multiplyRate( + WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE, &finalFreq); + + // Demotion for a word with excessive character + if (excessiveCount > 0) { + multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq); + if (!lastCharExceeded && !proximityInfoState->existsAdjacentProximityChars(excessivePos)) { + if (DEBUG_DICT_FULL) { + AKLOGI("Double excessive demotion"); + } + // If an excessive character is not adjacent to the left char or the right char, + // we will demote this word. + multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE, &finalFreq); + } + } + + const bool performTouchPositionCorrection = + CALIBRATE_SCORE_BY_TOUCH_COORDINATES + && proximityInfoState->touchPositionCorrectionEnabled() + && skippedCount == 0 && excessiveCount == 0 && transposedCount == 0; + // Score calibration by touch coordinates is being done only for pure-fat finger typing error + // cases. + int additionalProximityCount = 0; + // TODO: Remove this constraint. + if (performTouchPositionCorrection) { + for (int i = 0; i < outputLength; ++i) { + const int squaredDistance = correction->mDistances[i]; + if (i < adjustedProximityMatchedCount) { + multiplyIntCapped(typedLetterMultiplier, &finalFreq); + } + if (squaredDistance >= 0) { + // Promote or demote the score according to the distance from the sweet spot + static const float A = ZERO_DISTANCE_PROMOTION_RATE / 100.0f; + static const float B = 1.0f; + static const float C = 0.5f; + static const float MIN = 0.3f; + static const float R1 = NEUTRAL_SCORE_SQUARED_RADIUS; + static const float R2 = HALF_SCORE_SQUARED_RADIUS; + const float x = (float)squaredDistance + / ProximityInfoState::NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR; + const float factor = max((x < R1) + ? (A * (R1 - x) + B * x) / R1 + : (B * (R2 - x) + C * (x - R1)) / (R2 - R1), MIN); + // factor is piecewise linear function like: + // A -_ . + // ^-_ . + // B \ . + // \_ . + // C ------------. + // . + // 0 R1 R2 . + multiplyRate((int)(factor * 100), &finalFreq); + } else if (squaredDistance == PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO) { + multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq); + } else if (squaredDistance == ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO) { + ++additionalProximityCount; + multiplyRate(WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq); + } + } + } else { + // Demote additional proximity characters + for (int i = 0; i < outputLength; ++i) { + const int squaredDistance = correction->mDistances[i]; + if (squaredDistance == ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO) { + ++additionalProximityCount; + } + } + // Promotion for a word with proximity characters + for (int i = 0; i < adjustedProximityMatchedCount; ++i) { + // A word with proximity corrections + if (DEBUG_DICT_FULL) { + AKLOGI("Found a proximity correction."); + } + multiplyIntCapped(typedLetterMultiplier, &finalFreq); + if (i < additionalProximityCount) { + multiplyRate(WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq); + } else { + multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq); + } + } + } + + // If the user types too many(three or more) proximity characters with additional proximity + // character,do not treat as the same length word. + if (sameLength && additionalProximityCount > 0 && (adjustedProximityMatchedCount >= 3 + || transposedCount > 0 || skipped || excessiveCount > 0)) { + sameLength = false; + } + + const int errorCount = adjustedProximityMatchedCount > 0 + ? adjustedProximityMatchedCount + : (proximityMatchedCount + transposedCount); + multiplyRate( + 100 - CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE * errorCount / inputLength, &finalFreq); + + // Promotion for an exactly matched word + if (ed == 0) { + // Full exact match + if (sameLength && transposedCount == 0 && !skipped && excessiveCount == 0 + && quoteDiffCount == 0 && additionalProximityCount == 0) { + finalFreq = capped255MultForFullMatchAccentsOrCapitalizationDifference(finalFreq); + } + } + + // Promote a word with no correction + if (proximityMatchedCount == 0 && transposedCount == 0 && !skipped && excessiveCount == 0 + && additionalProximityCount == 0) { + multiplyRate(FULL_MATCHED_WORDS_PROMOTION_RATE, &finalFreq); + } + + // TODO: Check excessive count and transposed count + // TODO: Remove this if possible + /* + If the last character of the user input word is the same as the next character + of the output word, and also all of characters of the user input are matched + to the output word, we'll promote that word a bit because + that word can be considered the combination of skipped and matched characters. + This means that the 'sm' pattern wins over the 'ma' pattern. + e.g.) + shel -> shell [mmmma] or [mmmsm] + hel -> hello [mmmaa] or [mmsma] + m ... matching + s ... skipping + a ... traversing all + t ... transposing + e ... exceeding + p ... proximity matching + */ + if (matchCount == inputLength && matchCount >= 2 && !skipped + && word[matchCount] == word[matchCount - 1]) { + multiplyRate(WORDS_WITH_MATCH_SKIP_PROMOTION_RATE, &finalFreq); + } + + // TODO: Do not use sameLength? + if (sameLength) { + multiplyIntCapped(fullWordMultiplier, &finalFreq); + } + + if (useFullEditDistance && outputLength > inputLength + 1) { + const int diff = outputLength - inputLength - 1; + const int divider = diff < 31 ? 1 << diff : S_INT_MAX; + finalFreq = divider > finalFreq ? 1 : finalFreq / divider; + } + + if (DEBUG_DICT_FULL) { + AKLOGI("calc: %d, %d", outputLength, sameLength); + } + + if (DEBUG_CORRECTION_FREQ + && (INPUTLENGTH_FOR_DEBUG <= 0 || INPUTLENGTH_FOR_DEBUG == inputLength)) { + DUMP_WORD(correction->getPrimaryInputWord(), inputLength); + DUMP_WORD(correction->mWord, outputLength); + AKLOGI("FinalFreq: [P%d, S%d, T%d, E%d, A%d] %d, %d, %d, %d, %d, %d", proximityMatchedCount, + skippedCount, transposedCount, excessiveCount, additionalProximityCount, + outputLength, lastCharExceeded, sameLength, quoteDiffCount, ed, finalFreq); + } + + return finalFreq; +} + +/* static */ +int Correction::RankingAlgorithm::calcFreqForSplitMultipleWords( + const int *freqArray, const int *wordLengthArray, const int wordCount, + const Correction* correction, const bool isSpaceProximity, const unsigned short *word) { + const int typedLetterMultiplier = correction->TYPED_LETTER_MULTIPLIER; + + bool firstCapitalizedWordDemotion = false; + bool secondCapitalizedWordDemotion = false; + + { + // TODO: Handle multiple capitalized word demotion properly + const int firstWordLength = wordLengthArray[0]; + const int secondWordLength = wordLengthArray[1]; + if (firstWordLength >= 2) { + firstCapitalizedWordDemotion = isUpperCase(word[0]); + } + + if (secondWordLength >= 2) { + // FIXME: word[firstWordLength + 1] is incorrect. + secondCapitalizedWordDemotion = isUpperCase(word[firstWordLength + 1]); + } + } + + + const bool capitalizedWordDemotion = + firstCapitalizedWordDemotion ^ secondCapitalizedWordDemotion; + + int totalLength = 0; + int totalFreq = 0; + for (int i = 0; i < wordCount; ++i){ + const int wordLength = wordLengthArray[i]; + if (wordLength <= 0) { + return 0; + } + totalLength += wordLength; + const int demotionRate = 100 - TWO_WORDS_CORRECTION_DEMOTION_BASE / (wordLength + 1); + int tempFirstFreq = freqArray[i]; + multiplyRate(demotionRate, &tempFirstFreq); + totalFreq += tempFirstFreq; + } + + if (totalLength <= 0 || totalFreq <= 0) { + return 0; + } + + // TODO: Currently totalFreq is adjusted to two word metrix. + // Promote pairFreq with multiplying by 2, because the word length is the same as the typed + // length. + totalFreq = totalFreq * 2 / wordCount; + if (wordCount > 2) { + // Safety net for 3+ words -- Caveats: many heuristics and workarounds here. + int oneLengthCounter = 0; + int twoLengthCounter = 0; + for (int i = 0; i < wordCount; ++i) { + const int wordLength = wordLengthArray[i]; + // TODO: Use bigram instead of this safety net + if (i < wordCount - 1) { + const int nextWordLength = wordLengthArray[i + 1]; + if (wordLength == 1 && nextWordLength == 2) { + // Safety net to filter 1 length and 2 length sequential words + return 0; + } + } + const int freq = freqArray[i]; + // Demote too short weak words + if (wordLength <= 4 && freq <= SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ) { + multiplyRate(100 * freq / MAX_FREQ, &totalFreq); + } + if (wordLength == 1) { + ++oneLengthCounter; + } else if (wordLength == 2) { + ++twoLengthCounter; + } + if (oneLengthCounter >= 2 || (oneLengthCounter + twoLengthCounter) >= 4) { + // Safety net to filter too many short words + return 0; + } + } + multiplyRate(MULTIPLE_WORDS_DEMOTION_RATE, &totalFreq); + } + + // This is a workaround to try offsetting the not-enough-demotion which will be done in + // calcNormalizedScore in Utils.java. + // In calcNormalizedScore the score will be demoted by (1 - 1 / length) + // but we demoted only (1 - 1 / (length + 1)) so we will additionally adjust freq by + // (1 - 1 / length) / (1 - 1 / (length + 1)) = (1 - 1 / (length * length)) + const int normalizedScoreNotEnoughDemotionAdjustment = 100 - 100 / (totalLength * totalLength); + multiplyRate(normalizedScoreNotEnoughDemotionAdjustment, &totalFreq); + + // At this moment, totalFreq is calculated by the following formula: + // (firstFreq * (1 - 1 / (firstWordLength + 1)) + secondFreq * (1 - 1 / (secondWordLength + 1))) + // * (1 - 1 / totalLength) / (1 - 1 / (totalLength + 1)) + + multiplyIntCapped(powerIntCapped(typedLetterMultiplier, totalLength), &totalFreq); + + // This is another workaround to offset the demotion which will be done in + // calcNormalizedScore in Utils.java. + // In calcNormalizedScore the score will be demoted by (1 - 1 / length) so we have to promote + // the same amount because we already have adjusted the synthetic freq of this "missing or + // mistyped space" suggestion candidate above in this method. + const int normalizedScoreDemotionRateOffset = (100 + 100 / totalLength); + multiplyRate(normalizedScoreDemotionRateOffset, &totalFreq); + + if (isSpaceProximity) { + // A word pair with one space proximity correction + if (DEBUG_DICT) { + AKLOGI("Found a word pair with space proximity correction."); + } + multiplyIntCapped(typedLetterMultiplier, &totalFreq); + multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &totalFreq); + } + + if (isSpaceProximity) { + multiplyRate(WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE, &totalFreq); + } else { + multiplyRate(WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE, &totalFreq); + } + + if (capitalizedWordDemotion) { + multiplyRate(TWO_WORDS_CAPITALIZED_DEMOTION_RATE, &totalFreq); + } + + if (DEBUG_CORRECTION_FREQ) { + AKLOGI("Multiple words (%d, %d) (%d, %d) %d, %d", freqArray[0], freqArray[1], + wordLengthArray[0], wordLengthArray[1], capitalizedWordDemotion, totalFreq); + DUMP_WORD(word, wordLengthArray[0]); + } + + return totalFreq; +} + +/* Damerau-Levenshtein distance */ +inline static int editDistanceInternal( + int* editDistanceTable, const unsigned short* before, + const int beforeLength, const unsigned short* after, const int afterLength) { + // dp[li][lo] dp[a][b] = dp[ a * lo + b] + int* dp = editDistanceTable; + const int li = beforeLength + 1; + const int lo = afterLength + 1; + for (int i = 0; i < li; ++i) { + dp[lo * i] = i; + } + for (int i = 0; i < lo; ++i) { + dp[i] = i; + } + + for (int i = 0; i < li - 1; ++i) { + for (int j = 0; j < lo - 1; ++j) { + const uint32_t ci = toBaseLowerCase(before[i]); + const uint32_t co = toBaseLowerCase(after[j]); + const uint16_t cost = (ci == co) ? 0 : 1; + dp[(i + 1) * lo + (j + 1)] = min(dp[i * lo + (j + 1)] + 1, + min(dp[(i + 1) * lo + j] + 1, dp[i * lo + j] + cost)); + if (i > 0 && j > 0 && ci == toBaseLowerCase(after[j - 1]) + && co == toBaseLowerCase(before[i - 1])) { + dp[(i + 1) * lo + (j + 1)] = min( + dp[(i + 1) * lo + (j + 1)], dp[(i - 1) * lo + (j - 1)] + cost); + } + } + } + + if (DEBUG_EDIT_DISTANCE) { + AKLOGI("IN = %d, OUT = %d", beforeLength, afterLength); + for (int i = 0; i < li; ++i) { + for (int j = 0; j < lo; ++j) { + AKLOGI("EDIT[%d][%d], %d", i, j, dp[i * lo + j]); + } + } + } + return dp[li * lo - 1]; +} + +int Correction::RankingAlgorithm::editDistance(const unsigned short* before, + const int beforeLength, const unsigned short* after, const int afterLength) { + int table[(beforeLength + 1) * (afterLength + 1)]; + return editDistanceInternal(table, before, beforeLength, after, afterLength); +} + + +// In dictionary.cpp, getSuggestion() method, +// suggestion scores are computed using the below formula. +// original score +// := pow(mTypedLetterMultiplier (this is defined 2), +// (the number of matched characters between typed word and suggested word)) +// * (individual word's score which defined in the unigram dictionary, +// and this score is defined in range [0, 255].) +// Then, the following processing is applied. +// - If the dictionary word is matched up to the point of the user entry +// (full match up to min(before.length(), after.length()) +// => Then multiply by FULL_MATCHED_WORDS_PROMOTION_RATE (this is defined 1.2) +// - If the word is a true full match except for differences in accents or +// capitalization, then treat it as if the score was 255. +// - If before.length() == after.length() +// => multiply by mFullWordMultiplier (this is defined 2)) +// So, maximum original score is pow(2, min(before.length(), after.length())) * 255 * 2 * 1.2 +// For historical reasons we ignore the 1.2 modifier (because the measure for a good +// autocorrection threshold was done at a time when it didn't exist). This doesn't change +// the result. +// So, we can normalize original score by dividing pow(2, min(b.l(),a.l())) * 255 * 2. + +/* static */ +float Correction::RankingAlgorithm::calcNormalizedScore(const unsigned short* before, + const int beforeLength, const unsigned short* after, const int afterLength, + const int score) { + if (0 == beforeLength || 0 == afterLength) { + return 0; + } + const int distance = editDistance(before, beforeLength, after, afterLength); + int spaceCount = 0; + for (int i = 0; i < afterLength; ++i) { + if (after[i] == CODE_SPACE) { + ++spaceCount; + } + } + + if (spaceCount == afterLength) { + return 0; + } + + const float maxScore = score >= S_INT_MAX ? S_INT_MAX : MAX_INITIAL_SCORE + * pow((float)TYPED_LETTER_MULTIPLIER, + (float)min(beforeLength, afterLength - spaceCount)) * FULL_WORD_MULTIPLIER; + + // add a weight based on edit distance. + // distance <= max(afterLength, beforeLength) == afterLength, + // so, 0 <= distance / afterLength <= 1 + const float weight = 1.0 - (float) distance / afterLength; + return (score / maxScore) * weight; +} +} // namespace latinime diff --git a/native/jni/src/correction.h b/native/jni/src/correction.h new file mode 100644 index 000000000..ae7b3a5f8 --- /dev/null +++ b/native/jni/src/correction.h @@ -0,0 +1,261 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_CORRECTION_H +#define LATINIME_CORRECTION_H + +#include <assert.h> +#include <stdint.h> + +#include "correction_state.h" +#include "defines.h" +#include "proximity_info_state.h" + +namespace latinime { + +class ProximityInfo; + +class Correction { + public: + typedef enum { + TRAVERSE_ALL_ON_TERMINAL, + TRAVERSE_ALL_NOT_ON_TERMINAL, + UNRELATED, + ON_TERMINAL, + NOT_ON_TERMINAL + } CorrectionType; + + ///////////////////////// + // static inline utils // + ///////////////////////// + + static const int TWO_31ST_DIV_255 = S_INT_MAX / 255; + static inline int capped255MultForFullMatchAccentsOrCapitalizationDifference(const int num) { + return (num < TWO_31ST_DIV_255 ? 255 * num : S_INT_MAX); + } + + static const int TWO_31ST_DIV_2 = S_INT_MAX / 2; + inline static void multiplyIntCapped(const int multiplier, int *base) { + const int temp = *base; + if (temp != S_INT_MAX) { + // Branch if multiplier == 2 for the optimization + if (multiplier < 0) { + if (DEBUG_DICT) { + assert(false); + } + AKLOGI("--- Invalid multiplier: %d", multiplier); + } else if (multiplier == 0) { + *base = 0; + } else if (multiplier == 2) { + *base = TWO_31ST_DIV_2 >= temp ? temp << 1 : S_INT_MAX; + } else { + // TODO: This overflow check gives a wrong answer when, for example, + // temp = 2^16 + 1 and multiplier = 2^17 + 1. + // Fix this behavior. + const int tempRetval = temp * multiplier; + *base = tempRetval >= temp ? tempRetval : S_INT_MAX; + } + } + } + + inline static int powerIntCapped(const int base, const int n) { + if (n <= 0) return 1; + if (base == 2) { + return n < 31 ? 1 << n : S_INT_MAX; + } else { + int ret = base; + for (int i = 1; i < n; ++i) multiplyIntCapped(base, &ret); + return ret; + } + } + + inline static void multiplyRate(const int rate, int *freq) { + if (*freq != S_INT_MAX) { + if (*freq > 1000000) { + *freq /= 100; + multiplyIntCapped(rate, freq); + } else { + multiplyIntCapped(rate, freq); + *freq /= 100; + } + } + } + + Correction() {}; + void resetCorrection(); + void initCorrection( + const ProximityInfo *pi, const int inputLength, const int maxWordLength); + void initCorrectionState(const int rootPos, const int childCount, const bool traverseAll); + + // TODO: remove + void setCorrectionParams(const int skipPos, const int excessivePos, const int transposedPos, + const int spaceProximityPos, const int missingSpacePos, const bool useFullEditDistance, + const bool doAutoCompletion, const int maxErrors); + void checkState(); + bool initProcessState(const int index); + + int getInputIndex(); + + virtual ~Correction(); + int getSpaceProximityPos() const { + return mSpaceProximityPos; + } + int getMissingSpacePos() const { + return mMissingSpacePos; + } + + int getSkipPos() const { + return mSkipPos; + } + + int getExcessivePos() const { + return mExcessivePos; + } + + int getTransposedPos() const { + return mTransposedPos; + } + + bool needsToPrune() const; + + int pushAndGetTotalTraverseCount() { + return ++mTotalTraverseCount; + } + + int getFreqForSplitMultipleWords( + const int *freqArray, const int *wordLengthArray, const int wordCount, + const bool isSpaceProximity, const unsigned short *word); + int getFinalProbability(const int probability, unsigned short **word, int* wordLength); + int getFinalProbabilityForSubQueue(const int probability, unsigned short **word, + int* wordLength, const int inputLength); + + CorrectionType processCharAndCalcState(const int32_t c, const bool isTerminal); + + ///////////////////////// + // Tree helper methods + int goDownTree(const int parentIndex, const int childCount, const int firstChildPos); + + inline int getTreeSiblingPos(const int index) const { + return mCorrectionStates[index].mSiblingPos; + } + + inline void setTreeSiblingPos(const int index, const int pos) { + mCorrectionStates[index].mSiblingPos = pos; + } + + inline int getTreeParentIndex(const int index) const { + return mCorrectionStates[index].mParentIndex; + } + + class RankingAlgorithm { + public: + static int calculateFinalProbability(const int inputIndex, const int depth, + const int probability, int *editDistanceTable, const Correction* correction, + const int inputLength); + static int calcFreqForSplitMultipleWords(const int *freqArray, const int *wordLengthArray, + const int wordCount, const Correction* correction, const bool isSpaceProximity, + const unsigned short *word); + static float calcNormalizedScore(const unsigned short* before, const int beforeLength, + const unsigned short* after, const int afterLength, const int score); + static int editDistance(const unsigned short* before, + const int beforeLength, const unsigned short* after, const int afterLength); + private: + static const int CODE_SPACE = ' '; + static const int MAX_INITIAL_SCORE = 255; + }; + + // proximity info state + void initInputParams(const ProximityInfo *proximityInfo, const int32_t *inputCodes, + const int inputLength, const int *xCoordinates, const int *yCoordinates) { + mProximityInfoState.initInputParams( + proximityInfo, inputCodes, inputLength, xCoordinates, yCoordinates); + } + + const unsigned short* getPrimaryInputWord() const { + return mProximityInfoState.getPrimaryInputWord(); + } + + unsigned short getPrimaryCharAt(const int index) const { + return mProximityInfoState.getPrimaryCharAt(index); + } + + private: + DISALLOW_COPY_AND_ASSIGN(Correction); + inline void incrementInputIndex(); + inline void incrementOutputIndex(); + inline void startToTraverseAllNodes(); + inline bool isQuote(const unsigned short c); + inline CorrectionType processSkipChar( + const int32_t c, const bool isTerminal, const bool inputIndexIncremented); + inline CorrectionType processUnrelatedCorrectionType(); + inline void addCharToCurrentWord(const int32_t c); + inline int getFinalProbabilityInternal(const int probability, unsigned short **word, + int* wordLength, const int inputLength); + + static const int TYPED_LETTER_MULTIPLIER = 2; + static const int FULL_WORD_MULTIPLIER = 2; + const ProximityInfo *mProximityInfo; + + bool mUseFullEditDistance; + bool mDoAutoCompletion; + int mMaxEditDistance; + int mMaxDepth; + int mInputLength; + int mSpaceProximityPos; + int mMissingSpacePos; + int mTerminalInputIndex; + int mTerminalOutputIndex; + int mMaxErrors; + + uint8_t mTotalTraverseCount; + + // The following arrays are state buffer. + unsigned short mWord[MAX_WORD_LENGTH_INTERNAL]; + int mDistances[MAX_WORD_LENGTH_INTERNAL]; + + // Edit distance calculation requires a buffer with (N+1)^2 length for the input length N. + // Caveat: Do not create multiple tables per thread as this table eats up RAM a lot. + int mEditDistanceTable[(MAX_WORD_LENGTH_INTERNAL + 1) * (MAX_WORD_LENGTH_INTERNAL + 1)]; + + CorrectionState mCorrectionStates[MAX_WORD_LENGTH_INTERNAL]; + + // The following member variables are being used as cache values of the correction state. + bool mNeedsToTraverseAllNodes; + int mOutputIndex; + int mInputIndex; + + int mEquivalentCharCount; + int mProximityCount; + int mExcessiveCount; + int mTransposedCount; + int mSkippedCount; + + int mTransposedPos; + int mExcessivePos; + int mSkipPos; + + bool mLastCharExceeded; + + bool mMatching; + bool mProximityMatching; + bool mAdditionalProximityMatching; + bool mExceeding; + bool mTransposing; + bool mSkipping; + ProximityInfoState mProximityInfoState; +}; +} // namespace latinime +#endif // LATINIME_CORRECTION_H diff --git a/native/jni/src/correction_state.h b/native/jni/src/correction_state.h new file mode 100644 index 000000000..5b2cbd3a2 --- /dev/null +++ b/native/jni/src/correction_state.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_CORRECTION_STATE_H +#define LATINIME_CORRECTION_STATE_H + +#include <stdint.h> + +#include "defines.h" + +namespace latinime { + +struct CorrectionState { + int mParentIndex; + int mSiblingPos; + uint16_t mChildCount; + uint8_t mInputIndex; + + uint8_t mEquivalentCharCount; + uint8_t mProximityCount; + uint8_t mTransposedCount; + uint8_t mExcessiveCount; + uint8_t mSkippedCount; + + int8_t mTransposedPos; + int8_t mExcessivePos; + int8_t mSkipPos; // should be signed + + // TODO: int? + bool mLastCharExceeded; + + bool mMatching; + bool mTransposing; + bool mExceeding; + bool mSkipping; + bool mProximityMatching; + bool mAdditionalProximityMatching; + + bool mNeedsToTraverseAllNodes; +}; + +inline static void initCorrectionState(CorrectionState *state, const int rootPos, + const uint16_t childCount, const bool traverseAll) { + state->mParentIndex = -1; + state->mChildCount = childCount; + state->mInputIndex = 0; + state->mSiblingPos = rootPos; + state->mNeedsToTraverseAllNodes = traverseAll; + + state->mTransposedPos = -1; + state->mExcessivePos = -1; + state->mSkipPos = -1; + + state->mEquivalentCharCount = 0; + state->mProximityCount = 0; + state->mTransposedCount = 0; + state->mExcessiveCount = 0; + state->mSkippedCount = 0; + + state->mLastCharExceeded = false; + + state->mMatching = false; + state->mProximityMatching = false; + state->mTransposing = false; + state->mExceeding = false; + state->mSkipping = false; + state->mAdditionalProximityMatching = false; +} + +} // namespace latinime +#endif // LATINIME_CORRECTION_STATE_H diff --git a/native/src/debug.h b/native/jni/src/debug.h index 38b2f107a..376ba59d9 100644 --- a/native/src/debug.h +++ b/native/jni/src/debug.h @@ -22,7 +22,7 @@ static inline unsigned char* convertToUnibyteString(unsigned short* input, unsigned char* output, const unsigned int length) { - int i = 0; + unsigned int i = 0; for (; i <= length && input[i] != 0; ++i) output[i] = input[i] & 0xFF; output[i] = 0; @@ -31,10 +31,10 @@ static inline unsigned char* convertToUnibyteString(unsigned short* input, unsig static inline unsigned char* convertToUnibyteStringAndReplaceLastChar(unsigned short* input, unsigned char* output, const unsigned int length, unsigned char c) { - int i = 0; + unsigned int i = 0; for (; i <= length && input[i] != 0; ++i) output[i] = input[i] & 0xFF; - output[i-1] = c; + if (i > 0) output[i-1] = c; output[i] = 0; return output; } @@ -42,7 +42,7 @@ static inline unsigned char* convertToUnibyteStringAndReplaceLastChar(unsigned s static inline void LOGI_S16(unsigned short* string, const unsigned int length) { unsigned char tmp_buffer[length]; convertToUnibyteString(string, tmp_buffer, length); - LOGI(">> %s", tmp_buffer); + AKLOGI(">> %s", tmp_buffer); // The log facility is throwing out log that comes too fast. The following // is a dirty way of slowing down processing so that we can see all log. // TODO : refactor this in a blocking log or something. @@ -53,7 +53,7 @@ static inline void LOGI_S16_PLUS(unsigned short* string, const unsigned int leng unsigned char c) { unsigned char tmp_buffer[length+1]; convertToUnibyteStringAndReplaceLastChar(string, tmp_buffer, length, c); - LOGI(">> %s", tmp_buffer); + AKLOGI(">> %s", tmp_buffer); // Likewise // usleep(10); } @@ -64,7 +64,7 @@ static inline void printDebug(const char* tag, int* codes, int codesSize, int MA buf[codesSize] = 0; while (--codesSize >= 0) buf[codesSize] = (unsigned char)codes[codesSize * MAX_PROXIMITY_CHARS]; - LOGI("%s, WORD = %s", tag, buf); + AKLOGI("%s, WORD = %s", tag, buf); free(buf); } diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h new file mode 100644 index 000000000..8bcadcbe9 --- /dev/null +++ b/native/jni/src/defines.h @@ -0,0 +1,315 @@ +/* +** +** Copyright 2010, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#ifndef LATINIME_DEFINES_H +#define LATINIME_DEFINES_H + +#if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) +#include <cutils/log.h> +#define AKLOGE ALOGE +#define AKLOGI ALOGI + +#define DUMP_WORD(word, length) do { dumpWord(word, length); } while(0) +#define DUMP_WORD_INT(word, length) do { dumpWordInt(word, length); } while(0) + +static inline void dumpWord(const unsigned short* word, const int length) { + static char charBuf[50]; + + for (int i = 0; i < length; ++i) { + charBuf[i] = word[i]; + } + charBuf[length] = 0; + AKLOGI("[ %s ]", charBuf); +} + +static inline void dumpWordInt(const int* word, const int length) { + static char charBuf[50]; + + for (int i = 0; i < length; ++i) { + charBuf[i] = word[i]; + } + charBuf[length] = 0; + AKLOGI("i[ %s ]", charBuf); +} + +#else +#define AKLOGE(fmt, ...) +#define AKLOGI(fmt, ...) +#define DUMP_WORD(word, length) +#define DUMP_WORD_INT(word, length) +#endif + +#ifdef FLAG_DO_PROFILE +// Profiler +#include <time.h> + +#define PROF_BUF_SIZE 100 +static float profile_buf[PROF_BUF_SIZE]; +static float profile_old[PROF_BUF_SIZE]; +static unsigned int profile_counter[PROF_BUF_SIZE]; + +#define PROF_RESET prof_reset() +#define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id] +#define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while(0) +#define PROF_START(prof_buf_id) do { \ + PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while(0) +#define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while(0) +#define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id]) +#define PROF_CLOCKOUT(prof_buf_id) \ + AKLOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id])) +#define PROF_OUTALL do { AKLOGI("--- %s ---", __FUNCTION__); prof_out(); } while(0) + +static inline void prof_reset(void) { + for (int i = 0; i < PROF_BUF_SIZE; ++i) { + profile_buf[i] = 0; + profile_old[i] = 0; + profile_counter[i] = 0; + } +} + +static inline void prof_out(void) { + if (profile_counter[PROF_BUF_SIZE - 1] != 1) { + AKLOGI("Error: You must call PROF_OPEN before PROF_CLOSE."); + } + AKLOGI("Total time is %6.3f ms.", + profile_buf[PROF_BUF_SIZE - 1] * 1000 / (float)CLOCKS_PER_SEC); + float all = 0; + for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { + all += profile_buf[i]; + } + if (all == 0) all = 1; + for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { + if (profile_buf[i] != 0) { + AKLOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.", + i, (profile_buf[i] * 100 / all), + profile_buf[i] * 1000 / (float)CLOCKS_PER_SEC, profile_counter[i]); + } + } +} + +#else // FLAG_DO_PROFILE +#define PROF_BUF_SIZE 0 +#define PROF_RESET +#define PROF_COUNT(prof_buf_id) +#define PROF_OPEN +#define PROF_START(prof_buf_id) +#define PROF_CLOSE +#define PROF_END(prof_buf_id) +#define PROF_CLOCK_OUT(prof_buf_id) +#define PROF_CLOCKOUT(prof_buf_id) +#define PROF_OUTALL + +#endif // FLAG_DO_PROFILE + +#ifdef FLAG_DBG +#include <cutils/log.h> +#ifndef LOG_TAG +#define LOG_TAG "LatinIME: " +#endif +#define DEBUG_DICT true +#define DEBUG_DICT_FULL false +#define DEBUG_EDIT_DISTANCE false +#define DEBUG_SHOW_FOUND_WORD false +#define DEBUG_NODE DEBUG_DICT_FULL +#define DEBUG_TRACE DEBUG_DICT_FULL +#define DEBUG_PROXIMITY_INFO false +#define DEBUG_PROXIMITY_CHARS false +#define DEBUG_CORRECTION false +#define DEBUG_CORRECTION_FREQ false +#define DEBUG_WORDS_PRIORITY_QUEUE false + +#else // FLAG_DBG + +#define DEBUG_DICT false +#define DEBUG_DICT_FULL false +#define DEBUG_EDIT_DISTANCE false +#define DEBUG_SHOW_FOUND_WORD false +#define DEBUG_NODE false +#define DEBUG_TRACE false +#define DEBUG_PROXIMITY_INFO false +#define DEBUG_PROXIMITY_CHARS false +#define DEBUG_CORRECTION false +#define DEBUG_CORRECTION_FREQ false +#define DEBUG_WORDS_PRIORITY_QUEUE false + + +#endif // FLAG_DBG + +#ifndef U_SHORT_MAX +#define U_SHORT_MAX 65535 // ((1 << 16) - 1) +#endif +#ifndef S_INT_MAX +#define S_INT_MAX 2147483647 // ((1 << 31) - 1) +#endif + +// Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap(). +// We measured and compared performance of both, and found mmap() is fairly good in terms of +// loading time, and acceptable even for several initial lookups which involve page faults. +#define USE_MMAP_FOR_DICTIONARY + +// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words +#define ADDRESS_MASK 0x3FFFFF + +// The bit that decides if an address follows in the next 22 bits +#define FLAG_ADDRESS_MASK 0x40 +// The bit that decides if this is a terminal node for a word. The node could still have children, +// if the word has other endings. +#define FLAG_TERMINAL_MASK 0x80 + +#define FLAG_BIGRAM_READ 0x80 +#define FLAG_BIGRAM_CHILDEXIST 0x40 +#define FLAG_BIGRAM_CONTINUED 0x80 +#define FLAG_BIGRAM_FREQ 0x7F + +#define DICTIONARY_VERSION_MIN 200 +#define NOT_VALID_WORD -99 +#define NOT_A_CHARACTER -1 +#define NOT_A_DISTANCE -1 +#define NOT_A_COORDINATE -1 +#define EQUIVALENT_CHAR_WITHOUT_DISTANCE_INFO -2 +#define PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO -3 +#define ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO -4 +#define NOT_AN_INDEX -1 +#define NOT_A_PROBABILITY -1 + +#define KEYCODE_SPACE ' ' + +#define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true + +#define SUGGEST_WORDS_WITH_MISSING_CHARACTER true +#define SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER true +#define SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS true +#define SUGGEST_MULTIPLE_WORDS true + +// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent. +#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80 +#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12 +#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 58 +#define WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE 50 +#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75 +#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75 +#define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 70 +#define FULL_MATCHED_WORDS_PROMOTION_RATE 120 +#define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90 +#define WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE 70 +#define WORDS_WITH_MATCH_SKIP_PROMOTION_RATE 105 +#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE 148 +#define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER 3 +#define CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE 45 +#define INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE 70 +#define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96 +#define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50 +#define TWO_WORDS_CORRECTION_DEMOTION_BASE 80 +#define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1 +#define ZERO_DISTANCE_PROMOTION_RATE 110 +#define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f +#define HALF_SCORE_SQUARED_RADIUS 32.0f +#define MAX_FREQ 255 +#define MAX_BIGRAM_FREQ 15 + +// This must be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java +// This is only used for the size of array. Not to be used in c functions. +#define MAX_WORD_LENGTH_INTERNAL 48 + +// This must be the same as ProximityInfo#MAX_PROXIMITY_CHARS_SIZE, currently it's 16. +#define MAX_PROXIMITY_CHARS_SIZE_INTERNAL 16 + +// This must be equal to ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE in KeyDetector.java +#define ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE 2 + +// Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used +// for better performance. +// Holds up to 1 candidate for each word +#define SUB_QUEUE_MAX_WORDS 1 +#define SUB_QUEUE_MAX_COUNT 10 +#define SUB_QUEUE_MIN_WORD_LENGTH 4 +// TODO: Extend this limitation +#define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 5 +// TODO: Remove this limitation +#define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12 +// TODO: Remove this limitation +#define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 45 +#define MULTIPLE_WORDS_DEMOTION_RATE 80 +#define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6 + +#define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35 +#define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185 +/* heuristic... This should be changed if we change the unit of the frequency. */ +#define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_FREQ * 58 / 100) + +#define MAX_DEPTH_MULTIPLIER 3 + +#define FIRST_WORD_INDEX 0 + +// TODO: Reduce this constant if possible; check the maximum number of digraphs in the same +// word in the dictionary for languages with digraphs, like German and French +#define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5 + +// Minimum suggest depth for one word for all cases except for missing space suggestions. +#define MIN_SUGGEST_DEPTH 1 +#define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3 +#define MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION 3 + +// Size, in bytes, of the bloom filter index for bigrams +// 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k, +// where k is the number of hash functions, n the number of bigrams, and m the number of +// bits we can test. +// At the moment 100 is the maximum number of bigrams for a word with the current +// dictionaries, so n = 100. 1024 buckets give us m = 1024. +// With 1 hash function, our false positive rate is about 9.3%, which should be enough for +// our uses since we are only using this to increase average performance. For the record, +// k = 2 gives 3.1% and k = 3 gives 1.6%. With k = 1, making m = 2048 gives 4.8%, +// and m = 4096 gives 2.4%. +#define BIGRAM_FILTER_BYTE_SIZE 128 +// Must be smaller than BIGRAM_FILTER_BYTE_SIZE * 8, and preferably prime. 1021 is the largest +// prime under 128 * 8. +#define BIGRAM_FILTER_MODULO 1021 +#if BIGRAM_FILTER_BYTE_SIZE * 8 < BIGRAM_FILTER_MODULO +#error "BIGRAM_FILTER_MODULO is larger than BIGRAM_FILTER_BYTE_SIZE" +#endif + +template<typename T> inline T min(T a, T b) { return a < b ? a : b; } +template<typename T> inline T max(T a, T b) { return a > b ? a : b; } + +// The ratio of neutral area radius to sweet spot radius. +#define NEUTRAL_AREA_RADIUS_RATIO 1.3f + +// DEBUG +#define INPUTLENGTH_FOR_DEBUG -1 +#define MIN_OUTPUT_INDEX_FOR_DEBUG -1 + +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + +#define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ + TypeName(); \ + DISALLOW_COPY_AND_ASSIGN(TypeName) + +// Used as a return value for character comparison +typedef enum { + // Same char, possibly with different case or accent + EQUIVALENT_CHAR, + // It is a char located nearby on the keyboard + NEAR_PROXIMITY_CHAR, + // It is an unrelated char + UNRELATED_CHAR, + // Additional proximity char which can differ by language. + ADDITIONAL_PROXIMITY_CHAR +} ProximityType; + +#endif // LATINIME_DEFINES_H diff --git a/native/src/dictionary.cpp b/native/jni/src/dictionary.cpp index 9e32ee80f..83bb26731 100644 --- a/native/src/dictionary.cpp +++ b/native/jni/src/dictionary.cpp @@ -19,6 +19,8 @@ #define LOG_TAG "LatinIME: dictionary.cpp" +#include "binary_format.h" +#include "defines.h" #include "dictionary.h" namespace latinime { @@ -26,22 +28,21 @@ namespace latinime { // TODO: Change the type of all keyCodes to uint32_t Dictionary::Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultiplier, int fullWordMultiplier, - int maxWordLength, int maxWords, int maxAlternatives) + int maxWordLength, int maxWords) : mDict((unsigned char*) dict), mDictSize(dictSize), - mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust), - // Checks whether it has the latest dictionary or the old dictionary - IS_LATEST_DICT_VERSION((((unsigned char*) dict)[0] & 0xFF) >= DICTIONARY_VERSION_MIN) { + mMmapFd(mmapFd), mDictBufAdjust(dictBufAdjust) { if (DEBUG_DICT) { if (MAX_WORD_LENGTH_INTERNAL < maxWordLength) { - LOGI("Max word length (%d) is greater than %d", + AKLOGI("Max word length (%d) is greater than %d", maxWordLength, MAX_WORD_LENGTH_INTERNAL); - LOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF)); + AKLOGI("IN NATIVE SUGGEST Version: %d", (mDict[0] & 0xFF)); } } - mUnigramDictionary = new UnigramDictionary(mDict, typedLetterMultiplier, fullWordMultiplier, - maxWordLength, maxWords, maxAlternatives, IS_LATEST_DICT_VERSION); - mBigramDictionary = new BigramDictionary(mDict, maxWordLength, maxAlternatives, - IS_LATEST_DICT_VERSION, hasBigram(), this); + const unsigned int headerSize = BinaryFormat::getHeaderSize(mDict); + const unsigned int options = BinaryFormat::getFlags(mDict); + mUnigramDictionary = new UnigramDictionary(mDict + headerSize, typedLetterMultiplier, + fullWordMultiplier, maxWordLength, maxWords, options); + mBigramDictionary = new BigramDictionary(mDict + headerSize, maxWordLength); } Dictionary::~Dictionary() { @@ -49,20 +50,13 @@ Dictionary::~Dictionary() { delete mBigramDictionary; } -bool Dictionary::hasBigram() { - return ((mDict[1] & 0xFF) == 1); +int Dictionary::getFrequency(const int32_t *word, int length) const { + return mUnigramDictionary->getFrequency(word, length); } -bool Dictionary::isValidWord(unsigned short *word, int length) { - return mUnigramDictionary->isValidWord(word, length); -} - -int Dictionary::getBigramPosition(unsigned short *word, int length) { - if (IS_LATEST_DICT_VERSION) { - return mUnigramDictionary->getBigramPosition(DICTIONARY_HEADER_SIZE, word, 0, length); - } else { - return mUnigramDictionary->getBigramPosition(0, word, 0, length); - } +bool Dictionary::isValidBigram(const int32_t *word1, int length1, const int32_t *word2, + int length2) const { + return mBigramDictionary->isValidBigram(word1, length1, word2, length2); } } // namespace latinime diff --git a/native/jni/src/dictionary.h b/native/jni/src/dictionary.h new file mode 100644 index 000000000..fd69f79e3 --- /dev/null +++ b/native/jni/src/dictionary.h @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICTIONARY_H +#define LATINIME_DICTIONARY_H + +#include <map> + +#include "bigram_dictionary.h" +#include "char_utils.h" +#include "defines.h" +#include "proximity_info.h" +#include "unigram_dictionary.h" +#include "words_priority_queue_pool.h" + +namespace latinime { + +class Dictionary { + public: + Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler, + int fullWordMultiplier, int maxWordLength, int maxWords); + + int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates, + int *codes, int codesSize, const int32_t* prevWordChars, const int prevWordLength, + bool useFullEditDistance, unsigned short *outWords, int *frequencies) const { + std::map<int, int> bigramMap; + uint8_t bigramFilter[BIGRAM_FILTER_BYTE_SIZE]; + mBigramDictionary->fillBigramAddressToFrequencyMapAndFilter(prevWordChars, + prevWordLength, &bigramMap, bigramFilter); + return mUnigramDictionary->getSuggestions(proximityInfo, + xcoordinates, ycoordinates, codes, codesSize, &bigramMap, + bigramFilter, useFullEditDistance, outWords, frequencies); + } + + int getBigrams(const int32_t *word, int length, int *codes, int codesSize, + unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams) const { + return mBigramDictionary->getBigrams(word, length, codes, codesSize, outWords, frequencies, + maxWordLength, maxBigrams); + } + + int getFrequency(const int32_t *word, int length) const; + bool isValidBigram(const int32_t *word1, int length1, const int32_t *word2, int length2) const; + void *getDict() const { return (void *)mDict; } + int getDictSize() const { return mDictSize; } + int getMmapFd() const { return mMmapFd; } + int getDictBufAdjust() const { return mDictBufAdjust; } + ~Dictionary(); + + // public static utility methods + // static inline methods should be defined in the header file + static int wideStrLen(unsigned short *str); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); + const unsigned char *mDict; + + // Used only for the mmap version of dictionary loading, but we use these as dummy variables + // also for the malloc version. + const int mDictSize; + const int mMmapFd; + const int mDictBufAdjust; + + const UnigramDictionary *mUnigramDictionary; + const BigramDictionary *mBigramDictionary; +}; + +// public static utility methods +// static inline methods should be defined in the header file +inline int Dictionary::wideStrLen(unsigned short *str) { + if (!str) return 0; + unsigned short *end = str; + while (*end) + end++; + return end - str; +} +} // namespace latinime + +#endif // LATINIME_DICTIONARY_H diff --git a/native/jni/src/proximity_info.cpp b/native/jni/src/proximity_info.cpp new file mode 100644 index 000000000..2ba244a7c --- /dev/null +++ b/native/jni/src/proximity_info.cpp @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <assert.h> +#include <stdio.h> +#include <string> + +#define LOG_TAG "LatinIME: proximity_info.cpp" + +#include "additional_proximity_chars.h" +#include "defines.h" +#include "dictionary.h" +#include "proximity_info.h" +#include "proximity_info_state.h" + +namespace latinime { + +inline void copyOrFillZero(void *to, const void *from, size_t size) { + if (from) { + memcpy(to, from, size); + } else { + memset(to, 0, size); + } +} + +ProximityInfo::ProximityInfo(const std::string localeStr, const int maxProximityCharsSize, + const int keyboardWidth, const int keyboardHeight, const int gridWidth, + const int gridHeight, const int mostCommonKeyWidth, + const int32_t *proximityCharsArray, const int keyCount, const int32_t *keyXCoordinates, + const int32_t *keyYCoordinates, const int32_t *keyWidths, const int32_t *keyHeights, + const int32_t *keyCharCodes, const float *sweetSpotCenterXs, const float *sweetSpotCenterYs, + const float *sweetSpotRadii) + : MAX_PROXIMITY_CHARS_SIZE(maxProximityCharsSize), KEYBOARD_WIDTH(keyboardWidth), + KEYBOARD_HEIGHT(keyboardHeight), GRID_WIDTH(gridWidth), GRID_HEIGHT(gridHeight), + MOST_COMMON_KEY_WIDTH_SQUARE(mostCommonKeyWidth * mostCommonKeyWidth), + CELL_WIDTH((keyboardWidth + gridWidth - 1) / gridWidth), + CELL_HEIGHT((keyboardHeight + gridHeight - 1) / gridHeight), + KEY_COUNT(min(keyCount, MAX_KEY_COUNT_IN_A_KEYBOARD)), + HAS_TOUCH_POSITION_CORRECTION_DATA(keyCount > 0 && keyXCoordinates && keyYCoordinates + && keyWidths && keyHeights && keyCharCodes && sweetSpotCenterXs + && sweetSpotCenterYs && sweetSpotRadii), + mLocaleStr(localeStr) { + const int proximityGridLength = GRID_WIDTH * GRID_HEIGHT * MAX_PROXIMITY_CHARS_SIZE; + if (DEBUG_PROXIMITY_INFO) { + AKLOGI("Create proximity info array %d", proximityGridLength); + } + mProximityCharsArray = new int32_t[proximityGridLength]; + memcpy(mProximityCharsArray, proximityCharsArray, + proximityGridLength * sizeof(mProximityCharsArray[0])); + + copyOrFillZero(mKeyXCoordinates, keyXCoordinates, KEY_COUNT * sizeof(mKeyXCoordinates[0])); + copyOrFillZero(mKeyYCoordinates, keyYCoordinates, KEY_COUNT * sizeof(mKeyYCoordinates[0])); + copyOrFillZero(mKeyWidths, keyWidths, KEY_COUNT * sizeof(mKeyWidths[0])); + copyOrFillZero(mKeyHeights, keyHeights, KEY_COUNT * sizeof(mKeyHeights[0])); + copyOrFillZero(mKeyCharCodes, keyCharCodes, KEY_COUNT * sizeof(mKeyCharCodes[0])); + copyOrFillZero(mSweetSpotCenterXs, sweetSpotCenterXs, + KEY_COUNT * sizeof(mSweetSpotCenterXs[0])); + copyOrFillZero(mSweetSpotCenterYs, sweetSpotCenterYs, + KEY_COUNT * sizeof(mSweetSpotCenterYs[0])); + copyOrFillZero(mSweetSpotRadii, sweetSpotRadii, KEY_COUNT * sizeof(mSweetSpotRadii[0])); + + initializeCodeToKeyIndex(); +} + +// Build the reversed look up table from the char code to the index in mKeyXCoordinates, +// mKeyYCoordinates, mKeyWidths, mKeyHeights, mKeyCharCodes. +void ProximityInfo::initializeCodeToKeyIndex() { + memset(mCodeToKeyIndex, -1, (MAX_CHAR_CODE + 1) * sizeof(mCodeToKeyIndex[0])); + for (int i = 0; i < KEY_COUNT; ++i) { + const int code = mKeyCharCodes[i]; + if (0 <= code && code <= MAX_CHAR_CODE) { + mCodeToKeyIndex[code] = i; + } + } +} + +ProximityInfo::~ProximityInfo() { + delete[] mProximityCharsArray; +} + +inline int ProximityInfo::getStartIndexFromCoordinates(const int x, const int y) const { + return ((y / CELL_HEIGHT) * GRID_WIDTH + (x / CELL_WIDTH)) + * MAX_PROXIMITY_CHARS_SIZE; +} + +bool ProximityInfo::hasSpaceProximity(const int x, const int y) const { + if (x < 0 || y < 0) { + if (DEBUG_DICT) { + AKLOGI("HasSpaceProximity: Illegal coordinates (%d, %d)", x, y); + assert(false); + } + return false; + } + + const int startIndex = getStartIndexFromCoordinates(x, y); + if (DEBUG_PROXIMITY_INFO) { + AKLOGI("hasSpaceProximity: index %d, %d, %d", startIndex, x, y); + } + int32_t* proximityCharsArray = mProximityCharsArray; + for (int i = 0; i < MAX_PROXIMITY_CHARS_SIZE; ++i) { + if (DEBUG_PROXIMITY_INFO) { + AKLOGI("Index: %d", mProximityCharsArray[startIndex + i]); + } + if (proximityCharsArray[startIndex + i] == KEYCODE_SPACE) { + return true; + } + } + return false; +} + +int ProximityInfo::squaredDistanceToEdge(const int keyId, const int x, const int y) const { + if (keyId < 0) return true; // NOT_A_ID is -1, but return whenever < 0 just in case + const int left = mKeyXCoordinates[keyId]; + const int top = mKeyYCoordinates[keyId]; + const int right = left + mKeyWidths[keyId]; + const int bottom = top + mKeyHeights[keyId]; + const int edgeX = x < left ? left : (x > right ? right : x); + const int edgeY = y < top ? top : (y > bottom ? bottom : y); + const int dx = x - edgeX; + const int dy = y - edgeY; + return dx * dx + dy * dy; +} + +void ProximityInfo::calculateNearbyKeyCodes( + const int x, const int y, const int32_t primaryKey, int *inputCodes) const { + int32_t *proximityCharsArray = mProximityCharsArray; + int insertPos = 0; + inputCodes[insertPos++] = primaryKey; + const int startIndex = getStartIndexFromCoordinates(x, y); + if (startIndex >= 0) { + for (int i = 0; i < MAX_PROXIMITY_CHARS_SIZE; ++i) { + const int32_t c = proximityCharsArray[startIndex + i]; + if (c < KEYCODE_SPACE || c == primaryKey) { + continue; + } + const int keyIndex = getKeyIndex(c); + const bool onKey = isOnKey(keyIndex, x, y); + const int distance = squaredDistanceToEdge(keyIndex, x, y); + if (onKey || distance < MOST_COMMON_KEY_WIDTH_SQUARE) { + inputCodes[insertPos++] = c; + if (insertPos >= MAX_PROXIMITY_CHARS_SIZE) { + if (DEBUG_DICT) { + assert(false); + } + return; + } + } + } + const int additionalProximitySize = + AdditionalProximityChars::getAdditionalCharsSize(&mLocaleStr, primaryKey); + if (additionalProximitySize > 0) { + inputCodes[insertPos++] = ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE; + if (insertPos >= MAX_PROXIMITY_CHARS_SIZE) { + if (DEBUG_DICT) { + assert(false); + } + return; + } + + const int32_t* additionalProximityChars = + AdditionalProximityChars::getAdditionalChars(&mLocaleStr, primaryKey); + for (int j = 0; j < additionalProximitySize; ++j) { + const int32_t ac = additionalProximityChars[j]; + int k = 0; + for (; k < insertPos; ++k) { + if ((int)ac == inputCodes[k]) { + break; + } + } + if (k < insertPos) { + continue; + } + inputCodes[insertPos++] = ac; + if (insertPos >= MAX_PROXIMITY_CHARS_SIZE) { + if (DEBUG_DICT) { + assert(false); + } + return; + } + } + } + } + // Add a delimiter for the proximity characters + for (int i = insertPos; i < MAX_PROXIMITY_CHARS_SIZE; ++i) { + inputCodes[i] = NOT_A_CODE; + } +} + +int ProximityInfo::getKeyIndex(const int c) const { + if (KEY_COUNT == 0) { + // We do not have the coordinate data + return NOT_AN_INDEX; + } + const unsigned short baseLowerC = toBaseLowerCase(c); + if (baseLowerC > MAX_CHAR_CODE) { + return NOT_AN_INDEX; + } + return mCodeToKeyIndex[baseLowerC]; +} +} // namespace latinime diff --git a/native/jni/src/proximity_info.h b/native/jni/src/proximity_info.h new file mode 100644 index 000000000..fec6555ea --- /dev/null +++ b/native/jni/src/proximity_info.h @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_PROXIMITY_INFO_H +#define LATINIME_PROXIMITY_INFO_H + +#include <stdint.h> +#include <string> + +#include "defines.h" + +namespace latinime { + +class Correction; + +class ProximityInfo { + public: + ProximityInfo(const std::string localeStr, const int maxProximityCharsSize, + const int keyboardWidth, const int keyboardHeight, const int gridWidth, + const int gridHeight, const int mostCommonkeyWidth, + const int32_t *proximityCharsArray, const int keyCount, const int32_t *keyXCoordinates, + const int32_t *keyYCoordinates, const int32_t *keyWidths, const int32_t *keyHeights, + const int32_t *keyCharCodes, const float *sweetSpotCenterXs, + const float *sweetSpotCenterYs, const float *sweetSpotRadii); + ~ProximityInfo(); + bool hasSpaceProximity(const int x, const int y) const; + int getNormalizedSquaredDistance(const int inputIndex, const int proximityIndex) const; + bool sameAsTyped(const unsigned short *word, int length) const; + int squaredDistanceToEdge(const int keyId, const int x, const int y) const; + bool isOnKey(const int keyId, const int x, const int y) const { + if (keyId < 0) return true; // NOT_A_ID is -1, but return whenever < 0 just in case + const int left = mKeyXCoordinates[keyId]; + const int top = mKeyYCoordinates[keyId]; + const int right = left + mKeyWidths[keyId] + 1; + const int bottom = top + mKeyHeights[keyId]; + return left < right && top < bottom && x >= left && x < right && y >= top && y < bottom; + } + int getKeyIndex(const int c) const; + bool hasSweetSpotData(const int keyIndex) const { + // When there are no calibration data for a key, + // the radius of the key is assigned to zero. + return mSweetSpotRadii[keyIndex] > 0.0; + } + float getSweetSpotRadiiAt(int keyIndex) const { + return mSweetSpotRadii[keyIndex]; + } + float getSweetSpotCenterXAt(int keyIndex) const { + return mSweetSpotCenterXs[keyIndex]; + } + float getSweetSpotCenterYAt(int keyIndex) const { + return mSweetSpotCenterYs[keyIndex]; + } + void calculateNearbyKeyCodes( + const int x, const int y, const int32_t primaryKey, int *inputCodes) const; + + bool hasTouchPositionCorrectionData() const { + return HAS_TOUCH_POSITION_CORRECTION_DATA; + } + + int getMostCommonKeyWidthSquare() const { + return MOST_COMMON_KEY_WIDTH_SQUARE; + } + + std::string getLocaleStr() const { + return mLocaleStr; + } + + int getKeyCount() const { + return KEY_COUNT; + } + + int getCellHeight() const { + return CELL_HEIGHT; + } + + int getCellWidth() const { + return CELL_WIDTH; + } + + int getGridWidth() const { + return GRID_WIDTH; + } + + int getGridHeight() const { + return GRID_HEIGHT; + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(ProximityInfo); + // The max number of the keys in one keyboard layout + static const int MAX_KEY_COUNT_IN_A_KEYBOARD = 64; + // The upper limit of the char code in mCodeToKeyIndex + static const int MAX_CHAR_CODE = 127; + static const float NOT_A_DISTANCE_FLOAT = -1.0f; + static const int NOT_A_CODE = -1; + + int getStartIndexFromCoordinates(const int x, const int y) const; + void initializeCodeToKeyIndex(); + float calculateNormalizedSquaredDistance(const int keyIndex, const int inputIndex) const; + float calculateSquaredDistanceFromSweetSpotCenter( + const int keyIndex, const int inputIndex) const; + bool hasInputCoordinates() const; + + const int MAX_PROXIMITY_CHARS_SIZE; + const int KEYBOARD_WIDTH; + const int KEYBOARD_HEIGHT; + const int GRID_WIDTH; + const int GRID_HEIGHT; + const int MOST_COMMON_KEY_WIDTH_SQUARE; + const int CELL_WIDTH; + const int CELL_HEIGHT; + const int KEY_COUNT; + const bool HAS_TOUCH_POSITION_CORRECTION_DATA; + const std::string mLocaleStr; + int32_t *mProximityCharsArray; + int32_t mKeyXCoordinates[MAX_KEY_COUNT_IN_A_KEYBOARD]; + int32_t mKeyYCoordinates[MAX_KEY_COUNT_IN_A_KEYBOARD]; + int32_t mKeyWidths[MAX_KEY_COUNT_IN_A_KEYBOARD]; + int32_t mKeyHeights[MAX_KEY_COUNT_IN_A_KEYBOARD]; + int32_t mKeyCharCodes[MAX_KEY_COUNT_IN_A_KEYBOARD]; + float mSweetSpotCenterXs[MAX_KEY_COUNT_IN_A_KEYBOARD]; + float mSweetSpotCenterYs[MAX_KEY_COUNT_IN_A_KEYBOARD]; + float mSweetSpotRadii[MAX_KEY_COUNT_IN_A_KEYBOARD]; + int mCodeToKeyIndex[MAX_CHAR_CODE + 1]; + // TODO: move to correction.h +}; + +} // namespace latinime + +#endif // LATINIME_PROXIMITY_INFO_H diff --git a/native/jni/src/proximity_info_state.cpp b/native/jni/src/proximity_info_state.cpp new file mode 100644 index 000000000..149299eb6 --- /dev/null +++ b/native/jni/src/proximity_info_state.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <assert.h> +#include <stdint.h> +#include <string> + +#define LOG_TAG "LatinIME: proximity_info_state.cpp" + +#include "additional_proximity_chars.h" +#include "defines.h" +#include "dictionary.h" +#include "proximity_info.h" +#include "proximity_info_state.h" + +namespace latinime { +void ProximityInfoState::initInputParams( + const ProximityInfo* proximityInfo, const int32_t* inputCodes, const int inputLength, + const int* xCoordinates, const int* yCoordinates) { + mProximityInfo = proximityInfo; + mHasTouchPositionCorrectionData = proximityInfo->hasTouchPositionCorrectionData(); + mMostCommonKeyWidthSquare = proximityInfo->getMostCommonKeyWidthSquare(); + mLocaleStr = proximityInfo->getLocaleStr(); + mKeyCount = proximityInfo->getKeyCount(); + mCellHeight = proximityInfo->getCellHeight(); + mCellWidth = proximityInfo->getCellWidth(); + mGridHeight = proximityInfo->getGridWidth(); + mGridWidth = proximityInfo->getGridHeight(); + const int normalizedSquaredDistancesLength = + MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH_INTERNAL; + for (int i = 0; i < normalizedSquaredDistancesLength; ++i) { + mNormalizedSquaredDistances[i] = NOT_A_DISTANCE; + } + + memset(mInputCodes, 0, + MAX_WORD_LENGTH_INTERNAL * MAX_PROXIMITY_CHARS_SIZE_INTERNAL * sizeof(mInputCodes[0])); + + for (int i = 0; i < inputLength; ++i) { + const int32_t primaryKey = inputCodes[i]; + const int x = xCoordinates[i]; + const int y = yCoordinates[i]; + int *proximities = &mInputCodes[i * MAX_PROXIMITY_CHARS_SIZE_INTERNAL]; + mProximityInfo->calculateNearbyKeyCodes(x, y, primaryKey, proximities); + } + + if (DEBUG_PROXIMITY_CHARS) { + for (int i = 0; i < inputLength; ++i) { + AKLOGI("---"); + for (int j = 0; j < MAX_PROXIMITY_CHARS_SIZE_INTERNAL; ++j) { + int icc = mInputCodes[i * MAX_PROXIMITY_CHARS_SIZE_INTERNAL + j]; + int icfjc = inputCodes[i * MAX_PROXIMITY_CHARS_SIZE_INTERNAL + j]; + icc += 0; + icfjc += 0; + AKLOGI("--- (%d)%c,%c", i, icc, icfjc); AKLOGI("--- A<%d>,B<%d>", icc, icfjc); + } + } + } + mInputXCoordinates = xCoordinates; + mInputYCoordinates = yCoordinates; + mTouchPositionCorrectionEnabled = + mHasTouchPositionCorrectionData && xCoordinates && yCoordinates; + mInputLength = inputLength; + for (int i = 0; i < inputLength; ++i) { + mPrimaryInputWord[i] = getPrimaryCharAt(i); + } + mPrimaryInputWord[inputLength] = 0; + if (DEBUG_PROXIMITY_CHARS) { + AKLOGI("--- initInputParams"); + } + for (int i = 0; i < mInputLength; ++i) { + const int *proximityChars = getProximityCharsAt(i); + const int primaryKey = proximityChars[0]; + const int x = xCoordinates[i]; + const int y = yCoordinates[i]; + if (DEBUG_PROXIMITY_CHARS) { + int a = x + y + primaryKey; + a += 0; + AKLOGI("--- Primary = %c, x = %d, y = %d", primaryKey, x, y); + } + for (int j = 0; j < MAX_PROXIMITY_CHARS_SIZE_INTERNAL && proximityChars[j] > 0; ++j) { + const int currentChar = proximityChars[j]; + const float squaredDistance = + hasInputCoordinates() ? calculateNormalizedSquaredDistance( + mProximityInfo->getKeyIndex(currentChar), i) : + NOT_A_DISTANCE_FLOAT; + if (squaredDistance >= 0.0f) { + mNormalizedSquaredDistances[i * MAX_PROXIMITY_CHARS_SIZE_INTERNAL + j] = + (int) (squaredDistance * NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR); + } else { + mNormalizedSquaredDistances[i * MAX_PROXIMITY_CHARS_SIZE_INTERNAL + j] = + (j == 0) ? EQUIVALENT_CHAR_WITHOUT_DISTANCE_INFO : + PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO; + } + if (DEBUG_PROXIMITY_CHARS) { + AKLOGI("--- Proximity (%d) = %c", j, currentChar); + } + } + } +} + +float ProximityInfoState::calculateNormalizedSquaredDistance( + const int keyIndex, const int inputIndex) const { + if (keyIndex == NOT_AN_INDEX) { + return NOT_A_DISTANCE_FLOAT; + } + if (!mProximityInfo->hasSweetSpotData(keyIndex)) { + return NOT_A_DISTANCE_FLOAT; + } + if (NOT_A_COORDINATE == mInputXCoordinates[inputIndex]) { + return NOT_A_DISTANCE_FLOAT; + } + const float squaredDistance = calculateSquaredDistanceFromSweetSpotCenter( + keyIndex, inputIndex); + const float squaredRadius = square(mProximityInfo->getSweetSpotRadiiAt(keyIndex)); + return squaredDistance / squaredRadius; +} + +float ProximityInfoState::calculateSquaredDistanceFromSweetSpotCenter( + const int keyIndex, const int inputIndex) const { + const float sweetSpotCenterX = mProximityInfo->getSweetSpotCenterXAt(keyIndex); + const float sweetSpotCenterY = mProximityInfo->getSweetSpotCenterYAt(keyIndex); + const float inputX = (float)mInputXCoordinates[inputIndex]; + const float inputY = (float)mInputYCoordinates[inputIndex]; + return square(inputX - sweetSpotCenterX) + square(inputY - sweetSpotCenterY); +} +} // namespace latinime diff --git a/native/jni/src/proximity_info_state.h b/native/jni/src/proximity_info_state.h new file mode 100644 index 000000000..717871c90 --- /dev/null +++ b/native/jni/src/proximity_info_state.h @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_PROXIMITY_INFO_STATE_H +#define LATINIME_PROXIMITY_INFO_STATE_H + +#include <assert.h> +#include <stdint.h> +#include <string> + +#include "additional_proximity_chars.h" +#include "char_utils.h" +#include "defines.h" + +namespace latinime { + +class ProximityInfo; + +class ProximityInfoState { + public: + static const int NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR_LOG_2 = 10; + static const int NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR = + 1 << NORMALIZED_SQUARED_DISTANCE_SCALING_FACTOR_LOG_2; + // The upper limit of the char code in mCodeToKeyIndex + static const int MAX_CHAR_CODE = 127; + static const float NOT_A_DISTANCE_FLOAT = -1.0f; + static const int NOT_A_CODE = -1; + + ///////////////////////////////////////// + // Defined in proximity_info_state.cpp // + ///////////////////////////////////////// + void initInputParams( + const ProximityInfo* proximityInfo, const int32_t* inputCodes, const int inputLength, + const int* xCoordinates, const int* yCoordinates); + + ///////////////////////////////////////// + // Defined here // + ///////////////////////////////////////// + ProximityInfoState() {}; + inline const int* getProximityCharsAt(const int index) const { + return mInputCodes + (index * MAX_PROXIMITY_CHARS_SIZE_INTERNAL); + } + + inline unsigned short getPrimaryCharAt(const int index) const { + return getProximityCharsAt(index)[0]; + } + + inline bool existsCharInProximityAt(const int index, const int c) const { + const int *chars = getProximityCharsAt(index); + int i = 0; + while (chars[i] > 0 && i < MAX_PROXIMITY_CHARS_SIZE_INTERNAL) { + if (chars[i++] == c) { + return true; + } + } + return false; + } + + inline bool existsAdjacentProximityChars(const int index) const { + if (index < 0 || index >= mInputLength) return false; + const int currentChar = getPrimaryCharAt(index); + const int leftIndex = index - 1; + if (leftIndex >= 0 && existsCharInProximityAt(leftIndex, currentChar)) { + return true; + } + const int rightIndex = index + 1; + if (rightIndex < mInputLength && existsCharInProximityAt(rightIndex, currentChar)) { + return true; + } + return false; + } + + // In the following function, c is the current character of the dictionary word + // currently examined. + // currentChars is an array containing the keys close to the character the + // user actually typed at the same position. We want to see if c is in it: if so, + // then the word contains at that position a character close to what the user + // typed. + // What the user typed is actually the first character of the array. + // proximityIndex is a pointer to the variable where getMatchedProximityId returns + // the index of c in the proximity chars of the input index. + // Notice : accented characters do not have a proximity list, so they are alone + // in their list. The non-accented version of the character should be considered + // "close", but not the other keys close to the non-accented version. + inline ProximityType getMatchedProximityId(const int index, + const unsigned short c, const bool checkProximityChars, int *proximityIndex = 0) const { + const int *currentChars = getProximityCharsAt(index); + const int firstChar = currentChars[0]; + const unsigned short baseLowerC = toBaseLowerCase(c); + + // The first char in the array is what user typed. If it matches right away, + // that means the user typed that same char for this pos. + if (firstChar == baseLowerC || firstChar == c) { + return EQUIVALENT_CHAR; + } + + if (!checkProximityChars) return UNRELATED_CHAR; + + // If the non-accented, lowercased version of that first character matches c, + // then we have a non-accented version of the accented character the user + // typed. Treat it as a close char. + if (toBaseLowerCase(firstChar) == baseLowerC) + return NEAR_PROXIMITY_CHAR; + + // Not an exact nor an accent-alike match: search the list of close keys + int j = 1; + while (j < MAX_PROXIMITY_CHARS_SIZE_INTERNAL + && currentChars[j] > ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE) { + const bool matched = (currentChars[j] == baseLowerC || currentChars[j] == c); + if (matched) { + if (proximityIndex) { + *proximityIndex = j; + } + return NEAR_PROXIMITY_CHAR; + } + ++j; + } + if (j < MAX_PROXIMITY_CHARS_SIZE_INTERNAL + && currentChars[j] == ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE) { + ++j; + while (j < MAX_PROXIMITY_CHARS_SIZE_INTERNAL + && currentChars[j] > ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE) { + const bool matched = (currentChars[j] == baseLowerC || currentChars[j] == c); + if (matched) { + if (proximityIndex) { + *proximityIndex = j; + } + return ADDITIONAL_PROXIMITY_CHAR; + } + ++j; + } + } + + // Was not included, signal this as an unrelated character. + return UNRELATED_CHAR; + } + + inline int getNormalizedSquaredDistance( + const int inputIndex, const int proximityIndex) const { + return mNormalizedSquaredDistances[ + inputIndex * MAX_PROXIMITY_CHARS_SIZE_INTERNAL + proximityIndex]; + } + + inline const unsigned short* getPrimaryInputWord() const { + return mPrimaryInputWord; + } + + inline bool touchPositionCorrectionEnabled() const { + return mTouchPositionCorrectionEnabled; + } + + private: + DISALLOW_COPY_AND_ASSIGN(ProximityInfoState); + ///////////////////////////////////////// + // Defined in proximity_info_state.cpp // + ///////////////////////////////////////// + float calculateNormalizedSquaredDistance(const int keyIndex, const int inputIndex) const; + + float calculateSquaredDistanceFromSweetSpotCenter( + const int keyIndex, const int inputIndex) const; + + ///////////////////////////////////////// + // Defined here // + ///////////////////////////////////////// + inline float square(const float x) const { return x * x; } + + bool hasInputCoordinates() const { + return mInputXCoordinates && mInputYCoordinates; + } + + bool sameAsTyped(const unsigned short *word, int length) const { + if (length != mInputLength) { + return false; + } + const int *inputCodes = mInputCodes; + while (length--) { + if ((unsigned int) *inputCodes != (unsigned int) *word) { + return false; + } + inputCodes += MAX_PROXIMITY_CHARS_SIZE_INTERNAL; + word++; + } + return true; + } + + // const + const ProximityInfo *mProximityInfo; + bool mHasTouchPositionCorrectionData; + int mMostCommonKeyWidthSquare; + std::string mLocaleStr; + int mKeyCount; + int mCellHeight; + int mCellWidth; + int mGridHeight; + int mGridWidth; + + const int *mInputXCoordinates; + const int *mInputYCoordinates; + bool mTouchPositionCorrectionEnabled; + int32_t mInputCodes[MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH_INTERNAL]; + int mNormalizedSquaredDistances[MAX_PROXIMITY_CHARS_SIZE_INTERNAL * MAX_WORD_LENGTH_INTERNAL]; + int mInputLength; + unsigned short mPrimaryInputWord[MAX_WORD_LENGTH_INTERNAL]; +}; + +} // namespace latinime + +#endif // LATINIME_PROXIMITY_INFO_STATE_H diff --git a/native/jni/src/terminal_attributes.h b/native/jni/src/terminal_attributes.h new file mode 100644 index 000000000..c712f502d --- /dev/null +++ b/native/jni/src/terminal_attributes.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_TERMINAL_ATTRIBUTES_H +#define LATINIME_TERMINAL_ATTRIBUTES_H + +#include "unigram_dictionary.h" + +namespace latinime { + +/** + * This class encapsulates information about a terminal that allows to + * retrieve local node attributes like the list of shortcuts without + * exposing the format structure to the client. + */ +class TerminalAttributes { + public: + class ShortcutIterator { + const uint8_t* const mDict; + bool mHasNextShortcutTarget; + int mPos; + + public: + ShortcutIterator(const uint8_t* dict, const int pos, const uint8_t flags) : mDict(dict), + mPos(pos) { + mHasNextShortcutTarget = (0 != (flags & UnigramDictionary::FLAG_HAS_SHORTCUT_TARGETS)); + } + + inline bool hasNextShortcutTarget() const { + return mHasNextShortcutTarget; + } + + // Gets the shortcut target itself as a uint16_t string. For parameters and return value + // see BinaryFormat::getWordAtAddress. + // TODO: make the output an uint32_t* to handle the whole unicode range. + inline int getNextShortcutTarget(const int maxDepth, uint16_t* outWord) { + const int shortcutFlags = BinaryFormat::getFlagsAndForwardPointer(mDict, &mPos); + mHasNextShortcutTarget = + 0 != (shortcutFlags & UnigramDictionary::FLAG_ATTRIBUTE_HAS_NEXT); + unsigned int i; + for (i = 0; i < MAX_WORD_LENGTH_INTERNAL; ++i) { + const int charCode = BinaryFormat::getCharCodeAndForwardPointer(mDict, &mPos); + if (NOT_A_CHARACTER == charCode) break; + outWord[i] = (uint16_t)charCode; + } + mPos += BinaryFormat::CHARACTER_ARRAY_TERMINATOR_SIZE; + return i; + } + }; + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(TerminalAttributes); + const uint8_t* const mDict; + const uint8_t mFlags; + const int mStartPos; + + public: + TerminalAttributes(const uint8_t* const dict, const uint8_t flags, const int pos) : + mDict(dict), mFlags(flags), mStartPos(pos) { + } + + inline ShortcutIterator getShortcutIterator() const { + // The size of the shortcuts is stored here so that the whole shortcut chunk can be + // skipped quickly, so we ignore it. + return ShortcutIterator(mDict, mStartPos + BinaryFormat::SHORTCUT_LIST_SIZE_SIZE, mFlags); + } +}; +} // namespace latinime + +#endif // LATINIME_TERMINAL_ATTRIBUTES_H diff --git a/native/jni/src/unigram_dictionary.cpp b/native/jni/src/unigram_dictionary.cpp new file mode 100644 index 000000000..3417d2ba7 --- /dev/null +++ b/native/jni/src/unigram_dictionary.cpp @@ -0,0 +1,993 @@ +/* +** +** Copyright 2010, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#include <assert.h> +#include <string.h> + +#define LOG_TAG "LatinIME: unigram_dictionary.cpp" + +#include "char_utils.h" +#include "defines.h" +#include "dictionary.h" +#include "unigram_dictionary.h" + +#include "binary_format.h" +#include "terminal_attributes.h" + +namespace latinime { + +const UnigramDictionary::digraph_t UnigramDictionary::GERMAN_UMLAUT_DIGRAPHS[] = + { { 'a', 'e', 0x00E4 }, // U+00E4 : LATIN SMALL LETTER A WITH DIAERESIS + { 'o', 'e', 0x00F6 }, // U+00F6 : LATIN SMALL LETTER O WITH DIAERESIS + { 'u', 'e', 0x00FC } }; // U+00FC : LATIN SMALL LETTER U WITH DIAERESIS + +const UnigramDictionary::digraph_t UnigramDictionary::FRENCH_LIGATURES_DIGRAPHS[] = + { { 'a', 'e', 0x00E6 }, // U+00E6 : LATIN SMALL LETTER AE + { 'o', 'e', 0x0153 } }; // U+0153 : LATIN SMALL LIGATURE OE + +// TODO: check the header +UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier, + int fullWordMultiplier, int maxWordLength, int maxWords, const unsigned int flags) + : DICT_ROOT(streamStart), MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords), + TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier), + // TODO : remove this variable. + ROOT_POS(0), + BYTES_IN_ONE_CHAR(sizeof(int)), + MAX_DIGRAPH_SEARCH_DEPTH(DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH), FLAGS(flags) { + if (DEBUG_DICT) { + AKLOGI("UnigramDictionary - constructor"); + } +} + +UnigramDictionary::~UnigramDictionary() { +} + +static inline unsigned int getCodesBufferSize(const int *codes, const int codesSize) { + return sizeof(*codes) * codesSize; +} + +// TODO: This needs to take a const unsigned short* and not tinker with its contents +static inline void addWord( + unsigned short *word, int length, int frequency, WordsPriorityQueue *queue) { + queue->push(frequency, word, length); +} + +// Return the replacement code point for a digraph, or 0 if none. +int UnigramDictionary::getDigraphReplacement(const int *codes, const int i, const int codesSize, + const digraph_t* const digraphs, const unsigned int digraphsSize) const { + + // There can't be a digraph if we don't have at least 2 characters to examine + if (i + 2 > codesSize) return false; + + // Search for the first char of some digraph + int lastDigraphIndex = -1; + const int thisChar = codes[i]; + for (lastDigraphIndex = digraphsSize - 1; lastDigraphIndex >= 0; --lastDigraphIndex) { + if (thisChar == digraphs[lastDigraphIndex].first) break; + } + // No match: return early + if (lastDigraphIndex < 0) return 0; + + // It's an interesting digraph if the second char matches too. + if (digraphs[lastDigraphIndex].second == codes[i + 1]) { + return digraphs[lastDigraphIndex].replacement; + } else { + return 0; + } +} + +// Mostly the same arguments as the non-recursive version, except: +// codes is the original value. It points to the start of the work buffer, and gets passed as is. +// codesSize is the size of the user input (thus, it is the size of codesSrc). +// codesDest is the current point in the work buffer. +// codesSrc is the current point in the user-input, original, content-unmodified buffer. +// codesRemain is the remaining size in codesSrc. +void UnigramDictionary::getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codesBuffer, + int *xCoordinatesBuffer, int *yCoordinatesBuffer, + const int codesBufferSize, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, const int *codesSrc, + const int codesRemain, const int currentDepth, int *codesDest, Correction *correction, + WordsPriorityQueuePool *queuePool, + const digraph_t* const digraphs, const unsigned int digraphsSize) const { + + const int startIndex = codesDest - codesBuffer; + if (currentDepth < MAX_DIGRAPH_SEARCH_DEPTH) { + for (int i = 0; i < codesRemain; ++i) { + xCoordinatesBuffer[startIndex + i] = xcoordinates[codesBufferSize - codesRemain + i]; + yCoordinatesBuffer[startIndex + i] = ycoordinates[codesBufferSize - codesRemain + i]; + const int replacementCodePoint = + getDigraphReplacement(codesSrc, i, codesRemain, digraphs, digraphsSize); + if (0 != replacementCodePoint) { + // Found a digraph. We will try both spellings. eg. the word is "pruefen" + + // Copy the word up to the first char of the digraph, including proximity chars, + // and overwrite the primary code with the replacement code point. Then, continue + // processing on the remaining part of the word, skipping the second char of the + // digraph. + // In our example, copy "pru", replace "u" with the version with the diaeresis and + // continue running on "fen". + // Make i the index of the second char of the digraph for simplicity. Forgetting + // to do that results in an infinite recursion so take care! + ++i; + memcpy(codesDest, codesSrc, i * BYTES_IN_ONE_CHAR); + codesDest[(i - 1) * (BYTES_IN_ONE_CHAR / sizeof(codesDest[0]))] = + replacementCodePoint; + getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, + codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize, + bigramMap, bigramFilter, useFullEditDistance, codesSrc + i + 1, + codesRemain - i - 1, currentDepth + 1, codesDest + i, correction, + queuePool, digraphs, digraphsSize); + + // Copy the second char of the digraph in place, then continue processing on + // the remaining part of the word. + // In our example, after "pru" in the buffer copy the "e", and continue on "fen" + memcpy(codesDest + i, codesSrc + i, BYTES_IN_ONE_CHAR); + getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, + codesBuffer, xCoordinatesBuffer, yCoordinatesBuffer, codesBufferSize, + bigramMap, bigramFilter, useFullEditDistance, codesSrc + i, codesRemain - i, + currentDepth + 1, codesDest + i, correction, queuePool, digraphs, + digraphsSize); + return; + } + } + } + + // If we come here, we hit the end of the word: let's check it against the dictionary. + // In our example, we'll come here once for "prufen" and then once for "pruefen". + // If the word contains several digraphs, we'll come it for the product of them. + // eg. if the word is "ueberpruefen" we'll test, in order, against + // "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen". + const unsigned int remainingBytes = BYTES_IN_ONE_CHAR * codesRemain; + if (0 != remainingBytes) { + memcpy(codesDest, codesSrc, remainingBytes); + memcpy(&xCoordinatesBuffer[startIndex], &xcoordinates[codesBufferSize - codesRemain], + sizeof(int) * codesRemain); + memcpy(&yCoordinatesBuffer[startIndex], &ycoordinates[codesBufferSize - codesRemain], + sizeof(int) * codesRemain); + } + + getWordSuggestions(proximityInfo, xCoordinatesBuffer, yCoordinatesBuffer, codesBuffer, + startIndex + codesRemain, bigramMap, bigramFilter, useFullEditDistance, correction, + queuePool); +} + +// bigramMap contains the association <bigram address> -> <bigram frequency> +// bigramFilter is a bloom filter for fast rejection: see functions setInFilter and isInFilter +// in bigram_dictionary.cpp +int UnigramDictionary::getSuggestions(ProximityInfo *proximityInfo, + const int *xcoordinates, + const int *ycoordinates, const int *codes, const int codesSize, + const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, unsigned short *outWords, int *frequencies) const { + + WordsPriorityQueuePool queuePool(MAX_WORDS, SUB_QUEUE_MAX_WORDS, MAX_WORD_LENGTH); + queuePool.clearAll(); + Correction masterCorrection; + masterCorrection.resetCorrection(); + if (BinaryFormat::REQUIRES_GERMAN_UMLAUT_PROCESSING & FLAGS) + { // Incrementally tune the word and try all possibilities + int codesBuffer[getCodesBufferSize(codes, codesSize)]; + int xCoordinatesBuffer[codesSize]; + int yCoordinatesBuffer[codesSize]; + getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, + xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter, + useFullEditDistance, codes, codesSize, 0, codesBuffer, &masterCorrection, + &queuePool, GERMAN_UMLAUT_DIGRAPHS, + sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0])); + } else if (BinaryFormat::REQUIRES_FRENCH_LIGATURES_PROCESSING & FLAGS) { + int codesBuffer[getCodesBufferSize(codes, codesSize)]; + int xCoordinatesBuffer[codesSize]; + int yCoordinatesBuffer[codesSize]; + getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, + xCoordinatesBuffer, yCoordinatesBuffer, codesSize, bigramMap, bigramFilter, + useFullEditDistance, codes, codesSize, 0, codesBuffer, &masterCorrection, + &queuePool, FRENCH_LIGATURES_DIGRAPHS, + sizeof(FRENCH_LIGATURES_DIGRAPHS) / sizeof(FRENCH_LIGATURES_DIGRAPHS[0])); + } else { // Normal processing + getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, + bigramMap, bigramFilter, useFullEditDistance, &masterCorrection, &queuePool); + } + + PROF_START(20); + if (DEBUG_DICT) { + float ns = queuePool.getMasterQueue()->getHighestNormalizedScore( + masterCorrection.getPrimaryInputWord(), codesSize, 0, 0, 0); + ns += 0; + AKLOGI("Max normalized score = %f", ns); + } + const int suggestedWordsCount = + queuePool.getMasterQueue()->outputSuggestions( + masterCorrection.getPrimaryInputWord(), codesSize, frequencies, outWords); + + if (DEBUG_DICT) { + float ns = queuePool.getMasterQueue()->getHighestNormalizedScore( + masterCorrection.getPrimaryInputWord(), codesSize, 0, 0, 0); + ns += 0; + AKLOGI("Returning %d words", suggestedWordsCount); + /// Print the returned words + for (int j = 0; j < suggestedWordsCount; ++j) { + short unsigned int* w = outWords + j * MAX_WORD_LENGTH; + char s[MAX_WORD_LENGTH]; + for (int i = 0; i <= MAX_WORD_LENGTH; i++) s[i] = w[i]; + (void)s; + AKLOGI("%s %i", s, frequencies[j]); + } + } + PROF_END(20); + PROF_CLOSE; + return suggestedWordsCount; +} + +void UnigramDictionary::getWordSuggestions(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codes, + const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, Correction *correction, + WordsPriorityQueuePool *queuePool) const { + + PROF_OPEN; + PROF_START(0); + PROF_END(0); + + PROF_START(1); + getOneWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, bigramMap, bigramFilter, + useFullEditDistance, inputLength, correction, queuePool); + PROF_END(1); + + PROF_START(2); + // Note: This line is intentionally left blank + PROF_END(2); + + PROF_START(3); + // Note: This line is intentionally left blank + PROF_END(3); + + PROF_START(4); + bool hasAutoCorrectionCandidate = false; + WordsPriorityQueue* masterQueue = queuePool->getMasterQueue(); + if (masterQueue->size() > 0) { + float nsForMaster = masterQueue->getHighestNormalizedScore( + correction->getPrimaryInputWord(), inputLength, 0, 0, 0); + hasAutoCorrectionCandidate = (nsForMaster > START_TWO_WORDS_CORRECTION_THRESHOLD); + } + PROF_END(4); + + PROF_START(5); + // Multiple word suggestions + if (SUGGEST_MULTIPLE_WORDS + && inputLength >= MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION) { + getSplitMultipleWordsSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, + useFullEditDistance, inputLength, correction, queuePool, + hasAutoCorrectionCandidate); + } + PROF_END(5); + + PROF_START(6); + // Note: This line is intentionally left blank + PROF_END(6); + + if (DEBUG_DICT) { + queuePool->dumpSubQueue1TopSuggestions(); + for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { + WordsPriorityQueue* queue = queuePool->getSubQueue(FIRST_WORD_INDEX, i); + if (queue->size() > 0) { + WordsPriorityQueue::SuggestedWord* sw = queue->top(); + const int score = sw->mScore; + const unsigned short* word = sw->mWord; + const int wordLength = sw->mWordLength; + float ns = Correction::RankingAlgorithm::calcNormalizedScore( + correction->getPrimaryInputWord(), i, word, wordLength, score); + ns += 0; + AKLOGI("--- TOP SUB WORDS for %d --- %d %f [%d]", i, score, ns, + (ns > TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD)); + DUMP_WORD(correction->getPrimaryInputWord(), i); + DUMP_WORD(word, wordLength); + } + } + } +} + +void UnigramDictionary::initSuggestions(ProximityInfo *proximityInfo, const int *xCoordinates, + const int *yCoordinates, const int *codes, const int inputLength, + Correction *correction) const { + if (DEBUG_DICT) { + AKLOGI("initSuggest"); + DUMP_WORD_INT(codes, inputLength); + } + correction->initInputParams(proximityInfo, codes, inputLength, xCoordinates, yCoordinates); + const int maxDepth = min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH); + correction->initCorrection(proximityInfo, inputLength, maxDepth); +} + +static const char QUOTE = '\''; +static const char SPACE = ' '; + +void UnigramDictionary::getOneWordSuggestions(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codes, + const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, const int inputLength, + Correction *correction, WordsPriorityQueuePool *queuePool) const { + initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, inputLength, correction); + getSuggestionCandidates(useFullEditDistance, inputLength, bigramMap, bigramFilter, correction, + queuePool, true /* doAutoCompletion */, DEFAULT_MAX_ERRORS, FIRST_WORD_INDEX); +} + +void UnigramDictionary::getSuggestionCandidates(const bool useFullEditDistance, + const int inputLength, const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + Correction *correction, WordsPriorityQueuePool *queuePool, + const bool doAutoCompletion, const int maxErrors, const int currentWordIndex) const { + uint8_t totalTraverseCount = correction->pushAndGetTotalTraverseCount(); + if (DEBUG_DICT) { + AKLOGI("Traverse count %d", totalTraverseCount); + } + if (totalTraverseCount > MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT) { + if (DEBUG_DICT) { + AKLOGI("Abort traversing %d", totalTraverseCount); + } + return; + } + // TODO: Remove setCorrectionParams + correction->setCorrectionParams(0, 0, 0, + -1 /* spaceProximityPos */, -1 /* missingSpacePos */, useFullEditDistance, + doAutoCompletion, maxErrors); + int rootPosition = ROOT_POS; + // Get the number of children of root, then increment the position + int childCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &rootPosition); + int outputIndex = 0; + + correction->initCorrectionState(rootPosition, childCount, (inputLength <= 0)); + + // Depth first search + while (outputIndex >= 0) { + if (correction->initProcessState(outputIndex)) { + int siblingPos = correction->getTreeSiblingPos(outputIndex); + int firstChildPos; + + const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, + bigramMap, bigramFilter, correction, &childCount, &firstChildPos, &siblingPos, + queuePool, currentWordIndex); + // Update next sibling pos + correction->setTreeSiblingPos(outputIndex, siblingPos); + + if (needsToTraverseChildrenNodes) { + // Goes to child node + outputIndex = correction->goDownTree(outputIndex, childCount, firstChildPos); + } + } else { + // Goes to parent sibling node + outputIndex = correction->getTreeParentIndex(outputIndex); + } + } +} + +inline void UnigramDictionary::onTerminal(const int probability, + const TerminalAttributes& terminalAttributes, Correction *correction, + WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, + const int currentWordIndex) const { + const int inputIndex = correction->getInputIndex(); + const bool addToSubQueue = inputIndex < SUB_QUEUE_MAX_COUNT; + + int wordLength; + unsigned short* wordPointer; + + if ((currentWordIndex == FIRST_WORD_INDEX) && addToMasterQueue) { + WordsPriorityQueue *masterQueue = queuePool->getMasterQueue(); + const int finalProbability = + correction->getFinalProbability(probability, &wordPointer, &wordLength); + if (finalProbability != NOT_A_PROBABILITY) { + addWord(wordPointer, wordLength, finalProbability, masterQueue); + + const int shortcutProbability = finalProbability > 0 ? finalProbability - 1 : 0; + // Please note that the shortcut candidates will be added to the master queue only. + TerminalAttributes::ShortcutIterator iterator = + terminalAttributes.getShortcutIterator(); + while (iterator.hasNextShortcutTarget()) { + // TODO: addWord only supports weak ordering, meaning we have no means + // to control the order of the shortcuts relative to one another or to the word. + // We need to either modulate the probability of each shortcut according + // to its own shortcut probability or to make the queue + // so that the insert order is protected inside the queue for words + // with the same score. For the moment we use -1 to make sure the shortcut will + // never be in front of the word. + uint16_t shortcutTarget[MAX_WORD_LENGTH_INTERNAL]; + const int shortcutTargetStringLength = iterator.getNextShortcutTarget( + MAX_WORD_LENGTH_INTERNAL, shortcutTarget); + addWord(shortcutTarget, shortcutTargetStringLength, shortcutProbability, + masterQueue); + } + } + } + + // We only allow two words + other error correction for words with SUB_QUEUE_MIN_WORD_LENGTH + // or more length. + if (inputIndex >= SUB_QUEUE_MIN_WORD_LENGTH && addToSubQueue) { + WordsPriorityQueue *subQueue; + subQueue = queuePool->getSubQueue(currentWordIndex, inputIndex); + if (!subQueue) { + return; + } + const int finalProbability = correction->getFinalProbabilityForSubQueue( + probability, &wordPointer, &wordLength, inputIndex); + addWord(wordPointer, wordLength, finalProbability, subQueue); + } +} + +int UnigramDictionary::getSubStringSuggestion( + ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, + const int *codes, const bool useFullEditDistance, Correction *correction, + WordsPriorityQueuePool* queuePool, const int inputLength, + const bool hasAutoCorrectionCandidate, const int currentWordIndex, + const int inputWordStartPos, const int inputWordLength, + const int outputWordStartPos, const bool isSpaceProximity, int *freqArray, + int*wordLengthArray, unsigned short* outputWord, int *outputWordLength) const { + if (inputWordLength > MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH) { + return FLAG_MULTIPLE_SUGGEST_ABORT; + } + + ///////////////////////////////////////////// + // safety net for multiple word suggestion // + // TODO: Remove this safety net // + ///////////////////////////////////////////// + int smallWordCount = 0; + int singleLetterWordCount = 0; + if (inputWordLength == 1) { + ++singleLetterWordCount; + } + if (inputWordLength <= 2) { + // small word == single letter or 2-letter word + ++smallWordCount; + } + for (int i = 0; i < currentWordIndex; ++i) { + const int length = wordLengthArray[i]; + if (length == 1) { + ++singleLetterWordCount; + // Safety net to avoid suggesting sequential single letter words + if (i < (currentWordIndex - 1)) { + if (wordLengthArray[i + 1] == 1) { + return FLAG_MULTIPLE_SUGGEST_ABORT; + } + } else if (inputWordLength == 1) { + return FLAG_MULTIPLE_SUGGEST_ABORT; + } + } + if (length <= 2) { + ++smallWordCount; + } + // Safety net to avoid suggesting multiple words with many (4 or more, for now) small words + if (singleLetterWordCount >= 3 || smallWordCount >= 4) { + return FLAG_MULTIPLE_SUGGEST_ABORT; + } + } + ////////////////////////////////////////////// + // TODO: Remove the safety net above // + ////////////////////////////////////////////// + + unsigned short* tempOutputWord = 0; + int nextWordLength = 0; + // TODO: Optimize init suggestion + initSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, + inputLength, correction); + + unsigned short word[MAX_WORD_LENGTH_INTERNAL]; + int freq = getMostFrequentWordLike( + inputWordStartPos, inputWordLength, correction, word); + if (freq > 0) { + nextWordLength = inputWordLength; + tempOutputWord = word; + } else if (!hasAutoCorrectionCandidate) { + if (inputWordStartPos > 0) { + const int offset = inputWordStartPos; + initSuggestions(proximityInfo, &xcoordinates[offset], &ycoordinates[offset], + codes + offset, inputWordLength, correction); + queuePool->clearSubQueue(currentWordIndex); + // TODO: pass the bigram list for substring suggestion + getSuggestionCandidates(useFullEditDistance, inputWordLength, + 0 /* bigramMap */, 0 /* bigramFilter */, correction, queuePool, + false /* doAutoCompletion */, MAX_ERRORS_FOR_TWO_WORDS, currentWordIndex); + if (DEBUG_DICT) { + if (currentWordIndex < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) { + AKLOGI("Dump word candidates(%d) %d", currentWordIndex, inputWordLength); + for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { + queuePool->getSubQueue(currentWordIndex, i)->dumpTopWord(); + } + } + } + } + WordsPriorityQueue* queue = queuePool->getSubQueue(currentWordIndex, inputWordLength); + // TODO: Return the correct value depending on doAutoCompletion + if (!queue || queue->size() <= 0) { + return FLAG_MULTIPLE_SUGGEST_ABORT; + } + int score = 0; + const float ns = queue->getHighestNormalizedScore( + correction->getPrimaryInputWord(), inputWordLength, + &tempOutputWord, &score, &nextWordLength); + if (DEBUG_DICT) { + AKLOGI("NS(%d) = %f, Score = %d", currentWordIndex, ns, score); + } + // Two words correction won't be done if the score of the first word doesn't exceed the + // threshold. + if (ns < TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD + || nextWordLength < SUB_QUEUE_MIN_WORD_LENGTH) { + return FLAG_MULTIPLE_SUGGEST_SKIP; + } + freq = score >> (nextWordLength + TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER); + } + if (DEBUG_DICT) { + AKLOGI("Freq(%d): %d, length: %d, input length: %d, input start: %d (%d)" + , currentWordIndex, freq, nextWordLength, inputWordLength, inputWordStartPos, + wordLengthArray[0]); + } + if (freq <= 0 || nextWordLength <= 0 + || MAX_WORD_LENGTH <= (outputWordStartPos + nextWordLength)) { + return FLAG_MULTIPLE_SUGGEST_SKIP; + } + for (int i = 0; i < nextWordLength; ++i) { + outputWord[outputWordStartPos + i] = tempOutputWord[i]; + } + + // Put output values + freqArray[currentWordIndex] = freq; + // TODO: put output length instead of input length + wordLengthArray[currentWordIndex] = inputWordLength; + const int tempOutputWordLength = outputWordStartPos + nextWordLength; + if (outputWordLength) { + *outputWordLength = tempOutputWordLength; + } + + if ((inputWordStartPos + inputWordLength) < inputLength) { + if (outputWordStartPos + nextWordLength >= MAX_WORD_LENGTH) { + return FLAG_MULTIPLE_SUGGEST_SKIP; + } + outputWord[tempOutputWordLength] = SPACE; + if (outputWordLength) { + ++*outputWordLength; + } + } else if (currentWordIndex >= 1) { + // TODO: Handle 3 or more words + const int pairFreq = correction->getFreqForSplitMultipleWords( + freqArray, wordLengthArray, currentWordIndex + 1, isSpaceProximity, outputWord); + if (DEBUG_DICT) { + DUMP_WORD(outputWord, tempOutputWordLength); + for (int i = 0; i < currentWordIndex + 1; ++i) { + AKLOGI("Split %d,%d words: freq = %d, length = %d", i, currentWordIndex + 1, + freqArray[i], wordLengthArray[i]); + } + AKLOGI("Split two words: freq = %d, length = %d, %d, isSpace ? %d", pairFreq, + inputLength, tempOutputWordLength, isSpaceProximity); + } + addWord(outputWord, tempOutputWordLength, pairFreq, queuePool->getMasterQueue()); + } + return FLAG_MULTIPLE_SUGGEST_CONTINUE; +} + +void UnigramDictionary::getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codes, + const bool useFullEditDistance, const int inputLength, + Correction *correction, WordsPriorityQueuePool* queuePool, + const bool hasAutoCorrectionCandidate, const int startInputPos, const int startWordIndex, + const int outputWordLength, int *freqArray, int* wordLengthArray, + unsigned short* outputWord) const { + if (startWordIndex >= (MULTIPLE_WORDS_SUGGESTION_MAX_WORDS - 1)) { + // Return if the last word index + return; + } + if (startWordIndex >= 1 + && (hasAutoCorrectionCandidate + || inputLength < MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION)) { + // Do not suggest 3+ words if already has auto correction candidate + return; + } + for (int i = startInputPos + 1; i < inputLength; ++i) { + if (DEBUG_CORRECTION_FREQ) { + AKLOGI("Multi words(%d), start in %d sep %d start out %d", + startWordIndex, startInputPos, i, outputWordLength); + DUMP_WORD(outputWord, outputWordLength); + } + int tempOutputWordLength = 0; + // Current word + int inputWordStartPos = startInputPos; + int inputWordLength = i - startInputPos; + const int suggestionFlag = getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, + codes, useFullEditDistance, correction, queuePool, inputLength, + hasAutoCorrectionCandidate, startWordIndex, inputWordStartPos, inputWordLength, + outputWordLength, true /* not used */, freqArray, wordLengthArray, outputWord, + &tempOutputWordLength); + if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_ABORT) { + // TODO: break here + continue; + } else if (suggestionFlag == FLAG_MULTIPLE_SUGGEST_SKIP) { + continue; + } + + if (DEBUG_CORRECTION_FREQ) { + AKLOGI("Do missing space correction"); + } + // Next word + // Missing space + inputWordStartPos = i; + inputWordLength = inputLength - i; + if(getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes, + useFullEditDistance, correction, queuePool, inputLength, hasAutoCorrectionCandidate, + startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength, + false /* missing space */, freqArray, wordLengthArray, outputWord, 0) + != FLAG_MULTIPLE_SUGGEST_CONTINUE) { + getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes, + useFullEditDistance, inputLength, correction, queuePool, + hasAutoCorrectionCandidate, inputWordStartPos, startWordIndex + 1, + tempOutputWordLength, freqArray, wordLengthArray, outputWord); + } + + // Mistyped space + ++inputWordStartPos; + --inputWordLength; + + if (inputWordLength <= 0) { + continue; + } + + const int x = xcoordinates[inputWordStartPos - 1]; + const int y = ycoordinates[inputWordStartPos - 1]; + if (!proximityInfo->hasSpaceProximity(x, y)) { + continue; + } + + if (DEBUG_CORRECTION_FREQ) { + AKLOGI("Do mistyped space correction"); + } + getSubStringSuggestion(proximityInfo, xcoordinates, ycoordinates, codes, + useFullEditDistance, correction, queuePool, inputLength, hasAutoCorrectionCandidate, + startWordIndex + 1, inputWordStartPos, inputWordLength, tempOutputWordLength, + true /* mistyped space */, freqArray, wordLengthArray, outputWord, 0); + } +} + +void UnigramDictionary::getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codes, + const bool useFullEditDistance, const int inputLength, + Correction *correction, WordsPriorityQueuePool* queuePool, + const bool hasAutoCorrectionCandidate) const { + if (inputLength >= MAX_WORD_LENGTH) return; + if (DEBUG_DICT) { + AKLOGI("--- Suggest multiple words"); + } + + // Allocating fixed length array on stack + unsigned short outputWord[MAX_WORD_LENGTH]; + int freqArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS]; + int wordLengthArray[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS]; + const int outputWordLength = 0; + const int startInputPos = 0; + const int startWordIndex = 0; + getMultiWordsSuggestionRec(proximityInfo, xcoordinates, ycoordinates, codes, + useFullEditDistance, inputLength, correction, queuePool, hasAutoCorrectionCandidate, + startInputPos, startWordIndex, outputWordLength, freqArray, wordLengthArray, + outputWord); +} + +// Wrapper for getMostFrequentWordLikeInner, which matches it to the previous +// interface. +inline int UnigramDictionary::getMostFrequentWordLike(const int startInputIndex, + const int inputLength, Correction *correction, unsigned short *word) const { + uint16_t inWord[inputLength]; + + for (int i = 0; i < inputLength; ++i) { + inWord[i] = (uint16_t)correction->getPrimaryCharAt(startInputIndex + i); + } + return getMostFrequentWordLikeInner(inWord, inputLength, word); +} + +// This function will take the position of a character array within a CharGroup, +// and check it actually like-matches the word in inWord starting at startInputIndex, +// that is, it matches it with case and accents squashed. +// The function returns true if there was a full match, false otherwise. +// The function will copy on-the-fly the characters in the CharGroup to outNewWord. +// It will also place the end position of the array in outPos; in outInputIndex, +// it will place the index of the first char AFTER the match if there was a match, +// and the initial position if there was not. It makes sense because if there was +// a match we want to continue searching, but if there was not, we want to go to +// the next CharGroup. +// In and out parameters may point to the same location. This function takes care +// not to use any input parameters after it wrote into its outputs. +static inline bool testCharGroupForContinuedLikeness(const uint8_t flags, + const uint8_t* const root, const int startPos, + const uint16_t* const inWord, const int startInputIndex, + int32_t* outNewWord, int* outInputIndex, int* outPos) { + const bool hasMultipleChars = (0 != (UnigramDictionary::FLAG_HAS_MULTIPLE_CHARS & flags)); + int pos = startPos; + int32_t character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + int32_t baseChar = toBaseLowerCase(character); + const uint16_t wChar = toBaseLowerCase(inWord[startInputIndex]); + + if (baseChar != wChar) { + *outPos = hasMultipleChars ? BinaryFormat::skipOtherCharacters(root, pos) : pos; + *outInputIndex = startInputIndex; + return false; + } + int inputIndex = startInputIndex; + outNewWord[inputIndex] = character; + if (hasMultipleChars) { + character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + while (NOT_A_CHARACTER != character) { + baseChar = toBaseLowerCase(character); + if (toBaseLowerCase(inWord[++inputIndex]) != baseChar) { + *outPos = BinaryFormat::skipOtherCharacters(root, pos); + *outInputIndex = startInputIndex; + return false; + } + outNewWord[inputIndex] = character; + character = BinaryFormat::getCharCodeAndForwardPointer(root, &pos); + } + } + *outInputIndex = inputIndex + 1; + *outPos = pos; + return true; +} + +// This function is invoked when a word like the word searched for is found. +// It will compare the frequency to the max frequency, and if greater, will +// copy the word into the output buffer. In output value maxFreq, it will +// write the new maximum frequency if it changed. +static inline void onTerminalWordLike(const int freq, int32_t* newWord, const int length, + short unsigned int* outWord, int* maxFreq) { + if (freq > *maxFreq) { + for (int q = 0; q < length; ++q) + outWord[q] = newWord[q]; + outWord[length] = 0; + *maxFreq = freq; + } +} + +// Will find the highest frequency of the words like the one passed as an argument, +// that is, everything that only differs by case/accents. +int UnigramDictionary::getMostFrequentWordLikeInner(const uint16_t * const inWord, + const int length, short unsigned int* outWord) const { + int32_t newWord[MAX_WORD_LENGTH_INTERNAL]; + int depth = 0; + int maxFreq = -1; + const uint8_t* const root = DICT_ROOT; + int stackChildCount[MAX_WORD_LENGTH_INTERNAL]; + int stackInputIndex[MAX_WORD_LENGTH_INTERNAL]; + int stackSiblingPos[MAX_WORD_LENGTH_INTERNAL]; + + int startPos = 0; + stackChildCount[0] = BinaryFormat::getGroupCountAndForwardPointer(root, &startPos); + stackInputIndex[0] = 0; + stackSiblingPos[0] = startPos; + while (depth >= 0) { + const int charGroupCount = stackChildCount[depth]; + int pos = stackSiblingPos[depth]; + for (int charGroupIndex = charGroupCount - 1; charGroupIndex >= 0; --charGroupIndex) { + int inputIndex = stackInputIndex[depth]; + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + // Test whether all chars in this group match with the word we are searching for. If so, + // we want to traverse its children (or if the length match, evaluate its frequency). + // Note that this function will output the position regardless, but will only write + // into inputIndex if there is a match. + const bool isAlike = testCharGroupForContinuedLikeness(flags, root, pos, inWord, + inputIndex, newWord, &inputIndex, &pos); + if (isAlike && (FLAG_IS_TERMINAL & flags) && (inputIndex == length)) { + const int frequency = BinaryFormat::readFrequencyWithoutMovingPointer(root, pos); + onTerminalWordLike(frequency, newWord, inputIndex, outWord, &maxFreq); + } + pos = BinaryFormat::skipFrequency(flags, pos); + const int siblingPos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos); + const int childrenNodePos = BinaryFormat::readChildrenPosition(root, flags, pos); + // If we had a match and the word has children, we want to traverse them. We don't have + // to traverse words longer than the one we are searching for, since they will not match + // anyway, so don't traverse unless inputIndex < length. + if (isAlike && (-1 != childrenNodePos) && (inputIndex < length)) { + // Save position for this depth, to get back to this once children are done + stackChildCount[depth] = charGroupIndex; + stackSiblingPos[depth] = siblingPos; + // Prepare stack values for next depth + ++depth; + int childrenPos = childrenNodePos; + stackChildCount[depth] = + BinaryFormat::getGroupCountAndForwardPointer(root, &childrenPos); + stackSiblingPos[depth] = childrenPos; + stackInputIndex[depth] = inputIndex; + pos = childrenPos; + // Go to the next depth level. + ++depth; + break; + } else { + // No match, or no children, or word too long to ever match: go the next sibling. + pos = siblingPos; + } + } + --depth; + } + return maxFreq; +} + +int UnigramDictionary::getFrequency(const int32_t* const inWord, const int length) const { + const uint8_t* const root = DICT_ROOT; + int pos = BinaryFormat::getTerminalPosition(root, inWord, length); + if (NOT_VALID_WORD == pos) { + return NOT_A_PROBABILITY; + } + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); + const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags)); + if (hasMultipleChars) { + pos = BinaryFormat::skipOtherCharacters(root, pos); + } else { + BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos); + } + const int unigramFreq = BinaryFormat::readFrequencyWithoutMovingPointer(root, pos); + return unigramFreq; +} + +// TODO: remove this function. +int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offset, + int length) const { + return -1; +} + +// ProcessCurrentNode returns a boolean telling whether to traverse children nodes or not. +// If the return value is false, then the caller should read in the output "nextSiblingPosition" +// to find out the address of the next sibling node and pass it to a new call of processCurrentNode. +// It is worthy to note that when false is returned, the output values other than +// nextSiblingPosition are undefined. +// If the return value is true, then the caller must proceed to traverse the children of this +// node. processCurrentNode will output the information about the children: their count in +// newCount, their position in newChildrenPosition, the traverseAllNodes flag in +// newTraverseAllNodes, the match weight into newMatchRate, the input index into newInputIndex, the +// diffs into newDiffs, the sibling position in nextSiblingPosition, and the output index into +// newOutputIndex. Please also note the following caveat: processCurrentNode does not know when +// there aren't any more nodes at this level, it merely returns the address of the first byte after +// the current node in nextSiblingPosition. Thus, the caller must keep count of the nodes at any +// given level, as output into newCount when traversing this level's parent. +inline bool UnigramDictionary::processCurrentNode(const int initialPos, + const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, Correction *correction, + int *newCount, int *newChildrenPosition, int *nextSiblingPosition, + WordsPriorityQueuePool *queuePool, const int currentWordIndex) const { + if (DEBUG_DICT) { + correction->checkState(); + } + int pos = initialPos; + + // Flags contain the following information: + // - Address type (MASK_GROUP_ADDRESS_TYPE) on two bits: + // - FLAG_GROUP_ADDRESS_TYPE_{ONE,TWO,THREE}_BYTES means there are children and their address + // is on the specified number of bytes. + // - FLAG_GROUP_ADDRESS_TYPE_NOADDRESS means there are no children, and therefore no address. + // - FLAG_HAS_MULTIPLE_CHARS: whether this node has multiple char or not. + // - FLAG_IS_TERMINAL: whether this node is a terminal or not (it may still have children) + // - FLAG_HAS_BIGRAMS: whether this node has bigrams or not + const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(DICT_ROOT, &pos); + const bool hasMultipleChars = (0 != (FLAG_HAS_MULTIPLE_CHARS & flags)); + const bool isTerminalNode = (0 != (FLAG_IS_TERMINAL & flags)); + + bool needsToInvokeOnTerminal = false; + + // This gets only ONE character from the stream. Next there will be: + // if FLAG_HAS_MULTIPLE CHARS: the other characters of the same node + // else if FLAG_IS_TERMINAL: the frequency + // else if MASK_GROUP_ADDRESS_TYPE is not NONE: the children address + // Note that you can't have a node that both is not a terminal and has no children. + int32_t c = BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos); + assert(NOT_A_CHARACTER != c); + + // We are going to loop through each character and make it look like it's a different + // node each time. To do that, we will process characters in this node in order until + // we find the character terminator. This is signalled by getCharCode* returning + // NOT_A_CHARACTER. + // As a special case, if there is only one character in this node, we must not read the + // next bytes so we will simulate the NOT_A_CHARACTER return by testing the flags. + // This way, each loop run will look like a "virtual node". + do { + // We prefetch the next char. If 'c' is the last char of this node, we will have + // NOT_A_CHARACTER in the next char. From this we can decide whether this virtual node + // should behave as a terminal or not and whether we have children. + const int32_t nextc = hasMultipleChars + ? BinaryFormat::getCharCodeAndForwardPointer(DICT_ROOT, &pos) : NOT_A_CHARACTER; + const bool isLastChar = (NOT_A_CHARACTER == nextc); + // If there are more chars in this nodes, then this virtual node is not a terminal. + // If we are on the last char, this virtual node is a terminal if this node is. + const bool isTerminal = isLastChar && isTerminalNode; + + Correction::CorrectionType stateType = correction->processCharAndCalcState( + c, isTerminal); + if (stateType == Correction::TRAVERSE_ALL_ON_TERMINAL + || stateType == Correction::ON_TERMINAL) { + needsToInvokeOnTerminal = true; + } else if (stateType == Correction::UNRELATED || correction->needsToPrune()) { + // We found that this is an unrelated character, so we should give up traversing + // this node and its children entirely. + // However we may not be on the last virtual node yet so we skip the remaining + // characters in this node, the frequency if it's there, read the next sibling + // position to output it, then return false. + // We don't have to output other values because we return false, as in + // "don't traverse children". + if (!isLastChar) { + pos = BinaryFormat::skipOtherCharacters(DICT_ROOT, pos); + } + pos = BinaryFormat::skipFrequency(flags, pos); + *nextSiblingPosition = + BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + return false; + } + + // Prepare for the next character. Promote the prefetched char to current char - the loop + // will take care of prefetching the next. If we finally found our last char, nextc will + // contain NOT_A_CHARACTER. + c = nextc; + } while (NOT_A_CHARACTER != c); + + if (isTerminalNode) { + // The frequency should be here, because we come here only if this is actually + // a terminal node, and we are on its last char. + const int unigramFreq = BinaryFormat::readFrequencyWithoutMovingPointer(DICT_ROOT, pos); + const int childrenAddressPos = BinaryFormat::skipFrequency(flags, pos); + const int attributesPos = BinaryFormat::skipChildrenPosition(flags, childrenAddressPos); + TerminalAttributes terminalAttributes(DICT_ROOT, flags, attributesPos); + // bigramMap contains the bigram frequencies indexed by addresses for fast lookup. + // bigramFilter is a bloom filter of said frequencies for even faster rejection. + const int probability = BinaryFormat::getProbability(initialPos, bigramMap, bigramFilter, + unigramFreq); + onTerminal(probability, terminalAttributes, correction, queuePool, needsToInvokeOnTerminal, + currentWordIndex); + + // If there are more chars in this node, then this virtual node has children. + // If we are on the last char, this virtual node has children if this node has. + const bool hasChildren = BinaryFormat::hasChildrenInFlags(flags); + + // This character matched the typed character (enough to traverse the node at least) + // so we just evaluated it. Now we should evaluate this virtual node's children - that + // is, if it has any. If it has no children, we're done here - so we skip the end of + // the node, output the siblings position, and return false "don't traverse children". + // Note that !hasChildren implies isLastChar, so we know we don't have to skip any + // remaining char in this group for there can't be any. + if (!hasChildren) { + pos = BinaryFormat::skipFrequency(flags, pos); + *nextSiblingPosition = + BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + return false; + } + + // Optimization: Prune out words that are too long compared to how much was typed. + if (correction->needsToPrune()) { + pos = BinaryFormat::skipFrequency(flags, pos); + *nextSiblingPosition = + BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + if (DEBUG_DICT_FULL) { + AKLOGI("Traversing was pruned."); + } + return false; + } + } + + // Now we finished processing this node, and we want to traverse children. If there are no + // children, we can't come here. + assert(BinaryFormat::hasChildrenInFlags(flags)); + + // If this node was a terminal it still has the frequency under the pointer (it may have been + // read, but not skipped - see readFrequencyWithoutMovingPointer). + // Next come the children position, then possibly attributes (attributes are bigrams only for + // now, maybe something related to shortcuts in the future). + // Once this is read, we still need to output the number of nodes in the immediate children of + // this node, so we read and output it before returning true, as in "please traverse children". + pos = BinaryFormat::skipFrequency(flags, pos); + int childrenPos = BinaryFormat::readChildrenPosition(DICT_ROOT, flags, pos); + *nextSiblingPosition = BinaryFormat::skipChildrenPosAndAttributes(DICT_ROOT, flags, pos); + *newCount = BinaryFormat::getGroupCountAndForwardPointer(DICT_ROOT, &childrenPos); + *newChildrenPosition = childrenPos; + return true; +} + +} // namespace latinime diff --git a/native/jni/src/unigram_dictionary.h b/native/jni/src/unigram_dictionary.h new file mode 100644 index 000000000..8352c5494 --- /dev/null +++ b/native/jni/src/unigram_dictionary.h @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_UNIGRAM_DICTIONARY_H +#define LATINIME_UNIGRAM_DICTIONARY_H + +#include <map> +#include <stdint.h> +#include "correction.h" +#include "correction_state.h" +#include "defines.h" +#include "proximity_info.h" +#include "words_priority_queue.h" +#include "words_priority_queue_pool.h" + +namespace latinime { + +class TerminalAttributes; +class UnigramDictionary { + typedef struct { int first; int second; int replacement; } digraph_t; + + public: + // Mask and flags for children address type selection. + static const int MASK_GROUP_ADDRESS_TYPE = 0xC0; + static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; + static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; + static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; + static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; + + // Flag for single/multiple char group + static const int FLAG_HAS_MULTIPLE_CHARS = 0x20; + + // Flag for terminal groups + static const int FLAG_IS_TERMINAL = 0x10; + + // Flag for shortcut targets presence + static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08; + // Flag for bigram presence + static const int FLAG_HAS_BIGRAMS = 0x04; + + // Attribute (bigram/shortcut) related flags: + // Flag for presence of more attributes + static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; + // Flag for sign of offset. If this flag is set, the offset value must be negated. + static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; + + // Mask for attribute frequency, stored on 4 bits inside the flags byte. + static const int MASK_ATTRIBUTE_FREQUENCY = 0x0F; + + // Mask and flags for attribute address type selection. + static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; + static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; + static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; + static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; + + // Error tolerances + static const int DEFAULT_MAX_ERRORS = 2; + static const int MAX_ERRORS_FOR_TWO_WORDS = 1; + + static const int FLAG_MULTIPLE_SUGGEST_ABORT = 0; + static const int FLAG_MULTIPLE_SUGGEST_SKIP = 1; + static const int FLAG_MULTIPLE_SUGGEST_CONTINUE = 2; + UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler, + int fullWordMultiplier, int maxWordLength, int maxWords, const unsigned int flags); + int getFrequency(const int32_t* const inWord, const int length) const; + int getBigramPosition(int pos, unsigned short *word, int offset, int length) const; + int getSuggestions( + ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, + const int *codes, const int codesSize, const std::map<int, int> *bigramMap, + const uint8_t *bigramFilter, const bool useFullEditDistance, unsigned short *outWords, + int *frequencies) const; + virtual ~UnigramDictionary(); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(UnigramDictionary); + void getWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, + const int *ycoordinates, const int *codes, const int inputLength, + const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, Correction *correction, + WordsPriorityQueuePool *queuePool) const; + int getDigraphReplacement(const int *codes, const int i, const int codesSize, + const digraph_t* const digraphs, const unsigned int digraphsSize) const; + void getWordWithDigraphSuggestionsRec(ProximityInfo *proximityInfo, + const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, + int *xCoordinatesBuffer, int *yCoordinatesBuffer, const int codesBufferSize, + const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + const bool useFullEditDistance, const int* codesSrc, const int codesRemain, + const int currentDepth, int* codesDest, Correction *correction, + WordsPriorityQueuePool* queuePool, const digraph_t* const digraphs, + const unsigned int digraphsSize) const; + void initSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, + const int *ycoordinates, const int *codes, const int codesSize, + Correction *correction) const; + void getOneWordSuggestions(ProximityInfo *proximityInfo, const int *xcoordinates, + const int *ycoordinates, const int *codes, const std::map<int, int> *bigramMap, + const uint8_t *bigramFilter, const bool useFullEditDistance, const int inputLength, + Correction *correction, WordsPriorityQueuePool* queuePool) const; + void getSuggestionCandidates( + const bool useFullEditDistance, const int inputLength, + const std::map<int, int> *bigramMap, const uint8_t *bigramFilter, + Correction *correction, WordsPriorityQueuePool* queuePool, const bool doAutoCompletion, + const int maxErrors, const int currentWordIndex) const; + void getSplitMultipleWordsSuggestions(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codes, + const bool useFullEditDistance, const int inputLength, + Correction *correction, WordsPriorityQueuePool* queuePool, + const bool hasAutoCorrectionCandidate) const; + void onTerminal(const int freq, const TerminalAttributes& terminalAttributes, + Correction *correction, WordsPriorityQueuePool *queuePool, const bool addToMasterQueue, + const int currentWordIndex) const; + // Process a node by considering proximity, missing and excessive character + bool processCurrentNode(const int initialPos, const std::map<int, int> *bigramMap, + const uint8_t *bigramFilter, Correction *correction, int *newCount, + int *newChildPosition, int *nextSiblingPosition, WordsPriorityQueuePool *queuePool, + const int currentWordIndex) const; + int getMostFrequentWordLike(const int startInputIndex, const int inputLength, + Correction *correction, unsigned short *word) const; + int getMostFrequentWordLikeInner(const uint16_t* const inWord, const int length, + short unsigned int *outWord) const; + int getSubStringSuggestion( + ProximityInfo *proximityInfo, const int *xcoordinates, const int *ycoordinates, + const int *codes, const bool useFullEditDistance, Correction *correction, + WordsPriorityQueuePool* queuePool, const int inputLength, + const bool hasAutoCorrectionCandidate, const int currentWordIndex, + const int inputWordStartPos, const int inputWordLength, + const int outputWordStartPos, const bool isSpaceProximity, int *freqArray, + int *wordLengthArray, unsigned short* outputWord, int *outputWordLength) const; + void getMultiWordsSuggestionRec(ProximityInfo *proximityInfo, + const int *xcoordinates, const int *ycoordinates, const int *codes, + const bool useFullEditDistance, const int inputLength, + Correction *correction, WordsPriorityQueuePool* queuePool, + const bool hasAutoCorrectionCandidate, const int startPos, const int startWordIndex, + const int outputWordLength, int *freqArray, int* wordLengthArray, + unsigned short* outputWord) const; + + const uint8_t* const DICT_ROOT; + const int MAX_WORD_LENGTH; + const int MAX_WORDS; + const int TYPED_LETTER_MULTIPLIER; + const int FULL_WORD_MULTIPLIER; + const int ROOT_POS; + const unsigned int BYTES_IN_ONE_CHAR; + const int MAX_DIGRAPH_SEARCH_DEPTH; + const int FLAGS; + + static const digraph_t GERMAN_UMLAUT_DIGRAPHS[]; + static const digraph_t FRENCH_LIGATURES_DIGRAPHS[]; +}; +} // namespace latinime + +#endif // LATINIME_UNIGRAM_DICTIONARY_H diff --git a/native/jni/src/words_priority_queue.h b/native/jni/src/words_priority_queue.h new file mode 100644 index 000000000..9c6d28d60 --- /dev/null +++ b/native/jni/src/words_priority_queue.h @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_WORDS_PRIORITY_QUEUE_H +#define LATINIME_WORDS_PRIORITY_QUEUE_H + +#include <cstring> // for memcpy() +#include <iostream> +#include <queue> +#include "defines.h" + +namespace latinime { + +class WordsPriorityQueue { + public: + class SuggestedWord { + public: + int mScore; + unsigned short mWord[MAX_WORD_LENGTH_INTERNAL]; + int mWordLength; + bool mUsed; + + void setParams(int score, unsigned short* word, int wordLength) { + mScore = score; + mWordLength = wordLength; + memcpy(mWord, word, sizeof(unsigned short) * wordLength); + mUsed = true; + } + }; + + WordsPriorityQueue(int maxWords, int maxWordLength) : + MAX_WORDS((unsigned int) maxWords), MAX_WORD_LENGTH( + (unsigned int) maxWordLength) { + mSuggestedWords = new SuggestedWord[maxWordLength]; + for (int i = 0; i < maxWordLength; ++i) { + mSuggestedWords[i].mUsed = false; + } + mHighestSuggestedWord = 0; + } + + ~WordsPriorityQueue() { + delete[] mSuggestedWords; + } + + void push(int score, unsigned short* word, int wordLength) { + SuggestedWord* sw = 0; + if (mSuggestions.size() >= MAX_WORDS) { + sw = mSuggestions.top(); + const int minScore = sw->mScore; + if (minScore >= score) { + return; + } else { + sw->mUsed = false; + mSuggestions.pop(); + } + } + if (sw == 0) { + sw = getFreeSuggestedWord(score, word, wordLength); + } else { + sw->setParams(score, word, wordLength); + } + if (sw == 0) { + AKLOGE("SuggestedWord is accidentally null."); + return; + } + if (DEBUG_WORDS_PRIORITY_QUEUE) { + AKLOGI("Push word. %d, %d", score, wordLength); + DUMP_WORD(word, wordLength); + } + mSuggestions.push(sw); + if (!mHighestSuggestedWord || mHighestSuggestedWord->mScore < sw->mScore) { + mHighestSuggestedWord = sw; + } + } + + SuggestedWord* top() { + if (mSuggestions.empty()) return 0; + SuggestedWord* sw = mSuggestions.top(); + return sw; + } + + int outputSuggestions(const unsigned short* before, const int beforeLength, + int *frequencies, unsigned short *outputChars) { + mHighestSuggestedWord = 0; + const unsigned int size = min( + MAX_WORDS, static_cast<unsigned int>(mSuggestions.size())); + SuggestedWord* swBuffer[size]; + int index = size - 1; + while (!mSuggestions.empty() && index >= 0) { + SuggestedWord* sw = mSuggestions.top(); + if (DEBUG_WORDS_PRIORITY_QUEUE) { + AKLOGI("dump word. %d", sw->mScore); + DUMP_WORD(sw->mWord, sw->mWordLength); + } + swBuffer[index] = sw; + mSuggestions.pop(); + --index; + } + if (size >= 2) { + SuggestedWord* nsMaxSw = 0; + unsigned int maxIndex = 0; + float maxNs = 0; + for (unsigned int i = 0; i < size; ++i) { + SuggestedWord* tempSw = swBuffer[i]; + if (!tempSw) { + continue; + } + const float tempNs = getNormalizedScore(tempSw, before, beforeLength, 0, 0, 0); + if (tempNs >= maxNs) { + maxNs = tempNs; + maxIndex = i; + nsMaxSw = tempSw; + } + } + if (maxIndex > 0 && nsMaxSw) { + memmove(&swBuffer[1], &swBuffer[0], maxIndex * sizeof(SuggestedWord*)); + swBuffer[0] = nsMaxSw; + } + } + for (unsigned int i = 0; i < size; ++i) { + SuggestedWord* sw = swBuffer[i]; + if (!sw) { + AKLOGE("SuggestedWord is null %d", i); + continue; + } + const unsigned int wordLength = sw->mWordLength; + char* targetAdr = (char*) outputChars + i * MAX_WORD_LENGTH * sizeof(short); + frequencies[i] = sw->mScore; + memcpy(targetAdr, sw->mWord, (wordLength) * sizeof(short)); + if (wordLength < MAX_WORD_LENGTH) { + ((unsigned short*) targetAdr)[wordLength] = 0; + } + sw->mUsed = false; + } + return size; + } + + int size() const { + return mSuggestions.size(); + } + + void clear() { + mHighestSuggestedWord = 0; + while (!mSuggestions.empty()) { + SuggestedWord* sw = mSuggestions.top(); + if (DEBUG_WORDS_PRIORITY_QUEUE) { + AKLOGI("Clear word. %d", sw->mScore); + DUMP_WORD(sw->mWord, sw->mWordLength); + } + sw->mUsed = false; + mSuggestions.pop(); + } + } + + void dumpTopWord() { + if (size() <= 0) { + return; + } + DUMP_WORD(mHighestSuggestedWord->mWord, mHighestSuggestedWord->mWordLength); + } + + float getHighestNormalizedScore(const unsigned short* before, const int beforeLength, + unsigned short** outWord, int *outScore, int *outLength) { + if (!mHighestSuggestedWord) { + return 0.0; + } + return getNormalizedScore( + mHighestSuggestedWord, before, beforeLength, outWord, outScore, outLength); + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(WordsPriorityQueue); + struct wordComparator { + bool operator ()(SuggestedWord * left, SuggestedWord * right) { + return left->mScore > right->mScore; + } + }; + + SuggestedWord* getFreeSuggestedWord(int score, unsigned short* word, + int wordLength) { + for (unsigned int i = 0; i < MAX_WORD_LENGTH; ++i) { + if (!mSuggestedWords[i].mUsed) { + mSuggestedWords[i].setParams(score, word, wordLength); + return &mSuggestedWords[i]; + } + } + return 0; + } + + static float getNormalizedScore(SuggestedWord* sw, const unsigned short* before, + const int beforeLength, unsigned short** outWord, int *outScore, int *outLength) { + const int score = sw->mScore; + unsigned short* word = sw->mWord; + const int wordLength = sw->mWordLength; + if (outScore) { + *outScore = score; + } + if (outWord) { + *outWord = word; + } + if (outLength) { + *outLength = wordLength; + } + return Correction::RankingAlgorithm::calcNormalizedScore( + before, beforeLength, word, wordLength, score); + } + + typedef std::priority_queue<SuggestedWord*, std::vector<SuggestedWord*>, + wordComparator> Suggestions; + Suggestions mSuggestions; + const unsigned int MAX_WORDS; + const unsigned int MAX_WORD_LENGTH; + SuggestedWord* mSuggestedWords; + SuggestedWord* mHighestSuggestedWord; +}; +} + +#endif // LATINIME_WORDS_PRIORITY_QUEUE_H diff --git a/native/jni/src/words_priority_queue_pool.h b/native/jni/src/words_priority_queue_pool.h new file mode 100644 index 000000000..b4e2bed26 --- /dev/null +++ b/native/jni/src/words_priority_queue_pool.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_WORDS_PRIORITY_QUEUE_POOL_H +#define LATINIME_WORDS_PRIORITY_QUEUE_POOL_H + +#include <assert.h> +#include <new> +#include "words_priority_queue.h" + +namespace latinime { + +class WordsPriorityQueuePool { + public: + WordsPriorityQueuePool(int mainQueueMaxWords, int subQueueMaxWords, int maxWordLength) { + // Note: using placement new() requires the caller to call the destructor explicitly. + mMasterQueue = new(mMasterQueueBuf) WordsPriorityQueue(mainQueueMaxWords, maxWordLength); + for (int i = 0, subQueueBufOffset = 0; + i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT; + ++i, subQueueBufOffset += sizeof(WordsPriorityQueue)) { + mSubQueues[i] = new(mSubQueueBuf + subQueueBufOffset) + WordsPriorityQueue(subQueueMaxWords, maxWordLength); + } + } + + virtual ~WordsPriorityQueuePool() { + // Note: these explicit calls to the destructor match the calls to placement new() above. + if (mMasterQueue) mMasterQueue->~WordsPriorityQueue(); + for (int i = 0; i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS * SUB_QUEUE_MAX_COUNT; ++i) { + if (mSubQueues[i]) mSubQueues[i]->~WordsPriorityQueue(); + } + } + + WordsPriorityQueue* getMasterQueue() { + return mMasterQueue; + } + + WordsPriorityQueue* getSubQueue(const int wordIndex, const int inputWordLength) { + if (wordIndex >= MULTIPLE_WORDS_SUGGESTION_MAX_WORDS) { + return 0; + } + if (inputWordLength < 0 || inputWordLength >= SUB_QUEUE_MAX_COUNT) { + if (DEBUG_WORDS_PRIORITY_QUEUE) { + assert(false); + } + return 0; + } + return mSubQueues[wordIndex * SUB_QUEUE_MAX_COUNT + inputWordLength]; + } + + inline void clearAll() { + mMasterQueue->clear(); + for (int i = 0; i < MULTIPLE_WORDS_SUGGESTION_MAX_WORDS; ++i) { + clearSubQueue(i); + } + } + + inline void clearSubQueue(const int wordIndex) { + for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { + WordsPriorityQueue* queue = getSubQueue(wordIndex, i); + if (queue) { + queue->clear(); + } + } + } + + void dumpSubQueue1TopSuggestions() { + AKLOGI("DUMP SUBQUEUE1 TOP SUGGESTIONS"); + for (int i = 0; i < SUB_QUEUE_MAX_COUNT; ++i) { + getSubQueue(0, i)->dumpTopWord(); + } + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(WordsPriorityQueuePool); + WordsPriorityQueue* mMasterQueue; + WordsPriorityQueue* mSubQueues[SUB_QUEUE_MAX_COUNT * MULTIPLE_WORDS_SUGGESTION_MAX_WORDS]; + char mMasterQueueBuf[sizeof(WordsPriorityQueue)]; + char mSubQueueBuf[MULTIPLE_WORDS_SUGGESTION_MAX_WORDS + * SUB_QUEUE_MAX_COUNT * sizeof(WordsPriorityQueue)]; +}; +} + +#endif // LATINIME_WORDS_PRIORITY_QUEUE_POOL_H diff --git a/native/src/bigram_dictionary.cpp b/native/src/bigram_dictionary.cpp deleted file mode 100644 index 11e6dc250..000000000 --- a/native/src/bigram_dictionary.cpp +++ /dev/null @@ -1,265 +0,0 @@ -/* -** -** Copyright 2010, The Android Open Source Project -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -*/ - -#include <string.h> - -#define LOG_TAG "LatinIME: bigram_dictionary.cpp" - -#include "bigram_dictionary.h" -#include "dictionary.h" - -namespace latinime { - -BigramDictionary::BigramDictionary(const unsigned char *dict, int maxWordLength, - int maxAlternatives, const bool isLatestDictVersion, const bool hasBigram, - Dictionary *parentDictionary) - : DICT(dict), MAX_WORD_LENGTH(maxWordLength), - MAX_ALTERNATIVES(maxAlternatives), IS_LATEST_DICT_VERSION(isLatestDictVersion), - HAS_BIGRAM(hasBigram), mParentDictionary(parentDictionary) { - if (DEBUG_DICT) { - LOGI("BigramDictionary - constructor"); - LOGI("Has Bigram : %d", hasBigram); - } -} - -BigramDictionary::~BigramDictionary() { -} - -bool BigramDictionary::addWordBigram(unsigned short *word, int length, int frequency) { - word[length] = 0; - if (DEBUG_DICT) { - char s[length + 1]; - for (int i = 0; i <= length; i++) s[i] = word[i]; - LOGI("Bigram: Found word = %s, freq = %d :", s, frequency); - } - - // Find the right insertion point - int insertAt = 0; - while (insertAt < mMaxBigrams) { - if (frequency > mBigramFreq[insertAt] || (mBigramFreq[insertAt] == frequency - && length < Dictionary::wideStrLen(mBigramChars + insertAt * MAX_WORD_LENGTH))) { - break; - } - insertAt++; - } - if (DEBUG_DICT) { - LOGI("Bigram: InsertAt -> %d maxBigrams: %d", insertAt, mMaxBigrams); - } - if (insertAt < mMaxBigrams) { - memmove((char*) mBigramFreq + (insertAt + 1) * sizeof(mBigramFreq[0]), - (char*) mBigramFreq + insertAt * sizeof(mBigramFreq[0]), - (mMaxBigrams - insertAt - 1) * sizeof(mBigramFreq[0])); - mBigramFreq[insertAt] = frequency; - memmove((char*) mBigramChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short), - (char*) mBigramChars + (insertAt ) * MAX_WORD_LENGTH * sizeof(short), - (mMaxBigrams - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH); - unsigned short *dest = mBigramChars + (insertAt ) * MAX_WORD_LENGTH; - while (length--) { - *dest++ = *word++; - } - *dest = 0; // NULL terminate - if (DEBUG_DICT) { - LOGI("Bigram: Added word at %d", insertAt); - } - return true; - } - return false; -} - -int BigramDictionary::getBigramAddress(int *pos, bool advance) { - int address = 0; - - address += (DICT[*pos] & 0x3F) << 16; - address += (DICT[*pos + 1] & 0xFF) << 8; - address += (DICT[*pos + 2] & 0xFF); - - if (advance) { - *pos += 3; - } - - return address; -} - -int BigramDictionary::getBigramFreq(int *pos) { - int freq = DICT[(*pos)++] & FLAG_BIGRAM_FREQ; - - return freq; -} - - -int BigramDictionary::getBigrams(unsigned short *prevWord, int prevWordLength, int *codes, - int codesSize, unsigned short *bigramChars, int *bigramFreq, int maxWordLength, - int maxBigrams, int maxAlternatives) { - mBigramFreq = bigramFreq; - mBigramChars = bigramChars; - mInputCodes = codes; - mInputLength = codesSize; - mMaxBigrams = maxBigrams; - - if (HAS_BIGRAM && IS_LATEST_DICT_VERSION) { - int pos = mParentDictionary->getBigramPosition(prevWord, prevWordLength); - if (DEBUG_DICT) { - LOGI("Pos -> %d", pos); - } - if (pos < 0) { - return 0; - } - - int bigramCount = 0; - int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ); - if (bigramExist > 0) { - int nextBigramExist = 1; - while (nextBigramExist > 0 && bigramCount < maxBigrams) { - int bigramAddress = getBigramAddress(&pos, true); - int frequency = (FLAG_BIGRAM_FREQ & DICT[pos]); - // search for all bigrams and store them - searchForTerminalNode(bigramAddress, frequency); - nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED); - bigramCount++; - } - } - - return bigramCount; - } - return 0; -} - -void BigramDictionary::searchForTerminalNode(int addressLookingFor, int frequency) { - // track word with such address and store it in an array - unsigned short word[MAX_WORD_LENGTH]; - - int pos; - int followDownBranchAddress = DICTIONARY_HEADER_SIZE; - bool found = false; - char followingChar = ' '; - int depth = -1; - - while(!found) { - bool followDownAddressSearchStop = false; - bool firstAddress = true; - bool haveToSearchAll = true; - - if (depth < MAX_WORD_LENGTH && depth >= 0) { - word[depth] = (unsigned short) followingChar; - } - pos = followDownBranchAddress; // pos start at count - int count = DICT[pos] & 0xFF; - if (DEBUG_DICT) { - LOGI("count - %d",count); - } - pos++; - for (int i = 0; i < count; i++) { - // pos at data - pos++; - // pos now at flag - if (!getFirstBitOfByte(&pos)) { // non-terminal - if (!followDownAddressSearchStop) { - int addr = getBigramAddress(&pos, false); - if (addr > addressLookingFor) { - followDownAddressSearchStop = true; - if (firstAddress) { - firstAddress = false; - haveToSearchAll = true; - } else if (!haveToSearchAll) { - break; - } - } else { - followDownBranchAddress = addr; - followingChar = (char)(0xFF & DICT[pos-1]); - if (firstAddress) { - firstAddress = false; - haveToSearchAll = false; - } - } - } - pos += 3; - } else if (getFirstBitOfByte(&pos)) { // terminal - if (addressLookingFor == (pos-1)) { // found !! - depth++; - word[depth] = (0xFF & DICT[pos-1]); - found = true; - break; - } - if (getSecondBitOfByte(&pos)) { // address + freq (4 byte) - if (!followDownAddressSearchStop) { - int addr = getBigramAddress(&pos, false); - if (addr > addressLookingFor) { - followDownAddressSearchStop = true; - if (firstAddress) { - firstAddress = false; - haveToSearchAll = true; - } else if (!haveToSearchAll) { - break; - } - } else { - followDownBranchAddress = addr; - followingChar = (char)(0xFF & DICT[pos-1]); - if (firstAddress) { - firstAddress = false; - haveToSearchAll = true; - } - } - } - pos += 4; - } else { // freq only (2 byte) - pos += 2; - } - - // skipping bigram - int bigramExist = (DICT[pos] & FLAG_BIGRAM_READ); - if (bigramExist > 0) { - int nextBigramExist = 1; - while (nextBigramExist > 0) { - pos += 3; - nextBigramExist = (DICT[pos++] & FLAG_BIGRAM_CONTINUED); - } - } else { - pos++; - } - } - } - depth++; - if (followDownBranchAddress == 0) { - if (DEBUG_DICT) { - LOGI("ERROR!!! Cannot find bigram!!"); - } - break; - } - } - if (checkFirstCharacter(word)) { - addWordBigram(word, depth, frequency); - } -} - -bool BigramDictionary::checkFirstCharacter(unsigned short *word) { - // Checks whether this word starts with same character or neighboring characters of - // what user typed. - - int *inputCodes = mInputCodes; - int maxAlt = MAX_ALTERNATIVES; - while (maxAlt > 0) { - if ((unsigned int) *inputCodes == (unsigned int) *word) { - return true; - } - inputCodes++; - maxAlt--; - } - return false; -} - -// TODO: Move functions related to bigram to here -} // namespace latinime diff --git a/native/src/defines.h b/native/src/defines.h deleted file mode 100644 index 0a3240507..000000000 --- a/native/src/defines.h +++ /dev/null @@ -1,170 +0,0 @@ -/* -** -** Copyright 2010, The Android Open Source Project -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -*/ - -#ifndef LATINIME_DEFINES_H -#define LATINIME_DEFINES_H - -#ifdef FLAG_DBG -#include <cutils/log.h> -#ifndef LOG_TAG -#define LOG_TAG "LatinIME: " -#endif -#define DEBUG_DICT true -#define DEBUG_DICT_FULL false -#define DEBUG_SHOW_FOUND_WORD DEBUG_DICT_FULL -#define DEBUG_NODE DEBUG_DICT_FULL -#define DEBUG_TRACE DEBUG_DICT_FULL -#define DEBUG_PROXIMITY_INFO true - -// Profiler -#include <time.h> -#define PROF_BUF_SIZE 100 -static double profile_buf[PROF_BUF_SIZE]; -static double profile_old[PROF_BUF_SIZE]; -static unsigned int profile_counter[PROF_BUF_SIZE]; - -#define PROF_RESET prof_reset() -#define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id] -#define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while(0) -#define PROF_START(prof_buf_id) do { \ - PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while(0) -#define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while(0) -#define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id]) -#define PROF_CLOCKOUT(prof_buf_id) \ - LOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id])) -#define PROF_OUTALL do { LOGI("--- %s ---", __FUNCTION__); prof_out(); } while(0) - -static void prof_reset(void) { - for (int i = 0; i < PROF_BUF_SIZE; ++i) { - profile_buf[i] = 0; - profile_old[i] = 0; - profile_counter[i] = 0; - } -} - -static void prof_out(void) { - if (profile_counter[PROF_BUF_SIZE - 1] != 1) { - LOGI("Error: You must call PROF_OPEN before PROF_CLOSE."); - } - LOGI("Total time is %6.3f ms.", - profile_buf[PROF_BUF_SIZE - 1] * 1000 / (double)CLOCKS_PER_SEC); - double all = 0; - for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { - all += profile_buf[i]; - } - if (all == 0) all = 1; - for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { - if (profile_buf[i] != 0) { - LOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.", - i, (profile_buf[i] * 100 / all), - profile_buf[i] * 1000 / (double)CLOCKS_PER_SEC, profile_counter[i]); - } - } -} - -#else // FLAG_DBG -#define LOGE(fmt, ...) -#define LOGI(fmt, ...) -#define DEBUG_DICT false -#define DEBUG_DICT_FULL false -#define DEBUG_SHOW_FOUND_WORD false -#define DEBUG_NODE false -#define DEBUG_TRACE false -#define DEBUG_PROXIMITY_INFO false - -#define PROF_BUF_SIZE 0 -#define PROF_RESET -#define PROF_COUNT(prof_buf_id) -#define PROF_OPEN -#define PROF_START(prof_buf_id) -#define PROF_CLOSE -#define PROF_END(prof_buf_id) -#define PROF_CLOCK_OUT(prof_buf_id) -#define PROF_CLOCKOUT(prof_buf_id) -#define PROF_OUTALL - -#endif // FLAG_DBG - -#ifndef U_SHORT_MAX -#define U_SHORT_MAX 1 << 16 -#endif -#ifndef S_INT_MAX -#define S_INT_MAX 2147483647 // ((1 << 31) - 1) -#endif - -// Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap(). -// We measured and compared performance of both, and found mmap() is fairly good in terms of -// loading time, and acceptable even for several initial lookups which involve page faults. -#define USE_MMAP_FOR_DICTIONARY - -// 22-bit address = ~4MB dictionary size limit, which on average would be about 200k-300k words -#define ADDRESS_MASK 0x3FFFFF - -// The bit that decides if an address follows in the next 22 bits -#define FLAG_ADDRESS_MASK 0x40 -// The bit that decides if this is a terminal node for a word. The node could still have children, -// if the word has other endings. -#define FLAG_TERMINAL_MASK 0x80 - -#define FLAG_BIGRAM_READ 0x80 -#define FLAG_BIGRAM_CHILDEXIST 0x40 -#define FLAG_BIGRAM_CONTINUED 0x80 -#define FLAG_BIGRAM_FREQ 0x7F - -#define DICTIONARY_VERSION_MIN 200 -#define DICTIONARY_HEADER_SIZE 2 -#define NOT_VALID_WORD -99 - -#define KEYCODE_SPACE ' ' - -#define SUGGEST_WORDS_WITH_MISSING_CHARACTER true -#define SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER true -#define SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER true -#define SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS true -#define SUGGEST_WORDS_WITH_SPACE_PROXIMITY true - -// The following "rate"s are used as a multiplier before dividing by 100, so they are in percent. -#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80 -#define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12 -#define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 67 -#define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75 -#define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75 -#define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 60 -#define FULL_MATCHED_WORDS_PROMOTION_RATE 120 -#define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90 - -// This should be greater than or equal to MAX_WORD_LENGTH defined in BinaryDictionary.java -// This is only used for the size of array. Not to be used in c functions. -#define MAX_WORD_LENGTH_INTERNAL 48 - -#define MAX_DEPTH_MULTIPLIER 3 - -// TODO: Reduce this constant if possible; check the maximum number of umlauts in the same German -// word in the dictionary -#define DEFAULT_MAX_UMLAUT_SEARCH_DEPTH 5 - -// Minimum suggest depth for one word for all cases except for missing space suggestions. -#define MIN_SUGGEST_DEPTH 1 -#define MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION 3 -#define MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION 3 - -// The size of next letters frequency array. Zero will disable the feature. -#define NEXT_LETTERS_SIZE 0 - -#define min(a,b) ((a)<(b)?(a):(b)) - -#endif // LATINIME_DEFINES_H diff --git a/native/src/dictionary.h b/native/src/dictionary.h deleted file mode 100644 index 3dc577a56..000000000 --- a/native/src/dictionary.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (C) 2009 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DICTIONARY_H -#define LATINIME_DICTIONARY_H - -#include "bigram_dictionary.h" -#include "defines.h" -#include "proximity_info.h" -#include "unigram_dictionary.h" - -namespace latinime { - -class Dictionary { -public: - Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler, - int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives); - int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates, - int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) { - return mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, - codesSize, flags, outWords, frequencies); - } - - // TODO: Call mBigramDictionary instead of mUnigramDictionary - int getBigrams(unsigned short *word, int length, int *codes, int codesSize, - unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams, - int maxAlternatives) { - return mBigramDictionary->getBigrams(word, length, codes, codesSize, outWords, frequencies, - maxWordLength, maxBigrams, maxAlternatives); - } - - bool isValidWord(unsigned short *word, int length); - void *getDict() { return (void *)mDict; } - int getDictSize() { return mDictSize; } - int getMmapFd() { return mMmapFd; } - int getDictBufAdjust() { return mDictBufAdjust; } - ~Dictionary(); - - // public static utility methods - // static inline methods should be defined in the header file - static unsigned short getChar(const unsigned char *dict, int *pos); - static int getCount(const unsigned char *dict, int *pos); - static bool getTerminal(const unsigned char *dict, int *pos); - static int getAddress(const unsigned char *dict, int *pos); - static int getFreq(const unsigned char *dict, const bool isLatestDictVersion, int *pos); - static int wideStrLen(unsigned short *str); - // returns next sibling's position - static int setDictionaryValues(const unsigned char *dict, const bool isLatestDictVersion, - const int pos, unsigned short *c, int *childrenPosition, - bool *terminal, int *freq); - - // TODO: delete this - int getBigramPosition(unsigned short *word, int length); - -private: - bool hasBigram(); - - const unsigned char *mDict; - - // Used only for the mmap version of dictionary loading, but we use these as dummy variables - // also for the malloc version. - const int mDictSize; - const int mMmapFd; - const int mDictBufAdjust; - - const bool IS_LATEST_DICT_VERSION; - UnigramDictionary *mUnigramDictionary; - BigramDictionary *mBigramDictionary; -}; - -// public static utility methods -// static inline methods should be defined in the header file -inline unsigned short Dictionary::getChar(const unsigned char *dict, int *pos) { - unsigned short ch = (unsigned short) (dict[(*pos)++] & 0xFF); - // If the code is 255, then actual 16 bit code follows (in big endian) - if (ch == 0xFF) { - ch = ((dict[*pos] & 0xFF) << 8) | (dict[*pos + 1] & 0xFF); - (*pos) += 2; - } - return ch; -} - -inline int Dictionary::getCount(const unsigned char *dict, int *pos) { - return dict[(*pos)++] & 0xFF; -} - -inline bool Dictionary::getTerminal(const unsigned char *dict, int *pos) { - return (dict[*pos] & FLAG_TERMINAL_MASK) > 0; -} - -inline int Dictionary::getAddress(const unsigned char *dict, int *pos) { - int address = 0; - if ((dict[*pos] & FLAG_ADDRESS_MASK) == 0) { - *pos += 1; - } else { - address += (dict[*pos] & (ADDRESS_MASK >> 16)) << 16; - address += (dict[*pos + 1] & 0xFF) << 8; - address += (dict[*pos + 2] & 0xFF); - *pos += 3; - } - return address; -} - -inline int Dictionary::getFreq(const unsigned char *dict, - const bool isLatestDictVersion, int *pos) { - int freq = dict[(*pos)++] & 0xFF; - if (isLatestDictVersion) { - // skipping bigram - int bigramExist = (dict[*pos] & FLAG_BIGRAM_READ); - if (bigramExist > 0) { - int nextBigramExist = 1; - while (nextBigramExist > 0) { - (*pos) += 3; - nextBigramExist = (dict[(*pos)++] & FLAG_BIGRAM_CONTINUED); - } - } else { - (*pos)++; - } - } - return freq; -} - -inline int Dictionary::wideStrLen(unsigned short *str) { - if (!str) return 0; - unsigned short *end = str; - while (*end) - end++; - return end - str; -} - -inline int Dictionary::setDictionaryValues(const unsigned char *dict, - const bool isLatestDictVersion, const int pos, unsigned short *c,int *childrenPosition, - bool *terminal, int *freq) { - int position = pos; - // -- at char - *c = Dictionary::getChar(dict, &position); - // -- at flag/add - *terminal = Dictionary::getTerminal(dict, &position); - *childrenPosition = Dictionary::getAddress(dict, &position); - // -- after address or flag - *freq = (*terminal) ? Dictionary::getFreq(dict, isLatestDictVersion, &position) : 1; - // returns next sibling's position - return position; -} - -} // namespace latinime - -#endif // LATINIME_DICTIONARY_H diff --git a/native/src/proximity_info.cpp b/native/src/proximity_info.cpp deleted file mode 100644 index 209c31e6e..000000000 --- a/native/src/proximity_info.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdio.h> -#include <string.h> - -#define LOG_TAG "LatinIME: proximity_info.cpp" - -#include "proximity_info.h" - -namespace latinime { - -ProximityInfo::ProximityInfo(const int maxProximityCharsSize, const int keyboardWidth, - const int keyboardHeight, const int gridWidth, const int gridHeight, - const uint32_t *proximityCharsArray) - : MAX_PROXIMITY_CHARS_SIZE(maxProximityCharsSize), KEYBOARD_WIDTH(keyboardWidth), - KEYBOARD_HEIGHT(keyboardHeight), GRID_WIDTH(gridWidth), GRID_HEIGHT(gridHeight), - CELL_WIDTH((keyboardWidth + gridWidth - 1) / gridWidth), - CELL_HEIGHT((keyboardHeight + gridHeight - 1) / gridHeight) { - const int len = GRID_WIDTH * GRID_HEIGHT * MAX_PROXIMITY_CHARS_SIZE; - mProximityCharsArray = new uint32_t[len]; - if (DEBUG_PROXIMITY_INFO) { - LOGI("Create proximity info array %d", len); - } - memcpy(mProximityCharsArray, proximityCharsArray, len * sizeof(mProximityCharsArray[0])); -} - -ProximityInfo::~ProximityInfo() { - delete[] mProximityCharsArray; -} - -inline int ProximityInfo::getStartIndexFromCoordinates(const int x, const int y) const { - return ((y / CELL_HEIGHT) * GRID_WIDTH + (x / CELL_WIDTH)) - * MAX_PROXIMITY_CHARS_SIZE; -} - -bool ProximityInfo::hasSpaceProximity(const int x, const int y) const { - const int startIndex = getStartIndexFromCoordinates(x, y); - if (DEBUG_PROXIMITY_INFO) { - LOGI("hasSpaceProximity: index %d", startIndex); - } - for (int i = 0; i < MAX_PROXIMITY_CHARS_SIZE; ++i) { - if (DEBUG_PROXIMITY_INFO) { - LOGI("Index: %d", mProximityCharsArray[startIndex + i]); - } - if (mProximityCharsArray[startIndex + i] == KEYCODE_SPACE) { - return true; - } - } - return false; -} - -} // namespace latinime diff --git a/native/src/proximity_info.h b/native/src/proximity_info.h deleted file mode 100644 index 327cd0940..000000000 --- a/native/src/proximity_info.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_PROXIMITY_INFO_H -#define LATINIME_PROXIMITY_INFO_H - -#include <stdint.h> - -#include "defines.h" - -namespace latinime { - -class ProximityInfo { -public: - ProximityInfo(const int maxProximityCharsSize, const int keyboardWidth, - const int keybaordHeight, const int gridWidth, const int gridHeight, - const uint32_t *proximityCharsArray); - ~ProximityInfo(); - bool hasSpaceProximity(const int x, const int y) const; -private: - int getStartIndexFromCoordinates(const int x, const int y) const; - const int MAX_PROXIMITY_CHARS_SIZE; - const int KEYBOARD_WIDTH; - const int KEYBOARD_HEIGHT; - const int GRID_WIDTH; - const int GRID_HEIGHT; - const int CELL_WIDTH; - const int CELL_HEIGHT; - uint32_t *mProximityCharsArray; -}; - -} // namespace latinime - -#endif // LATINIME_PROXIMITY_INFO_H diff --git a/native/src/unigram_dictionary.cpp b/native/src/unigram_dictionary.cpp deleted file mode 100644 index e3296f12a..000000000 --- a/native/src/unigram_dictionary.cpp +++ /dev/null @@ -1,1127 +0,0 @@ -/* -** -** Copyright 2010, The Android Open Source Project -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -*/ - -#include <assert.h> -#include <string.h> - -#define LOG_TAG "LatinIME: unigram_dictionary.cpp" - -#include "basechars.h" -#include "char_utils.h" -#include "dictionary.h" -#include "unigram_dictionary.h" - -namespace latinime { - -const UnigramDictionary::digraph_t UnigramDictionary::GERMAN_UMLAUT_DIGRAPHS[] = - { { 'a', 'e' }, - { 'o', 'e' }, - { 'u', 'e' } }; - -// TODO: check the header -UnigramDictionary::UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultiplier, - int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars, - const bool isLatestDictVersion) - : DICT_ROOT(streamStart), - MAX_WORD_LENGTH(maxWordLength), MAX_WORDS(maxWords), - MAX_PROXIMITY_CHARS(maxProximityChars), IS_LATEST_DICT_VERSION(isLatestDictVersion), - TYPED_LETTER_MULTIPLIER(typedLetterMultiplier), FULL_WORD_MULTIPLIER(fullWordMultiplier), - ROOT_POS(isLatestDictVersion ? DICTIONARY_HEADER_SIZE : 0), - BYTES_IN_ONE_CHAR(MAX_PROXIMITY_CHARS * sizeof(*mInputCodes)), - MAX_UMLAUT_SEARCH_DEPTH(DEFAULT_MAX_UMLAUT_SEARCH_DEPTH) { - if (DEBUG_DICT) { - LOGI("UnigramDictionary - constructor"); - } -} - -UnigramDictionary::~UnigramDictionary() {} - -static inline unsigned int getCodesBufferSize(const int* codes, const int codesSize, - const int MAX_PROXIMITY_CHARS) { - return sizeof(*codes) * MAX_PROXIMITY_CHARS * codesSize; -} - -bool UnigramDictionary::isDigraph(const int* codes, const int i, const int codesSize) const { - - // There can't be a digraph if we don't have at least 2 characters to examine - if (i + 2 > codesSize) return false; - - // Search for the first char of some digraph - int lastDigraphIndex = -1; - const int thisChar = codes[i * MAX_PROXIMITY_CHARS]; - for (lastDigraphIndex = sizeof(GERMAN_UMLAUT_DIGRAPHS) / sizeof(GERMAN_UMLAUT_DIGRAPHS[0]) - 1; - lastDigraphIndex >= 0; --lastDigraphIndex) { - if (thisChar == GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].first) break; - } - // No match: return early - if (lastDigraphIndex < 0) return false; - - // It's an interesting digraph if the second char matches too. - return GERMAN_UMLAUT_DIGRAPHS[lastDigraphIndex].second == codes[(i + 1) * MAX_PROXIMITY_CHARS]; -} - -// Mostly the same arguments as the non-recursive version, except: -// codes is the original value. It points to the start of the work buffer, and gets passed as is. -// codesSize is the size of the user input (thus, it is the size of codesSrc). -// codesDest is the current point in the work buffer. -// codesSrc is the current point in the user-input, original, content-unmodified buffer. -// codesRemain is the remaining size in codesSrc. -void UnigramDictionary::getWordWithDigraphSuggestionsRec(const ProximityInfo *proximityInfo, - const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, - const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain, - const int currentDepth, int* codesDest, unsigned short* outWords, int* frequencies) { - - if (currentDepth < MAX_UMLAUT_SEARCH_DEPTH) { - for (int i = 0; i < codesRemain; ++i) { - if (isDigraph(codesSrc, i, codesRemain)) { - // Found a digraph. We will try both spellings. eg. the word is "pruefen" - - // Copy the word up to the first char of the digraph, then continue processing - // on the remaining part of the word, skipping the second char of the digraph. - // In our example, copy "pru" and continue running on "fen" - // Make i the index of the second char of the digraph for simplicity. Forgetting - // to do that results in an infinite recursion so take care! - ++i; - memcpy(codesDest, codesSrc, i * BYTES_IN_ONE_CHAR); - getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, - codesBuffer, codesBufferSize, flags, - codesSrc + (i + 1) * MAX_PROXIMITY_CHARS, codesRemain - i - 1, - currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, outWords, - frequencies); - - // Copy the second char of the digraph in place, then continue processing on - // the remaining part of the word. - // In our example, after "pru" in the buffer copy the "e", and continue on "fen" - memcpy(codesDest + i * MAX_PROXIMITY_CHARS, codesSrc + i * MAX_PROXIMITY_CHARS, - BYTES_IN_ONE_CHAR); - getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, - codesBuffer, codesBufferSize, flags, codesSrc + i * MAX_PROXIMITY_CHARS, - codesRemain - i, currentDepth + 1, codesDest + i * MAX_PROXIMITY_CHARS, - outWords, frequencies); - return; - } - } - } - - // If we come here, we hit the end of the word: let's check it against the dictionary. - // In our example, we'll come here once for "prufen" and then once for "pruefen". - // If the word contains several digraphs, we'll come it for the product of them. - // eg. if the word is "ueberpruefen" we'll test, in order, against - // "uberprufen", "uberpruefen", "ueberprufen", "ueberpruefen". - const unsigned int remainingBytes = BYTES_IN_ONE_CHAR * codesRemain; - if (0 != remainingBytes) - memcpy(codesDest, codesSrc, remainingBytes); - - getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codesBuffer, - (codesDest - codesBuffer) / MAX_PROXIMITY_CHARS + codesRemain, outWords, frequencies); -} - -int UnigramDictionary::getSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const int codesSize, const int flags, - unsigned short *outWords, int *frequencies) { - - if (REQUIRES_GERMAN_UMLAUT_PROCESSING & flags) - { // Incrementally tune the word and try all possibilities - int codesBuffer[getCodesBufferSize(codes, codesSize, MAX_PROXIMITY_CHARS)]; - getWordWithDigraphSuggestionsRec(proximityInfo, xcoordinates, ycoordinates, codesBuffer, - codesSize, flags, codes, codesSize, 0, codesBuffer, outWords, frequencies); - } else { // Normal processing - getWordSuggestions(proximityInfo, xcoordinates, ycoordinates, codes, codesSize, - outWords, frequencies); - } - - PROF_START(20); - // Get the word count - int suggestedWordsCount = 0; - while (suggestedWordsCount < MAX_WORDS && mFrequencies[suggestedWordsCount] > 0) { - suggestedWordsCount++; - } - - if (DEBUG_DICT) { - LOGI("Returning %d words", suggestedWordsCount); - LOGI("Next letters: "); - for (int k = 0; k < NEXT_LETTERS_SIZE; k++) { - if (mNextLettersFrequency[k] > 0) { - LOGI("%c = %d,", k, mNextLettersFrequency[k]); - } - } - } - PROF_END(20); - PROF_CLOSE; - return suggestedWordsCount; -} - -void UnigramDictionary::getWordSuggestions(const ProximityInfo *proximityInfo, - const int *xcoordinates, const int *ycoordinates, const int *codes, const int codesSize, - unsigned short *outWords, int *frequencies) { - - PROF_OPEN; - PROF_START(0); - initSuggestions(codes, codesSize, outWords, frequencies); - if (DEBUG_DICT) assert(codesSize == mInputLength); - - const int MAX_DEPTH = min(mInputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH); - PROF_END(0); - - PROF_START(1); - getSuggestionCandidates(-1, -1, -1, mNextLettersFrequency, NEXT_LETTERS_SIZE, MAX_DEPTH); - PROF_END(1); - - PROF_START(2); - // Suggestion with missing character - if (SUGGEST_WORDS_WITH_MISSING_CHARACTER) { - for (int i = 0; i < codesSize; ++i) { - if (DEBUG_DICT) { - LOGI("--- Suggest missing characters %d", i); - } - getSuggestionCandidates(i, -1, -1, NULL, 0, MAX_DEPTH); - } - } - PROF_END(2); - - PROF_START(3); - // Suggestion with excessive character - if (SUGGEST_WORDS_WITH_EXCESSIVE_CHARACTER - && mInputLength >= MIN_USER_TYPED_LENGTH_FOR_EXCESSIVE_CHARACTER_SUGGESTION) { - for (int i = 0; i < codesSize; ++i) { - if (DEBUG_DICT) { - LOGI("--- Suggest excessive characters %d", i); - } - getSuggestionCandidates(-1, i, -1, NULL, 0, MAX_DEPTH); - } - } - PROF_END(3); - - PROF_START(4); - // Suggestion with transposed characters - // Only suggest words that length is mInputLength - if (SUGGEST_WORDS_WITH_TRANSPOSED_CHARACTERS) { - for (int i = 0; i < codesSize; ++i) { - if (DEBUG_DICT) { - LOGI("--- Suggest transposed characters %d", i); - } - getSuggestionCandidates(-1, -1, i, NULL, 0, mInputLength - 1); - } - } - PROF_END(4); - - PROF_START(5); - // Suggestions with missing space - if (SUGGEST_WORDS_WITH_MISSING_SPACE_CHARACTER - && mInputLength >= MIN_USER_TYPED_LENGTH_FOR_MISSING_SPACE_SUGGESTION) { - for (int i = 1; i < codesSize; ++i) { - if (DEBUG_DICT) { - LOGI("--- Suggest missing space characters %d", i); - } - getMissingSpaceWords(mInputLength, i); - } - } - PROF_END(5); - - PROF_START(6); - if (SUGGEST_WORDS_WITH_SPACE_PROXIMITY && proximityInfo) { - // The first and last "mistyped spaces" are taken care of by excessive character handling - for (int i = 1; i < codesSize - 1; ++i) { - if (DEBUG_DICT) { - LOGI("--- Suggest words with proximity space %d", i); - } - const int x = xcoordinates[i]; - const int y = ycoordinates[i]; - if (DEBUG_PROXIMITY_INFO) { - LOGI("Input[%d] x = %d, y = %d, has space proximity = %d", - i, x, y, proximityInfo->hasSpaceProximity(x, y)); - } - if (proximityInfo->hasSpaceProximity(x, y)) { - getMistypedSpaceWords(mInputLength, i); - } - } - } - PROF_END(6); -} - -void UnigramDictionary::initSuggestions(const int *codes, const int codesSize, - unsigned short *outWords, int *frequencies) { - if (DEBUG_DICT) { - LOGI("initSuggest"); - } - mFrequencies = frequencies; - mOutputChars = outWords; - mInputCodes = codes; - mInputLength = codesSize; - mMaxEditDistance = mInputLength < 5 ? 2 : mInputLength / 2; -} - -static inline void registerNextLetter(unsigned short c, int *nextLetters, int nextLettersSize) { - if (c < nextLettersSize) { - nextLetters[c]++; - } -} - -// TODO: We need to optimize addWord by using STL or something -// TODO: This needs to take an const unsigned short* and not tinker with its contents -bool UnigramDictionary::addWord(unsigned short *word, int length, int frequency) { - word[length] = 0; - if (DEBUG_DICT && DEBUG_SHOW_FOUND_WORD) { - char s[length + 1]; - for (int i = 0; i <= length; i++) s[i] = word[i]; - LOGI("Found word = %s, freq = %d", s, frequency); - } - if (length > MAX_WORD_LENGTH) { - if (DEBUG_DICT) { - LOGI("Exceeded max word length."); - } - return false; - } - - // Find the right insertion point - int insertAt = 0; - while (insertAt < MAX_WORDS) { - // TODO: How should we sort words with the same frequency? - if (frequency > mFrequencies[insertAt]) { - break; - } - insertAt++; - } - if (insertAt < MAX_WORDS) { - if (DEBUG_DICT) { - char s[length + 1]; - for (int i = 0; i <= length; i++) s[i] = word[i]; - LOGI("Added word = %s, freq = %d, %d", s, frequency, S_INT_MAX); - } - memmove((char*) mFrequencies + (insertAt + 1) * sizeof(mFrequencies[0]), - (char*) mFrequencies + insertAt * sizeof(mFrequencies[0]), - (MAX_WORDS - insertAt - 1) * sizeof(mFrequencies[0])); - mFrequencies[insertAt] = frequency; - memmove((char*) mOutputChars + (insertAt + 1) * MAX_WORD_LENGTH * sizeof(short), - (char*) mOutputChars + insertAt * MAX_WORD_LENGTH * sizeof(short), - (MAX_WORDS - insertAt - 1) * sizeof(short) * MAX_WORD_LENGTH); - unsigned short *dest = mOutputChars + insertAt * MAX_WORD_LENGTH; - while (length--) { - *dest++ = *word++; - } - *dest = 0; // NULL terminate - if (DEBUG_DICT) { - LOGI("Added word at %d", insertAt); - } - return true; - } - return false; -} - -inline void UnigramDictionary::addWordAlternatesSpellings(const uint8_t* const root, int pos, - int depth, int finalFreq) { - // TODO: actually add alternates when the format supports it. -} - -static inline bool hasAlternateSpellings(uint8_t flags) { - // TODO: when the format supports it, return the actual value. - return false; -} - -static inline unsigned short toBaseLowerCase(unsigned short c) { - if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) { - c = BASE_CHARS[c]; - } - if (c >='A' && c <= 'Z') { - c |= 32; - } else if (c > 127) { - c = latin_tolower(c); - } - return c; -} - -bool UnigramDictionary::sameAsTyped(const unsigned short *word, int length) const { - if (length != mInputLength) { - return false; - } - const int *inputCodes = mInputCodes; - while (length--) { - if ((unsigned int) *inputCodes != (unsigned int) *word) { - return false; - } - inputCodes += MAX_PROXIMITY_CHARS; - word++; - } - return true; -} - -static const char QUOTE = '\''; -static const char SPACE = ' '; - -void UnigramDictionary::getSuggestionCandidates(const int skipPos, - const int excessivePos, const int transposedPos, int *nextLetters, - const int nextLettersSize, const int maxDepth) { - if (DEBUG_DICT) { - LOGI("getSuggestionCandidates %d", maxDepth); - assert(transposedPos + 1 < mInputLength); - assert(excessivePos < mInputLength); - assert(missingPos < mInputLength); - } - int rootPosition = ROOT_POS; - // Get the number of child of root, then increment the position - int childCount = Dictionary::getCount(DICT_ROOT, &rootPosition); - int depth = 0; - - mStackChildCount[0] = childCount; - mStackTraverseAll[0] = (mInputLength <= 0); - mStackNodeFreq[0] = 1; - mStackInputIndex[0] = 0; - mStackDiffs[0] = 0; - mStackSiblingPos[0] = rootPosition; - mStackOutputIndex[0] = 0; - - // Depth first search - while (depth >= 0) { - if (mStackChildCount[depth] > 0) { - --mStackChildCount[depth]; - bool traverseAllNodes = mStackTraverseAll[depth]; - int matchWeight = mStackNodeFreq[depth]; - int inputIndex = mStackInputIndex[depth]; - int diffs = mStackDiffs[depth]; - int siblingPos = mStackSiblingPos[depth]; - int outputIndex = mStackOutputIndex[depth]; - int firstChildPos; - // depth will never be greater than maxDepth because in that case, - // needsToTraverseChildrenNodes should be false - const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, outputIndex, - maxDepth, traverseAllNodes, matchWeight, inputIndex, diffs, skipPos, - excessivePos, transposedPos, nextLetters, nextLettersSize, &childCount, - &firstChildPos, &traverseAllNodes, &matchWeight, &inputIndex, &diffs, - &siblingPos, &outputIndex); - // Update next sibling pos - mStackSiblingPos[depth] = siblingPos; - if (needsToTraverseChildrenNodes) { - // Goes to child node - ++depth; - mStackChildCount[depth] = childCount; - mStackTraverseAll[depth] = traverseAllNodes; - mStackNodeFreq[depth] = matchWeight; - mStackInputIndex[depth] = inputIndex; - mStackDiffs[depth] = diffs; - mStackSiblingPos[depth] = firstChildPos; - mStackOutputIndex[depth] = outputIndex; - } - } else { - // Goes to parent sibling node - --depth; - } - } -} - -static const int TWO_31ST_DIV_255 = S_INT_MAX / 255; -static inline int capped255MultForFullMatchAccentsOrCapitalizationDifference(const int num) { - return (num < TWO_31ST_DIV_255 ? 255 * num : S_INT_MAX); -} - -static const int TWO_31ST_DIV_2 = S_INT_MAX / 2; -inline static void multiplyIntCapped(const int multiplier, int *base) { - const int temp = *base; - if (temp != S_INT_MAX) { - // Branch if multiplier == 2 for the optimization - if (multiplier == 2) { - *base = TWO_31ST_DIV_2 >= temp ? temp << 1 : S_INT_MAX; - } else { - const int tempRetval = temp * multiplier; - *base = tempRetval >= temp ? tempRetval : S_INT_MAX; - } - } -} - -inline static int powerIntCapped(const int base, const int n) { - if (base == 2) { - return n < 31 ? 1 << n : S_INT_MAX; - } else { - int ret = base; - for (int i = 1; i < n; ++i) multiplyIntCapped(base, &ret); - return ret; - } -} - -inline static void multiplyRate(const int rate, int *freq) { - if (*freq != S_INT_MAX) { - if (*freq > 1000000) { - *freq /= 100; - multiplyIntCapped(rate, freq); - } else { - multiplyIntCapped(rate, freq); - *freq /= 100; - } - } -} - -inline static int calcFreqForSplitTwoWords( - const int typedLetterMultiplier, const int firstWordLength, const int secondWordLength, - const int firstFreq, const int secondFreq, const bool isSpaceProximity) { - if (firstWordLength == 0 || secondWordLength == 0) { - return 0; - } - const int firstDemotionRate = 100 - 100 / (firstWordLength + 1); - int tempFirstFreq = firstFreq; - multiplyRate(firstDemotionRate, &tempFirstFreq); - - const int secondDemotionRate = 100 - 100 / (secondWordLength + 1); - int tempSecondFreq = secondFreq; - multiplyRate(secondDemotionRate, &tempSecondFreq); - - const int totalLength = firstWordLength + secondWordLength; - - // Promote pairFreq with multiplying by 2, because the word length is the same as the typed - // length. - int totalFreq = tempFirstFreq + tempSecondFreq; - - // This is a workaround to try offsetting the not-enough-demotion which will be done in - // calcNormalizedScore in Utils.java. - // In calcNormalizedScore the score will be demoted by (1 - 1 / length) - // but we demoted only (1 - 1 / (length + 1)) so we will additionally adjust freq by - // (1 - 1 / length) / (1 - 1 / (length + 1)) = (1 - 1 / (length * length)) - const int normalizedScoreNotEnoughDemotionAdjustment = 100 - 100 / (totalLength * totalLength); - multiplyRate(normalizedScoreNotEnoughDemotionAdjustment, &totalFreq); - - // At this moment, totalFreq is calculated by the following formula: - // (firstFreq * (1 - 1 / (firstWordLength + 1)) + secondFreq * (1 - 1 / (secondWordLength + 1))) - // * (1 - 1 / totalLength) / (1 - 1 / (totalLength + 1)) - - multiplyIntCapped(powerIntCapped(typedLetterMultiplier, totalLength), &totalFreq); - - // This is another workaround to offset the demotion which will be done in - // calcNormalizedScore in Utils.java. - // In calcNormalizedScore the score will be demoted by (1 - 1 / length) so we have to promote - // the same amount because we already have adjusted the synthetic freq of this "missing or - // mistyped space" suggestion candidate above in this method. - const int normalizedScoreDemotionRateOffset = (100 + 100 / totalLength); - multiplyRate(normalizedScoreDemotionRateOffset, &totalFreq); - - if (isSpaceProximity) { - // A word pair with one space proximity correction - if (DEBUG_DICT) { - LOGI("Found a word pair with space proximity correction."); - } - multiplyIntCapped(typedLetterMultiplier, &totalFreq); - multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &totalFreq); - } - - multiplyRate(WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE, &totalFreq); - return totalFreq; -} - -bool UnigramDictionary::getMissingSpaceWords(const int inputLength, const int missingSpacePos) { - return getSplitTwoWordsSuggestion( - inputLength, 0, missingSpacePos, missingSpacePos, inputLength - missingSpacePos, false); -} - -bool UnigramDictionary::getMistypedSpaceWords(const int inputLength, const int spaceProximityPos) { - return getSplitTwoWordsSuggestion( - inputLength, 0, spaceProximityPos, spaceProximityPos + 1, - inputLength - spaceProximityPos - 1, true); -} - -inline int UnigramDictionary::calculateFinalFreq(const int inputIndex, const int depth, - const int matchWeight, const int skipPos, const int excessivePos, const int transposedPos, - const int freq, const bool sameLength) const { - // TODO: Demote by edit distance - int finalFreq = freq * matchWeight; - if (skipPos >= 0) { - if (mInputLength >= 2) { - const int demotionRate = WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE - * (10 * mInputLength - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X) - / (10 * mInputLength - - WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X + 10); - if (DEBUG_DICT_FULL) { - LOGI("Demotion rate for missing character is %d.", demotionRate); - } - multiplyRate(demotionRate, &finalFreq); - } else { - finalFreq = 0; - } - } - if (transposedPos >= 0) multiplyRate( - WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE, &finalFreq); - if (excessivePos >= 0) { - multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE, &finalFreq); - if (!existsAdjacentProximityChars(inputIndex, mInputLength)) { - multiplyRate(WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE, &finalFreq); - } - } - int lengthFreq = TYPED_LETTER_MULTIPLIER; - multiplyIntCapped(powerIntCapped(TYPED_LETTER_MULTIPLIER, depth), &lengthFreq); - if (lengthFreq == matchWeight) { - // Full exact match - if (depth > 1) { - if (DEBUG_DICT) { - LOGI("Found full matched word."); - } - multiplyRate(FULL_MATCHED_WORDS_PROMOTION_RATE, &finalFreq); - } - if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0) { - finalFreq = capped255MultForFullMatchAccentsOrCapitalizationDifference(finalFreq); - } - } else if (sameLength && transposedPos < 0 && skipPos < 0 && excessivePos < 0 && depth > 0) { - // A word with proximity corrections - if (DEBUG_DICT) { - LOGI("Found one proximity correction."); - } - multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &finalFreq); - multiplyRate(WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE, &finalFreq); - } - if (DEBUG_DICT) { - LOGI("calc: %d, %d", depth, sameLength); - } - if (sameLength) multiplyIntCapped(FULL_WORD_MULTIPLIER, &finalFreq); - return finalFreq; -} - -inline bool UnigramDictionary::needsToSkipCurrentNode(const unsigned short c, - const int inputIndex, const int skipPos, const int depth) { - const unsigned short userTypedChar = getInputCharsAt(inputIndex)[0]; - // Skip the ' or other letter and continue deeper - return (c == QUOTE && userTypedChar != QUOTE) || skipPos == depth; -} - -inline bool UnigramDictionary::existsAdjacentProximityChars(const int inputIndex, - const int inputLength) const { - if (inputIndex < 0 || inputIndex >= inputLength) return false; - const int currentChar = *getInputCharsAt(inputIndex); - const int leftIndex = inputIndex - 1; - if (leftIndex >= 0) { - const int *leftChars = getInputCharsAt(leftIndex); - int i = 0; - while (leftChars[i] > 0 && i < MAX_PROXIMITY_CHARS) { - if (leftChars[i++] == currentChar) return true; - } - } - const int rightIndex = inputIndex + 1; - if (rightIndex < inputLength) { - const int *rightChars = getInputCharsAt(rightIndex); - int i = 0; - while (rightChars[i] > 0 && i < MAX_PROXIMITY_CHARS) { - if (rightChars[i++] == currentChar) return true; - } - } - return false; -} - -// In the following function, c is the current character of the dictionary word -// currently examined. -// currentChars is an array containing the keys close to the character the -// user actually typed at the same position. We want to see if c is in it: if so, -// then the word contains at that position a character close to what the user -// typed. -// What the user typed is actually the first character of the array. -// Notice : accented characters do not have a proximity list, so they are alone -// in their list. The non-accented version of the character should be considered -// "close", but not the other keys close to the non-accented version. -inline UnigramDictionary::ProximityType UnigramDictionary::getMatchedProximityId( - const int *currentChars, const unsigned short c, const int skipPos, - const int excessivePos, const int transposedPos) { - const unsigned short baseLowerC = toBaseLowerCase(c); - - // The first char in the array is what user typed. If it matches right away, - // that means the user typed that same char for this pos. - if (currentChars[0] == baseLowerC || currentChars[0] == c) - return SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR; - - // If one of those is true, we should not check for close characters at all. - if (skipPos >= 0 || excessivePos >= 0 || transposedPos >= 0) - return UNRELATED_CHAR; - - // If the non-accented, lowercased version of that first character matches c, - // then we have a non-accented version of the accented character the user - // typed. Treat it as a close char. - if (toBaseLowerCase(currentChars[0]) == baseLowerC) - return NEAR_PROXIMITY_CHAR; - - // Not an exact nor an accent-alike match: search the list of close keys - int j = 1; - while (currentChars[j] > 0 && j < MAX_PROXIMITY_CHARS) { - const bool matched = (currentChars[j] == baseLowerC || currentChars[j] == c); - if (matched) return NEAR_PROXIMITY_CHAR; - ++j; - } - - // Was not included, signal this as an unrelated character. - return UNRELATED_CHAR; -} - -inline void UnigramDictionary::onTerminal(unsigned short int* word, const int depth, - const uint8_t* const root, const uint8_t flags, int pos, - const int inputIndex, const int matchWeight, const int skipPos, - const int excessivePos, const int transposedPos, const int freq, const bool sameLength, - int* nextLetters, const int nextLettersSize) { - - const bool isSameAsTyped = sameLength ? sameAsTyped(word, depth + 1) : false; - const bool hasAlternates = hasAlternateSpellings(flags); - if (isSameAsTyped && !hasAlternates) return; - - if (depth >= MIN_SUGGEST_DEPTH) { - const int finalFreq = calculateFinalFreq(inputIndex, depth, matchWeight, skipPos, - excessivePos, transposedPos, freq, sameLength); - if (!isSameAsTyped) - addWord(word, depth + 1, finalFreq); - if (hasAlternates) - addWordAlternatesSpellings(DICT_ROOT, pos, flags, finalFreq); - } - - if (sameLength && depth >= mInputLength && skipPos < 0) { - registerNextLetter(word[mInputLength], nextLetters, nextLettersSize); - } -} - -#ifndef NEW_DICTIONARY_FORMAT -// TODO: Don't forget to bring inline functions back to over where they are used. - -// The following functions will be entirely replaced with new implementations. -void UnigramDictionary::getWordsOld(const int initialPos, const int inputLength, const int skipPos, - const int excessivePos, const int transposedPos,int *nextLetters, - const int nextLettersSize) { - int initialPosition = initialPos; - const int count = Dictionary::getCount(DICT_ROOT, &initialPosition); - getWordsRec(count, initialPosition, 0, - min(inputLength * MAX_DEPTH_MULTIPLIER, MAX_WORD_LENGTH), - mInputLength <= 0, 1, 0, 0, skipPos, excessivePos, transposedPos, nextLetters, - nextLettersSize); -} - -void UnigramDictionary::getWordsRec(const int childrenCount, const int pos, const int depth, - const int maxDepth, const bool traverseAllNodes, const int matchWeight, - const int inputIndex, const int diffs, const int skipPos, const int excessivePos, - const int transposedPos, int *nextLetters, const int nextLettersSize) { - int siblingPos = pos; - for (int i = 0; i < childrenCount; ++i) { - int newCount; - int newChildPosition; - bool newTraverseAllNodes; - int newMatchRate; - int newInputIndex; - int newDiffs; - int newSiblingPos; - int newOutputIndex; - const bool needsToTraverseChildrenNodes = processCurrentNode(siblingPos, depth, maxDepth, - traverseAllNodes, matchWeight, inputIndex, diffs, - skipPos, excessivePos, transposedPos, - nextLetters, nextLettersSize, - &newCount, &newChildPosition, &newTraverseAllNodes, &newMatchRate, - &newInputIndex, &newDiffs, &newSiblingPos, &newOutputIndex); - siblingPos = newSiblingPos; - - if (needsToTraverseChildrenNodes) { - getWordsRec(newCount, newChildPosition, newOutputIndex, maxDepth, newTraverseAllNodes, - newMatchRate, newInputIndex, newDiffs, skipPos, excessivePos, transposedPos, - nextLetters, nextLettersSize); - } - } -} - -inline int UnigramDictionary::getBestWordFreq(const int startInputIndex, const int inputLength, - unsigned short *word) { - int pos = ROOT_POS; - int count = Dictionary::getCount(DICT_ROOT, &pos); - int maxFreq = 0; - int depth = 0; - unsigned short newWord[MAX_WORD_LENGTH_INTERNAL]; - bool terminal = false; - - mStackChildCount[0] = count; - mStackSiblingPos[0] = pos; - - while (depth >= 0) { - if (mStackChildCount[depth] > 0) { - --mStackChildCount[depth]; - int firstChildPos; - int newFreq; - int siblingPos = mStackSiblingPos[depth]; - const bool needsToTraverseChildrenNodes = processCurrentNodeForExactMatch(siblingPos, - startInputIndex, depth, newWord, &firstChildPos, &count, &terminal, &newFreq, - &siblingPos); - mStackSiblingPos[depth] = siblingPos; - if (depth == (inputLength - 1)) { - // Traverse sibling node - if (terminal) { - if (newFreq > maxFreq) { - for (int i = 0; i < inputLength; ++i) word[i] = newWord[i]; - if (DEBUG_DICT && DEBUG_NODE) { - char s[inputLength + 1]; - for (int i = 0; i < inputLength; ++i) s[i] = word[i]; - s[inputLength] = 0; - LOGI("New missing space word found: %d > %d (%s), %d, %d", - newFreq, maxFreq, s, inputLength, depth); - } - maxFreq = newFreq; - } - } - } else if (needsToTraverseChildrenNodes) { - // Traverse children nodes - ++depth; - mStackChildCount[depth] = count; - mStackSiblingPos[depth] = firstChildPos; - } - } else { - // Traverse parent node - --depth; - } - } - - word[inputLength] = 0; - return maxFreq; -} - -inline bool UnigramDictionary::processCurrentNodeForExactMatch(const int firstChildPos, - const int startInputIndex, const int depth, unsigned short *word, int *newChildPosition, - int *newCount, bool *newTerminal, int *newFreq, int *siblingPos) { - const int inputIndex = startInputIndex + depth; - const int *currentChars = getInputCharsAt(inputIndex); - unsigned short c; - *siblingPos = Dictionary::setDictionaryValues(DICT_ROOT, IS_LATEST_DICT_VERSION, firstChildPos, - &c, newChildPosition, newTerminal, newFreq); - const unsigned int inputC = currentChars[0]; - if (DEBUG_DICT) { - assert(inputC <= U_SHORT_MAX); - } - const unsigned short baseLowerC = toBaseLowerCase(c); - const bool matched = (inputC == baseLowerC || inputC == c); - const bool hasChild = *newChildPosition != 0; - if (matched) { - word[depth] = c; - if (DEBUG_DICT && DEBUG_NODE) { - LOGI("Node(%c, %c)<%d>, %d, %d", inputC, c, matched, hasChild, *newFreq); - if (*newTerminal) { - LOGI("Terminal %d", *newFreq); - } - } - if (hasChild) { - *newCount = Dictionary::getCount(DICT_ROOT, newChildPosition); - return true; - } else { - return false; - } - } else { - // If this node is not user typed character, this method treats this word as unmatched. - // Thus newTerminal shouldn't be true. - *newTerminal = false; - return false; - } -} - -// TODO: use uint32_t instead of unsigned short -bool UnigramDictionary::isValidWord(unsigned short *word, int length) { - if (IS_LATEST_DICT_VERSION) { - return (getBigramPosition(DICTIONARY_HEADER_SIZE, word, 0, length) != NOT_VALID_WORD); - } else { - return (getBigramPosition(0, word, 0, length) != NOT_VALID_WORD); - } -} - - -// Require strict exact match. -int UnigramDictionary::getBigramPosition(int pos, unsigned short *word, int offset, - int length) const { - // returns address of bigram data of that word - // return -99 if not found - - int count = Dictionary::getCount(DICT_ROOT, &pos); - unsigned short currentChar = (unsigned short) word[offset]; - for (int j = 0; j < count; j++) { - unsigned short c = Dictionary::getChar(DICT_ROOT, &pos); - int terminal = Dictionary::getTerminal(DICT_ROOT, &pos); - int childPos = Dictionary::getAddress(DICT_ROOT, &pos); - if (c == currentChar) { - if (offset == length - 1) { - if (terminal) { - return (pos+1); - } - } else { - if (childPos != 0) { - int t = getBigramPosition(childPos, word, offset + 1, length); - if (t > 0) { - return t; - } - } - } - } - if (terminal) { - Dictionary::getFreq(DICT_ROOT, IS_LATEST_DICT_VERSION, &pos); - } - // There could be two instances of each alphabet - upper and lower case. So continue - // looking ... - } - return NOT_VALID_WORD; -} - - -// The following functions will be modified. -bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength, - const int firstWordStartPos, const int firstWordLength, const int secondWordStartPos, - const int secondWordLength, const bool isSpaceProximity) { - if (inputLength >= MAX_WORD_LENGTH) return false; - if (0 >= firstWordLength || 0 >= secondWordLength || firstWordStartPos >= secondWordStartPos - || firstWordStartPos < 0 || secondWordStartPos + secondWordLength > inputLength) - return false; - const int newWordLength = firstWordLength + secondWordLength + 1; - // Allocating variable length array on stack - unsigned short word[newWordLength]; - const int firstFreq = getBestWordFreq(firstWordStartPos, firstWordLength, mWord); - if (DEBUG_DICT) { - LOGI("First freq: %d", firstFreq); - } - if (firstFreq <= 0) return false; - - for (int i = 0; i < firstWordLength; ++i) { - word[i] = mWord[i]; - } - - const int secondFreq = getBestWordFreq(secondWordStartPos, secondWordLength, mWord); - if (DEBUG_DICT) { - LOGI("Second freq: %d", secondFreq); - } - if (secondFreq <= 0) return false; - - word[firstWordLength] = SPACE; - for (int i = (firstWordLength + 1); i < newWordLength; ++i) { - word[i] = mWord[i - firstWordLength - 1]; - } - - int pairFreq = calcFreqForSplitTwoWords(TYPED_LETTER_MULTIPLIER, firstWordLength, - secondWordLength, firstFreq, secondFreq, isSpaceProximity); - if (DEBUG_DICT) { - LOGI("Split two words: %d, %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength, - TYPED_LETTER_MULTIPLIER); - } - addWord(word, newWordLength, pairFreq); - return true; -} - -inline bool UnigramDictionary::processCurrentNode(const int pos, const int depth, - const int maxDepth, const bool traverseAllNodes, int matchWeight, int inputIndex, - const int diffs, const int skipPos, const int excessivePos, const int transposedPos, - int *nextLetters, const int nextLettersSize, int *newCount, int *newChildPosition, - bool *newTraverseAllNodes, int *newMatchRate, int *newInputIndex, int *newDiffs, - int *nextSiblingPosition, int *nextOutputIndex) { - if (DEBUG_DICT) { - int inputCount = 0; - if (skipPos >= 0) ++inputCount; - if (excessivePos >= 0) ++inputCount; - if (transposedPos >= 0) ++inputCount; - assert(inputCount <= 1); - } - unsigned short c; - int childPosition; - bool terminal; - int freq; - bool isSameAsUserTypedLength = false; - - const uint8_t flags = 0; // No flags for now - - if (excessivePos == depth && inputIndex < mInputLength - 1) ++inputIndex; - - *nextSiblingPosition = Dictionary::setDictionaryValues(DICT_ROOT, IS_LATEST_DICT_VERSION, pos, - &c, &childPosition, &terminal, &freq); - *nextOutputIndex = depth + 1; - - const bool needsToTraverseChildrenNodes = childPosition != 0; - - // If we are only doing traverseAllNodes, no need to look at the typed characters. - if (traverseAllNodes || needsToSkipCurrentNode(c, inputIndex, skipPos, depth)) { - mWord[depth] = c; - if (traverseAllNodes && terminal) { - onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos, - excessivePos, transposedPos, freq, false, nextLetters, nextLettersSize); - } - if (!needsToTraverseChildrenNodes) return false; - *newTraverseAllNodes = traverseAllNodes; - *newMatchRate = matchWeight; - *newDiffs = diffs; - *newInputIndex = inputIndex; - } else { - const int *currentChars = getInputCharsAt(inputIndex); - - if (transposedPos >= 0) { - if (inputIndex == transposedPos) currentChars += MAX_PROXIMITY_CHARS; - if (inputIndex == (transposedPos + 1)) currentChars -= MAX_PROXIMITY_CHARS; - } - - int matchedProximityCharId = getMatchedProximityId(currentChars, c, skipPos, excessivePos, - transposedPos); - if (UNRELATED_CHAR == matchedProximityCharId) return false; - mWord[depth] = c; - // If inputIndex is greater than mInputLength, that means there is no - // proximity chars. So, we don't need to check proximity. - if (SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR == matchedProximityCharId) { - multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &matchWeight); - } - bool isSameAsUserTypedLength = mInputLength == inputIndex + 1 - || (excessivePos == mInputLength - 1 && inputIndex == mInputLength - 2); - if (isSameAsUserTypedLength && terminal) { - onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos, - excessivePos, transposedPos, freq, true, nextLetters, nextLettersSize); - } - if (!needsToTraverseChildrenNodes) return false; - // Start traversing all nodes after the index exceeds the user typed length - *newTraverseAllNodes = isSameAsUserTypedLength; - *newMatchRate = matchWeight; - *newDiffs = diffs + ((NEAR_PROXIMITY_CHAR == matchedProximityCharId) ? 1 : 0); - *newInputIndex = inputIndex + 1; - } - // Optimization: Prune out words that are too long compared to how much was typed. - if (depth >= maxDepth || *newDiffs > mMaxEditDistance) { - return false; - } - - // If inputIndex is greater than mInputLength, that means there are no proximity chars. - // TODO: Check if this can be isSameAsUserTypedLength only. - if (isSameAsUserTypedLength || mInputLength <= *newInputIndex) { - *newTraverseAllNodes = true; - } - // get the count of nodes and increment childAddress. - *newCount = Dictionary::getCount(DICT_ROOT, &childPosition); - *newChildPosition = childPosition; - if (DEBUG_DICT) assert(needsToTraverseChildrenNodes); - return needsToTraverseChildrenNodes; -} - -#else // NEW_DICTIONARY_FORMAT - -bool UnigramDictionary::getSplitTwoWordsSuggestion(const int inputLength, - const int firstWordStartPos, const int firstWordLength, const int secondWordStartPos, - const int secondWordLength, const bool isSpaceProximity) { - if (inputLength >= MAX_WORD_LENGTH) return false; - if (0 >= firstWordLength || 0 >= secondWordLength || firstWordStartPos >= secondWordStartPos - || firstWordStartPos < 0 || secondWordStartPos + secondWordLength > inputLength) - return false; - const int newWordLength = firstWordLength + secondWordLength + 1; - // Allocating variable length array on stack - unsigned short word[newWordLength]; - const int firstFreq = getBestWordFreq(firstWordStartPos, firstWordLength, mWord); - if (DEBUG_DICT) { - LOGI("First freq: %d", firstFreq); - } - if (firstFreq <= 0) return false; - - for (int i = 0; i < firstWordLength; ++i) { - word[i] = mWord[i]; - } - - const int secondFreq = getBestWordFreq(secondWordStartPos, secondWordLength, mWord); - if (DEBUG_DICT) { - LOGI("Second freq: %d", secondFreq); - } - if (secondFreq <= 0) return false; - - word[firstWordLength] = SPACE; - for (int i = (firstWordLength + 1); i < newWordLength; ++i) { - word[i] = mWord[i - firstWordLength - 1]; - } - - int pairFreq = calcFreqForSplitTwoWords(TYPED_LETTER_MULTIPLIER, firstWordLength, - secondWordLength, firstFreq, secondFreq, isSpaceProximity); - if (DEBUG_DICT) { - LOGI("Split two words: %d, %d, %d, %d, %d", firstFreq, secondFreq, pairFreq, inputLength, - TYPED_LETTER_MULTIPLIER); - } - addWord(word, newWordLength, pairFreq); - return true; -} - -inline bool UnigramDictionary::processCurrentNode(const int pos, const int depth, - const int maxDepth, const bool traverseAllNodes, int matchWeight, int inputIndex, - const int diffs, const int skipPos, const int excessivePos, const int transposedPos, - int *nextLetters, const int nextLettersSize, int *newCount, int *newChildPosition, - bool *newTraverseAllNodes, int *newMatchRate, int *newInputIndex, int *newDiffs, - int *nextSiblingPosition, int *nextOutputIndex) { - if (DEBUG_DICT) { - int inputCount = 0; - if (skipPos >= 0) ++inputCount; - if (excessivePos >= 0) ++inputCount; - if (transposedPos >= 0) ++inputCount; - assert(inputCount <= 1); - } - unsigned short c; - int childPosition; - bool terminal; - int freq; - bool isSameAsUserTypedLength = false; - - const uint8_t flags = 0; // No flags for now - - if (excessivePos == depth && inputIndex < mInputLength - 1) ++inputIndex; - - *nextSiblingPosition = Dictionary::setDictionaryValues(DICT_ROOT, IS_LATEST_DICT_VERSION, pos, - &c, &childPosition, &terminal, &freq); - *nextOutputIndex = depth + 1; - - const bool needsToTraverseChildrenNodes = childPosition != 0; - - // If we are only doing traverseAllNodes, no need to look at the typed characters. - if (traverseAllNodes || needsToSkipCurrentNode(c, inputIndex, skipPos, depth)) { - mWord[depth] = c; - if (traverseAllNodes && terminal) { - onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos, - excessivePos, transposedPos, freq, false, nextLetters, nextLettersSize); - } - if (!needsToTraverseChildrenNodes) return false; - *newTraverseAllNodes = traverseAllNodes; - *newMatchRate = matchWeight; - *newDiffs = diffs; - *newInputIndex = inputIndex; - } else { - const int *currentChars = getInputCharsAt(inputIndex); - - if (transposedPos >= 0) { - if (inputIndex == transposedPos) currentChars += MAX_PROXIMITY_CHARS; - if (inputIndex == (transposedPos + 1)) currentChars -= MAX_PROXIMITY_CHARS; - } - - int matchedProximityCharId = getMatchedProximityId(currentChars, c, skipPos, excessivePos, - transposedPos); - if (UNRELATED_CHAR == matchedProximityCharId) return false; - mWord[depth] = c; - // If inputIndex is greater than mInputLength, that means there is no - // proximity chars. So, we don't need to check proximity. - if (SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR == matchedProximityCharId) { - multiplyIntCapped(TYPED_LETTER_MULTIPLIER, &matchWeight); - } - bool isSameAsUserTypedLength = mInputLength == inputIndex + 1 - || (excessivePos == mInputLength - 1 && inputIndex == mInputLength - 2); - if (isSameAsUserTypedLength && terminal) { - onTerminal(mWord, depth, DICT_ROOT, flags, pos, inputIndex, matchWeight, skipPos, - excessivePos, transposedPos, freq, true, nextLetters, nextLettersSize); - } - if (!needsToTraverseChildrenNodes) return false; - // Start traversing all nodes after the index exceeds the user typed length - *newTraverseAllNodes = isSameAsUserTypedLength; - *newMatchRate = matchWeight; - *newDiffs = diffs + ((NEAR_PROXIMITY_CHAR == matchedProximityCharId) ? 1 : 0); - *newInputIndex = inputIndex + 1; - } - // Optimization: Prune out words that are too long compared to how much was typed. - if (depth >= maxDepth || *newDiffs > mMaxEditDistance) { - return false; - } - - // If inputIndex is greater than mInputLength, that means there are no proximity chars. - // TODO: Check if this can be isSameAsUserTypedLength only. - if (isSameAsUserTypedLength || mInputLength <= *newInputIndex) { - *newTraverseAllNodes = true; - } - // get the count of nodes and increment childAddress. - *newCount = Dictionary::getCount(DICT_ROOT, &childPosition); - *newChildPosition = childPosition; - if (DEBUG_DICT) assert(needsToTraverseChildrenNodes); - return needsToTraverseChildrenNodes; -} - -#endif // NEW_DICTIONARY_FORMAT - -} // namespace latinime diff --git a/native/src/unigram_dictionary.h b/native/src/unigram_dictionary.h deleted file mode 100644 index 154ac9b36..000000000 --- a/native/src/unigram_dictionary.h +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_UNIGRAM_DICTIONARY_H -#define LATINIME_UNIGRAM_DICTIONARY_H - -#include <stdint.h> -#include "defines.h" -#include "proximity_info.h" - -#ifndef NULL -#define NULL 0 -#endif - -namespace latinime { - -class UnigramDictionary { - - typedef enum { // Used as a return value for character comparison - SAME_OR_ACCENTED_OR_CAPITALIZED_CHAR, // Same char, possibly with different case or accent - NEAR_PROXIMITY_CHAR, // It is a char located nearby on the keyboard - UNRELATED_CHAR // It is an unrelated char - } ProximityType; - -public: - UnigramDictionary(const uint8_t* const streamStart, int typedLetterMultipler, - int fullWordMultiplier, int maxWordLength, int maxWords, int maxProximityChars, - const bool isLatestDictVersion); - bool isValidWord(unsigned short *word, int length); - int getBigramPosition(int pos, unsigned short *word, int offset, int length) const; - int getSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const int codesSize, const int flags, - unsigned short *outWords, int *frequencies); - ~UnigramDictionary(); - -private: - void getWordSuggestions(const ProximityInfo *proximityInfo, const int *xcoordinates, - const int *ycoordinates, const int *codes, const int codesSize, - unsigned short *outWords, int *frequencies); - bool isDigraph(const int* codes, const int i, const int codesSize) const; - void getWordWithDigraphSuggestionsRec(const ProximityInfo *proximityInfo, - const int *xcoordinates, const int* ycoordinates, const int *codesBuffer, - const int codesBufferSize, const int flags, const int* codesSrc, const int codesRemain, - const int currentDepth, int* codesDest, unsigned short* outWords, int* frequencies); - void initSuggestions(const int *codes, const int codesSize, unsigned short *outWords, - int *frequencies); - void getSuggestionCandidates(const int skipPos, const int excessivePos, - const int transposedPos, int *nextLetters, const int nextLettersSize, - const int maxDepth); - void getVersionNumber(); - bool checkIfDictVersionIsLatest(); - int getAddress(int *pos); - int getFreq(int *pos); - bool sameAsTyped(const unsigned short *word, int length) const; - bool addWord(unsigned short *word, int length, int frequency); - void addWordAlternatesSpellings(const uint8_t* const root, int pos, int depth, int finalFreq); - void getWordsRec(const int childrenCount, const int pos, const int depth, const int maxDepth, - const bool traverseAllNodes, const int snr, const int inputIndex, const int diffs, - const int skipPos, const int excessivePos, const int transposedPos, int *nextLetters, - const int nextLettersSize); - bool getSplitTwoWordsSuggestion(const int inputLength, - const int firstWordStartPos, const int firstWordLength, - const int secondWordStartPos, const int secondWordLength, const bool isSpaceProximity); - bool getMissingSpaceWords(const int inputLength, const int missingSpacePos); - bool getMistypedSpaceWords(const int inputLength, const int spaceProximityPos); - // Keep getWordsOld for comparing performance between getWords and getWordsOld - void getWordsOld(const int initialPos, const int inputLength, const int skipPos, - const int excessivePos, const int transposedPos, int *nextLetters, - const int nextLettersSize); - int calculateFinalFreq(const int inputIndex, const int depth, const int snr, const int skipPos, - const int excessivePos, const int transposedPos, const int freq, - const bool sameLength) const; - void onTerminal(unsigned short int* word, const int depth, - const uint8_t* const root, const uint8_t flags, int pos, - const int inputIndex, const int matchWeight, const int skipPos, - const int excessivePos, const int transposedPos, const int freq, const bool sameLength, - int *nextLetters, const int nextLettersSize); - bool needsToSkipCurrentNode(const unsigned short c, - const int inputIndex, const int skipPos, const int depth); - ProximityType getMatchedProximityId(const int *currentChars, const unsigned short c, - const int skipPos, const int excessivePos, const int transposedPos); - // Process a node by considering proximity, missing and excessive character - bool processCurrentNode(const int pos, const int depth, - const int maxDepth, const bool traverseAllNodes, const int snr, int inputIndex, - const int diffs, const int skipPos, const int excessivePos, const int transposedPos, - int *nextLetters, const int nextLettersSize, int *newCount, int *newChildPosition, - bool *newTraverseAllNodes, int *newSnr, int*newInputIndex, int *newDiffs, - int *nextSiblingPosition, int *nextOutputIndex); - int getBestWordFreq(const int startInputIndex, const int inputLength, unsigned short *word); - // Process a node by considering missing space - bool processCurrentNodeForExactMatch(const int firstChildPos, - const int startInputIndex, const int depth, unsigned short *word, - int *newChildPosition, int *newCount, bool *newTerminal, int *newFreq, int *siblingPos); - bool existsAdjacentProximityChars(const int inputIndex, const int inputLength) const; - inline const int* getInputCharsAt(const int index) const { - return mInputCodes + (index * MAX_PROXIMITY_CHARS); - } - - const uint8_t* const DICT_ROOT; - const int MAX_WORD_LENGTH; - const int MAX_WORDS; - const int MAX_PROXIMITY_CHARS; - const bool IS_LATEST_DICT_VERSION; - const int TYPED_LETTER_MULTIPLIER; - const int FULL_WORD_MULTIPLIER; - const int ROOT_POS; - const unsigned int BYTES_IN_ONE_CHAR; - const int MAX_UMLAUT_SEARCH_DEPTH; - - // Flags for special processing - // Those *must* match the flags in BinaryDictionary.Flags.ALL_FLAGS in BinaryDictionary.java - // or something very bad (like, the apocalypse) will happen. - // Please update both at the same time. - enum { - REQUIRES_GERMAN_UMLAUT_PROCESSING = 0x1 - }; - static const struct digraph_t { int first; int second; } GERMAN_UMLAUT_DIGRAPHS[]; - - int *mFrequencies; - unsigned short *mOutputChars; - const int *mInputCodes; - int mInputLength; - // MAX_WORD_LENGTH_INTERNAL must be bigger than MAX_WORD_LENGTH - unsigned short mWord[MAX_WORD_LENGTH_INTERNAL]; - int mMaxEditDistance; - - int mStackChildCount[MAX_WORD_LENGTH_INTERNAL]; - bool mStackTraverseAll[MAX_WORD_LENGTH_INTERNAL]; - int mStackNodeFreq[MAX_WORD_LENGTH_INTERNAL]; - int mStackInputIndex[MAX_WORD_LENGTH_INTERNAL]; - int mStackDiffs[MAX_WORD_LENGTH_INTERNAL]; - int mStackSiblingPos[MAX_WORD_LENGTH_INTERNAL]; - int mStackOutputIndex[MAX_WORD_LENGTH_INTERNAL]; - int mNextLettersFrequency[NEXT_LETTERS_SIZE]; -}; - -} // namespace latinime - -#endif // LATINIME_UNIGRAM_DICTIONARY_H |