diff options
39 files changed, 206 insertions, 256 deletions
diff --git a/java/res/drawable-hdpi/btn_center_default.9.png b/java/res/drawable-hdpi/btn_center_default.9.png Binary files differdeleted file mode 100644 index 4f5f01cb8..000000000 --- a/java/res/drawable-hdpi/btn_center_default.9.png +++ /dev/null diff --git a/java/res/drawable-hdpi/btn_center_pressed.9.png b/java/res/drawable-hdpi/btn_center_pressed.9.png Binary files differdeleted file mode 100644 index 213b482d4..000000000 --- a/java/res/drawable-hdpi/btn_center_pressed.9.png +++ /dev/null diff --git a/java/res/drawable-hdpi/btn_center_selected.9.png b/java/res/drawable-hdpi/btn_center_selected.9.png Binary files differdeleted file mode 100644 index 213b482d4..000000000 --- a/java/res/drawable-hdpi/btn_center_selected.9.png +++ /dev/null diff --git a/java/res/drawable-mdpi/btn_center_default.9.png b/java/res/drawable-mdpi/btn_center_default.9.png Binary files differdeleted file mode 100644 index d5ec36ba4..000000000 --- a/java/res/drawable-mdpi/btn_center_default.9.png +++ /dev/null diff --git a/java/res/drawable-mdpi/btn_center_pressed.9.png b/java/res/drawable-mdpi/btn_center_pressed.9.png Binary files differdeleted file mode 100644 index 593a679d0..000000000 --- a/java/res/drawable-mdpi/btn_center_pressed.9.png +++ /dev/null diff --git a/java/res/drawable-mdpi/btn_center_selected.9.png b/java/res/drawable-mdpi/btn_center_selected.9.png Binary files differdeleted file mode 100644 index f1914a886..000000000 --- a/java/res/drawable-mdpi/btn_center_selected.9.png +++ /dev/null diff --git a/java/res/drawable-xhdpi/btn_center_default.9.png b/java/res/drawable-xhdpi/btn_center_default.9.png Binary files differdeleted file mode 100644 index e847425f8..000000000 --- a/java/res/drawable-xhdpi/btn_center_default.9.png +++ /dev/null diff --git a/java/res/drawable-xhdpi/btn_center_pressed.9.png b/java/res/drawable-xhdpi/btn_center_pressed.9.png Binary files differdeleted file mode 100644 index facfd4323..000000000 --- a/java/res/drawable-xhdpi/btn_center_pressed.9.png +++ /dev/null diff --git a/java/res/drawable-xhdpi/btn_center_selected.9.png b/java/res/drawable-xhdpi/btn_center_selected.9.png Binary files differdeleted file mode 100644 index facfd4323..000000000 --- a/java/res/drawable-xhdpi/btn_center_selected.9.png +++ /dev/null diff --git a/java/res/drawable/btn_center.xml b/java/res/drawable/btn_center.xml deleted file mode 100644 index 3ac21297c..000000000 --- a/java/res/drawable/btn_center.xml +++ /dev/null @@ -1,40 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- -/* -** -** Copyright 2011, The Android Open Source Project -** -** Licensed under the Apache License, Version 2.0 (the "License"); -** you may not use this file except in compliance with the License. -** You may obtain a copy of the License at -** -** http://www.apache.org/licenses/LICENSE-2.0 -** -** Unless required by applicable law or agreed to in writing, software -** distributed under the License is distributed on an "AS IS" BASIS, -** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -** See the License for the specific language governing permissions and -** limitations under the License. -*/ ---> - -<selector - xmlns:android="http://schemas.android.com/apk/res/android" - android:exitFadeDuration="@android:integer/config_mediumAnimTime"> - <item - android:state_window_focused="false" - android:state_enabled="true" - android:drawable="@drawable/btn_center_default" /> - <item - android:state_pressed="true" - android:drawable="@drawable/btn_center_pressed" /> - <item - android:state_focused="true" - android:state_enabled="true" - android:drawable="@drawable/btn_center_selected" /> - <item - android:state_enabled="true" - android:drawable="@drawable/btn_center_default" /> - <item - android:drawable="@drawable/btn_center_default" /> -</selector> diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 89e9f28fb..ffe317161 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1357,10 +1357,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } - private static boolean isAlphabet(final int code) { - return Character.isLetter(code); - } - private void onSettingsKeyPressed() { if (isShowingOptionDialog()) return; showSubtypeSelectorAndSettings(); @@ -1850,23 +1846,23 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // When we exit this if-clause, mWordComposer.isComposingWord() will return false. } if (mWordComposer.isComposingWord()) { - final int length = mWordComposer.size(); - if (length > 0) { - if (mWordComposer.isBatchMode()) { - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - final String word = mWordComposer.getTypedWord(); - ResearchLogger.latinIME_handleBackspace_batch(word, 1); - } - final String rejectedSuggestion = mWordComposer.getTypedWord(); - mWordComposer.reset(); - mWordComposer.setRejectedBatchModeSuggestion(rejectedSuggestion); - } else { - mWordComposer.deleteLast(); + if (mWordComposer.isBatchMode()) { + if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { + final String word = mWordComposer.getTypedWord(); + ResearchLogger.latinIME_handleBackspace_batch(word, 1); } - mConnection.setComposingText(getTextWithUnderline(mWordComposer.getTypedWord()), 1); - mHandler.postUpdateSuggestionStrip(); + final String rejectedSuggestion = mWordComposer.getTypedWord(); + mWordComposer.reset(); + mWordComposer.setRejectedBatchModeSuggestion(rejectedSuggestion); } else { - mConnection.deleteSurroundingText(1, 0); + mWordComposer.deleteLast(); + } + mConnection.setComposingText(getTextWithUnderline(mWordComposer.getTypedWord()), 1); + mHandler.postUpdateSuggestionStrip(); + if (!mWordComposer.isComposingWord()) { + // If we just removed the last character, auto-caps mode may have changed so we + // need to re-evaluate. + mKeyboardSwitcher.updateShiftState(); } } else { final SettingsValues currentSettings = mSettings.getCurrent(); @@ -1881,8 +1877,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Cancel multi-character input: remove the text we just entered. // This is triggered on backspace after a key that inputs multiple characters, // like the smiley key or the .com key. - final int length = mEnteredText.length(); - mConnection.deleteSurroundingText(length, 0); + mConnection.deleteSurroundingText(mEnteredText.length(), 0); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_handleBackspace_cancelTextInput(mEnteredText); } @@ -1928,6 +1923,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This should never happen. Log.e(TAG, "Backspace when we don't know the selection position"); } + final int lengthToDelete = Character.isSupplementaryCodePoint( + mConnection.getCodePointBeforeCursor()) ? 2 : 1; if (mAppWorkAroundsUtils.isBeforeJellyBean()) { // Backward compatibility mode. Before Jelly bean, the keyboard would simulate // a hardware keyboard event on pressing enter or delete. This is bad for many @@ -1935,15 +1932,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // relying on this behavior so we continue to support it for older apps. sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_DEL); } else { - mConnection.deleteSurroundingText(1, 0); + mConnection.deleteSurroundingText(lengthToDelete, 0); } if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(1, true /* shouldUncommitLogUnit */); + ResearchLogger.latinIME_handleBackspace(lengthToDelete, + true /* shouldUncommitLogUnit */); } if (mDeleteCount > DELETE_ACCELERATE_AT) { - mConnection.deleteSurroundingText(1, 0); + final int lengthToDeleteAgain = Character.isSupplementaryCodePoint( + mConnection.getCodePointBeforeCursor()) ? 2 : 1; + mConnection.deleteSurroundingText(lengthToDeleteAgain, 0); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(1, + ResearchLogger.latinIME_handleBackspace(lengthToDeleteAgain, true /* shouldUncommitLogUnit */); } } @@ -1951,6 +1951,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) { restartSuggestionsOnWordBeforeCursorIfAtEndOfWord(); } + // We just removed a character. We need to update the auto-caps state. + mKeyboardSwitcher.updateShiftState(); } } @@ -1997,8 +1999,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // NOTE: isCursorTouchingWord() is a blocking IPC call, so it often takes several // dozen milliseconds. Avoid calling it as much as possible, since we are on the UI // thread here. - if (!isComposingWord && (isAlphabet(primaryCode) - || currentSettings.isWordConnector(primaryCode)) + if (!isComposingWord && currentSettings.isWordCodePoint(primaryCode) && currentSettings.isSuggestionsRequested(mDisplayOrientation) && !mConnection.isCursorTouchingWord(currentSettings)) { // Reset entirely the composing state anyway, then start composing a new word unless @@ -2560,8 +2561,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } mWordComposer.setComposingWord(typedWord, mKeyboardSwitcher.getKeyboard()); - // TODO: this is in chars but the callee expects code points! - mWordComposer.setCursorPositionWithinWord(numberOfCharsInWordBeforeCursor); + mWordComposer.setCursorPositionWithinWord( + typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor)); mConnection.setComposingRegion( mLastSelectionStart - numberOfCharsInWordBeforeCursor, mLastSelectionEnd + range.getNumberOfCharsInWordAfterCursor()); @@ -2751,17 +2752,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen break; } } - - if (Constants.CODE_DELETE == primaryCode) { - // This is a stopgap solution to avoid leaving a high surrogate alone in a text view. - // In the future, we need to deprecate deteleSurroundingText() and have a surrogate - // pair-friendly way of deleting characters in InputConnection. - // TODO: use getCodePointBeforeCursor instead to improve performance - final CharSequence lastChar = mConnection.getTextBeforeCursor(1, 0); - if (!TextUtils.isEmpty(lastChar) && Character.isHighSurrogate(lastChar.charAt(0))) { - mConnection.deleteSurroundingText(1, 0); - } - } } // Hooks for hardware keyboard diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index a25cf620c..195f9f8ef 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -22,6 +22,7 @@ import android.content.res.Resources; import android.util.Log; import android.view.inputmethod.EditorInfo; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.internal.KeySpecParser; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.InputAttributes; @@ -170,6 +171,55 @@ public final class SettingsValues { mIsInternal = Settings.isInternal(prefs); } + // Only for tests + private SettingsValues(final Locale locale) { + // TODO: locale is saved, but not used yet. May have to change this if tests require. + mLocale = locale; + mDelayUpdateOldSuggestions = 0; + mSymbolsPrecededBySpace = new int[] { '(', '[', '{', '&' }; + Arrays.sort(mSymbolsPrecededBySpace); + mSymbolsFollowedBySpace = new int[] { '.', ',', ';', ':', '!', '?', ')', ']', '}', '&' }; + Arrays.sort(mSymbolsFollowedBySpace); + mWordConnectors = new int[] { '\'', '-' }; + Arrays.sort(mWordConnectors); + final String[] suggestPuncsSpec = new String[] { "!", "?", ",", ":", ";" }; + mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec); + mWordSeparators = "&\t \n()[]{}*&<>+=|.,;:!?/_\""; + mHintToSaveText = "Touch again to save"; + mInputAttributes = new InputAttributes(null, false /* isFullscreenMode */); + mAutoCap = true; + mVibrateOn = true; + mSoundOn = true; + mKeyPreviewPopupOn = true; + mSlidingKeyInputPreviewEnabled = true; + mVoiceMode = "0"; + mIncludesOtherImesInLanguageSwitchList = false; + mShowsLanguageSwitchKey = true; + mUseContactsDict = true; + mUseDoubleSpacePeriod = true; + mBlockPotentiallyOffensive = true; + mAutoCorrectEnabled = true; + mBigramPredictionEnabled = true; + mKeyLongpressTimeout = 300; + mKeypressVibrationDuration = 5; + mKeypressSoundVolume = 1; + mKeyPreviewPopupDismissDelay = 70; + mAutoCorrectionThreshold = 1; + mVoiceKeyEnabled = true; + mVoiceKeyOnMain = true; + mGestureInputEnabled = true; + mGestureTrailEnabled = true; + mGestureFloatingPreviewTextEnabled = true; + mCorrectionEnabled = mAutoCorrectEnabled && !mInputAttributes.mInputTypeNoAutoCorrect; + mSuggestionVisibility = 0; + mIsInternal = false; + } + + @UsedForTesting + public static SettingsValues makeDummySettingsValuesForTest(final Locale locale) { + return new SettingsValues(locale); + } + public boolean isApplicationSpecifiedCompletionsOn() { return mInputAttributes.mApplicationSpecifiedCompletionOn; } @@ -194,6 +244,10 @@ public final class SettingsValues { return Arrays.binarySearch(mWordConnectors, code) >= 0; } + public boolean isWordCodePoint(final int code) { + return Character.isLetter(code) || isWordConnector(code); + } + public boolean isUsuallyPrecededBySpace(final int code) { return Arrays.binarySearch(mSymbolsPrecededBySpace, code) >= 0; } diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index 7406d855a..f88f2cca7 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin.utils; import android.text.TextUtils; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.settings.SettingsValues; import java.util.ArrayList; import java.util.Locale; @@ -193,27 +194,55 @@ public final class StringUtils { } public static boolean isIdenticalAfterUpcase(final String text) { - final int len = text.length(); - for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) { + final int length = text.length(); + int i = 0; + while (i < length) { final int codePoint = text.codePointAt(i); if (Character.isLetter(codePoint) && !Character.isUpperCase(codePoint)) { return false; } + i += Character.charCount(codePoint); } return true; } public static boolean isIdenticalAfterDowncase(final String text) { - final int len = text.length(); - for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) { + final int length = text.length(); + int i = 0; + while (i < length) { final int codePoint = text.codePointAt(i); if (Character.isLetter(codePoint) && !Character.isLowerCase(codePoint)) { return false; } + i += Character.charCount(codePoint); } return true; } + public static boolean looksValidForDictionaryInsertion(final CharSequence text, + final SettingsValues settings) { + if (TextUtils.isEmpty(text)) return false; + final int length = text.length(); + int i = 0; + int digitCount = 0; + while (i < length) { + final int codePoint = Character.codePointAt(text, i); + final int charCount = Character.charCount(codePoint); + i += charCount; + if (Character.isDigit(codePoint)) { + // Count digits: see below + digitCount += charCount; + continue; + } + if (!settings.isWordCodePoint(codePoint)) return false; + } + // We reject strings entirely comprised of digits to avoid using PIN codes or credit + // card numbers. It would come in handy for word prediction though; a good example is + // when writing one's address where the street number is usually quite discriminative, + // as well as the postal code. + return digitCount < length; + } + public static boolean isIdenticalAfterCapitalizeEachWord(final String text, final String separators) { boolean needCapsNext = true; diff --git a/native/jni/Android.mk b/native/jni/Android.mk index a0c1c68ff..6e1d765b3 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -53,9 +53,7 @@ LATIN_IME_CORE_SRC_FILES := \ dic_nodes_cache.cpp) \ $(addprefix suggest/core/dictionary/, \ bigram_dictionary.cpp \ - binary_dictionary_format_utils.cpp \ bloom_filter.cpp \ - byte_array_utils.cpp \ dictionary.cpp \ digraph_utils.cpp \ multi_bigram_map.cpp) \ @@ -72,6 +70,8 @@ LATIN_IME_CORE_SRC_FILES := \ header/header_policy.cpp \ header/header_reading_utils.cpp \ shortcut/shortcut_list_reading_utils.cpp \ + utils/byte_array_utils.cpp \ + utils/format_utils.cpp \ dictionary_structure_with_buffer_policy_factory.cpp \ dynamic_patricia_trie_node_reader.cpp \ dynamic_patricia_trie_policy.cpp \ diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 8b46c2644..a25cef555 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -27,10 +27,10 @@ #include "defines.h" #include "jni.h" #include "jni_common.h" -#include "suggest/core/dictionary/binary_dictionary_format_utils.h" #include "suggest/core/dictionary/binary_dictionary_info.h" #include "suggest/core/dictionary/dictionary.h" #include "suggest/core/suggest_options.h" +#include "suggest/policyimpl/dictionary/utils/format_utils.h" #include "utils/autocorrection_threshold_utils.h" namespace latinime { @@ -87,8 +87,8 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return 0; } Dictionary *dictionary = 0; - if (BinaryDictionaryFormatUtils::UNKNOWN_VERSION - == BinaryDictionaryFormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf), + if (FormatUtils::UNKNOWN_VERSION + == FormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf), static_cast<int>(dictSize))) { AKLOGE("DICT: dictionary format is unknown, bad magic number"); releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd); diff --git a/native/jni/src/suggest/core/dicnode/dic_node_proximity_filter.h b/native/jni/src/suggest/core/dicnode/dic_node_proximity_filter.h deleted file mode 100644 index c7ab571de..000000000 --- a/native/jni/src/suggest/core/dicnode/dic_node_proximity_filter.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DIC_NODE_PROXIMITY_FILTER_H -#define LATINIME_DIC_NODE_PROXIMITY_FILTER_H - -#include "defines.h" -#include "suggest/core/layout/proximity_info_state.h" -#include "suggest/core/layout/proximity_info_utils.h" -#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" - -namespace latinime { - -class DicNodeProximityFilter : public DictionaryStructureWithBufferPolicy::NodeFilter { - public: - DicNodeProximityFilter(const ProximityInfoState *const pInfoState, - const int pointIndex, const bool exactOnly) - : mProximityInfoState(pInfoState), mPointIndex(pointIndex), mExactOnly(exactOnly) {} - - bool isFilteredOut(const int codePoint) const { - return !isProximityCodePoint(codePoint); - } - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DicNodeProximityFilter); - - const ProximityInfoState *const mProximityInfoState; - const int mPointIndex; - const bool mExactOnly; - - // TODO: Move to proximity info state - bool isProximityCodePoint(const int codePoint) const { - if (!mProximityInfoState) { - return true; - } - if (mExactOnly) { - return mProximityInfoState->getPrimaryCodePointAt(mPointIndex) == codePoint; - } - const ProximityType matchedId = mProximityInfoState->getProximityType( - mPointIndex, codePoint, true /* checkProximityChars */); - return ProximityInfoUtils::isMatchOrProximityChar(matchedId); - } -}; -} // namespace latinime -#endif // LATINIME_DIC_NODE_PROXIMITY_FILTER_H diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp index 150eb6762..bb54e608e 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp @@ -19,7 +19,6 @@ #include <cstring> #include "suggest/core/dicnode/dic_node.h" -#include "suggest/core/dicnode/dic_node_proximity_filter.h" #include "suggest/core/dicnode/dic_node_vector.h" #include "suggest/core/dictionary/multi_bigram_map.h" #include "suggest/core/dictionary/probability_utils.h" @@ -52,38 +51,16 @@ namespace latinime { /////////////////////////////////// // Traverse node expansion utils // /////////////////////////////////// - -/* static */ void DicNodeUtils::createAndGetPassingChildNode(DicNode *dicNode, - const DicNodeProximityFilter *const childrenFilter, - DicNodeVector *childDicNodes) { - // Passing multiple chars node. No need to traverse child - const int codePoint = dicNode->getNodeTypedCodePoint(); - const int baseLowerCaseCodePoint = CharUtils::toBaseLowerCase(codePoint); - if (!childrenFilter->isFilteredOut(codePoint) - || CharUtils::isIntentionalOmissionCodePoint(baseLowerCaseCodePoint)) { - childDicNodes->pushPassingChild(dicNode); - } -} - /* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode, const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, DicNodeVector *childDicNodes) { - getProximityChildDicNodes(dicNode, dictionaryStructurePolicy, 0, 0, false, childDicNodes); -} - -/* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode, - const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, - const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly, - DicNodeVector *childDicNodes) { if (dicNode->isTotalInputSizeExceedingLimit()) { return; } - const DicNodeProximityFilter childrenFilter(pInfoState, pointIndex, exactOnly); if (!dicNode->isLeavingNode()) { - DicNodeUtils::createAndGetPassingChildNode(dicNode, &childrenFilter, childDicNodes); + childDicNodes->pushPassingChild(dicNode); } else { - dictionaryStructurePolicy->createAndGetAllChildNodes(dicNode, - &childrenFilter, childDicNodes); + dictionaryStructurePolicy->createAndGetAllChildNodes(dicNode, childDicNodes); } } diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.h b/native/jni/src/suggest/core/dicnode/dic_node_utils.h index 8dc984fe1..3fb351a61 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.h @@ -24,10 +24,8 @@ namespace latinime { class DicNode; -class DicNodeProximityFilter; class DicNodeVector; class DictionaryStructureWithBufferPolicy; -class ProximityInfoState; class MultiBigramMap; class DicNodeUtils { @@ -47,11 +45,6 @@ class DicNodeUtils { static float getBigramNodeImprobability( const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, const DicNode *const node, MultiBigramMap *const multiBigramMap); - // TODO: Move to private - static void getProximityChildDicNodes(DicNode *dicNode, - const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, - const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly, - DicNodeVector *childDicNodes); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DicNodeUtils); @@ -61,8 +54,6 @@ class DicNodeUtils { static int getBigramNodeProbability( const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy, const DicNode *const node, MultiBigramMap *multiBigramMap); - static void createAndGetPassingChildNode(DicNode *dicNode, - const DicNodeProximityFilter *const childrenFilter, DicNodeVector *childDicNodes); }; } // namespace latinime #endif // LATINIME_DIC_NODE_UTILS_H diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index 1b34f03f0..37daef98a 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -33,25 +33,12 @@ class DictionaryShortcutsStructurePolicy; */ class DictionaryStructureWithBufferPolicy { public: - // This provides a filtering method for filtering new node. - class NodeFilter { - public: - virtual bool isFilteredOut(const int codePoint) const = 0; - - protected: - NodeFilter() {} - virtual ~NodeFilter() {} - - private: - DISALLOW_COPY_AND_ASSIGN(NodeFilter); - }; - virtual ~DictionaryStructureWithBufferPolicy() {} virtual int getRootPosition() const = 0; virtual void createAndGetAllChildNodes(const DicNode *const dicNode, - const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const = 0; + DicNodeVector *const childDicNodes) const = 0; virtual int getCodePointsAndProbabilityAndReturnCodePointCount( const int nodePos, const int maxCodePointCount, int *const outCodePoints, diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index 3b77227a0..7d8dd21c5 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -456,7 +456,6 @@ void Suggest::processDicNodeAsOmission( Weighting::addCostAndForwardInputIndex(WEIGHTING, CT_OMISSION, traverseSession, dicNode, childDicNode, 0 /* multiBigramMap */); weightChildNode(traverseSession, childDicNode); - if (!TRAVERSAL->isPossibleOmissionChildNode(traverseSession, dicNode, childDicNode)) { continue; } @@ -472,11 +471,14 @@ void Suggest::processDicNodeAsInsertion(DicTraverseSession *traverseSession, DicNode *dicNode) const { const int16_t pointIndex = dicNode->getInputIndex(0); DicNodeVector childDicNodes; - DicNodeUtils::getProximityChildDicNodes(dicNode, - traverseSession->getDictionaryStructurePolicy(), - traverseSession->getProximityInfoState(0), pointIndex + 1, true, &childDicNodes); + DicNodeUtils::getAllChildDicNodes(dicNode, traverseSession->getDictionaryStructurePolicy(), + &childDicNodes); const int size = childDicNodes.getSizeAndLock(); for (int i = 0; i < size; i++) { + if (traverseSession->getProximityInfoState(0)->getPrimaryCodePointAt(pointIndex + 1) + != childDicNodes[i]->getNodeCodePoint()) { + continue; + } DicNode *const childDicNode = childDicNodes[i]; Weighting::addCostAndForwardInputIndex(WEIGHTING, CT_INSERTION, traverseSession, dicNode, childDicNode, 0 /* multiBigramMap */); @@ -491,19 +493,29 @@ void Suggest::processDicNodeAsTransposition(DicTraverseSession *traverseSession, DicNode *dicNode) const { const int16_t pointIndex = dicNode->getInputIndex(0); DicNodeVector childDicNodes1; - DicNodeUtils::getProximityChildDicNodes(dicNode, - traverseSession->getDictionaryStructurePolicy(), - traverseSession->getProximityInfoState(0), pointIndex + 1, false, &childDicNodes1); + DicNodeUtils::getAllChildDicNodes(dicNode, traverseSession->getDictionaryStructurePolicy(), + &childDicNodes1); const int childSize1 = childDicNodes1.getSizeAndLock(); for (int i = 0; i < childSize1; i++) { + const ProximityType matchedId1 = traverseSession->getProximityInfoState(0) + ->getProximityType(pointIndex + 1, childDicNodes1[i]->getNodeCodePoint(), + true /* checkProximityChars */); + if (!ProximityInfoUtils::isMatchOrProximityChar(matchedId1)) { + continue; + } if (childDicNodes1[i]->hasChildren()) { DicNodeVector childDicNodes2; - DicNodeUtils::getProximityChildDicNodes( - childDicNodes1[i], traverseSession->getDictionaryStructurePolicy(), - traverseSession->getProximityInfoState(0), pointIndex, false, &childDicNodes2); + DicNodeUtils::getAllChildDicNodes(childDicNodes1[i], + traverseSession->getDictionaryStructurePolicy(), &childDicNodes2); const int childSize2 = childDicNodes2.getSizeAndLock(); for (int j = 0; j < childSize2; j++) { DicNode *const childDicNode2 = childDicNodes2[j]; + const ProximityType matchedId2 = traverseSession->getProximityInfoState(0) + ->getProximityType(pointIndex, childDicNode2->getNodeCodePoint(), + true /* checkProximityChars */); + if (!ProximityInfoUtils::isMatchOrProximityChar(matchedId2)) { + continue; + } Weighting::addCostAndForwardInputIndex(WEIGHTING, CT_TRANSPOSITION, traverseSession, childDicNodes1[i], childDicNode2, 0 /* multiBigramMap */); processExpandedDicNode(traverseSession, childDicNode2); diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp index 6f4fcbfc2..6da0e8b85 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.cpp @@ -16,7 +16,7 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h index 6b2bfe8c9..d0c584bd5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_reading_utils.h @@ -20,7 +20,7 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp index 3054e4ea6..34f092f49 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp @@ -19,19 +19,19 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/binary_dictionary_format_utils.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/patricia_trie_policy.h" +#include "suggest/policyimpl/dictionary/utils/format_utils.h" namespace latinime { /* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory ::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf, const int dictSize) { - switch (BinaryDictionaryFormatUtils::detectFormatVersion(dictBuf, dictSize)) { - case BinaryDictionaryFormatUtils::VERSION_2: + switch (FormatUtils::detectFormatVersion(dictBuf, dictSize)) { + case FormatUtils::VERSION_2: return new PatriciaTriePolicy(dictBuf); - case BinaryDictionaryFormatUtils::VERSION_3: + case FormatUtils::VERSION_3: return new DynamicPatriciaTriePolicy(dictBuf); default: ASSERT(false); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index 7d3b2e28e..cca801098 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -29,7 +29,7 @@ namespace latinime { const int DynamicPatriciaTriePolicy::MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP = 100000; void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, - const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const { + DicNodeVector *const childDicNodes) const { if (!dicNode->hasChildren()) { return; } @@ -52,8 +52,8 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d for (int i = 0; i < childCount; i++) { nodeReader.fetchNodeInfoFromBufferAndGetNodeCodePoints(nextPos, MAX_WORD_LENGTH, mergedNodeCodePoints); - if (!nodeReader.isDeleted() && !nodeFilter->isFilteredOut(mergedNodeCodePoints[0])) { - // Push child node when the node is not deleted and not filtered out. + if (!nodeReader.isDeleted()) { + // Push child node when the node is not a deleted node. childDicNodes->pushLeavingChild(dicNode, nodeReader.getNodePos(), nodeReader.getChildrenPos(), nodeReader.getProbability(), nodeReader.isTerminal(), nodeReader.hasChildren(), diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index 56475b137..ad8911c80 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -43,7 +43,7 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { } void createAndGetAllChildNodes(const DicNode *const dicNode, - const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const; + DicNodeVector *const childDicNodes) const; int getCodePointsAndProbabilityAndReturnCodePointCount( const int terminalNodePos, const int maxCodePointCount, int *const outCodePoints, diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp index 0de6341b0..1ef3b65c3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp @@ -17,7 +17,7 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h" #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h index 5398d7e37..a6cb46d39 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h @@ -20,7 +20,7 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp index 70f45dfa8..23b88ecdf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp @@ -20,7 +20,7 @@ #include <cstdlib> #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp index 8ce2b3ea0..3e664a29b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp @@ -26,7 +26,7 @@ namespace latinime { void PatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, - const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const { + DicNodeVector *const childDicNodes) const { if (!dicNode->hasChildren()) { return; } @@ -34,7 +34,7 @@ void PatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, const int childCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition( mDictRoot, &nextPos); for (int i = 0; i < childCount; i++) { - nextPos = createAndGetLeavingChildNode(dicNode, nextPos, nodeFilter, childDicNodes); + nextPos = createAndGetLeavingChildNode(dicNode, nextPos, childDicNodes); } } @@ -116,8 +116,7 @@ int PatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const { } int PatriciaTriePolicy::createAndGetLeavingChildNode(const DicNode *const dicNode, - const int nodePos, const NodeFilter *const childrenFilter, - DicNodeVector *childDicNodes) const { + const int nodePos, DicNodeVector *childDicNodes) const { int pos = nodePos; const PatriciaTrieReadingUtils::NodeFlags flags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos); @@ -136,14 +135,12 @@ int PatriciaTriePolicy::createAndGetLeavingChildNode(const DicNode *const dicNod if (PatriciaTrieReadingUtils::hasBigrams(flags)) { getBigramsStructurePolicy()->skipAllBigrams(&pos); } - if (!childrenFilter->isFilteredOut(mergedNodeCodePoints[0])) { - childDicNodes->pushLeavingChild(dicNode, nodePos, childrenPos, probability, - PatriciaTrieReadingUtils::isTerminal(flags), - PatriciaTrieReadingUtils::hasChildrenInFlags(flags), - PatriciaTrieReadingUtils::isBlacklisted(flags) || - PatriciaTrieReadingUtils::isNotAWord(flags), - mergedNodeCodePointCount, mergedNodeCodePoints); - } + childDicNodes->pushLeavingChild(dicNode, nodePos, childrenPos, probability, + PatriciaTrieReadingUtils::isTerminal(flags), + PatriciaTrieReadingUtils::hasChildrenInFlags(flags), + PatriciaTrieReadingUtils::isBlacklisted(flags) || + PatriciaTrieReadingUtils::isNotAWord(flags), + mergedNodeCodePointCount, mergedNodeCodePoints); return pos; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index bebe1bfff..2e34480aa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -43,7 +43,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { } void createAndGetAllChildNodes(const DicNode *const dicNode, - const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const; + DicNodeVector *const childDicNodes) const; int getCodePointsAndProbabilityAndReturnCodePointCount( const int terminalNodePos, const int maxCodePointCount, int *const outCodePoints, @@ -79,7 +79,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { const ShortcutListPolicy mShortcutListPolicy; int createAndGetLeavingChildNode(const DicNode *const dicNode, const int nodePos, - const NodeFilter *const nodeFilter, DicNodeVector *const childDicNodes) const; + DicNodeVector *const childDicNodes) const; }; } // namespace latinime #endif // LATINIME_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp index 89e981df8..003b9435b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp @@ -17,7 +17,7 @@ #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h index 002c3f19b..9f2fc2059 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h @@ -20,7 +20,7 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h index 3c7fab033..d73f73953 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h @@ -33,6 +33,9 @@ class ShortcutListPolicy : public DictionaryShortcutsStructurePolicy { ~ShortcutListPolicy() {} int getStartPos(const int pos) const { + if (pos == NOT_A_DICT_POS) { + return NOT_A_DICT_POS; + } int listPos = pos; ShortcutListReadingUtils::getShortcutListSizeAndForwardPointer(mShortcutsBuf, &listPos); return listPos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h index e92fa5f9f..b5bb964d5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h @@ -20,7 +20,7 @@ #include <stdint.h> #include "defines.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.cpp index 68b1d5d15..a84cfb9d5 100644 --- a/native/jni/src/suggest/core/dictionary/byte_array_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { diff --git a/native/jni/src/suggest/core/dictionary/byte_array_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h index 75ccfc766..75ccfc766 100644 --- a/native/jni/src/suggest/core/dictionary/byte_array_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp index 15f2aa630..3796c7b7b 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.cpp @@ -14,9 +14,9 @@ * limitations under the License. */ -#include "suggest/core/dictionary/binary_dictionary_format_utils.h" +#include "suggest/policyimpl/dictionary/utils/format_utils.h" -#include "suggest/core/dictionary/byte_array_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { @@ -24,21 +24,19 @@ namespace latinime { * Dictionary size */ // Any file smaller than this is not a dictionary. -const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4; +const int FormatUtils::DICTIONARY_MINIMUM_SIZE = 4; /** * Format versions */ -// The versions of Latin IME that only handle format version 1 only test for the magic -// number, so we had to change it so that version 2 files would be rejected by older -// implementations. On this occasion, we made the magic number 32 bits long. -const uint32_t BinaryDictionaryFormatUtils::HEADER_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; +// 32 bit magic number is stored at the beginning of the dictionary header to reject unsupported +// or obsolete dictionary formats. +const uint32_t FormatUtils::HEADER_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; // Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12 -const int BinaryDictionaryFormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12; +const int FormatUtils::HEADER_VERSION_2_MINIMUM_SIZE = 12; -/* static */ BinaryDictionaryFormatUtils::FORMAT_VERSION - BinaryDictionaryFormatUtils::detectFormatVersion(const uint8_t *const dict, - const int dictSize) { +/* static */ FormatUtils::FORMAT_VERSION FormatUtils::detectFormatVersion( + const uint8_t *const dict, const int dictSize) { // The magic number is stored big-endian. // If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't // understand this format. diff --git a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h index 62c7376e1..f84321577 100644 --- a/native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/format_utils.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_BINARY_DICTIONARY_FORMAT_UTILS_H -#define LATINIME_BINARY_DICTIONARY_FORMAT_UTILS_H +#ifndef LATINIME_FORMAT_UTILS_H +#define LATINIME_FORMAT_UTILS_H #include <stdint.h> @@ -25,12 +25,8 @@ namespace latinime { /** * Methods to handle binary dictionary format version. - * - * Currently, we have a file with a similar name, binary_format.h. binary_format.h contains binary - * reading methods and utility methods for various purposes. - * On the other hand, this file deals with only about dictionary format version. */ -class BinaryDictionaryFormatUtils { +class FormatUtils { public: enum FORMAT_VERSION { VERSION_2, @@ -41,11 +37,11 @@ class BinaryDictionaryFormatUtils { static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize); private: - DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils); + DISALLOW_IMPLICIT_CONSTRUCTORS(FormatUtils); static const int DICTIONARY_MINIMUM_SIZE; static const uint32_t HEADER_VERSION_2_MAGIC_NUMBER; static const int HEADER_VERSION_2_MINIMUM_SIZE; }; } // namespace latinime -#endif /* LATINIME_BINARY_DICTIONARY_FORMAT_UTILS_H */ +#endif /* LATINIME_FORMAT_UTILS_H */ diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java index 9ee8e387b..175e511b0 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java @@ -16,6 +16,8 @@ package com.android.inputmethod.latin.utils; +import com.android.inputmethod.latin.settings.SettingsValues; + import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; @@ -183,6 +185,18 @@ public class StringUtilsTests extends AndroidTestCase { assertTrue(StringUtils.isIdenticalAfterDowncase("")); } + public void testLooksValidForDictionaryInsertion() { + final SettingsValues settings = + SettingsValues.makeDummySettingsValuesForTest(Locale.ENGLISH); + assertTrue(StringUtils.looksValidForDictionaryInsertion("aochaueo", settings)); + assertFalse(StringUtils.looksValidForDictionaryInsertion("", settings)); + assertTrue(StringUtils.looksValidForDictionaryInsertion("ao-ch'aueo", settings)); + assertFalse(StringUtils.looksValidForDictionaryInsertion("2908743256", settings)); + assertTrue(StringUtils.looksValidForDictionaryInsertion("31aochaueo", settings)); + assertFalse(StringUtils.looksValidForDictionaryInsertion("akeo raeoch oerch .", settings)); + assertFalse(StringUtils.looksValidForDictionaryInsertion("!!!", settings)); + } + private static void checkCapitalize(final String src, final String dst, final String separators, final Locale locale) { assertEquals(dst, StringUtils.capitalizeEachWord(src, separators, locale)); |
