diff options
36 files changed, 320 insertions, 932 deletions
diff --git a/java/res/values-ar/config-spacing-and-punctuations.xml b/java/res/values-ar/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..9e637e458 --- /dev/null +++ b/java/res/values-ar/config-spacing-and-punctuations.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- The all letters need to be mirrored are found at + http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt --> + <!-- Symbols that are suggested between words --> + <string name="suggested_punctuations" translatable="false">!,?,\\,,:,;,\",(|),)|(,\',-,/,@,_</string> +</resources> diff --git a/java/res/values-fa/config-spacing-and-punctuations.xml b/java/res/values-fa/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..9e637e458 --- /dev/null +++ b/java/res/values-fa/config-spacing-and-punctuations.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- The all letters need to be mirrored are found at + http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt --> + <!-- Symbols that are suggested between words --> + <string name="suggested_punctuations" translatable="false">!,?,\\,,:,;,\",(|),)|(,\',-,/,@,_</string> +</resources> diff --git a/java/res/values-fr-rCA/config-spacing-and-punctuations.xml b/java/res/values-fr-rCA/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..06254800b --- /dev/null +++ b/java/res/values-fr-rCA/config-spacing-and-punctuations.xml @@ -0,0 +1,31 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2009, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Symbols that are normally preceded by a space (used to add an auto-space before these) --> + <!-- This is similar to French with the exception of "!" "?" and ";" which do not take a space before in Canadian French. Note that ":" does take a space before according to Canadian rules. --> + <string name="symbols_preceded_by_space" translatable="false">([{&:</string> + <!-- Symbols that are normally followed by a space (used to add an auto-space after these) --> + <string name="symbols_followed_by_space" translatable="false">.,;:!?)]}&</string> + <!-- Symbols that separate words --> + <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) --> + <string name="symbols_word_separators" translatable="false">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string> + <!-- Word connectors --> + <string name="symbols_word_connectors" translatable="false">\'-</string> +</resources> diff --git a/java/res/values-fr/config-spacing-and-punctuations.xml b/java/res/values-fr/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..33e0236b0 --- /dev/null +++ b/java/res/values-fr/config-spacing-and-punctuations.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2009, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Symbols that are normally preceded by a space (used to add an auto-space before these) --> + <string name="symbols_preceded_by_space" translatable="false">([{&;:!?</string> + <!-- Symbols that are normally followed by a space (used to add an auto-space after these) --> + <string name="symbols_followed_by_space" translatable="false">.,;:!?)]}&</string> + <!-- Symbols that separate words --> + <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) --> + <string name="symbols_word_separators" translatable="false">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string> + <!-- Word connectors --> + <string name="symbols_word_connectors" translatable="false">\'-</string> +</resources> diff --git a/java/res/values-hy-rAM/config-spacing-and-punctuations.xml b/java/res/values-hy-rAM/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..f26a30def --- /dev/null +++ b/java/res/values-hy-rAM/config-spacing-and-punctuations.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Same list as in English, but add armenian period and comma: --> + <!-- U+055D: "՝" ARMENIAN COMMA --> + <!-- U+0589: "։" ARMENIAN FULL STOP --> + <!-- Symbols that are normally followed by a space (used to add an auto-space after these) --> + <string name="symbols_followed_by_space" translatable="false">.,;:!?)]}&։՝</string> + <!-- Symbols that separate words. Adding armenian period and comma. --> + <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) --> + <string name="symbols_word_separators" translatable="false">"	 \n"()[]{}*&<>+=|.,;:!?/_\"։՝</string> + <!-- The sentence separator code point, for capitalization --> + <!-- U+0589: "։" ARMENIAN FULL STOP ; 589h = 1417d --> + <integer name="sentence_separator" translatable="false">1417</integer> +</resources> diff --git a/java/res/values-iw/config-spacing-and-punctuations.xml b/java/res/values-iw/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..9e637e458 --- /dev/null +++ b/java/res/values-iw/config-spacing-and-punctuations.xml @@ -0,0 +1,25 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2012, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- The all letters need to be mirrored are found at + http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt --> + <!-- Symbols that are suggested between words --> + <string name="suggested_punctuations" translatable="false">!,?,\\,,:,;,\",(|),)|(,\',-,/,@,_</string> +</resources> diff --git a/java/res/values-km-rKH/config-spacing-and-punctuations.xml b/java/res/values-km-rKH/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..a9893feec --- /dev/null +++ b/java/res/values-km-rKH/config-spacing-and-punctuations.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Whether this language uses spaces between words --> + <bool name="current_language_has_spaces">false</bool> +</resources> diff --git a/java/res/values-lo-rLA/config-spacing-and-punctuations.xml b/java/res/values-lo-rLA/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..a9893feec --- /dev/null +++ b/java/res/values-lo-rLA/config-spacing-and-punctuations.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Whether this language uses spaces between words --> + <bool name="current_language_has_spaces">false</bool> +</resources> diff --git a/java/res/values-nb/strings.xml b/java/res/values-nb/strings.xml index 337358c25..6d9f64fb3 100644 --- a/java/res/values-nb/strings.xml +++ b/java/res/values-nb/strings.xml @@ -180,7 +180,7 @@ <string name="setup_step3_title" msgid="3154757183631490281">"Gratulerer, du er klar!"</string> <string name="setup_step3_instruction" msgid="8025981829605426000">"Nå kan du skrive inn alle favorittappene dine med <xliff:g id="APPLICATION_NAME">%s</xliff:g>."</string> <string name="setup_step3_action" msgid="600879797256942259">"Konfigurer flere språk"</string> - <string name="setup_finish_action" msgid="276559243409465389">"Ferdig"</string> + <string name="setup_finish_action" msgid="276559243409465389">"Fullført"</string> <string name="show_setup_wizard_icon" msgid="5008028590593710830">"Vis app-ikonet"</string> <string name="show_setup_wizard_icon_summary" msgid="4119998322536880213">"Vis app-ikonet i appvelgeren"</string> <string name="app_name" msgid="6320102637491234792">"Ordlisteleverandør"</string> diff --git a/java/res/values-th/config-spacing-and-punctuations.xml b/java/res/values-th/config-spacing-and-punctuations.xml new file mode 100644 index 000000000..a9893feec --- /dev/null +++ b/java/res/values-th/config-spacing-and-punctuations.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Whether this language uses spaces between words --> + <bool name="current_language_has_spaces">false</bool> +</resources> diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 4fa682d66..174bbfb8e 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -376,7 +376,10 @@ public final class BinaryDictionary extends Dictionary { private void reopen() { close(); final File dictFile = new File(mDictFilePath); - mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, + // WARNING: Because we pass 0 as the offstet and file.length() as the length, this can + // only be called for actual files. Right now it's only called by the flush() family of + // functions, which require an updatable dictionary, so it's okay. But beware. + loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, dictFile.length(), true /* isUpdatable */); } diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index 366f3d4fe..b70362f90 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -72,8 +72,8 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { private final boolean mUseFirstLastBigrams; public ContactsBinaryDictionary(final Context context, final Locale locale) { - super(context, getFilenameWithLocale(NAME, locale.toString()), Dictionary.TYPE_CONTACTS, - false /* isUpdatable */); + super(context, getFilenameWithLocale(NAME, locale), locale, + Dictionary.TYPE_CONTACTS, false /* isUpdatable */); mLocale = locale; mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale); registerObserver(context); diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 17cb7151d..154e9b58b 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -33,8 +33,10 @@ import com.android.inputmethod.latin.utils.StringUtils; import java.io.File; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -99,6 +101,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private final String mFilename; + /** Dictionary locale */ + private final Locale mLocale; + /** Whether to support dynamically updating the dictionary */ private final boolean mIsUpdatable; @@ -183,15 +188,17 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * @param context The application context of the parent. * @param filename The filename for this binary dictionary. Multiple dictionaries with the same * filename is supported. + * @param locale the dictionary locale. * @param dictType the dictionary type, as a human-readable string * @param isUpdatable whether to support dynamically updating the dictionary. Please note that * dynamic dictionary has negative effects on memory space and computation time. */ public ExpandableBinaryDictionary(final Context context, final String filename, - final String dictType, final boolean isUpdatable) { + final Locale locale, final String dictType, final boolean isUpdatable) { super(dictType); mFilename = filename; mContext = context; + mLocale = locale; mIsUpdatable = isUpdatable; mBinaryDictionary = null; mFilenameDictionaryUpdateController = getDictionaryUpdateController(filename); @@ -199,8 +206,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); } - protected static String getFilenameWithLocale(final String name, final String localeStr) { - return name + "." + localeStr + DICT_FILE_EXTENSION; + protected static String getFilenameWithLocale(final String name, final Locale locale) { + return name + "." + locale.toString() + DICT_FILE_EXTENSION; } /** @@ -237,9 +244,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - SUPPORTS_DYNAMIC_UPDATE); attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString()); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE, + String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); return attributeMap; } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 6f9dd6757..8ce1e38f4 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -614,7 +614,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Locale switcherSubtypeLocale = mSubtypeSwitcher.getCurrentSubtypeLocale(); final String switcherLocaleStr = switcherSubtypeLocale.toString(); final Locale subtypeLocale; - final String localeStr; if (TextUtils.isEmpty(switcherLocaleStr)) { // This happens in very rare corner cases - for example, immediately after a switch // to LatinIME has been requested, about a frame later another switch happens. In this @@ -624,10 +623,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // of knowing anyway. Log.e(TAG, "System is reporting no current subtype."); subtypeLocale = getResources().getConfiguration().locale; - localeStr = subtypeLocale.toString(); } else { subtypeLocale = switcherSubtypeLocale; - localeStr = switcherLocaleStr; } final Suggest newSuggest = new Suggest(this /* Context */, subtypeLocale, @@ -642,19 +639,20 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.getInstance().initSuggest(newSuggest); } - mUserDictionary = new UserBinaryDictionary(this, localeStr); + mUserDictionary = new UserBinaryDictionary(this, subtypeLocale); mIsUserDictionaryAvailable = mUserDictionary.isEnabled(); newSuggest.setUserDictionary(mUserDictionary); final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); - mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary(this, localeStr); + mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary( + this, subtypeLocale); newSuggest.setUserHistoryDictionary(mUserHistoryDictionary); mPersonalizationDictionary = - PersonalizationHelper.getPersonalizationDictionary(this, localeStr); + PersonalizationHelper.getPersonalizationDictionary(this, subtypeLocale); newSuggest.setPersonalizationDictionary(mPersonalizationDictionary); mPersonalizationPredictionDictionary = - PersonalizationHelper.getPersonalizationPredictionDictionary(this, localeStr); + PersonalizationHelper.getPersonalizationPredictionDictionary(this, subtypeLocale); newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary); final Suggest oldSuggest = mSuggest; diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java index 6405b5e46..9cb2f5bc4 100644 --- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java @@ -22,14 +22,15 @@ import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.util.ArrayList; +import java.util.Locale; public final class SynchronouslyLoadedUserBinaryDictionary extends UserBinaryDictionary { - public SynchronouslyLoadedUserBinaryDictionary(final Context context, final String locale) { + public SynchronouslyLoadedUserBinaryDictionary(final Context context, final Locale locale) { this(context, locale, false); } - public SynchronouslyLoadedUserBinaryDictionary(final Context context, final String locale, + public SynchronouslyLoadedUserBinaryDictionary(final Context context, final Locale locale, final boolean alsoUseMoreRestrictiveLocales) { super(context, locale, alsoUseMoreRestrictiveLocales); } diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index 15b3d8d02..cc7687b62 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -75,20 +75,21 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { final private String mLocale; final private boolean mAlsoUseMoreRestrictiveLocales; - public UserBinaryDictionary(final Context context, final String locale) { + public UserBinaryDictionary(final Context context, final Locale locale) { this(context, locale, false); } - public UserBinaryDictionary(final Context context, final String locale, + public UserBinaryDictionary(final Context context, final Locale locale, final boolean alsoUseMoreRestrictiveLocales) { - super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_USER, + super(context, getFilenameWithLocale(NAME, locale), locale, Dictionary.TYPE_USER, false /* isUpdatable */); if (null == locale) throw new NullPointerException(); // Catch the error earlier - if (SubtypeLocaleUtils.NO_LANGUAGE.equals(locale)) { + final String localeStr = locale.toString(); + if (SubtypeLocaleUtils.NO_LANGUAGE.equals(localeStr)) { // If we don't have a locale, insert into the "all locales" user dictionary. mLocale = USER_DICTIONARY_ALL_LANGUAGES; } else { - mLocale = locale; + mLocale = localeStr; } mAlsoUseMoreRestrictiveLocales = alsoUseMoreRestrictiveLocales; // Perform a managed query. The Activity will handle closing and re-querying the cursor diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index dbf6d0046..8f26f8442 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -35,7 +35,9 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * This class is a base class of a dictionary that supports decaying for the personalized language @@ -55,7 +57,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4; /** Locale for which this user history dictionary is storing words */ - private final String mLocale; + private final Locale mLocale; private final String mFileName; @@ -66,11 +68,11 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB @UsedForTesting boolean mIsTest = false; /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, - final String locale, final String dictionaryType, final String fileName) { - super(context, fileName, dictionaryType, true); + final Locale locale, final String dictionaryType, final String fileName) { + super(context, fileName, locale, dictionaryType, true); mLocale = locale; mFileName = fileName; - if (mLocale != null && mLocale.length() > 1) { + if (mLocale != null && mLocale.toString().length() > 1) { reloadDictionaryIfRequired(); } } @@ -93,7 +95,9 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString()); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE, + String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); return attributeMap; } @@ -164,10 +168,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB // Never loaded to memory in Java side. } - protected String getLocale() { - return mLocale; - } - public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { session.setPredictionDictionary(this); mSessions.add(session); diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java index 67015f491..8d3dcc31c 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -21,9 +21,9 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.utils.CollectionUtils; import android.content.Context; -import android.content.SharedPreferences; import java.util.ArrayList; +import java.util.Locale; /** * This class is a dictionary for the personalized language model that uses binary dictionary. @@ -33,14 +33,10 @@ public class PersonalizationDictionary extends ExpandableBinaryDictionary { private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = CollectionUtils.newArrayList(); - /** Locale for which this user history dictionary is storing words */ - private final String mLocale; - - public PersonalizationDictionary(final Context context, final String locale) { + public PersonalizationDictionary(final Context context, final Locale locale) { // TODO: Make isUpdatable true. - super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION, - false /* isUpdatable */); - mLocale = locale; + super(context, getFilenameWithLocale(NAME, locale), locale, + Dictionary.TYPE_PERSONALIZATION, false /* isUpdatable */); // TODO: Restore last updated time loadDictionary(); } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index a47cc4db9..f73bb28af 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -24,6 +24,7 @@ import android.preference.PreferenceManager; import android.util.Log; import java.lang.ref.SoftReference; +import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; public class PersonalizationHelper { @@ -41,11 +42,12 @@ public class PersonalizationHelper { CollectionUtils.newConcurrentHashMap(); public static UserHistoryDictionary getUserHistoryDictionary( - final Context context, final String locale) { + final Context context, final Locale locale) { + final String localeStr = locale.toString(); synchronized (sLangUserHistoryDictCache) { - if (sLangUserHistoryDictCache.containsKey(locale)) { + if (sLangUserHistoryDictCache.containsKey(localeStr)) { final SoftReference<UserHistoryDictionary> ref = - sLangUserHistoryDictCache.get(locale); + sLangUserHistoryDictCache.get(localeStr); final UserHistoryDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -56,7 +58,8 @@ public class PersonalizationHelper { } } final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale); - sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict)); + sLangUserHistoryDictCache.put(localeStr, + new SoftReference<UserHistoryDictionary>(dict)); return dict; } } @@ -74,7 +77,7 @@ public class PersonalizationHelper { } public static void registerPersonalizationDictionaryUpdateSession(final Context context, - final PersonalizationDictionaryUpdateSession session, String locale) { + final PersonalizationDictionaryUpdateSession session, Locale locale) { final PersonalizationPredictionDictionary predictionDictionary = getPersonalizationPredictionDictionary(context, locale); predictionDictionary.registerUpdateSession(session); @@ -83,11 +86,12 @@ public class PersonalizationHelper { } public static PersonalizationDictionary getPersonalizationDictionary( - final Context context, final String locale) { + final Context context, final Locale locale) { + final String localeStr = locale.toString(); synchronized (sLangPersonalizationDictCache) { - if (sLangPersonalizationDictCache.containsKey(locale)) { + if (sLangPersonalizationDictCache.containsKey(localeStr)) { final SoftReference<PersonalizationDictionary> ref = - sLangPersonalizationDictCache.get(locale); + sLangPersonalizationDictCache.get(localeStr); final PersonalizationDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -98,17 +102,18 @@ public class PersonalizationHelper { } final PersonalizationDictionary dict = new PersonalizationDictionary(context, locale); sLangPersonalizationDictCache.put( - locale, new SoftReference<PersonalizationDictionary>(dict)); + localeStr, new SoftReference<PersonalizationDictionary>(dict)); return dict; } } public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( - final Context context, final String locale) { + final Context context, final Locale locale) { + final String localeStr = locale.toString(); synchronized (sLangPersonalizationPredictionDictCache) { - if (sLangPersonalizationPredictionDictCache.containsKey(locale)) { + if (sLangPersonalizationPredictionDictCache.containsKey(localeStr)) { final SoftReference<PersonalizationPredictionDictionary> ref = - sLangPersonalizationPredictionDictCache.get(locale); + sLangPersonalizationPredictionDictCache.get(localeStr); final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -120,7 +125,7 @@ public class PersonalizationHelper { final PersonalizationPredictionDictionary dict = new PersonalizationPredictionDictionary(context, locale); sLangPersonalizationPredictionDictCache.put( - locale, new SoftReference<PersonalizationPredictionDictionary>(dict)); + localeStr, new SoftReference<PersonalizationPredictionDictionary>(dict)); return dict; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java index 16107e29f..6d947162d 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -19,14 +19,16 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import java.util.Locale; + import android.content.Context; public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); - /* package */ PersonalizationPredictionDictionary(final Context context, final String locale) { + /* package */ PersonalizationPredictionDictionary(final Context context, final Locale locale) { super(context, locale, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, - getDictionaryFileName(locale)); + getDictionaryFileName(locale.toString())); } private static String getDictionaryFileName(final String locale) { diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 95a6fe142..60370d84e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -19,6 +19,8 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import java.util.Locale; + import android.content.Context; /** @@ -28,8 +30,9 @@ import android.content.Context; public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase { /* package for tests */ static final String NAME = UserHistoryDictionary.class.getSimpleName(); - /* package */ UserHistoryDictionary(final Context context, final String locale) { - super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); + /* package */ UserHistoryDictionary(final Context context, final Locale locale) { + super(context, locale, Dictionary.TYPE_USER_HISTORY, + getDictionaryFileName(locale.toString())); } private static String getDictionaryFileName(final String locale) { diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java index 503b18b1b..c108b20ed 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java @@ -428,7 +428,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService final String localeStr = locale.toString(); UserBinaryDictionary userDictionary = mUserDictionaries.get(localeStr); if (null == userDictionary) { - userDictionary = new SynchronouslyLoadedUserBinaryDictionary(this, localeStr, true); + userDictionary = new SynchronouslyLoadedUserBinaryDictionary(this, locale, true); mUserDictionaries.put(localeStr, userDictionary); } dictionaryCollection.addDictionary(userDictionary); diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 35620a43d..3c8b85617 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -80,8 +80,6 @@ LATIN_IME_CORE_SRC_FILES := \ patricia_trie_reading_utils.cpp) \ $(addprefix suggest/policyimpl/dictionary/structure/v3/, \ dynamic_patricia_trie_gc_event_listeners.cpp \ - dynamic_patricia_trie_node_reader.cpp \ - dynamic_patricia_trie_policy.cpp \ dynamic_patricia_trie_reading_helper.cpp \ dynamic_patricia_trie_reading_utils.cpp \ dynamic_patricia_trie_updating_helper.cpp \ diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp index de9fc9bbc..0ef6ccf37 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp @@ -38,7 +38,6 @@ const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::FLAG_ATTRI // Mask for attribute probability, stored on 4 bits inside the flags byte. const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F; -const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; /* static */ void BigramListReadWriteUtils::getBigramEntryPropertiesAndAdvancePosition( const uint8_t *const bigramsBuf, BigramFlags *const outBigramFlags, @@ -91,92 +90,4 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; } } -/* static */ bool BigramListReadWriteUtils::setHasNextFlag( - BufferWithExtendableBuffer *const buffer, const bool hasNext, const int entryPos) { - const bool usesAdditionalBuffer = buffer->isInAdditionalBuffer(entryPos); - int readingPos = entryPos; - if (usesAdditionalBuffer) { - readingPos -= buffer->getOriginalBufferSize(); - } - BigramFlags bigramFlags = ByteArrayUtils::readUint8AndAdvancePosition( - buffer->getBuffer(usesAdditionalBuffer), &readingPos); - if (hasNext) { - bigramFlags = bigramFlags | FLAG_ATTRIBUTE_HAS_NEXT; - } else { - bigramFlags = bigramFlags & (~FLAG_ATTRIBUTE_HAS_NEXT); - } - int writingPos = entryPos; - return buffer->writeUintAndAdvancePosition(bigramFlags, 1 /* size */, &writingPos); -} - -/* static */ bool BigramListReadWriteUtils::createAndWriteBigramEntry( - BufferWithExtendableBuffer *const buffer, const int targetPos, const int probability, - const bool hasNext, int *const writingPos) { - BigramFlags flags; - if (!createAndGetBigramFlags(*writingPos, targetPos, probability, hasNext, &flags)) { - return false; - } - return writeBigramEntry(buffer, flags, targetPos, writingPos); -} - -/* static */ bool BigramListReadWriteUtils::writeBigramEntry( - BufferWithExtendableBuffer *const bufferToWrite, const BigramFlags flags, - const int targetPtNodePos, int *const writingPos) { - const int offset = getBigramTargetOffset(targetPtNodePos, *writingPos); - const BigramFlags flagsToWrite = (offset < 0) ? - (flags | FLAG_ATTRIBUTE_OFFSET_NEGATIVE) : (flags & ~FLAG_ATTRIBUTE_OFFSET_NEGATIVE); - if (!bufferToWrite->writeUintAndAdvancePosition(flagsToWrite, 1 /* size */, writingPos)) { - return false; - } - const uint32_t absOffest = abs(offset); - const int bigramTargetFieldSize = attributeAddressSize(flags); - return bufferToWrite->writeUintAndAdvancePosition(absOffest, bigramTargetFieldSize, - writingPos); -} - -// Returns true if the bigram entry is valid and put entry flags into out*. -/* static */ bool BigramListReadWriteUtils::createAndGetBigramFlags(const int entryPos, - const int targetPtNodePos, const int probability, const bool hasNext, - BigramFlags *const outBigramFlags) { - BigramFlags flags = probability & MASK_ATTRIBUTE_PROBABILITY; - if (hasNext) { - flags |= FLAG_ATTRIBUTE_HAS_NEXT; - } - const int offset = getBigramTargetOffset(targetPtNodePos, entryPos); - if (offset < 0) { - flags |= FLAG_ATTRIBUTE_OFFSET_NEGATIVE; - } - const uint32_t absOffest = abs(offset); - if ((absOffest >> 24) != 0) { - // Offset is too large. - return false; - } else if ((absOffest >> 16) != 0) { - flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; - } else if ((absOffest >> 8) != 0) { - flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; - } else { - flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; - } - // Currently, all newly written bigram position fields are 3 bytes to simplify dictionary - // writing. - // TODO: Remove following 2 lines and optimize memory space. - flags = (flags & (~MASK_ATTRIBUTE_ADDRESS_TYPE)) | FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; - *outBigramFlags = flags; - return true; -} - -/* static */ int BigramListReadWriteUtils::getBigramTargetOffset(const int targetPtNodePos, - const int entryPos) { - if (targetPtNodePos == NOT_A_DICT_POS) { - return DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; - } else { - const int offset = targetPtNodePos - (entryPos + 1 /* bigramFlagsField */); - if (offset == 0) { - return DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; - } else { - return offset; - } - } -} - } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h index eabe4e099..7e1038300 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h @@ -45,34 +45,6 @@ public: // Bigrams reading methods static void skipExistingBigrams(const uint8_t *const bigramsBuf, int *const bigramListPos); - // Returns the size of the bigram position field that is stored in bigram flags. - static AK_FORCE_INLINE int attributeAddressSize(const BigramFlags flags) { - return (flags & MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT; - /* Note: this is a value-dependant optimization of what may probably be - more readably written this way: - switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1; - case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2; - case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3; - default: return 0; - } - */ - } - - static bool setHasNextFlag(BufferWithExtendableBuffer *const buffer, - const bool hasNext, const int entryPos); - - static AK_FORCE_INLINE BigramFlags setProbabilityInFlags(const BigramFlags flags, - const int probability) { - return (flags & (~MASK_ATTRIBUTE_PROBABILITY)) | (probability & MASK_ATTRIBUTE_PROBABILITY); - } - - static bool createAndWriteBigramEntry(BufferWithExtendableBuffer *const buffer, - const int targetPos, const int probability, const bool hasNext, int *const writingPos); - - static bool writeBigramEntry(BufferWithExtendableBuffer *const buffer, const BigramFlags flags, - const int targetOffset, int *const writingPos); - private: DISALLOW_IMPLICIT_CONSTRUCTORS(BigramListReadWriteUtils); @@ -83,11 +55,6 @@ private: static const BigramFlags FLAG_ATTRIBUTE_OFFSET_NEGATIVE; static const BigramFlags FLAG_ATTRIBUTE_HAS_NEXT; static const BigramFlags MASK_ATTRIBUTE_PROBABILITY; - static const int ATTRIBUTE_ADDRESS_SHIFT; - - // Returns true if the bigram entry is valid and put entry flags into out*. - static bool createAndGetBigramFlags(const int entryPos, const int targetPos, - const int probability, const bool hasNext, BigramFlags *const outBigramFlags); static AK_FORCE_INLINE bool isOffsetNegative(const BigramFlags flags) { return (flags & FLAG_ATTRIBUTE_OFFSET_NEGATIVE) != 0; @@ -95,8 +62,6 @@ private: static int getBigramAddressAndAdvancePosition(const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos); - - static int getBigramTargetOffset(const int targetPtNodePos, const int entryPos); }; } // namespace latinime #endif // LATINIME_BIGRAM_LIST_READ_WRITE_UTILS_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp deleted file mode 100644 index 768a91eb2..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" - -#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" -#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" - -namespace latinime { - -const int DynamicBigramListPolicy::CONTINUING_BIGRAM_LINK_COUNT_LIMIT = 10000; -const int DynamicBigramListPolicy::BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT = 100000; - -void DynamicBigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability, - bool *const outHasNext, int *const bigramEntryPos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramEntryPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *bigramEntryPos -= mBuffer->getOriginalBufferSize(); - } - BigramListReadWriteUtils::BigramFlags bigramFlags; - int originalBigramPos; - BigramListReadWriteUtils::getBigramEntryPropertiesAndAdvancePosition(buffer, &bigramFlags, - &originalBigramPos, bigramEntryPos); - if (usesAdditionalBuffer && originalBigramPos != NOT_A_DICT_POS) { - originalBigramPos += mBuffer->getOriginalBufferSize(); - } - *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags); - *outHasNext = BigramListReadWriteUtils::hasNext(bigramFlags); - if (mIsDecayingDict && !ForgettingCurveUtils::isValidEncodedProbability(*outProbability)) { - // This bigram is too weak to output. - *outBigramPos = NOT_A_DICT_POS; - } else { - *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); - } - if (usesAdditionalBuffer) { - *bigramEntryPos += mBuffer->getOriginalBufferSize(); - } -} - -void DynamicBigramListPolicy::skipAllBigrams(int *const bigramListPos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *bigramListPos -= mBuffer->getOriginalBufferSize(); - } - BigramListReadWriteUtils::skipExistingBigrams(buffer, bigramListPos); - if (usesAdditionalBuffer) { - *bigramListPos += mBuffer->getOriginalBufferSize(); - } -} - -int DynamicBigramListPolicy::followBigramLinkAndGetCurrentBigramPtNodePos( - const int originalBigramPos) const { - if (originalBigramPos == NOT_A_DICT_POS) { - return NOT_A_DICT_POS; - } - DynamicPatriciaTrieNodeReader nodeReader(mBuffer, this /* bigramsPolicy */, mShortcutPolicy); - int currentPos = NOT_A_DICT_POS; - int bigramLinkCount = 0; - int bigramLinkedNodePos = originalBigramPos; - do { - currentPos = bigramLinkedNodePos; - const PtNodeParams ptNodeParams(nodeReader.fetchNodeInfoInBufferFromPtNodePos(currentPos)); - bigramLinkedNodePos = ptNodeParams.getBigramLinkedNodePos(); - bigramLinkCount++; - if (bigramLinkCount > CONTINUING_BIGRAM_LINK_COUNT_LIMIT) { - AKLOGE("Bigram link is invalid. start position: %d", originalBigramPos); - ASSERT(false); - return NOT_A_DICT_POS; - } - bigramLinkedNodePos = ptNodeParams.getBigramLinkedNodePos(); - } while (bigramLinkedNodePos != NOT_A_DICT_POS); - return currentPos; -} - -} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h deleted file mode 100644 index f48c3a0b6..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H -#define LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H - -#include <stdint.h> - -#include "defines.h" -#include "suggest/core/policy/dictionary_bigrams_structure_policy.h" -#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_gc_event_listeners.h" - -namespace latinime { - -class BufferWithExtendableBuffer; -class DictionaryHeaderStructurePolicy; -class DictionaryShortcutsStructurePolicy; - -/* - * This is a dynamic version of BigramListPolicy and supports an additional buffer. - */ -class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { - public: - DynamicBigramListPolicy(const DictionaryHeaderStructurePolicy *const headerPolicy, - BufferWithExtendableBuffer *const buffer, - const DictionaryShortcutsStructurePolicy *const shortcutPolicy, - const bool isDecayingDict) - : mHeaderPolicy(headerPolicy), mBuffer(buffer), mShortcutPolicy(shortcutPolicy), - mIsDecayingDict(isDecayingDict) {} - - ~DynamicBigramListPolicy() {} - - void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, - int *const bigramEntryPos) const; - - void skipAllBigrams(int *const bigramListPos) const; - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy); - - static const int CONTINUING_BIGRAM_LINK_COUNT_LIMIT; - static const int BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT; - - const DictionaryHeaderStructurePolicy *const mHeaderPolicy; - BufferWithExtendableBuffer *const mBuffer; - const DictionaryShortcutsStructurePolicy *const mShortcutPolicy; - const bool mIsDecayingDict; - - // Follow bigram link and return the position of bigram target PtNode that is currently valid. - int followBigramLinkAndGetCurrentBigramPtNodePos(const int originalBigramPos) const; -}; -} // namespace latinime -#endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h deleted file mode 100644 index bd3211f6a..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H -#define LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H - -#include <stdint.h> - -#include "defines.h" -#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h" -#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" - -namespace latinime { - -/* - * This is a dynamic version of ShortcutListPolicy and supports an additional buffer. - */ -class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { - public: - explicit DynamicShortcutListPolicy(const BufferWithExtendableBuffer *const buffer) - : mBuffer(buffer) {} - - ~DynamicShortcutListPolicy() {} - - int getStartPos(const int pos) const { - if (pos == NOT_A_DICT_POS) { - return NOT_A_DICT_POS; - } - return pos + ShortcutListReadingUtils::getShortcutListSizeFieldSize(); - } - - void getNextShortcut(const int maxCodePointCount, int *const outCodePoint, - int *const outCodePointCount, bool *const outIsWhitelist, bool *const outHasNext, - int *const pos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *pos -= mBuffer->getOriginalBufferSize(); - } - const ShortcutListReadingUtils::ShortcutFlags flags = - ShortcutListReadingUtils::getFlagsAndForwardPointer(buffer, pos); - if (outHasNext) { - *outHasNext = ShortcutListReadingUtils::hasNext(flags); - } - if (outIsWhitelist) { - *outIsWhitelist = ShortcutListReadingUtils::isWhitelist(flags); - } - if (outCodePoint) { - *outCodePointCount = ShortcutListReadingUtils::readShortcutTarget( - buffer, maxCodePointCount, outCodePoint, pos); - } - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); - } - } - - void skipAllShortcuts(int *const pos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *pos -= mBuffer->getOriginalBufferSize(); - } - const int shortcutListSize = ShortcutListReadingUtils - ::getShortcutListSizeAndForwardPointer(buffer, pos); - *pos += shortcutListSize; - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); - } - } - - // Copy shortcuts from the shortcut list that starts at fromPos in mBuffer to toPos in - // bufferToWrite and advance these positions after the shortcut lists. This returns whether - // the copy was succeeded or not. - bool copyAllShortcutsAndReturnIfSucceededOrNot(BufferWithExtendableBuffer *const bufferToWrite, - int *const fromPos, int *const toPos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); - if (usesAdditionalBuffer) { - *fromPos -= mBuffer->getOriginalBufferSize(); - } - const int shortcutListSize = ShortcutListReadingUtils - ::getShortcutListSizeAndForwardPointer(mBuffer->getBuffer(usesAdditionalBuffer), - fromPos); - // Copy shortcut list size. - if (!bufferToWrite->writeUintAndAdvancePosition( - shortcutListSize + ShortcutListReadingUtils::getShortcutListSizeFieldSize(), - ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos)) { - return false; - } - // Copy shortcut list. - for (int i = 0; i < shortcutListSize; ++i) { - const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition( - mBuffer->getBuffer(usesAdditionalBuffer), fromPos); - if (!bufferToWrite->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) { - return false; - } - } - if (usesAdditionalBuffer) { - *fromPos += mBuffer->getOriginalBufferSize(); - } - return true; - } - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicShortcutListPolicy); - - const BufferWithExtendableBuffer *const mBuffer; -}; -} // namespace latinime -#endif // LATINIME_DYNAMIC_SHORTCUT_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp index 903d55307..f3d90f81c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/dictionary_structure_with_buffer_policy_factory.cpp @@ -20,7 +20,6 @@ #include "defines.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/utils/file_utils.h" @@ -45,9 +44,6 @@ namespace latinime { case FormatUtils::VERSION_2: return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr( new PatriciaTriePolicy(mmappedBuffer)); - case FormatUtils::VERSION_3: - return DictionaryStructureWithBufferPolicy::StructurePoilcyPtr( - new DynamicPatriciaTriePolicy(mmappedBuffer)); case FormatUtils::VERSION_4: { const int dictDirPathBufSize = strlen(path) + 1 /* terminator */; char dictDirPath[dictDirPathBufSize]; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h index d105a3e49..b5ef61b51 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h @@ -53,23 +53,6 @@ class PtNodeParams { memcpy(mCodePoints, ptNodeParams.getCodePoints(), sizeof(int) * mCodePointCount); } - // PtNode without terminal id. - PtNodeParams(const int headPos, const PatriciaTrieReadingUtils::NodeFlags flags, - const int parentPos, const int codePointCount, const int *const codePoints, - const int probabilityFieldPos, const int probability, const int childrenPosFieldPos, - const int childrenPos, const int bigramLinkedNodePos, const int shortcutPos, - const int bigramPos, const int siblingPos) - : mHeadPos(headPos), mFlags(flags), mParentPos(parentPos), - mCodePointCount(codePointCount), mCodePoints(), - mTerminalIdFieldPos(NOT_A_DICT_POS), - mTerminalId(Ver4DictConstants::NOT_A_TERMINAL_ID), - mProbabilityFieldPos(probabilityFieldPos), mProbability(probability), - mChildrenPosFieldPos(childrenPosFieldPos), mChildrenPos(childrenPos), - mBigramLinkedNodePos(bigramLinkedNodePos), mShortcutPos(shortcutPos), - mBigramPos(bigramPos), mSiblingPos(siblingPos) { - memcpy(mCodePoints, codePoints, sizeof(int) * mCodePointCount); - } - // PtNode with a terminal id. PtNodeParams(const int headPos, const PatriciaTrieReadingUtils::NodeFlags flags, const int parentPos, const int codePointCount, const int *const codePoints, diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.cpp deleted file mode 100644 index 3393ce662..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (C) 2013, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h" - -#include "suggest/core/policy/dictionary_bigrams_structure_policy.h" -#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" -#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" - -namespace latinime { - -const PtNodeParams DynamicPatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNode( - const int ptNodePos, const int siblingNodePos, const int bigramLinkedNodePos) const { - if (ptNodePos < 0 || ptNodePos >= mBuffer->getTailPosition()) { - // Reading invalid position because of bug or broken dictionary. - AKLOGE("Fetching PtNode info from invalid dictionary position: %d, dictionary size: %d", - ptNodePos, mBuffer->getTailPosition()); - ASSERT(false); - return PtNodeParams(); - } - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(ptNodePos); - const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); - int pos = ptNodePos; - const int headPos = ptNodePos; - if (usesAdditionalBuffer) { - pos -= mBuffer->getOriginalBufferSize(); - } - const PatriciaTrieReadingUtils::NodeFlags flags = - PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); - const int parentPosOffset = - DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(dictBuf, - &pos); - const int parentPos = - DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, headPos); - int codePoints[MAX_WORD_LENGTH]; - const int codePonitCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( - dictBuf, flags, MAX_WORD_LENGTH, codePoints, &pos); - int probability = NOT_A_PROBABILITY; - int probabilityFieldPos = NOT_A_DICT_POS; - if (PatriciaTrieReadingUtils::isTerminal(flags)) { - probabilityFieldPos = pos; - if (usesAdditionalBuffer) { - probabilityFieldPos += mBuffer->getOriginalBufferSize(); - } - probability = PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(dictBuf, &pos); - } - int childrenPosFieldPos = pos; - if (usesAdditionalBuffer) { - childrenPosFieldPos += mBuffer->getOriginalBufferSize(); - } - int childrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( - dictBuf, &pos); - if (usesAdditionalBuffer && childrenPos != NOT_A_DICT_POS) { - childrenPos += mBuffer->getOriginalBufferSize(); - } - int newBigramLinkedNodePos = bigramLinkedNodePos; - if (siblingNodePos == NOT_A_DICT_POS) { - if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) { - newBigramLinkedNodePos = childrenPos; - } - } - if (usesAdditionalBuffer) { - pos += mBuffer->getOriginalBufferSize(); - } - int shortcutsPos = NOT_A_DICT_POS; - if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) { - shortcutsPos = pos; - mShortcutsPolicy->skipAllShortcuts(&pos); - } - int bigramsPos = NOT_A_DICT_POS; - if (PatriciaTrieReadingUtils::hasBigrams(flags)) { - bigramsPos = pos; - mBigramsPolicy->skipAllBigrams(&pos); - } - int newSiblingNodePos = siblingNodePos; - if (siblingNodePos == NOT_A_DICT_POS) { - // Sibling position is the tail position of current node. - newSiblingNodePos = pos; - } - // Read destination node if the read node is a moved node. - if (DynamicPatriciaTrieReadingUtils::isMoved(flags)) { - // The destination position is stored at the same place as the parent position. - return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(parentPos, newSiblingNodePos, - newBigramLinkedNodePos); - } else { - return PtNodeParams(headPos, flags, parentPos, codePonitCount, codePoints, - probabilityFieldPos, probability, childrenPosFieldPos, childrenPos, - newBigramLinkedNodePos, shortcutsPos, bigramsPos, newSiblingNodePos); - } -} - -} diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h deleted file mode 100644 index ef0067b48..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2013, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_NODE_READER_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_NODE_READER_H - -#include <stdint.h> - -#include "defines.h" -#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_params.h" -#include "suggest/policyimpl/dictionary/structure/pt_common/pt_node_reader.h" - -namespace latinime { - -class BufferWithExtendableBuffer; -class DictionaryBigramsStructurePolicy; -class DictionaryShortcutsStructurePolicy; - -/* - * This class is used for helping to read nodes of dynamic patricia trie. This class handles moved - * node and reads node attributes. - */ -class DynamicPatriciaTrieNodeReader : public PtNodeReader { - public: - DynamicPatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer, - const DictionaryBigramsStructurePolicy *const bigramsPolicy, - const DictionaryShortcutsStructurePolicy *const shortcutsPolicy) - : mBuffer(buffer), mBigramsPolicy(bigramsPolicy), - mShortcutsPolicy(shortcutsPolicy) {} - - ~DynamicPatriciaTrieNodeReader() {} - - virtual const PtNodeParams fetchNodeInfoInBufferFromPtNodePos(const int ptNodePos) const { - return fetchPtNodeInfoFromBufferAndProcessMovedPtNode(ptNodePos, - NOT_A_DICT_POS /* siblingNodePos */, NOT_A_DICT_POS /* bigramLinkedNodePos */); - } - - private: - DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeReader); - - const BufferWithExtendableBuffer *const mBuffer; - const DictionaryBigramsStructurePolicy *const mBigramsPolicy; - const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy; - - const PtNodeParams fetchPtNodeInfoFromBufferAndProcessMovedPtNode(const int ptNodePos, - const int siblingNodePos, const int bigramLinkedNodePos) const; -}; -} // namespace latinime -#endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_NODE_READER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp deleted file mode 100644 index 88af67404..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (C) 2013, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h" - -#include <cstdio> -#include <cstring> -#include <ctime> - -#include "defines.h" -#include "suggest/core/dicnode/dic_node.h" -#include "suggest/core/dicnode/dic_node_vector.h" -#include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_helper.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_reading_utils.h" -#include "suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h" -#include "suggest/policyimpl/dictionary/utils/probability_utils.h" - -namespace latinime { - -void DynamicPatriciaTriePolicy::createAndGetAllChildDicNodes(const DicNode *const dicNode, - DicNodeVector *const childDicNodes) const { - if (!dicNode->hasChildren()) { - return; - } - DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, &mNodeReader); - readingHelper.initWithPtNodeArrayPos(dicNode->getChildrenPtNodeArrayPos()); - while (!readingHelper.isEnd()) { - const PtNodeParams ptNodeParams(readingHelper.getPtNodeParams()); - if (!ptNodeParams.isValid()) { - break; - } - bool isTerminal = ptNodeParams.isTerminal() && !ptNodeParams.isDeleted(); - if (isTerminal && mHeaderPolicy.isDecayingDict()) { - // A DecayingDict may have a terminal PtNode that has a terminal DicNode whose - // probability is NOT_A_PROBABILITY. In such case, we don't want to treat it as a - // valid terminal DicNode. - isTerminal = getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY) - != NOT_A_PROBABILITY; - } - childDicNodes->pushLeavingChild(dicNode, ptNodeParams.getHeadPos(), - ptNodeParams.getChildrenPos(), ptNodeParams.getProbability(), isTerminal, - ptNodeParams.hasChildren(), - ptNodeParams.isBlacklisted() || ptNodeParams.isNotAWord(), - ptNodeParams.getCodePointCount(), ptNodeParams.getCodePoints()); - readingHelper.readNextSiblingNode(ptNodeParams); - } -} - -int DynamicPatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( - const int ptNodePos, const int maxCodePointCount, int *const outCodePoints, - int *const outUnigramProbability) const { - DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, &mNodeReader); - readingHelper.initWithPtNodePos(ptNodePos); - return readingHelper.getCodePointsAndProbabilityAndReturnCodePointCount(maxCodePointCount, - outCodePoints, outUnigramProbability); -} - -int DynamicPatriciaTriePolicy::getTerminalPtNodePositionOfWord(const int *const inWord, - const int length, const bool forceLowerCaseSearch) const { - DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, &mNodeReader); - readingHelper.initWithPtNodeArrayPos(getRootPosition()); - return readingHelper.getTerminalPtNodePositionOfWord(inWord, length, forceLowerCaseSearch); -} - -int DynamicPatriciaTriePolicy::getProbability(const int unigramProbability, - const int bigramProbability) const { - if (mHeaderPolicy.isDecayingDict()) { - return ForgettingCurveUtils::getProbability(unigramProbability, bigramProbability); - } else { - if (unigramProbability == NOT_A_PROBABILITY) { - return NOT_A_PROBABILITY; - } else if (bigramProbability == NOT_A_PROBABILITY) { - return ProbabilityUtils::backoff(unigramProbability); - } else { - return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, - bigramProbability); - } - } -} - -int DynamicPatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int ptNodePos) const { - if (ptNodePos == NOT_A_DICT_POS) { - return NOT_A_PROBABILITY; - } - const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos)); - if (ptNodeParams.isDeleted() || ptNodeParams.isBlacklisted() || ptNodeParams.isNotAWord()) { - return NOT_A_PROBABILITY; - } - return getProbability(ptNodeParams.getProbability(), NOT_A_PROBABILITY); -} - -int DynamicPatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) const { - if (ptNodePos == NOT_A_DICT_POS) { - return NOT_A_DICT_POS; - } - const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos)); - if (ptNodeParams.isDeleted()) { - return NOT_A_DICT_POS; - } - return ptNodeParams.getShortcutPos(); -} - -int DynamicPatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) const { - if (ptNodePos == NOT_A_DICT_POS) { - return NOT_A_DICT_POS; - } - const PtNodeParams ptNodeParams(mNodeReader.fetchNodeInfoInBufferFromPtNodePos(ptNodePos)); - if (ptNodeParams.isDeleted()) { - return NOT_A_DICT_POS; - } - return ptNodeParams.getBigramsPos(); -} - -} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h deleted file mode 100644 index 3965e31a0..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_policy.h +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2013, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H -#define LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H - -#include "defines.h" -#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h" -#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" -#include "suggest/policyimpl/dictionary/header/header_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/structure/v3/dynamic_patricia_trie_node_reader.h" -#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" -#include "suggest/policyimpl/dictionary/utils/format_utils.h" -#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" - -namespace latinime { - -class DicNode; -class DicNodeVector; - -class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { - public: - DynamicPatriciaTriePolicy(const MmappedBuffer::MmappedBufferPtr &mmappedBuffer) - : mMmappedBuffer(mmappedBuffer), - mHeaderPolicy(mMmappedBuffer.get()->getBuffer(), FormatUtils::VERSION_3), - mBufferWithExtendableBuffer(mMmappedBuffer.get()->getBuffer() - + mHeaderPolicy.getSize(), mMmappedBuffer.get()->getBufferSize() - - mHeaderPolicy.getSize(), - BufferWithExtendableBuffer - ::DEFAULT_MAX_ADDITIONAL_BUFFER_SIZE), - mShortcutListPolicy(&mBufferWithExtendableBuffer), - mBigramListPolicy(&mHeaderPolicy, &mBufferWithExtendableBuffer, &mShortcutListPolicy, - mHeaderPolicy.isDecayingDict()), - mNodeReader(&mBufferWithExtendableBuffer, &mBigramListPolicy, &mShortcutListPolicy) {} - - AK_FORCE_INLINE int getRootPosition() const { - return 0; - } - - void createAndGetAllChildDicNodes(const DicNode *const dicNode, - DicNodeVector *const childDicNodes) const; - - int getCodePointsAndProbabilityAndReturnCodePointCount( - const int terminalPtNodePos, const int maxCodePointCount, int *const outCodePoints, - int *const outUnigramProbability) const; - - int getTerminalPtNodePositionOfWord(const int *const inWord, - const int length, const bool forceLowerCaseSearch) const; - - int getProbability(const int unigramProbability, const int bigramProbability) const; - - int getUnigramProbabilityOfPtNode(const int ptNodePos) const; - - int getShortcutPositionOfPtNode(const int ptNodePos) const; - - int getBigramsPositionOfPtNode(const int ptNodePos) const; - - const DictionaryHeaderStructurePolicy *getHeaderStructurePolicy() const { - return &mHeaderPolicy; - } - - const DictionaryBigramsStructurePolicy *getBigramsStructurePolicy() const { - return &mBigramListPolicy; - } - - const DictionaryShortcutsStructurePolicy *getShortcutsStructurePolicy() const { - return &mShortcutListPolicy; - } - - bool addUnigramWord(const int *const word, const int length, const int probability, - const int timestamp) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); - return false; - } - - bool addBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1, const int probability, const int timestamp) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); - return false; - } - - bool removeBigramWords(const int *const word0, const int length0, const int *const word1, - const int length1) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary."); - return false; - } - - void flush(const char *const filePath) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: flush() is called for non-updatable dictionary."); - } - - void flushWithGC(const char *const filePath) { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); - } - - bool needsToRunGC(const bool mindsBlockByGC) const { - // This method should not be called for non-updatable dictionary. - AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); - return false; - } - - void getProperty(const char *const query, const int queryLength, char *const outResult, - const int maxResultLength) { - // getProperty is not supported for this class. - if (maxResultLength > 0) { - outResult[0] = '\0'; - } - } - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); - - const MmappedBuffer::MmappedBufferPtr mMmappedBuffer; - const HeaderPolicy mHeaderPolicy; - BufferWithExtendableBuffer mBufferWithExtendableBuffer; - DynamicShortcutListPolicy mShortcutListPolicy; - DynamicBigramListPolicy mBigramListPolicy; - DynamicPatriciaTrieNodeReader mNodeReader; -}; -} // namespace latinime -#endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 9aa6e60c5..15a3b2461 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -17,7 +17,7 @@ #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.h" #include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h" -#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/shortcut/ver4_shortcut_list_policy.h" #include "suggest/policyimpl/dictionary/structure/v2/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/structure/v4/content/probability_entry.h" #include "suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_reader.h" diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 9c93b8b25..beac57b2d 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -26,6 +26,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -84,7 +85,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final List<String> words = generateWords(numberOfWords, random); final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix /* locale */); + new Locale(testFilenameSuffix)); // Add random words to the user history dictionary. addToDict(dict, words); if (checkContents) { @@ -108,7 +109,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private void clearHistory(final String testFilenameSuffix) { final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix /* locale */); + new Locale(testFilenameSuffix)); dict.clearAndFlushDictionary(); dict.close(); } @@ -121,7 +122,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { try { final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix); + new Locale(testFilenameSuffix)); dict.shutdownExecutorForTests(); while (!dict.isTerminatedForTests()) { Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); @@ -134,7 +135,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { public void testRandomWords() { Log.d(TAG, "This test can be used for profiling."); Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); - final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final String testFilenameSuffix = "test_random_words" + System.currentTimeMillis(); final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; @@ -169,7 +170,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { - testFilenameSuffixes[i] = "testSwitchingLanguages" + i; + testFilenameSuffixes[i] = "test_switching_languages" + i; final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFiles[i] = new File(getContext().getFilesDir(), fileName); @@ -205,7 +206,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } public void testAddManyWords() { - final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final String testFilenameSuffix = "test_random_words" + System.currentTimeMillis(); final int numberOfWords = 10000; final Random random = new Random(123456); clearHistory(testFilenameSuffix); |