diff options
3 files changed, 208 insertions, 32 deletions
diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index 47aaeadcc..e1acf3da8 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -24,6 +24,7 @@ import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback; import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.ContextualDictionary; @@ -36,7 +37,6 @@ import com.android.inputmethod.latin.utils.DistracterFilter; import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions; import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; import com.android.inputmethod.latin.utils.ExecutorUtils; -import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.SuggestionResults; import java.io.File; @@ -67,6 +67,7 @@ public class DictionaryFacilitator { // To synchronize assigning mDictionaryGroup to ensure closing dictionaries. private final Object mLock = new Object(); private final DistracterFilter mDistracterFilter; + private final PersonalizationDictionaryFacilitator mPersonalizationDictionaryFacilitator; private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS = new String[] { @@ -174,14 +175,18 @@ public class DictionaryFacilitator { public DictionaryFacilitator() { mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER; + mPersonalizationDictionaryFacilitator = null; } public DictionaryFacilitator(final Context context) { mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context); + mPersonalizationDictionaryFacilitator = + new PersonalizationDictionaryFacilitator(context, mDistracterFilter); } public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); + mPersonalizationDictionaryFacilitator.updateEnabledSubtypes(enabledSubtypes); } public Locale getLocale() { @@ -353,6 +358,9 @@ public class DictionaryFacilitator { dictionaryGroup.closeDict(dictType); } mDistracterFilter.close(); + if (mPersonalizationDictionaryFacilitator != null) { + mPersonalizationDictionaryFacilitator.close(); + } } @UsedForTesting @@ -372,11 +380,11 @@ public class DictionaryFacilitator { } public void flushPersonalizationDictionary() { - final ExpandableBinaryDictionary personalizationDict = + final ExpandableBinaryDictionary personalizationDictUsedForSuggestion = mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION); - if (personalizationDict != null) { - personalizationDict.asyncFlushBinaryDictionary(); - } + mPersonalizationDictionaryFacilitator.flushPersonalizationDictionariesToUpdate( + personalizationDictUsedForSuggestion); + mDistracterFilter.close(); } public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit) @@ -580,6 +588,7 @@ public class DictionaryFacilitator { // personalization dictionary. public void clearPersonalizationDictionary() { clearSubDictionary(Dictionary.TYPE_PERSONALIZATION); + mPersonalizationDictionaryFacilitator.clearDictionariesToUpdate(); } public void clearContextualDictionary() { @@ -589,30 +598,9 @@ public class DictionaryFacilitator { public void addEntriesToPersonalizationDictionary( final PersonalizationDataChunk personalizationDataChunk, final SpacingAndPunctuations spacingAndPunctuations, - final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { - final ExpandableBinaryDictionary personalizationDict = - mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION); - if (personalizationDict == null) { - if (callback != null) { - callback.onFinished(); - } - return; - } - // TODO: Get locale from personalizationDataChunk.mDetectedLanguage. - final Locale dataChunkLocale = getLocale(); - final ArrayList<LanguageModelParam> languageModelParams = - LanguageModelParam.createLanguageModelParamsFrom( - personalizationDataChunk.mTokens, - personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations, - dataChunkLocale, new DistracterFilterCheckingIsInDictionary( - mDistracterFilter, personalizationDict)); - if (languageModelParams == null || languageModelParams.isEmpty()) { - if (callback != null) { - callback.onFinished(); - } - return; - } - personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback); + final AddMultipleDictionaryEntriesCallback callback) { + mPersonalizationDictionaryFacilitator.addEntriesToPersonalizationDictionariesToUpdate( + personalizationDataChunk, spacingAndPunctuations, callback); } public void addPhraseToContextualDictionary(final String[] phrase, final int probability, diff --git a/java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java new file mode 100644 index 000000000..03c108f0e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; + +import android.content.Context; +import android.view.inputmethod.InputMethodSubtype; + +import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback; +import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; +import com.android.inputmethod.latin.personalization.PersonalizationDictionary; +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; +import com.android.inputmethod.latin.utils.DistracterFilter; +import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; +import com.android.inputmethod.latin.utils.LanguageModelParam; +import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; + +/** + * Class for managing and updating personalization dictionaries. + */ +public class PersonalizationDictionaryFacilitator { + private final Context mContext; + private final DistracterFilter mDistracterFilter; + private final HashMap<String, HashSet<Locale>> mLangToLocalesMap = new HashMap<>(); + private final HashMap<Locale, ExpandableBinaryDictionary> mPersonalizationDictsToUpdate = + new HashMap<>(); + + PersonalizationDictionaryFacilitator(final Context context, + final DistracterFilter distracterFilter) { + mContext = context; + mDistracterFilter = distracterFilter; + } + + public void close() { + mLangToLocalesMap.clear(); + for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) { + dict.close(); + } + mPersonalizationDictsToUpdate.clear(); + } + + public void clearDictionariesToUpdate() { + for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) { + dict.clear(); + } + mPersonalizationDictsToUpdate.clear(); + } + + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { + for (final InputMethodSubtype subtype : enabledSubtypes) { + final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); + final String language = locale.getLanguage(); + final HashSet<Locale> locales = mLangToLocalesMap.get(language); + if (locales != null) { + locales.add(locale); + } else { + final HashSet<Locale> localeSet = new HashSet<>(); + localeSet.add(locale); + mLangToLocalesMap.put(language, localeSet); + } + } + } + + /** + * Flush personalization dictionaries to dictionary files. Close dictionaries after writing + * files except the dictionary that is used for generating suggestions. + * + * @param personalizationDictUsedForSuggestion the personalization dictionary used for + * generating suggestions that won't be closed. + */ + public void flushPersonalizationDictionariesToUpdate( + final ExpandableBinaryDictionary personalizationDictUsedForSuggestion) { + for (final ExpandableBinaryDictionary personalizationDict : + mPersonalizationDictsToUpdate.values()) { + personalizationDict.asyncFlushBinaryDictionary(); + if (personalizationDict != personalizationDictUsedForSuggestion) { + // Close if the dictionary is not being used for suggestion. + personalizationDict.close(); + } + } + mDistracterFilter.close(); + mPersonalizationDictsToUpdate.clear(); + } + + private ExpandableBinaryDictionary getPersonalizationDictToUpdate(final Context context, + final Locale locale) { + ExpandableBinaryDictionary personalizationDict = mPersonalizationDictsToUpdate.get(locale); + if (personalizationDict != null) { + return personalizationDict; + } + personalizationDict = PersonalizationDictionary.getDictionary(context, locale, + null /* dictFile */, "" /* dictNamePrefix */); + mPersonalizationDictsToUpdate.put(locale, personalizationDict); + return personalizationDict; + } + + private void addEntriesToPersonalizationDictionariesForLocale(final Locale locale, + final PersonalizationDataChunk personalizationDataChunk, + final SpacingAndPunctuations spacingAndPunctuations, + final AddMultipleDictionaryEntriesCallback callback) { + final ExpandableBinaryDictionary personalizationDict = + getPersonalizationDictToUpdate(mContext, locale); + if (personalizationDict == null) { + if (callback != null) { + callback.onFinished(); + } + return; + } + final ArrayList<LanguageModelParam> languageModelParams = + LanguageModelParam.createLanguageModelParamsFrom( + personalizationDataChunk.mTokens, + personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations, + locale, new DistracterFilterCheckingIsInDictionary( + mDistracterFilter, personalizationDict)); + if (languageModelParams == null || languageModelParams.isEmpty()) { + if (callback != null) { + callback.onFinished(); + } + return; + } + personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback); + } + + public void addEntriesToPersonalizationDictionariesToUpdate( + final PersonalizationDataChunk personalizationDataChunk, + final SpacingAndPunctuations spacingAndPunctuations, + final AddMultipleDictionaryEntriesCallback callback) { + final HashSet<Locale> locales = + mLangToLocalesMap.get(personalizationDataChunk.mDetectedLanguage); + if (locales == null || locales.isEmpty()) { + if (callback != null) { + callback.onFinished(); + } + return; + } + final AtomicInteger remainingTaskCount = new AtomicInteger(locales.size()); + final AddMultipleDictionaryEntriesCallback callbackForLocales = + new AddMultipleDictionaryEntriesCallback() { + @Override + public void onFinished() { + if (remainingTaskCount.decrementAndGet() == 0) { + // Update tasks for all locales have been finished. + if (callback != null) { + callback.onFinished(); + } + } + } + }; + for (final Locale locale : locales) { + addEntriesToPersonalizationDictionariesForLocale(locale, personalizationDataChunk, + spacingAndPunctuations, callbackForLocales); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java index e9a97ff92..4e7e8140a 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java @@ -29,6 +29,7 @@ import com.android.inputmethod.latin.BinaryDictionary; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.RichInputMethodManager; import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.settings.SpacingAndPunctuations; @@ -36,6 +37,7 @@ import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; +import android.view.inputmethod.InputMethodSubtype; /** * Unit tests for personalization dictionary @@ -55,16 +57,28 @@ public class PersonalizationDictionaryTests extends AndroidTestCase { final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(getContext()); dictionaryFacilitator.resetDictionariesForTesting(getContext(), LOCALE_EN_US, dictTypes, new HashMap<String, File>(), new HashMap<String, Map<String, String>>()); + // Set subtypes. + RichInputMethodManager.init(getContext()); + final RichInputMethodManager richImm = RichInputMethodManager.getInstance(); + final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>(); + subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet( + LOCALE_EN_US.toString(), "qwerty")); + dictionaryFacilitator.updateEnabledSubtypes(subtypes); return dictionaryFacilitator; } public void testAddManyTokens() { final DictionaryFacilitator dictionaryFacilitator = getDictionaryFacilitator(); dictionaryFacilitator.clearPersonalizationDictionary(); - final int dataChunkCount = 20; - final int wordCountInOneChunk = 2000; + final int dataChunkCount = 100; + final int wordCountInOneChunk = 200; + final int uniqueWordCount = 100; final Random random = new Random(System.currentTimeMillis()); final int[] codePointSet = CodePointUtils.LATIN_ALPHABETS_LOWER; + final ArrayList<String> words = new ArrayList<>(); + for (int i = 0; i < uniqueWordCount; i++) { + words.add(CodePointUtils.generateWord(random, codePointSet)); + } final SpacingAndPunctuations spacingAndPunctuations = new SpacingAndPunctuations(getContext().getResources()); @@ -75,7 +89,7 @@ public class PersonalizationDictionaryTests extends AndroidTestCase { for (int i = 0; i < dataChunkCount; i++) { final ArrayList<String> tokens = new ArrayList<>(); for (int j = 0; j < wordCountInOneChunk; j++) { - tokens.add(CodePointUtils.generateWord(random, codePointSet)); + tokens.add(words.get(random.nextInt(words.size()))); } final PersonalizationDataChunk personalizationDataChunk = new PersonalizationDataChunk( true /* inputByUser */, tokens, timeStampInSeconds, DUMMY_PACKAGE_NAME, |