diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/personalization')
9 files changed, 152 insertions, 717 deletions
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 1de15a333..8f7378c58 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -17,26 +17,20 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; -import android.content.SharedPreferences; -import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; -import com.android.inputmethod.latin.LatinImeLogger; -import com.android.inputmethod.latin.makedict.DictDecoder; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.settings.Settings; -import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; +import com.android.inputmethod.latin.makedict.DictionaryHeader; +import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * This class is a base class of a dictionary that supports decaying for the personalized language @@ -44,9 +38,7 @@ import java.util.Map; */ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary { private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName(); - public static final boolean DBG_SAVE_RESTORE = false; - private static final boolean DBG_STRESS_TEST = false; - private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; + private static final boolean DBG_DUMP_ON_CLOSE = false; /** Any pair being typed or picked */ public static final int FREQUENCY_FOR_TYPED = 2; @@ -54,52 +46,42 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED; public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY; - /** Locale for which this user history dictionary is storing words */ - private final String mLocale; + /** The locale for this dictionary. */ + public final Locale mLocale; - private final String mFileName; + private final String mDictName; - private final SharedPreferences mPrefs; - - private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = - CollectionUtils.newArrayList(); - - // Should always be false except when we use this class for test - @UsedForTesting boolean mIsTest = false; - - /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, - final String locale, final SharedPreferences sp, final String dictionaryType, - final String fileName) { - super(context, fileName, dictionaryType, true); + protected DecayingExpandableBinaryDictionaryBase(final Context context, + final String dictName, final Locale locale, final String dictionaryType, + final File dictFile) { + super(context, dictName, locale, dictionaryType, true /* isUpdatable */, dictFile); mLocale = locale; - mFileName = fileName; - mPrefs = sp; - if (mLocale != null && mLocale.length() > 1) { - asyncLoadDictionaryToMemory(); + mDictName = dictName; + if (mLocale != null && mLocale.toString().length() > 1) { reloadDictionaryIfRequired(); } } @Override public void close() { - if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - closeBinaryDictionary(); + if (DBG_DUMP_ON_CLOSE) { + dumpAllWordsForDebug(); } // Flush pending writes. - // TODO: Remove after this class become to use a dynamic binary dictionary. - asyncFlashAllBinaryDictionary(); - Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); + asyncFlushBinaryDictionary(); } @Override protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale); + attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, + DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); + attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); + attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, + String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); return attributeMap; } @@ -113,6 +95,18 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB return false; } + public void addMultipleDictionaryEntriesToDictionary( + final ArrayList<LanguageModelParam> languageModelParams, + final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { + if (languageModelParams == null || languageModelParams.isEmpty()) { + if (callback != null) { + callback.onFinished(); + } + return; + } + addMultipleDictionaryEntriesDynamically(languageModelParams, callback); + } + /** * Pair will be added to the decaying dictionary. * @@ -121,104 +115,28 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public void addToDictionary(final String word0, final String word1, final boolean isValid) { + public void addToDictionary(final String word0, final String word1, final boolean isValid, + final int timestamp) { if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { return; } - final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? - (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) : - FREQUENCY_FOR_TYPED; - addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */, - false /* isNotAWord */); + final int frequency = isValid ? + FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; + addWordDynamically(word1, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */, + false /* isNotAWord */, false /* isBlacklisted */, timestamp); // Do not insert a word as a bigram of itself if (word1.equals(word0)) { return; } if (null != word0) { - addBigramDynamically(word0, word1, frequency, isValid); + addBigramDynamically(word0, word1, frequency, timestamp); } } - public void cancelAddingUserHistory(final String word0, final String word1) { - removeBigramDynamically(word0, word1); - } - @Override protected void loadDictionaryAsync() { - final int[] profTotalCount = { 0 }; - final String locale = getLocale(); - if (DBG_STRESS_TEST) { - try { - Log.w(TAG, "Start stress in loading: " + locale); - Thread.sleep(15000); - Log.w(TAG, "End stress in loading"); - } catch (InterruptedException e) { - } - } - final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale); - final long now = System.currentTimeMillis(); - final ExpandableBinaryDictionary dictionary = this; - final OnAddWordListener listener = new OnAddWordListener() { - @Override - public void setUnigram(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq) { - if (DBG_SAVE_RESTORE) { - Log.d(TAG, "load unigram: " + word + "," + frequency); - } - addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */); - ++profTotalCount[0]; - } - - @Override - public void setBigram(final String word0, final String word1, final int frequency) { - if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH - && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { - if (DBG_SAVE_RESTORE) { - Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); - } - ++profTotalCount[0]; - addBigram(word0, word1, frequency, last); - } - } - }; - - // Load the dictionary from binary file - final File dictFile = new File(mContext.getFilesDir(), mFileName); - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, - DictDecoder.USE_BYTEARRAY); - if (dictDecoder == null) { - // This is an expected condition: we don't have a user history dictionary for this - // language yet. It will be created sometime later. - return; - } - - try { - dictDecoder.openDictBuffer(); - UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); - } catch (IOException e) { - Log.d(TAG, "IOException on opening a bytebuffer", e); - } finally { - if (PROFILE_SAVE_RESTORE) { - final long diff = System.currentTimeMillis() - now; - Log.d(TAG, "PROF: Load UserHistoryDictionary: " - + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries."); - } - } - } - - protected String getLocale() { - return mLocale; - } - - public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { - session.setPredictionDictionary(this); - mSessions.add(session); - session.onDictionaryReady(); - } - - public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { - mSessions.remove(session); + // Never loaded to memory in Java side. } @UsedForTesting @@ -226,7 +144,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB // Clear the node structure on memory clear(); // Then flush the cleared state of the dictionary on disk. - asyncFlashAllBinaryDictionary(); + asyncFlushBinaryDictionary(); } /* package */ void decayIfNeeded() { diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java deleted file mode 100644 index 6f152bb91..000000000 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.personalization; - -import android.content.Context; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.compat.ActivityManagerCompatUtils; -import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.AbstractDictionaryWriter; -import com.android.inputmethod.latin.ExpandableDictionary; -import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.ExpandableDictionary.NextWord; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Map; - -// Currently this class is used to implement dynamic prodiction dictionary. -// TODO: Move to native code. -public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { - private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); - /** Maximum number of pairs. Pruning will start when databases goes above this number. */ - public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000; - public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000; - - /** Any pair being typed or picked */ - private static final int FREQUENCY_FOR_TYPED = 2; - - private static final int BINARY_DICT_VERSION = 3; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); - - private final UserHistoryDictionaryBigramList mBigramList = - new UserHistoryDictionaryBigramList(); - private final ExpandableDictionary mExpandableDictionary; - private final int mMaxHistoryBigrams; - - public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { - super(context, dictType); - mExpandableDictionary = new ExpandableDictionary(dictType); - final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context); - mMaxHistoryBigrams = isLowRamDevice ? - LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS; - } - - @Override - public void clear() { - mBigramList.evictAll(); - mExpandableDictionary.clearDictionary(); - } - - /** - * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes - * are done to update the binary dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - @Override - public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, - final int shortcutFreq, final boolean isNotAWord) { - if (mBigramList.size() > mMaxHistoryBigrams * 2) { - // Too many entries: just stop adding new vocabulary and wait next refresh. - return; - } - mExpandableDictionary.addWord(word, shortcutTarget, frequency, shortcutFreq); - mBigramList.addBigram(null, word, (byte)frequency); - } - - @Override - public void addBigramWords(final String word0, final String word1, final int frequency, - final boolean isValid, final long lastModifiedTime) { - if (mBigramList.size() > mMaxHistoryBigrams * 2) { - // Too many entries: just stop adding new vocabulary and wait next refresh. - return; - } - if (lastModifiedTime > 0) { - mExpandableDictionary.setBigramAndGetFrequency(word0, word1, - new ForgettingCurveParams(frequency, System.currentTimeMillis(), - lastModifiedTime)); - mBigramList.addBigram(word0, word1, (byte)frequency); - } else { - mExpandableDictionary.setBigramAndGetFrequency(word0, word1, - new ForgettingCurveParams(isValid)); - mBigramList.addBigram(word0, word1, (byte)frequency); - } - } - - @Override - public void removeBigramWords(final String word0, final String word1) { - if (mBigramList.removeBigram(word0, word1)) { - mExpandableDictionary.removeBigram(word0, word1); - } - } - - @Override - protected void writeDictionary(final DictEncoder dictEncoder, - final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException { - UserHistoryDictIOUtils.writeDictionary(dictEncoder, - new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams), - mBigramList, FORMAT_OPTIONS); - } - - private static class FrequencyProvider implements BigramDictionaryInterface { - private final UserHistoryDictionaryBigramList mBigramList; - private final ExpandableDictionary mExpandableDictionary; - private final int mMaxHistoryBigrams; - - public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, - final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) { - mBigramList = bigramList; - mExpandableDictionary = expandableDictionary; - mMaxHistoryBigrams = maxHistoryBigrams; - } - - @Override - public int getFrequency(final String word0, final String word1) { - final int freq; - if (word0 == null) { // unigram - freq = FREQUENCY_FOR_TYPED; - } else { // bigram - final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1); - if (nw != null) { - final ForgettingCurveParams forgettingCurveParams = nw.getFcParams(); - final byte prevFc = mBigramList.getBigrams(word0).get(word1); - final byte fc = forgettingCurveParams.getFc(); - final boolean isValid = forgettingCurveParams.isValid(); - if (prevFc > 0 && prevFc == fc) { - freq = fc & 0xFF; - } else if (UserHistoryForgettingCurveUtils. - needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) { - freq = fc & 0xFF; - } else { - // Delete this entry - freq = -1; - } - } else { - // Delete this entry - freq = -1; - } - } - return freq; - } - } - - @Override - public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, - boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { - return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords, additionalFeaturesOptions); - } - - @Override - public boolean isValidWord(final String word) { - return mExpandableDictionary.isValidWord(word); - } - - @UsedForTesting - public boolean isInBigramListForTests(final String word) { - // TODO: Use native method to determine whether the word is in dictionary or not - return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word); - } -} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java index f257165cb..4afd5b4c9 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -16,58 +16,29 @@ package com.android.inputmethod.latin.personalization; -import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.ExpandableBinaryDictionary; -import com.android.inputmethod.latin.utils.CollectionUtils; - import android.content.Context; -import android.content.SharedPreferences; -import java.util.ArrayList; +import com.android.inputmethod.latin.Dictionary; -/** - * This class is a dictionary for the personalized language model that uses binary dictionary. - */ -public class PersonalizationDictionary extends ExpandableBinaryDictionary { - private static final String NAME = "personalization"; - private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = - CollectionUtils.newArrayList(); +import java.io.File; +import java.util.Locale; - /** Locale for which this user history dictionary is storing words */ - private final String mLocale; +public class PersonalizationDictionary extends DecayingExpandableBinaryDictionaryBase { + /* package */ static final String NAME = PersonalizationDictionary.class.getSimpleName(); - public PersonalizationDictionary(final Context context, final String locale, - final SharedPreferences prefs) { - // TODO: Make isUpdatable true. - super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION, - false /* isUpdatable */); - mLocale = locale; - // TODO: Restore last updated time - loadDictionary(); + /* package */ PersonalizationDictionary(final Context context, final Locale locale) { + this(context, locale, null /* dictFile */); } - @Override - protected void loadDictionaryAsync() { - // TODO: Implement + public PersonalizationDictionary(final Context context, final Locale locale, + final File dictFile) { + super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_PERSONALIZATION, + dictFile); } @Override - protected boolean hasContentChanged() { + public boolean isValidWord(final String word) { + // Strings out of this dictionary should not be considered existing words. return false; } - - @Override - protected boolean needsToReloadBeforeWriting() { - return false; - } - - public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { - session.setDictionary(this); - mSessions.add(session); - session.onDictionaryReady(); - } - - public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { - mSessions.remove(session); - } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java index c1833ff14..d6c0dc0dc 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java @@ -19,19 +19,26 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; import android.content.res.Configuration; -public class PersonalizationDictionarySessionRegister { - public static void init(Context context) { +import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; + +public class PersonalizationDictionarySessionRegistrar { + public static void init(final Context context, + final DictionaryFacilitatorForSuggest dictionaryFacilitator) { + } + + public static void onConfigurationChanged(final Context context, final Configuration conf, + final DictionaryFacilitatorForSuggest dictionaryFacilitator) { } - public static void onConfigurationChanged(final Context context, final Configuration conf) { + public static void onUpdateData(final Context context, final String type) { } - public static void onUpdateData(Context context, String type) { + public static void onRemoveData(final Context context, final String type) { } - public static void onRemoveData(Context context, String type) { + public static void resetAll(final Context context) { } - public static void onDestroy(Context context) { + public static void close(final Context context) { } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java deleted file mode 100644 index a86f6e584..000000000 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.personalization; - -import android.content.Context; - -import java.lang.ref.WeakReference; -import java.util.ArrayList; - -/** - * This class is a session where a data provider can communicate with a personalization - * dictionary. - */ -public abstract class PersonalizationDictionaryUpdateSession { - /** - * This class is a parameter for a new unigram or bigram word which will be added - * to the personalization dictionary. - */ - public static class PersonalizationLanguageModelParam { - public final String mWord0; - public final String mWord1; - public final boolean mIsValid; - public final int mFrequency; - public PersonalizationLanguageModelParam(String word0, String word1, boolean isValid, - int frequency) { - mWord0 = word0; - mWord1 = word1; - mIsValid = isValid; - mFrequency = frequency; - } - } - - // TODO: Use a dynamic binary dictionary instead - public WeakReference<PersonalizationDictionary> mDictionary; - public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary; - public final String mSystemLocale; - public PersonalizationDictionaryUpdateSession(String locale) { - mSystemLocale = locale; - } - - public abstract void onDictionaryReady(); - - public abstract void onDictionaryClosed(Context context); - - public void setDictionary(PersonalizationDictionary dictionary) { - mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); - } - - public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) { - mPredictionDictionary = - new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary); - } - - protected PersonalizationDictionary getDictionary() { - return mDictionary == null ? null : mDictionary.get(); - } - - protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() { - return mPredictionDictionary == null ? null : mPredictionDictionary.get(); - } - - private void unsetDictionary() { - final PersonalizationDictionary dictionary = getDictionary(); - if (dictionary == null) { - return; - } - dictionary.unRegisterUpdateSession(this); - } - - private void unsetPredictionDictionary() { - final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); - if (dictionary == null) { - return; - } - dictionary.unRegisterUpdateSession(this); - } - - public void clearAndFlushPredictionDictionary(Context context) { - final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); - if (dictionary == null) { - return; - } - dictionary.clearAndFlushDictionary(); - } - - public void closeSession(Context context) { - unsetDictionary(); - unsetPredictionDictionary(); - onDictionaryClosed(context); - } - - // TODO: Support multi locale to add bigram - public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, - int frequency) { - final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); - if (dictionary == null) { - return; - } - dictionary.addToDictionary(word0, word1, isValid); - } - - // Bulk import - // TODO: Support multi locale to add bigram - public void addBigramsToPersonalizationDictionary( - final ArrayList<PersonalizationLanguageModelParam> lmParams) { - final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); - if (dictionary == null) { - return; - } - for (final PersonalizationLanguageModelParam lmParam : lmParams) { - dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid); - } - } -} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index 221ddeeba..5ae2fb6f8 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -17,13 +17,15 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.FileUtils; import android.content.Context; -import android.content.SharedPreferences; -import android.preference.PreferenceManager; import android.util.Log; +import java.io.File; +import java.io.FilenameFilter; import java.lang.ref.SoftReference; +import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; public class PersonalizationHelper { @@ -31,21 +33,16 @@ public class PersonalizationHelper { private static final boolean DEBUG = false; private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>> sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); - private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); - private static final ConcurrentHashMap<String, - SoftReference<PersonalizationPredictionDictionary>> - sLangPersonalizationPredictionDictCache = - CollectionUtils.newConcurrentHashMap(); - public static UserHistoryDictionary getUserHistoryDictionary( - final Context context, final String locale, final SharedPreferences sp) { + final Context context, final Locale locale) { + final String localeStr = locale.toString(); synchronized (sLangUserHistoryDictCache) { - if (sLangUserHistoryDictCache.containsKey(locale)) { + if (sLangUserHistoryDictCache.containsKey(localeStr)) { final SoftReference<UserHistoryDictionary> ref = - sLangUserHistoryDictCache.get(locale); + sLangUserHistoryDictCache.get(localeStr); final UserHistoryDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -55,8 +52,9 @@ public class PersonalizationHelper { return dict; } } - final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp); - sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict)); + final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale); + sLangUserHistoryDictCache.put(localeStr, + new SoftReference<UserHistoryDictionary>(dict)); return dict; } } @@ -73,59 +71,71 @@ public class PersonalizationHelper { } } - public static void registerPersonalizationDictionaryUpdateSession(final Context context, - final PersonalizationDictionaryUpdateSession session, String locale) { - final PersonalizationPredictionDictionary predictionDictionary = - getPersonalizationPredictionDictionary(context, locale, - PreferenceManager.getDefaultSharedPreferences(context)); - predictionDictionary.registerUpdateSession(session); - final PersonalizationDictionary dictionary = - getPersonalizationDictionary(context, locale, - PreferenceManager.getDefaultSharedPreferences(context)); - dictionary.registerUpdateSession(session); - } - public static PersonalizationDictionary getPersonalizationDictionary( - final Context context, final String locale, final SharedPreferences sp) { + final Context context, final Locale locale) { + final String localeStr = locale.toString(); synchronized (sLangPersonalizationDictCache) { - if (sLangPersonalizationDictCache.containsKey(locale)) { + if (sLangPersonalizationDictCache.containsKey(localeStr)) { final SoftReference<PersonalizationDictionary> ref = - sLangPersonalizationDictCache.get(locale); + sLangPersonalizationDictCache.get(localeStr); final PersonalizationDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { - Log.w(TAG, "Use cached PersonalizationDictCache for " + locale); + Log.w(TAG, "Use cached PersonalizationDictionary for " + locale); } return dict; } } - final PersonalizationDictionary dict = - new PersonalizationDictionary(context, locale, sp); + final PersonalizationDictionary dict = new PersonalizationDictionary(context, locale); sLangPersonalizationDictCache.put( - locale, new SoftReference<PersonalizationDictionary>(dict)); + localeStr, new SoftReference<PersonalizationDictionary>(dict)); return dict; } } - public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( - final Context context, final String locale, final SharedPreferences sp) { - synchronized (sLangPersonalizationPredictionDictCache) { - if (sLangPersonalizationPredictionDictCache.containsKey(locale)) { - final SoftReference<PersonalizationPredictionDictionary> ref = - sLangPersonalizationPredictionDictCache.get(locale); - final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get(); - if (dict != null) { - if (DEBUG) { - Log.w(TAG, "Use cached PersonalizationPredictionDictionary for " + locale); + public static void removeAllPersonalizationDictionaries(final Context context) { + removeAllDictionaries(context, sLangPersonalizationDictCache, + PersonalizationDictionary.NAME); + } + + public static void removeAllUserHistoryDictionaries(final Context context) { + removeAllDictionaries(context, sLangUserHistoryDictCache, + UserHistoryDictionary.NAME); + } + + private static <T extends DecayingExpandableBinaryDictionaryBase> void removeAllDictionaries( + final Context context, final ConcurrentHashMap<String, SoftReference<T>> dictionaryMap, + final String dictNamePrefix) { + synchronized (dictionaryMap) { + for (final ConcurrentHashMap.Entry<String, SoftReference<T>> entry + : dictionaryMap.entrySet()) { + if (entry.getValue() != null) { + final DecayingExpandableBinaryDictionaryBase dict = entry.getValue().get(); + if (dict != null) { + dict.clearAndFlushDictionary(); } - return dict; } } - final PersonalizationPredictionDictionary dict = - new PersonalizationPredictionDictionary(context, locale, sp); - sLangPersonalizationPredictionDictCache.put( - locale, new SoftReference<PersonalizationPredictionDictionary>(dict)); - return dict; + dictionaryMap.clear(); + if (!FileUtils.deleteFilteredFiles( + context.getFilesDir(), new DictFilter(dictNamePrefix))) { + Log.e(TAG, "Cannot remove all existing dictionary files. filesDir: " + + context.getFilesDir().getAbsolutePath() + ", dictNamePrefix: " + + dictNamePrefix); + } + } + } + + private static class DictFilter implements FilenameFilter { + private final String mName; + + DictFilter(final String name) { + mName = name; + } + + @Override + public boolean accept(final File dir, final String name) { + return name.startsWith(mName); } } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java deleted file mode 100644 index 432954453..000000000 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.personalization; - -import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.ExpandableBinaryDictionary; - -import android.content.Context; -import android.content.SharedPreferences; - -public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { - private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); - - /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, - final SharedPreferences sp) { - super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, - getDictionaryFileName(locale)); - } - - private static String getDictionaryFileName(final String locale) { - return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; - } -} diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index a60226d7e..504e9b2f3 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -16,25 +16,37 @@ package com.android.inputmethod.latin.personalization; +import android.content.Context; + import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.ExpandableBinaryDictionary; -import android.content.Context; -import android.content.SharedPreferences; +import java.io.File; +import java.util.Locale; /** * Locally gathers stats about the words user types and various other signals like auto-correction * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. */ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase { - /* package for tests */ static final String NAME = - UserHistoryDictionary.class.getSimpleName(); - /* package */ UserHistoryDictionary(final Context context, final String locale, - final SharedPreferences sp) { - super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); + /* package */ static final String NAME = UserHistoryDictionary.class.getSimpleName(); + + /* package */ UserHistoryDictionary(final Context context, final Locale locale) { + this(context, locale, null /* dictFile */); + } + + public UserHistoryDictionary(final Context context, final Locale locale, + final File dictFile) { + super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER_HISTORY, + dictFile); + } + + public void cancelAddingUserHistory(final String word0, final String word1) { + removeBigramDynamically(word0, word1); } - private static String getDictionaryFileName(final String locale) { - return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + @Override + public boolean isValidWord(final String word) { + // Strings out of this dictionary should not be considered existing words. + return false; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java deleted file mode 100644 index 55a90ee51..000000000 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.personalization; - -import android.util.Log; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.util.HashMap; -import java.util.Set; - -/** - * A store of bigrams which will be updated when the user history dictionary is closed - * All bigrams including stale ones in SQL DB should be stored in this class to avoid adding stale - * bigrams when we write to the SQL DB. - */ -@UsedForTesting -public final class UserHistoryDictionaryBigramList { - public static final byte FORGETTING_CURVE_INITIAL_VALUE = 0; - private static final String TAG = UserHistoryDictionaryBigramList.class.getSimpleName(); - private static final HashMap<String, Byte> EMPTY_BIGRAM_MAP = CollectionUtils.newHashMap(); - private final HashMap<String, HashMap<String, Byte>> mBigramMap = CollectionUtils.newHashMap(); - private int mSize = 0; - - public void evictAll() { - mSize = 0; - mBigramMap.clear(); - } - - /** - * Called when the user typed a word. - */ - @UsedForTesting - public void addBigram(String word1, String word2) { - addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE); - } - - /** - * Called when loaded from the SQL DB. - */ - public void addBigram(String word1, String word2, byte fcValue) { - if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { - Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); - } - final HashMap<String, Byte> map; - if (mBigramMap.containsKey(word1)) { - map = mBigramMap.get(word1); - } else { - map = CollectionUtils.newHashMap(); - mBigramMap.put(word1, map); - } - if (!map.containsKey(word2)) { - ++mSize; - map.put(word2, fcValue); - } - } - - /** - * Called when inserted to the SQL DB. - */ - public void updateBigram(String word1, String word2, byte fcValue) { - if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { - Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); - } - final HashMap<String, Byte> map; - if (mBigramMap.containsKey(word1)) { - map = mBigramMap.get(word1); - } else { - return; - } - if (!map.containsKey(word2)) { - return; - } - map.put(word2, fcValue); - } - - public int size() { - return mSize; - } - - public boolean isEmpty() { - return mBigramMap.isEmpty(); - } - - public boolean containsKey(String word) { - return mBigramMap.containsKey(word); - } - - public Set<String> keySet() { - return mBigramMap.keySet(); - } - - public HashMap<String, Byte> getBigrams(String word1) { - if (mBigramMap.containsKey(word1)) return mBigramMap.get(word1); - // TODO: lower case according to locale - final String lowerWord1 = word1.toLowerCase(); - if (mBigramMap.containsKey(lowerWord1)) return mBigramMap.get(lowerWord1); - return EMPTY_BIGRAM_MAP; - } - - public boolean removeBigram(String word1, String word2) { - final HashMap<String, Byte> set = getBigrams(word1); - if (set.isEmpty()) { - return false; - } - if (set.containsKey(word2)) { - set.remove(word2); - --mSize; - return true; - } - return false; - } -} |