diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/personalization')
9 files changed, 980 insertions, 0 deletions
diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java new file mode 100644 index 000000000..66517a800 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; +import android.content.SharedPreferences; +import android.util.Log; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.LatinImeLogger; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.settings.Settings; +import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +/** + * This class is a base class of a dictionary that supports decaying for the personalized language + * model. + */ +public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary { + private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName(); + public static final boolean DBG_SAVE_RESTORE = false; + private static final boolean DBG_STRESS_TEST = false; + private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; + + /** Any pair being typed or picked */ + public static final int FREQUENCY_FOR_TYPED = 2; + + /** Locale for which this user history dictionary is storing words */ + private final String mLocale; + + private final String mFileName; + + private final SharedPreferences mPrefs; + + private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = + CollectionUtils.newArrayList(); + + // Should always be false except when we use this class for test + @UsedForTesting boolean mIsTest = false; + + /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, + final String locale, final SharedPreferences sp, final String dictionaryType, + final String fileName) { + super(context, fileName, dictionaryType, true); + mLocale = locale; + mFileName = fileName; + mPrefs = sp; + if (mLocale != null && mLocale.length() > 1) { + asyncLoadDictionaryToMemory(); + reloadDictionaryIfRequired(); + } + } + + @Override + public void close() { + if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + closeBinaryDictionary(); + } + // Flush pending writes. + // TODO: Remove after this class become to use a dynamic binary dictionary. + asyncFlashAllBinaryDictionary(); + Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); + } + + @Override + protected Map<String, String> getHeaderAttributeMap() { + HashMap<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + return attributeMap; + } + + @Override + protected boolean hasContentChanged() { + return false; + } + + @Override + protected boolean needsToReloadBeforeWriting() { + return false; + } + + /** + * Return whether the passed charsequence is in the dictionary. + */ + @Override + public boolean isValidWord(final String word) { + // Words included only in the user history should be treated as not in dictionary words. + return false; + } + + /** + * Pair will be added to the personalization prediction dictionary. + * + * The first word may be null. That means we don't know the context, in other words, + * it's only a unigram. The first word may also be an empty string : this means start + * context, as in beginning of a sentence for example. + * The second word may not be null (a NullPointerException would be thrown). + */ + public void addToPersonalizationPredictionDictionary( + final String word0, final String word1, final boolean isValid) { + if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || + (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { + return; + } + addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED, + false /* isNotAWord */); + // Do not insert a word as a bigram of itself + if (word1.equals(word0)) { + return; + } + if (null != word0) { + addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid); + } + } + + public void cancelAddingUserHistory(final String word0, final String word1) { + removeBigramDynamically(word0, word1); + } + + @Override + protected void loadDictionaryAsync() { + final int[] profTotalCount = { 0 }; + final String locale = getLocale(); + if (DBG_STRESS_TEST) { + try { + Log.w(TAG, "Start stress in loading: " + locale); + Thread.sleep(15000); + Log.w(TAG, "End stress in loading"); + } catch (InterruptedException e) { + } + } + final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale); + final long now = System.currentTimeMillis(); + final ExpandableBinaryDictionary dictionary = this; + final OnAddWordListener listener = new OnAddWordListener() { + @Override + public void setUnigram(final String word, final String shortcutTarget, + final int frequency) { + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "load unigram: " + word + "," + frequency); + } + addWord(word, shortcutTarget, frequency, false /* isNotAWord */); + ++profTotalCount[0]; + } + + @Override + public void setBigram(final String word0, final String word1, final int frequency) { + if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH + && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); + } + ++profTotalCount[0]; + addBigram(word0, word1, frequency, last); + } + } + }; + + // Load the dictionary from binary file + final File dictFile = new File(mContext.getFilesDir(), mFileName); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, + DictDecoder.USE_BYTEARRAY); + if (dictDecoder == null) { + // This is an expected condition: we don't have a user history dictionary for this + // language yet. It will be created sometime later. + return; + } + + try { + dictDecoder.openDictBuffer(); + UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); + } catch (IOException e) { + Log.d(TAG, "IOException on opening a bytebuffer", e); + } finally { + if (PROFILE_SAVE_RESTORE) { + final long diff = System.currentTimeMillis() - now; + Log.d(TAG, "PROF: Load UserHistoryDictionary: " + + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries."); + } + } + } + + protected String getLocale() { + return mLocale; + } + + public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { + session.setPredictionDictionary(this); + mSessions.add(session); + session.onDictionaryReady(); + } + + public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { + mSessions.remove(session); + } + + public void clearAndFlushDictionary() { + // Clear the node structure on memory + clear(); + // Then flush the cleared state of the dictionary on disk. + asyncFlashAllBinaryDictionary(); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java new file mode 100644 index 000000000..0af028a9e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.compat.ActivityManagerCompatUtils; +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.AbstractDictionaryWriter; +import com.android.inputmethod.latin.ExpandableDictionary; +import com.android.inputmethod.latin.WordComposer; +import com.android.inputmethod.latin.ExpandableDictionary.NextWord; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + +import java.io.IOException; +import java.util.ArrayList; + +// Currently this class is used to implement dynamic prodiction dictionary. +// TODO: Move to native code. +public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { + private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); + /** Maximum number of pairs. Pruning will start when databases goes above this number. */ + public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000; + public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000; + + /** Any pair being typed or picked */ + private static final int FREQUENCY_FOR_TYPED = 2; + + private static final int BINARY_DICT_VERSION = 3; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); + + private final UserHistoryDictionaryBigramList mBigramList = + new UserHistoryDictionaryBigramList(); + private final ExpandableDictionary mExpandableDictionary; + private final int mMaxHistoryBigrams; + + public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { + super(context, dictType); + mExpandableDictionary = new ExpandableDictionary(dictType); + final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context); + mMaxHistoryBigrams = isLowRamDevice ? + LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS; + } + + @Override + public void clear() { + mBigramList.evictAll(); + mExpandableDictionary.clearDictionary(); + } + + /** + * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes + * are done to update the binary dictionary. + */ + @Override + public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, + final boolean isNotAWord) { + if (mBigramList.size() > mMaxHistoryBigrams * 2) { + // Too many entries: just stop adding new vocabrary and wait next refresh. + return; + } + mExpandableDictionary.addWord(word, shortcutTarget, frequency); + mBigramList.addBigram(null, word, (byte)frequency); + } + + @Override + public void addBigramWords(final String word0, final String word1, final int frequency, + final boolean isValid, final long lastModifiedTime) { + if (mBigramList.size() > mMaxHistoryBigrams * 2) { + // Too many entries: just stop adding new vocabrary and wait next refresh. + return; + } + if (lastModifiedTime > 0) { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(frequency, System.currentTimeMillis(), + lastModifiedTime)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } else { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(isValid)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } + } + + @Override + public void removeBigramWords(final String word0, final String word1) { + if (mBigramList.removeBigram(word0, word1)) { + mExpandableDictionary.removeBigram(word0, word1); + } + } + + @Override + protected void writeDictionary(final DictEncoder dictEncoder) + throws IOException, UnsupportedFormatException { + UserHistoryDictIOUtils.writeDictionary(dictEncoder, + new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams), + mBigramList, FORMAT_OPTIONS); + } + + private static class FrequencyProvider implements BigramDictionaryInterface { + private final UserHistoryDictionaryBigramList mBigramList; + private final ExpandableDictionary mExpandableDictionary; + private final int mMaxHistoryBigrams; + + public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, + final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) { + mBigramList = bigramList; + mExpandableDictionary = expandableDictionary; + mMaxHistoryBigrams = maxHistoryBigrams; + } + + @Override + public int getFrequency(final String word0, final String word1) { + final int freq; + if (word0 == null) { // unigram + freq = FREQUENCY_FOR_TYPED; + } else { // bigram + final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1); + if (nw != null) { + final ForgettingCurveParams forgettingCurveParams = nw.getFcParams(); + final byte prevFc = mBigramList.getBigrams(word0).get(word1); + final byte fc = forgettingCurveParams.getFc(); + final boolean isValid = forgettingCurveParams.isValid(); + if (prevFc > 0 && prevFc == fc) { + freq = fc & 0xFF; + } else if (UserHistoryForgettingCurveUtils. + needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) { + freq = fc & 0xFF; + } else { + // Delete this entry + freq = -1; + } + } else { + // Delete this entry + freq = -1; + } + } + return freq; + } + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo, + blockOffensiveWords, additionalFeaturesOptions); + } + + @Override + public boolean isValidWord(final String word) { + return mExpandableDictionary.isValidWord(word); + } + + @UsedForTesting + public boolean isInDictionaryForTests(final String word) { + // TODO: Use native method to determine whether the word is in dictionary or not + return mBigramList.containsKey(word); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java new file mode 100644 index 000000000..f257165cb --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.utils.CollectionUtils; + +import android.content.Context; +import android.content.SharedPreferences; + +import java.util.ArrayList; + +/** + * This class is a dictionary for the personalized language model that uses binary dictionary. + */ +public class PersonalizationDictionary extends ExpandableBinaryDictionary { + private static final String NAME = "personalization"; + private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = + CollectionUtils.newArrayList(); + + /** Locale for which this user history dictionary is storing words */ + private final String mLocale; + + public PersonalizationDictionary(final Context context, final String locale, + final SharedPreferences prefs) { + // TODO: Make isUpdatable true. + super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION, + false /* isUpdatable */); + mLocale = locale; + // TODO: Restore last updated time + loadDictionary(); + } + + @Override + protected void loadDictionaryAsync() { + // TODO: Implement + } + + @Override + protected boolean hasContentChanged() { + return false; + } + + @Override + protected boolean needsToReloadBeforeWriting() { + return false; + } + + public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { + session.setDictionary(this); + mSessions.add(session); + session.onDictionaryReady(); + } + + public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { + mSessions.remove(session); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java new file mode 100644 index 000000000..c1833ff14 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; +import android.content.res.Configuration; + +public class PersonalizationDictionarySessionRegister { + public static void init(Context context) { + } + + public static void onConfigurationChanged(final Context context, final Configuration conf) { + } + + public static void onUpdateData(Context context, String type) { + } + + public static void onRemoveData(Context context, String type) { + } + + public static void onDestroy(Context context) { + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java new file mode 100644 index 000000000..c616a296c --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; + +/** + * This class is a session where a data provider can communicate with a personalization + * dictionary. + */ +public abstract class PersonalizationDictionaryUpdateSession { + /** + * This class is a parameter for a new unigram or bigram word which will be added + * to the personalization dictionary. + */ + public static class PersonalizationLanguageModelParam { + public final String mWord0; + public final String mWord1; + public final boolean mIsValid; + public final int mFrequency; + public PersonalizationLanguageModelParam(String word0, String word1, boolean isValid, + int frequency) { + mWord0 = word0; + mWord1 = word1; + mIsValid = isValid; + mFrequency = frequency; + } + } + + // TODO: Use a dynamic binary dictionary instead + public WeakReference<PersonalizationDictionary> mDictionary; + public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary; + public final String mSystemLocale; + public PersonalizationDictionaryUpdateSession(String locale) { + mSystemLocale = locale; + } + + public abstract void onDictionaryReady(); + + public abstract void onDictionaryClosed(Context context); + + public void setDictionary(PersonalizationDictionary dictionary) { + mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); + } + + public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) { + mPredictionDictionary = + new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary); + } + + protected PersonalizationDictionary getDictionary() { + return mDictionary == null ? null : mDictionary.get(); + } + + protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() { + return mPredictionDictionary == null ? null : mPredictionDictionary.get(); + } + + private void unsetDictionary() { + final PersonalizationDictionary dictionary = getDictionary(); + if (dictionary == null) { + return; + } + dictionary.unRegisterUpdateSession(this); + } + + private void unsetPredictionDictionary() { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.unRegisterUpdateSession(this); + } + + public void clearAndFlushPredictionDictionary(Context context) { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.clearAndFlushDictionary(); + } + + public void closeSession(Context context) { + unsetDictionary(); + unsetPredictionDictionary(); + onDictionaryClosed(context); + } + + // TODO: Support multi locale to add bigram + public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, + int frequency) { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.addToPersonalizationPredictionDictionary(word0, word1, isValid); + } + + // Bulk import + // TODO: Support multi locale to add bigram + public void addBigramsToPersonalizationDictionary( + final ArrayList<PersonalizationLanguageModelParam> lmParams) { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + for (final PersonalizationLanguageModelParam lmParam : lmParams) { + dictionary.addToPersonalizationPredictionDictionary( + lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid); + } + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java new file mode 100644 index 000000000..5f702ee3f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import com.android.inputmethod.latin.utils.CollectionUtils; + +import android.content.Context; +import android.content.SharedPreferences; +import android.preference.PreferenceManager; +import android.util.Log; + +import java.lang.ref.SoftReference; +import java.util.concurrent.ConcurrentHashMap; + +public class PersonalizationHelper { + private static final String TAG = PersonalizationHelper.class.getSimpleName(); + private static final boolean DEBUG = false; + + private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>> + sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); + + private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> + sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); + + private static final ConcurrentHashMap<String, + SoftReference<PersonalizationPredictionDictionary>> + sLangPersonalizationPredictionDictCache = + CollectionUtils.newConcurrentHashMap(); + + public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary( + final Context context, final String locale, final SharedPreferences sp) { + synchronized (sLangUserHistoryDictCache) { + if (sLangUserHistoryDictCache.containsKey(locale)) { + final SoftReference<UserHistoryPredictionDictionary> ref = + sLangUserHistoryDictCache.get(locale); + final UserHistoryPredictionDictionary dict = ref == null ? null : ref.get(); + if (dict != null) { + if (DEBUG) { + Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale); + } + dict.reloadDictionaryIfRequired(); + return dict; + } + } + final UserHistoryPredictionDictionary dict = + new UserHistoryPredictionDictionary(context, locale, sp); + sLangUserHistoryDictCache.put( + locale, new SoftReference<UserHistoryPredictionDictionary>(dict)); + return dict; + } + } + + public static void registerPersonalizationDictionaryUpdateSession(final Context context, + final PersonalizationDictionaryUpdateSession session, String locale) { + final PersonalizationPredictionDictionary predictionDictionary = + getPersonalizationPredictionDictionary(context, locale, + PreferenceManager.getDefaultSharedPreferences(context)); + predictionDictionary.registerUpdateSession(session); + final PersonalizationDictionary dictionary = + getPersonalizationDictionary(context, locale, + PreferenceManager.getDefaultSharedPreferences(context)); + dictionary.registerUpdateSession(session); + } + + public static PersonalizationDictionary getPersonalizationDictionary( + final Context context, final String locale, final SharedPreferences sp) { + synchronized (sLangPersonalizationDictCache) { + if (sLangPersonalizationDictCache.containsKey(locale)) { + final SoftReference<PersonalizationDictionary> ref = + sLangPersonalizationDictCache.get(locale); + final PersonalizationDictionary dict = ref == null ? null : ref.get(); + if (dict != null) { + if (DEBUG) { + Log.w(TAG, "Use cached PersonalizationDictCache for " + locale); + } + return dict; + } + } + final PersonalizationDictionary dict = + new PersonalizationDictionary(context, locale, sp); + sLangPersonalizationDictCache.put( + locale, new SoftReference<PersonalizationDictionary>(dict)); + return dict; + } + } + + public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( + final Context context, final String locale, final SharedPreferences sp) { + synchronized (sLangPersonalizationPredictionDictCache) { + if (sLangPersonalizationPredictionDictCache.containsKey(locale)) { + final SoftReference<PersonalizationPredictionDictionary> ref = + sLangPersonalizationPredictionDictCache.get(locale); + final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get(); + if (dict != null) { + if (DEBUG) { + Log.w(TAG, "Use cached PersonalizationPredictionDictionary for " + locale); + } + return dict; + } + } + final PersonalizationPredictionDictionary dict = + new PersonalizationPredictionDictionary(context, locale, sp); + sLangPersonalizationPredictionDictCache.put( + locale, new SoftReference<PersonalizationPredictionDictionary>(dict)); + return dict; + } + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java new file mode 100644 index 000000000..432954453 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; + +import android.content.Context; +import android.content.SharedPreferences; + +public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { + private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); + + /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, + final SharedPreferences sp) { + super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, + getDictionaryFileName(locale)); + } + + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java new file mode 100644 index 000000000..55a90ee51 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.util.Log; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.utils.CollectionUtils; + +import java.util.HashMap; +import java.util.Set; + +/** + * A store of bigrams which will be updated when the user history dictionary is closed + * All bigrams including stale ones in SQL DB should be stored in this class to avoid adding stale + * bigrams when we write to the SQL DB. + */ +@UsedForTesting +public final class UserHistoryDictionaryBigramList { + public static final byte FORGETTING_CURVE_INITIAL_VALUE = 0; + private static final String TAG = UserHistoryDictionaryBigramList.class.getSimpleName(); + private static final HashMap<String, Byte> EMPTY_BIGRAM_MAP = CollectionUtils.newHashMap(); + private final HashMap<String, HashMap<String, Byte>> mBigramMap = CollectionUtils.newHashMap(); + private int mSize = 0; + + public void evictAll() { + mSize = 0; + mBigramMap.clear(); + } + + /** + * Called when the user typed a word. + */ + @UsedForTesting + public void addBigram(String word1, String word2) { + addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE); + } + + /** + * Called when loaded from the SQL DB. + */ + public void addBigram(String word1, String word2, byte fcValue) { + if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { + Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); + } + final HashMap<String, Byte> map; + if (mBigramMap.containsKey(word1)) { + map = mBigramMap.get(word1); + } else { + map = CollectionUtils.newHashMap(); + mBigramMap.put(word1, map); + } + if (!map.containsKey(word2)) { + ++mSize; + map.put(word2, fcValue); + } + } + + /** + * Called when inserted to the SQL DB. + */ + public void updateBigram(String word1, String word2, byte fcValue) { + if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { + Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); + } + final HashMap<String, Byte> map; + if (mBigramMap.containsKey(word1)) { + map = mBigramMap.get(word1); + } else { + return; + } + if (!map.containsKey(word2)) { + return; + } + map.put(word2, fcValue); + } + + public int size() { + return mSize; + } + + public boolean isEmpty() { + return mBigramMap.isEmpty(); + } + + public boolean containsKey(String word) { + return mBigramMap.containsKey(word); + } + + public Set<String> keySet() { + return mBigramMap.keySet(); + } + + public HashMap<String, Byte> getBigrams(String word1) { + if (mBigramMap.containsKey(word1)) return mBigramMap.get(word1); + // TODO: lower case according to locale + final String lowerWord1 = word1.toLowerCase(); + if (mBigramMap.containsKey(lowerWord1)) return mBigramMap.get(lowerWord1); + return EMPTY_BIGRAM_MAP; + } + + public boolean removeBigram(String word1, String word2) { + final HashMap<String, Byte> set = getBigrams(word1); + if (set.isEmpty()) { + return false; + } + if (set.containsKey(word2)) { + set.remove(word2); + --mSize; + return true; + } + return false; + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java new file mode 100644 index 000000000..38e308a4e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; + +import android.content.Context; +import android.content.SharedPreferences; + +/** + * Locally gathers stats about the words user types and various other signals like auto-correction + * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. + */ +public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { + /* package for tests */ static final String NAME = + UserHistoryPredictionDictionary.class.getSimpleName(); + /* package */ UserHistoryPredictionDictionary(final Context context, final String locale, + final SharedPreferences sp) { + super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); + } + + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + } +} |