Merge branch 'master' of https://googleplex-android.googlesource.com/_direct/platform/packages/inputmethods/LatinIME

8 files changed, 471 insertions, 160 deletions

diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index 9b573b4b8..1de15a333 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -17,16 +17,18 @@
 package com.android.inputmethod.latin.personalization;
 
 import android.content.Context;
+import android.content.SharedPreferences;
 import android.util.Log;
 
 import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
 import com.android.inputmethod.latin.Constants;
 import com.android.inputmethod.latin.Dictionary;
 import com.android.inputmethod.latin.ExpandableBinaryDictionary;
+import com.android.inputmethod.latin.LatinImeLogger;
 import com.android.inputmethod.latin.makedict.DictDecoder;
 import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.settings.Settings;
+import com.android.inputmethod.latin.utils.CollectionUtils;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
 
@@ -34,9 +36,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
 
 /**
  * This class is a base class of a dictionary that supports decaying for the personalized language
@@ -45,7 +45,8 @@ import java.util.concurrent.TimeUnit;
 public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
     private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
     public static final boolean DBG_SAVE_RESTORE = false;
-    private static final boolean DBG_DUMP_ON_CLOSE = false;
+    private static final boolean DBG_STRESS_TEST = false;
+    private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
 
     /** Any pair being typed or picked */
     public static final int FREQUENCY_FOR_TYPED = 2;
@@ -53,56 +54,52 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
     public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
     public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
 
-    public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4;
-
     /** Locale for which this user history dictionary is storing words */
-    private final Locale mLocale;
+    private final String mLocale;
 
-    private final String mDictName;
+    private final String mFileName;
 
-    /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
-            final Locale locale, final String dictionaryType, final String dictName) {
-        super(context, dictName, locale, dictionaryType, true);
-        mLocale = locale;
-        mDictName = dictName;
-        if (mLocale != null && mLocale.toString().length() > 1) {
-            reloadDictionaryIfRequired();
-        }
-    }
+    private final SharedPreferences mPrefs;
+
+    private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
+            CollectionUtils.newArrayList();
+
+    // Should always be false except when we use this class for test
+    @UsedForTesting boolean mIsTest = false;
 
-    // Creates an instance that uses a given dictionary file for testing.
-    @UsedForTesting
     /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
-            final Locale locale, final String dictionaryType, final String dictName,
-            final File dictFile) {
-        super(context, dictName, locale, dictionaryType, true, dictFile);
+            final String locale, final SharedPreferences sp, final String dictionaryType,
+            final String fileName) {
+        super(context, fileName, dictionaryType, true);
         mLocale = locale;
-        mDictName = dictName;
-        if (mLocale != null && mLocale.toString().length() > 1) {
+        mFileName = fileName;
+        mPrefs = sp;
+        if (mLocale != null && mLocale.length() > 1) {
+            asyncLoadDictionaryToMemory();
             reloadDictionaryIfRequired();
         }
     }
 
     @Override
     public void close() {
-        if (DBG_DUMP_ON_CLOSE) {
-            dumpAllWordsForDebug();
+        if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+            closeBinaryDictionary();
         }
         // Flush pending writes.
-        asyncFlushBinaryDictionary();
+        // TODO: Remove after this class become to use a dynamic binary dictionary.
+        asyncFlashAllBinaryDictionary();
+        Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
     }
 
     @Override
     protected Map<String, String> getHeaderAttributeMap() {
         HashMap<String, String> attributeMap = new HashMap<String, String>();
-        attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
+        attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
-        attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
+        attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName);
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString());
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE,
-                String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
         return attributeMap;
     }
 
@@ -116,25 +113,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
         return false;
     }
 
-    @Override
-    protected boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) {
-        // This class is using format 4 because it's used by all version 4 dictionaries.
-        // TODO: remove this when all dynamically generated dicts use version 4.
-        return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION;
-    }
-
-    public void addMultipleDictionaryEntriesToDictionary(
-            final ArrayList<LanguageModelParam> languageModelParams,
-            final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
-        if (languageModelParams == null || languageModelParams.isEmpty()) {
-            if (callback != null) {
-                callback.onFinished();
-            }
-            return;
-        }
-        addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
-    }
-
     /**
      * Pair will be added to the decaying dictionary.
      *
@@ -143,63 +121,70 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
      * context, as in beginning of a sentence for example.
      * The second word may not be null (a NullPointerException would be thrown).
      */
-    public void addToDictionary(final String word0, final String word1, final boolean isValid,
-            final int timestamp) {
+    public void addToDictionary(final String word0, final String word1, final boolean isValid) {
         if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
                 (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
             return;
         }
-        final int frequency = isValid ?
-                FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
-        addWordDynamically(word1, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */,
-                false /* isNotAWord */, false /* isBlacklisted */, timestamp);
+        final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+                (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) :
+                        FREQUENCY_FOR_TYPED;
+        addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */,
+                false /* isNotAWord */);
         // Do not insert a word as a bigram of itself
         if (word1.equals(word0)) {
             return;
         }
         if (null != word0) {
-            addBigramDynamically(word0, word1, frequency, timestamp);
+            addBigramDynamically(word0, word1, frequency, isValid);
         }
     }
 
-    @Override
-    protected void loadDictionaryAsync() {
-        // Never loaded to memory in Java side.
+    public void cancelAddingUserHistory(final String word0, final String word1) {
+        removeBigramDynamically(word0, word1);
     }
 
-    @UsedForTesting
-    public void dumpAllWordsForDebug() {
-        runAfterGcForDebug(new Runnable() {
-            @Override
-            public void run() {
-                dumpAllWordsForDebugLocked();
+    @Override
+    protected void loadDictionaryAsync() {
+        final int[] profTotalCount = { 0 };
+        final String locale = getLocale();
+        if (DBG_STRESS_TEST) {
+            try {
+                Log.w(TAG, "Start stress in loading: " + locale);
+                Thread.sleep(15000);
+                Log.w(TAG, "End stress in loading");
+            } catch (InterruptedException e) {
             }
-        });
-    }
-
-    private void dumpAllWordsForDebugLocked() {
-        Log.d(TAG, "dumpAllWordsForDebug started.");
+        }
+        final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
+        final long now = System.currentTimeMillis();
+        final ExpandableBinaryDictionary dictionary = this;
         final OnAddWordListener listener = new OnAddWordListener() {
             @Override
             public void setUnigram(final String word, final String shortcutTarget,
                     final int frequency, final int shortcutFreq) {
-                Log.d(TAG, "load unigram: " + word + "," + frequency);
+                if (DBG_SAVE_RESTORE) {
+                    Log.d(TAG, "load unigram: " + word + "," + frequency);
+                }
+                addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */);
+                ++profTotalCount[0];
             }
 
             @Override
             public void setBigram(final String word0, final String word1, final int frequency) {
                 if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
                         && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
-                    Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
-                } else {
-                    Log.d(TAG, "Skip inserting a too long bigram: " + word0 + "," + word1 + ","
-                            + frequency);
+                    if (DBG_SAVE_RESTORE) {
+                        Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
+                    }
+                    ++profTotalCount[0];
+                    addBigram(word0, word1, frequency, last);
                 }
             }
         };
 
         // Load the dictionary from binary file
-        final File dictFile = new File(mContext.getFilesDir(), mDictName);
+        final File dictFile = new File(mContext.getFilesDir(), mFileName);
         final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
                 DictDecoder.USE_BYTEARRAY);
         if (dictDecoder == null) {
@@ -213,17 +198,35 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
             UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
         } catch (IOException e) {
             Log.d(TAG, "IOException on opening a bytebuffer", e);
-        } catch (UnsupportedFormatException e) {
-            Log.d(TAG, "Unsupported format, can't read the dictionary", e);
+        } finally {
+            if (PROFILE_SAVE_RESTORE) {
+                final long diff = System.currentTimeMillis() - now;
+                Log.d(TAG, "PROF: Load UserHistoryDictionary: "
+                        + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries.");
+            }
         }
     }
 
+    protected String getLocale() {
+        return mLocale;
+    }
+
+    public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
+        session.setPredictionDictionary(this);
+        mSessions.add(session);
+        session.onDictionaryReady();
+    }
+
+    public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) {
+        mSessions.remove(session);
+    }
+
     @UsedForTesting
     public void clearAndFlushDictionary() {
         // Clear the node structure on memory
         clear();
         // Then flush the cleared state of the dictionary on disk.
-        asyncFlushBinaryDictionary();
+        asyncFlashAllBinaryDictionary();
     }
 
     /* package */ void decayIfNeeded() {
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
new file mode 100644
index 000000000..6f152bb91
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.personalization;
+
+import android.content.Context;
+
+import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.compat.ActivityManagerCompatUtils;
+import com.android.inputmethod.keyboard.ProximityInfo;
+import com.android.inputmethod.latin.AbstractDictionaryWriter;
+import com.android.inputmethod.latin.ExpandableDictionary;
+import com.android.inputmethod.latin.WordComposer;
+import com.android.inputmethod.latin.ExpandableDictionary.NextWord;
+import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
+import com.android.inputmethod.latin.makedict.DictEncoder;
+import com.android.inputmethod.latin.makedict.FormatSpec;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
+import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
+import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
+import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
+// Currently this class is used to implement dynamic prodiction dictionary.
+// TODO: Move to native code.
+public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter {
+    private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName();
+    /** Maximum number of pairs. Pruning will start when databases goes above this number. */
+    public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000;
+    public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000;
+
+    /** Any pair being typed or picked */
+    private static final int FREQUENCY_FOR_TYPED = 2;
+
+    private static final int BINARY_DICT_VERSION = 3;
+    private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
+            new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */);
+
+    private final UserHistoryDictionaryBigramList mBigramList =
+            new UserHistoryDictionaryBigramList();
+    private final ExpandableDictionary mExpandableDictionary;
+    private final int mMaxHistoryBigrams;
+
+    public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) {
+        super(context, dictType);
+        mExpandableDictionary = new ExpandableDictionary(dictType);
+        final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context);
+        mMaxHistoryBigrams = isLowRamDevice ?
+                LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS;
+    }
+
+    @Override
+    public void clear() {
+        mBigramList.evictAll();
+        mExpandableDictionary.clearDictionary();
+    }
+
+    /**
+     * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes
+     * are done to update the binary dictionary.
+     * @param word The word to add.
+     * @param shortcutTarget A shortcut target for this word, or null if none.
+     * @param frequency The frequency for this unigram.
+     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
+     *   if shortcutTarget is null.
+     * @param isNotAWord true if this is not a word, i.e. shortcut only.
+     */
+    @Override
+    public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
+            final int shortcutFreq, final boolean isNotAWord) {
+        if (mBigramList.size() > mMaxHistoryBigrams * 2) {
+            // Too many entries: just stop adding new vocabulary and wait next refresh.
+            return;
+        }
+        mExpandableDictionary.addWord(word, shortcutTarget, frequency, shortcutFreq);
+        mBigramList.addBigram(null, word, (byte)frequency);
+    }
+
+    @Override
+    public void addBigramWords(final String word0, final String word1, final int frequency,
+            final boolean isValid, final long lastModifiedTime) {
+        if (mBigramList.size() > mMaxHistoryBigrams * 2) {
+            // Too many entries: just stop adding new vocabulary and wait next refresh.
+            return;
+        }
+        if (lastModifiedTime > 0) {
+            mExpandableDictionary.setBigramAndGetFrequency(word0, word1,
+                    new ForgettingCurveParams(frequency, System.currentTimeMillis(),
+                            lastModifiedTime));
+            mBigramList.addBigram(word0, word1, (byte)frequency);
+        } else {
+            mExpandableDictionary.setBigramAndGetFrequency(word0, word1,
+                    new ForgettingCurveParams(isValid));
+            mBigramList.addBigram(word0, word1, (byte)frequency);
+        }
+    }
+
+    @Override
+    public void removeBigramWords(final String word0, final String word1) {
+        if (mBigramList.removeBigram(word0, word1)) {
+            mExpandableDictionary.removeBigram(word0, word1);
+        }
+    }
+
+    @Override
+    protected void writeDictionary(final DictEncoder dictEncoder,
+            final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
+        UserHistoryDictIOUtils.writeDictionary(dictEncoder,
+                new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams),
+                mBigramList, FORMAT_OPTIONS);
+    }
+
+    private static class FrequencyProvider implements BigramDictionaryInterface {
+        private final UserHistoryDictionaryBigramList mBigramList;
+        private final ExpandableDictionary mExpandableDictionary;
+        private final int mMaxHistoryBigrams;
+
+        public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList,
+                final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) {
+            mBigramList = bigramList;
+            mExpandableDictionary = expandableDictionary;
+            mMaxHistoryBigrams = maxHistoryBigrams;
+        }
+
+        @Override
+        public int getFrequency(final String word0, final String word1) {
+            final int freq;
+            if (word0 == null) { // unigram
+                freq = FREQUENCY_FOR_TYPED;
+            } else { // bigram
+                final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1);
+                if (nw != null) {
+                    final ForgettingCurveParams forgettingCurveParams = nw.getFcParams();
+                    final byte prevFc = mBigramList.getBigrams(word0).get(word1);
+                    final byte fc = forgettingCurveParams.getFc();
+                    final boolean isValid = forgettingCurveParams.isValid();
+                    if (prevFc > 0 && prevFc == fc) {
+                        freq = fc & 0xFF;
+                    } else if (UserHistoryForgettingCurveUtils.
+                            needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) {
+                        freq = fc & 0xFF;
+                    } else {
+                        // Delete this entry
+                        freq = -1;
+                    }
+                } else {
+                    // Delete this entry
+                    freq = -1;
+                }
+            }
+            return freq;
+        }
+    }
+
+    @Override
+    public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
+            final String prevWord, final ProximityInfo proximityInfo,
+            boolean blockOffensiveWords, final int[] additionalFeaturesOptions) {
+        return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo,
+                blockOffensiveWords, additionalFeaturesOptions);
+    }
+
+    @Override
+    public boolean isValidWord(final String word) {
+        return mExpandableDictionary.isValidWord(word);
+    }
+
+    @UsedForTesting
+    public boolean isInBigramListForTests(final String word) {
+        // TODO: Use native method to determine whether the word is in dictionary or not
+        return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word);
+    }
+}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
index 596562f1d..f257165cb 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
@@ -16,37 +16,53 @@
 
 package com.android.inputmethod.latin.personalization;
 
-import com.android.inputmethod.annotations.UsedForTesting;
 import com.android.inputmethod.latin.Dictionary;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Locale;
-
 import android.content.Context;
+import android.content.SharedPreferences;
 
-public class PersonalizationDictionary extends DecayingExpandableBinaryDictionaryBase {
-    private static final String NAME = PersonalizationDictionary.class.getSimpleName();
+import java.util.ArrayList;
 
+/**
+ * This class is a dictionary for the personalized language model that uses binary dictionary.
+ */
+public class PersonalizationDictionary extends ExpandableBinaryDictionary {
+    private static final String NAME = "personalization";
     private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
             CollectionUtils.newArrayList();
 
-    /* package */ PersonalizationDictionary(final Context context, final Locale locale) {
-        super(context, locale, Dictionary.TYPE_PERSONALIZATION,
-                getDictNameWithLocale(NAME, locale));
+    /** Locale for which this user history dictionary is storing words */
+    private final String mLocale;
+
+    public PersonalizationDictionary(final Context context, final String locale,
+            final SharedPreferences prefs) {
+        // TODO: Make isUpdatable true.
+        super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION,
+                false /* isUpdatable */);
+        mLocale = locale;
+        // TODO: Restore last updated time
+        loadDictionary();
+    }
+
+    @Override
+    protected void loadDictionaryAsync() {
+        // TODO: Implement
+    }
+
+    @Override
+    protected boolean hasContentChanged() {
+        return false;
     }
 
-    // Creates an instance that uses a given dictionary file for testing.
-    @UsedForTesting
-    public PersonalizationDictionary(final Context context, final Locale locale,
-            final File dictFile) {
-        super(context, locale, Dictionary.TYPE_PERSONALIZATION, getDictNameWithLocale(NAME, locale),
-                dictFile);
+    @Override
+    protected boolean needsToReloadBeforeWriting() {
+        return false;
     }
 
     public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
-        session.setPredictionDictionary(this);
+        session.setDictionary(this);
         mSessions.add(session);
         session.onDictionaryReady();
     }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java
index 542bda621..c1833ff14 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java
@@ -20,18 +20,18 @@ import android.content.Context;
 import android.content.res.Configuration;
 
 public class PersonalizationDictionarySessionRegister {
-    public static void init(final Context context) {
+    public static void init(Context context) {
     }
 
     public static void onConfigurationChanged(final Context context, final Configuration conf) {
     }
 
-    public static void onUpdateData(final Context context, final String type) {
+    public static void onUpdateData(Context context, String type) {
     }
 
-    public static void onRemoveData(final Context context, final String type) {
+    public static void onRemoveData(Context context, String type) {
     }
 
-    public static void onDestroy(final Context context) {
+    public static void onDestroy(Context context) {
     }
 }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index 61354762b..a86f6e584 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -18,37 +18,61 @@ package com.android.inputmethod.latin.personalization;
 
 import android.content.Context;
 
-import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
-import com.android.inputmethod.latin.ExpandableBinaryDictionary;
-
 import java.lang.ref.WeakReference;
 import java.util.ArrayList;
-import java.util.Locale;
 
 /**
  * This class is a session where a data provider can communicate with a personalization
  * dictionary.
  */
 public abstract class PersonalizationDictionaryUpdateSession {
-    public WeakReference<PersonalizationDictionary> mDictionary;
-    public final Locale mSystemLocale;
+    /**
+     * This class is a parameter for a new unigram or bigram word which will be added
+     * to the personalization dictionary.
+     */
+    public static class PersonalizationLanguageModelParam {
+        public final String mWord0;
+        public final String mWord1;
+        public final boolean mIsValid;
+        public final int mFrequency;
+        public PersonalizationLanguageModelParam(String word0, String word1, boolean isValid,
+                int frequency) {
+            mWord0 = word0;
+            mWord1 = word1;
+            mIsValid = isValid;
+            mFrequency = frequency;
+        }
+    }
 
-    public PersonalizationDictionaryUpdateSession(final Locale locale) {
+    // TODO: Use a dynamic binary dictionary instead
+    public WeakReference<PersonalizationDictionary> mDictionary;
+    public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;
+    public final String mSystemLocale;
+    public PersonalizationDictionaryUpdateSession(String locale) {
         mSystemLocale = locale;
     }
 
     public abstract void onDictionaryReady();
 
-    public abstract void onDictionaryClosed(final Context context);
+    public abstract void onDictionaryClosed(Context context);
 
-    public void setPredictionDictionary(final PersonalizationDictionary dictionary) {
+    public void setDictionary(PersonalizationDictionary dictionary) {
         mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
     }
 
+    public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {
+        mPredictionDictionary =
+                new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);
+    }
+
     protected PersonalizationDictionary getDictionary() {
         return mDictionary == null ? null : mDictionary.get();
     }
 
+    protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {
+        return mPredictionDictionary == null ? null : mPredictionDictionary.get();
+    }
+
     private void unsetDictionary() {
         final PersonalizationDictionary dictionary = getDictionary();
         if (dictionary == null) {
@@ -57,30 +81,48 @@ public abstract class PersonalizationDictionaryUpdateSession {
         dictionary.unRegisterUpdateSession(this);
     }
 
-    public void clearAndFlushDictionary(final Context context) {
-        final PersonalizationDictionary dictionary = getDictionary();
+    private void unsetPredictionDictionary() {
+        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
+        if (dictionary == null) {
+            return;
+        }
+        dictionary.unRegisterUpdateSession(this);
+    }
+
+    public void clearAndFlushPredictionDictionary(Context context) {
+        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
         if (dictionary == null) {
             return;
         }
         dictionary.clearAndFlushDictionary();
     }
 
-    public void closeSession(final Context context) {
+    public void closeSession(Context context) {
         unsetDictionary();
+        unsetPredictionDictionary();
         onDictionaryClosed(context);
     }
 
-    // TODO: Support multi locale.
-    public void addMultipleDictionaryEntriesToDictionary(
-            final ArrayList<LanguageModelParam> languageModelParams,
-            final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
-        final PersonalizationDictionary dictionary = getDictionary();
+    // TODO: Support multi locale to add bigram
+    public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
+            int frequency) {
+        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
         if (dictionary == null) {
-            if (callback != null) {
-                callback.onFinished();
-            }
             return;
         }
-        dictionary.addMultipleDictionaryEntriesToDictionary(languageModelParams, callback);
+        dictionary.addToDictionary(word0, word1, isValid);
+    }
+
+    // Bulk import
+    // TODO: Support multi locale to add bigram
+    public void addBigramsToPersonalizationDictionary(
+            final ArrayList<PersonalizationLanguageModelParam> lmParams) {
+        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
+        if (dictionary == null) {
+            return;
+        }
+        for (final PersonalizationLanguageModelParam lmParam : lmParams) {
+            dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
+        }
     }
 }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
index d55cae132..221ddeeba 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
@@ -19,10 +19,11 @@ package com.android.inputmethod.latin.personalization;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 
 import android.content.Context;
+import android.content.SharedPreferences;
+import android.preference.PreferenceManager;
 import android.util.Log;
 
 import java.lang.ref.SoftReference;
-import java.util.Locale;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class PersonalizationHelper {
@@ -30,16 +31,21 @@ public class PersonalizationHelper {
     private static final boolean DEBUG = false;
     private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
             sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap();
+
     private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>>
             sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap();
 
+    private static final ConcurrentHashMap<String,
+            SoftReference<PersonalizationPredictionDictionary>>
+                    sLangPersonalizationPredictionDictCache =
+                            CollectionUtils.newConcurrentHashMap();
+
     public static UserHistoryDictionary getUserHistoryDictionary(
-            final Context context, final Locale locale) {
-        final String localeStr = locale.toString();
+            final Context context, final String locale, final SharedPreferences sp) {
         synchronized (sLangUserHistoryDictCache) {
-            if (sLangUserHistoryDictCache.containsKey(localeStr)) {
+            if (sLangUserHistoryDictCache.containsKey(locale)) {
                 final SoftReference<UserHistoryDictionary> ref =
-                        sLangUserHistoryDictCache.get(localeStr);
+                        sLangUserHistoryDictCache.get(locale);
                 final UserHistoryDictionary dict = ref == null ? null : ref.get();
                 if (dict != null) {
                     if (DEBUG) {
@@ -49,9 +55,8 @@ public class PersonalizationHelper {
                     return dict;
                 }
             }
-            final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale);
-            sLangUserHistoryDictCache.put(localeStr,
-                    new SoftReference<UserHistoryDictionary>(dict));
+            final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp);
+            sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
             return dict;
         }
     }
@@ -69,30 +74,57 @@ public class PersonalizationHelper {
     }
 
     public static void registerPersonalizationDictionaryUpdateSession(final Context context,
-            final PersonalizationDictionaryUpdateSession session, final Locale locale) {
-        final PersonalizationDictionary personalizationDictionary =
-                getPersonalizationDictionary(context, locale);
-        personalizationDictionary.registerUpdateSession(session);
+            final PersonalizationDictionaryUpdateSession session, String locale) {
+        final PersonalizationPredictionDictionary predictionDictionary =
+                getPersonalizationPredictionDictionary(context, locale,
+                        PreferenceManager.getDefaultSharedPreferences(context));
+        predictionDictionary.registerUpdateSession(session);
+        final PersonalizationDictionary dictionary =
+                getPersonalizationDictionary(context, locale,
+                        PreferenceManager.getDefaultSharedPreferences(context));
+        dictionary.registerUpdateSession(session);
     }
 
     public static PersonalizationDictionary getPersonalizationDictionary(
-            final Context context, final Locale locale) {
-        final String localeStr = locale.toString();
+            final Context context, final String locale, final SharedPreferences sp) {
         synchronized (sLangPersonalizationDictCache) {
-            if (sLangPersonalizationDictCache.containsKey(localeStr)) {
+            if (sLangPersonalizationDictCache.containsKey(locale)) {
                 final SoftReference<PersonalizationDictionary> ref =
-                        sLangPersonalizationDictCache.get(localeStr);
+                        sLangPersonalizationDictCache.get(locale);
                 final PersonalizationDictionary dict = ref == null ? null : ref.get();
                 if (dict != null) {
                     if (DEBUG) {
-                        Log.w(TAG, "Use cached PersonalizationDictionary for " + locale);
+                        Log.w(TAG, "Use cached PersonalizationDictCache for " + locale);
                     }
                     return dict;
                 }
             }
-            final PersonalizationDictionary dict = new PersonalizationDictionary(context, locale);
+            final PersonalizationDictionary dict =
+                    new PersonalizationDictionary(context, locale, sp);
             sLangPersonalizationDictCache.put(
-                    localeStr, new SoftReference<PersonalizationDictionary>(dict));
+                    locale, new SoftReference<PersonalizationDictionary>(dict));
+            return dict;
+        }
+    }
+
+    public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary(
+            final Context context, final String locale, final SharedPreferences sp) {
+        synchronized (sLangPersonalizationPredictionDictCache) {
+            if (sLangPersonalizationPredictionDictCache.containsKey(locale)) {
+                final SoftReference<PersonalizationPredictionDictionary> ref =
+                        sLangPersonalizationPredictionDictCache.get(locale);
+                final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get();
+                if (dict != null) {
+                    if (DEBUG) {
+                        Log.w(TAG, "Use cached PersonalizationPredictionDictionary for " + locale);
+                    }
+                    return dict;
+                }
+            }
+            final PersonalizationPredictionDictionary dict =
+                    new PersonalizationPredictionDictionary(context, locale, sp);
+            sLangPersonalizationPredictionDictCache.put(
+                    locale, new SoftReference<PersonalizationPredictionDictionary>(dict));
             return dict;
         }
     }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
new file mode 100644
index 000000000..432954453
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin.personalization;
+
+import com.android.inputmethod.latin.Dictionary;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
+
+import android.content.Context;
+import android.content.SharedPreferences;
+
+public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
+    private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
+
+    /* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
+            final SharedPreferences sp) {
+        super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA,
+                getDictionaryFileName(locale));
+    }
+
+    private static String getDictionaryFileName(final String locale) {
+        return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
+    }
+}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
index 868f21cbc..a60226d7e 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
@@ -16,13 +16,11 @@
 
 package com.android.inputmethod.latin.personalization;
 
-import com.android.inputmethod.annotations.UsedForTesting;
 import com.android.inputmethod.latin.Dictionary;
-
-import java.io.File;
-import java.util.Locale;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
 
 import android.content.Context;
+import android.content.SharedPreferences;
 
 /**
  * Locally gathers stats about the words user types and various other signals like auto-correction
@@ -31,19 +29,12 @@ import android.content.Context;
 public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase {
     /* package for tests */ static final String NAME =
             UserHistoryDictionary.class.getSimpleName();
-    /* package */ UserHistoryDictionary(final Context context, final Locale locale) {
-        super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale));
-    }
-
-    // Creates an instance that uses a given dictionary file for testing.
-    @UsedForTesting
-    public UserHistoryDictionary(final Context context, final Locale locale,
-            final File dictFile) {
-        super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale),
-                dictFile);
+    /* package */ UserHistoryDictionary(final Context context, final String locale,
+            final SharedPreferences sp) {
+        super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
     }
 
-    public void cancelAddingUserHistory(final String word0, final String word1) {
-        removeBigramDynamically(word0, word1);
+    private static String getDictionaryFileName(final String locale) {
+        return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
     }
 }