Reset to 9bd6dac4708ad94fd0257c53e977df62b152e20c

The bulk merge from -bayo to klp-dev should not have been merged to master.

Change-Id: I527a03a76f5247e4939a672f27c314dc11cbb854

8 files changed, 160 insertions, 469 deletions

diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index 1de15a333..296733fad 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -17,18 +17,16 @@
 package com.android.inputmethod.latin.personalization;
 
 import android.content.Context;
-import android.content.SharedPreferences;
 import android.util.Log;
 
 import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
 import com.android.inputmethod.latin.Constants;
 import com.android.inputmethod.latin.Dictionary;
 import com.android.inputmethod.latin.ExpandableBinaryDictionary;
-import com.android.inputmethod.latin.LatinImeLogger;
 import com.android.inputmethod.latin.makedict.DictDecoder;
 import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.settings.Settings;
-import com.android.inputmethod.latin.utils.CollectionUtils;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
 
@@ -36,7 +34,9 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 /**
  * This class is a base class of a dictionary that supports decaying for the personalized language
@@ -45,8 +45,7 @@ import java.util.Map;
 public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
     private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
     public static final boolean DBG_SAVE_RESTORE = false;
-    private static final boolean DBG_STRESS_TEST = false;
-    private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
+    private static final boolean DBG_DUMP_ON_CLOSE = false;
 
     /** Any pair being typed or picked */
     public static final int FREQUENCY_FOR_TYPED = 2;
@@ -54,41 +53,43 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
     public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
     public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
 
-    /** Locale for which this user history dictionary is storing words */
-    private final String mLocale;
-
-    private final String mFileName;
+    public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4;
 
-    private final SharedPreferences mPrefs;
+    /** Locale for which this user history dictionary is storing words */
+    private final Locale mLocale;
 
-    private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
-            CollectionUtils.newArrayList();
+    private final String mDictName;
 
-    // Should always be false except when we use this class for test
-    @UsedForTesting boolean mIsTest = false;
+    /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
+            final Locale locale, final String dictionaryType, final String dictName) {
+        super(context, dictName, locale, dictionaryType, true);
+        mLocale = locale;
+        mDictName = dictName;
+        if (mLocale != null && mLocale.toString().length() > 1) {
+            reloadDictionaryIfRequired();
+        }
+    }
 
+    // Creates an instance that uses a given dictionary file for testing.
+    @UsedForTesting
     /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
-            final String locale, final SharedPreferences sp, final String dictionaryType,
-            final String fileName) {
-        super(context, fileName, dictionaryType, true);
+            final Locale locale, final String dictionaryType, final String dictName,
+            final File dictFile) {
+        super(context, dictName, locale, dictionaryType, true, dictFile);
         mLocale = locale;
-        mFileName = fileName;
-        mPrefs = sp;
-        if (mLocale != null && mLocale.length() > 1) {
-            asyncLoadDictionaryToMemory();
+        mDictName = dictName;
+        if (mLocale != null && mLocale.toString().length() > 1) {
             reloadDictionaryIfRequired();
         }
     }
 
     @Override
     public void close() {
-        if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
-            closeBinaryDictionary();
+        if (DBG_DUMP_ON_CLOSE) {
+            dumpAllWordsForDebug();
         }
         // Flush pending writes.
-        // TODO: Remove after this class become to use a dynamic binary dictionary.
-        asyncFlashAllBinaryDictionary();
-        Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
+        asyncFlushBinaryDictionary();
     }
 
     @Override
@@ -98,8 +99,12 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
         attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
+        attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
+                FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName);
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString());
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE,
+                String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
         return attributeMap;
     }
 
@@ -113,6 +118,25 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
         return false;
     }
 
+    @Override
+    protected boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) {
+        // This class is using format 4 because it's used by all version 4 dictionaries.
+        // TODO: remove this when all dynamically generated dicts use version 4.
+        return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION;
+    }
+
+    public void addMultipleDictionaryEntriesToDictionary(
+            final ArrayList<LanguageModelParam> languageModelParams,
+            final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
+        if (languageModelParams == null || languageModelParams.isEmpty()) {
+            if (callback != null) {
+                callback.onFinished();
+            }
+            return;
+        }
+        addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
+    }
+
     /**
      * Pair will be added to the decaying dictionary.
      *
@@ -121,70 +145,63 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
      * context, as in beginning of a sentence for example.
      * The second word may not be null (a NullPointerException would be thrown).
      */
-    public void addToDictionary(final String word0, final String word1, final boolean isValid) {
+    public void addToDictionary(final String word0, final String word1, final boolean isValid,
+            final int timestamp) {
         if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
                 (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
             return;
         }
-        final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
-                (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) :
-                        FREQUENCY_FOR_TYPED;
-        addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */,
-                false /* isNotAWord */);
+        final int frequency = isValid ?
+                FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
+        addWordDynamically(word1, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */,
+                false /* isNotAWord */, false /* isBlacklisted */, timestamp);
         // Do not insert a word as a bigram of itself
         if (word1.equals(word0)) {
             return;
         }
         if (null != word0) {
-            addBigramDynamically(word0, word1, frequency, isValid);
+            addBigramDynamically(word0, word1, frequency, timestamp);
         }
     }
 
-    public void cancelAddingUserHistory(final String word0, final String word1) {
-        removeBigramDynamically(word0, word1);
-    }
-
     @Override
     protected void loadDictionaryAsync() {
-        final int[] profTotalCount = { 0 };
-        final String locale = getLocale();
-        if (DBG_STRESS_TEST) {
-            try {
-                Log.w(TAG, "Start stress in loading: " + locale);
-                Thread.sleep(15000);
-                Log.w(TAG, "End stress in loading");
-            } catch (InterruptedException e) {
+        // Never loaded to memory in Java side.
+    }
+
+    @UsedForTesting
+    public void dumpAllWordsForDebug() {
+        runAfterGcForDebug(new Runnable() {
+            @Override
+            public void run() {
+                dumpAllWordsForDebugLocked();
             }
-        }
-        final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
-        final long now = System.currentTimeMillis();
-        final ExpandableBinaryDictionary dictionary = this;
+        });
+    }
+
+    private void dumpAllWordsForDebugLocked() {
+        Log.d(TAG, "dumpAllWordsForDebug started.");
         final OnAddWordListener listener = new OnAddWordListener() {
             @Override
             public void setUnigram(final String word, final String shortcutTarget,
                     final int frequency, final int shortcutFreq) {
-                if (DBG_SAVE_RESTORE) {
-                    Log.d(TAG, "load unigram: " + word + "," + frequency);
-                }
-                addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */);
-                ++profTotalCount[0];
+                Log.d(TAG, "load unigram: " + word + "," + frequency);
             }
 
             @Override
             public void setBigram(final String word0, final String word1, final int frequency) {
                 if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
                         && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
-                    if (DBG_SAVE_RESTORE) {
-                        Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
-                    }
-                    ++profTotalCount[0];
-                    addBigram(word0, word1, frequency, last);
+                    Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
+                } else {
+                    Log.d(TAG, "Skip inserting a too long bigram: " + word0 + "," + word1 + ","
+                            + frequency);
                 }
             }
         };
 
         // Load the dictionary from binary file
-        final File dictFile = new File(mContext.getFilesDir(), mFileName);
+        final File dictFile = new File(mContext.getFilesDir(), mDictName);
         final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
                 DictDecoder.USE_BYTEARRAY);
         if (dictDecoder == null) {
@@ -198,35 +215,17 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
             UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
         } catch (IOException e) {
             Log.d(TAG, "IOException on opening a bytebuffer", e);
-        } finally {
-            if (PROFILE_SAVE_RESTORE) {
-                final long diff = System.currentTimeMillis() - now;
-                Log.d(TAG, "PROF: Load UserHistoryDictionary: "
-                        + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries.");
-            }
+        } catch (UnsupportedFormatException e) {
+            Log.d(TAG, "Unsupported format, can't read the dictionary", e);
         }
     }
 
-    protected String getLocale() {
-        return mLocale;
-    }
-
-    public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
-        session.setPredictionDictionary(this);
-        mSessions.add(session);
-        session.onDictionaryReady();
-    }
-
-    public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) {
-        mSessions.remove(session);
-    }
-
     @UsedForTesting
     public void clearAndFlushDictionary() {
         // Clear the node structure on memory
         clear();
         // Then flush the cleared state of the dictionary on disk.
-        asyncFlashAllBinaryDictionary();
+        asyncFlushBinaryDictionary();
     }
 
     /* package */ void decayIfNeeded() {
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
deleted file mode 100644
index 6f152bb91..000000000
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin.personalization;
-
-import android.content.Context;
-
-import com.android.inputmethod.annotations.UsedForTesting;
-import com.android.inputmethod.compat.ActivityManagerCompatUtils;
-import com.android.inputmethod.keyboard.ProximityInfo;
-import com.android.inputmethod.latin.AbstractDictionaryWriter;
-import com.android.inputmethod.latin.ExpandableDictionary;
-import com.android.inputmethod.latin.WordComposer;
-import com.android.inputmethod.latin.ExpandableDictionary.NextWord;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.makedict.DictEncoder;
-import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
-import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
-import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
-import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
-import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-
-// Currently this class is used to implement dynamic prodiction dictionary.
-// TODO: Move to native code.
-public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter {
-    private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName();
-    /** Maximum number of pairs. Pruning will start when databases goes above this number. */
-    public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000;
-    public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000;
-
-    /** Any pair being typed or picked */
-    private static final int FREQUENCY_FOR_TYPED = 2;
-
-    private static final int BINARY_DICT_VERSION = 3;
-    private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
-            new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */);
-
-    private final UserHistoryDictionaryBigramList mBigramList =
-            new UserHistoryDictionaryBigramList();
-    private final ExpandableDictionary mExpandableDictionary;
-    private final int mMaxHistoryBigrams;
-
-    public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) {
-        super(context, dictType);
-        mExpandableDictionary = new ExpandableDictionary(dictType);
-        final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context);
-        mMaxHistoryBigrams = isLowRamDevice ?
-                LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS;
-    }
-
-    @Override
-    public void clear() {
-        mBigramList.evictAll();
-        mExpandableDictionary.clearDictionary();
-    }
-
-    /**
-     * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes
-     * are done to update the binary dictionary.
-     * @param word The word to add.
-     * @param shortcutTarget A shortcut target for this word, or null if none.
-     * @param frequency The frequency for this unigram.
-     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored
-     *   if shortcutTarget is null.
-     * @param isNotAWord true if this is not a word, i.e. shortcut only.
-     */
-    @Override
-    public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
-            final int shortcutFreq, final boolean isNotAWord) {
-        if (mBigramList.size() > mMaxHistoryBigrams * 2) {
-            // Too many entries: just stop adding new vocabulary and wait next refresh.
-            return;
-        }
-        mExpandableDictionary.addWord(word, shortcutTarget, frequency, shortcutFreq);
-        mBigramList.addBigram(null, word, (byte)frequency);
-    }
-
-    @Override
-    public void addBigramWords(final String word0, final String word1, final int frequency,
-            final boolean isValid, final long lastModifiedTime) {
-        if (mBigramList.size() > mMaxHistoryBigrams * 2) {
-            // Too many entries: just stop adding new vocabulary and wait next refresh.
-            return;
-        }
-        if (lastModifiedTime > 0) {
-            mExpandableDictionary.setBigramAndGetFrequency(word0, word1,
-                    new ForgettingCurveParams(frequency, System.currentTimeMillis(),
-                            lastModifiedTime));
-            mBigramList.addBigram(word0, word1, (byte)frequency);
-        } else {
-            mExpandableDictionary.setBigramAndGetFrequency(word0, word1,
-                    new ForgettingCurveParams(isValid));
-            mBigramList.addBigram(word0, word1, (byte)frequency);
-        }
-    }
-
-    @Override
-    public void removeBigramWords(final String word0, final String word1) {
-        if (mBigramList.removeBigram(word0, word1)) {
-            mExpandableDictionary.removeBigram(word0, word1);
-        }
-    }
-
-    @Override
-    protected void writeDictionary(final DictEncoder dictEncoder,
-            final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {
-        UserHistoryDictIOUtils.writeDictionary(dictEncoder,
-                new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams),
-                mBigramList, FORMAT_OPTIONS);
-    }
-
-    private static class FrequencyProvider implements BigramDictionaryInterface {
-        private final UserHistoryDictionaryBigramList mBigramList;
-        private final ExpandableDictionary mExpandableDictionary;
-        private final int mMaxHistoryBigrams;
-
-        public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList,
-                final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) {
-            mBigramList = bigramList;
-            mExpandableDictionary = expandableDictionary;
-            mMaxHistoryBigrams = maxHistoryBigrams;
-        }
-
-        @Override
-        public int getFrequency(final String word0, final String word1) {
-            final int freq;
-            if (word0 == null) { // unigram
-                freq = FREQUENCY_FOR_TYPED;
-            } else { // bigram
-                final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1);
-                if (nw != null) {
-                    final ForgettingCurveParams forgettingCurveParams = nw.getFcParams();
-                    final byte prevFc = mBigramList.getBigrams(word0).get(word1);
-                    final byte fc = forgettingCurveParams.getFc();
-                    final boolean isValid = forgettingCurveParams.isValid();
-                    if (prevFc > 0 && prevFc == fc) {
-                        freq = fc & 0xFF;
-                    } else if (UserHistoryForgettingCurveUtils.
-                            needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) {
-                        freq = fc & 0xFF;
-                    } else {
-                        // Delete this entry
-                        freq = -1;
-                    }
-                } else {
-                    // Delete this entry
-                    freq = -1;
-                }
-            }
-            return freq;
-        }
-    }
-
-    @Override
-    public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer,
-            final String prevWord, final ProximityInfo proximityInfo,
-            boolean blockOffensiveWords, final int[] additionalFeaturesOptions) {
-        return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo,
-                blockOffensiveWords, additionalFeaturesOptions);
-    }
-
-    @Override
-    public boolean isValidWord(final String word) {
-        return mExpandableDictionary.isValidWord(word);
-    }
-
-    @UsedForTesting
-    public boolean isInBigramListForTests(final String word) {
-        // TODO: Use native method to determine whether the word is in dictionary or not
-        return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word);
-    }
-}
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
index f257165cb..596562f1d 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java
@@ -16,53 +16,37 @@
 
 package com.android.inputmethod.latin.personalization;
 
+import com.android.inputmethod.annotations.UsedForTesting;
 import com.android.inputmethod.latin.Dictionary;
-import com.android.inputmethod.latin.ExpandableBinaryDictionary;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Locale;
+
 import android.content.Context;
-import android.content.SharedPreferences;
 
-import java.util.ArrayList;
+public class PersonalizationDictionary extends DecayingExpandableBinaryDictionaryBase {
+    private static final String NAME = PersonalizationDictionary.class.getSimpleName();
 
-/**
- * This class is a dictionary for the personalized language model that uses binary dictionary.
- */
-public class PersonalizationDictionary extends ExpandableBinaryDictionary {
-    private static final String NAME = "personalization";
     private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
             CollectionUtils.newArrayList();
 
-    /** Locale for which this user history dictionary is storing words */
-    private final String mLocale;
-
-    public PersonalizationDictionary(final Context context, final String locale,
-            final SharedPreferences prefs) {
-        // TODO: Make isUpdatable true.
-        super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION,
-                false /* isUpdatable */);
-        mLocale = locale;
-        // TODO: Restore last updated time
-        loadDictionary();
-    }
-
-    @Override
-    protected void loadDictionaryAsync() {
-        // TODO: Implement
-    }
-
-    @Override
-    protected boolean hasContentChanged() {
-        return false;
+    /* package */ PersonalizationDictionary(final Context context, final Locale locale) {
+        super(context, locale, Dictionary.TYPE_PERSONALIZATION,
+                getDictNameWithLocale(NAME, locale));
     }
 
-    @Override
-    protected boolean needsToReloadBeforeWriting() {
-        return false;
+    // Creates an instance that uses a given dictionary file for testing.
+    @UsedForTesting
+    public PersonalizationDictionary(final Context context, final Locale locale,
+            final File dictFile) {
+        super(context, locale, Dictionary.TYPE_PERSONALIZATION, getDictNameWithLocale(NAME, locale),
+                dictFile);
     }
 
     public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
-        session.setDictionary(this);
+        session.setPredictionDictionary(this);
         mSessions.add(session);
         session.onDictionaryReady();
     }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java
index c1833ff14..542bda621 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java
@@ -20,18 +20,18 @@ import android.content.Context;
 import android.content.res.Configuration;
 
 public class PersonalizationDictionarySessionRegister {
-    public static void init(Context context) {
+    public static void init(final Context context) {
     }
 
     public static void onConfigurationChanged(final Context context, final Configuration conf) {
     }
 
-    public static void onUpdateData(Context context, String type) {
+    public static void onUpdateData(final Context context, final String type) {
     }
 
-    public static void onRemoveData(Context context, String type) {
+    public static void onRemoveData(final Context context, final String type) {
     }
 
-    public static void onDestroy(Context context) {
+    public static void onDestroy(final Context context) {
     }
 }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
index a86f6e584..61354762b 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java
@@ -18,61 +18,37 @@ package com.android.inputmethod.latin.personalization;
 
 import android.content.Context;
 
+import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
+import com.android.inputmethod.latin.ExpandableBinaryDictionary;
+
 import java.lang.ref.WeakReference;
 import java.util.ArrayList;
+import java.util.Locale;
 
 /**
  * This class is a session where a data provider can communicate with a personalization
  * dictionary.
  */
 public abstract class PersonalizationDictionaryUpdateSession {
-    /**
-     * This class is a parameter for a new unigram or bigram word which will be added
-     * to the personalization dictionary.
-     */
-    public static class PersonalizationLanguageModelParam {
-        public final String mWord0;
-        public final String mWord1;
-        public final boolean mIsValid;
-        public final int mFrequency;
-        public PersonalizationLanguageModelParam(String word0, String word1, boolean isValid,
-                int frequency) {
-            mWord0 = word0;
-            mWord1 = word1;
-            mIsValid = isValid;
-            mFrequency = frequency;
-        }
-    }
-
-    // TODO: Use a dynamic binary dictionary instead
     public WeakReference<PersonalizationDictionary> mDictionary;
-    public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;
-    public final String mSystemLocale;
-    public PersonalizationDictionaryUpdateSession(String locale) {
+    public final Locale mSystemLocale;
+
+    public PersonalizationDictionaryUpdateSession(final Locale locale) {
         mSystemLocale = locale;
     }
 
     public abstract void onDictionaryReady();
 
-    public abstract void onDictionaryClosed(Context context);
+    public abstract void onDictionaryClosed(final Context context);
 
-    public void setDictionary(PersonalizationDictionary dictionary) {
+    public void setPredictionDictionary(final PersonalizationDictionary dictionary) {
         mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);
     }
 
-    public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {
-        mPredictionDictionary =
-                new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);
-    }
-
     protected PersonalizationDictionary getDictionary() {
         return mDictionary == null ? null : mDictionary.get();
     }
 
-    protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {
-        return mPredictionDictionary == null ? null : mPredictionDictionary.get();
-    }
-
     private void unsetDictionary() {
         final PersonalizationDictionary dictionary = getDictionary();
         if (dictionary == null) {
@@ -81,48 +57,30 @@ public abstract class PersonalizationDictionaryUpdateSession {
         dictionary.unRegisterUpdateSession(this);
     }
 
-    private void unsetPredictionDictionary() {
-        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
-        if (dictionary == null) {
-            return;
-        }
-        dictionary.unRegisterUpdateSession(this);
-    }
-
-    public void clearAndFlushPredictionDictionary(Context context) {
-        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
+    public void clearAndFlushDictionary(final Context context) {
+        final PersonalizationDictionary dictionary = getDictionary();
         if (dictionary == null) {
             return;
         }
         dictionary.clearAndFlushDictionary();
     }
 
-    public void closeSession(Context context) {
+    public void closeSession(final Context context) {
         unsetDictionary();
-        unsetPredictionDictionary();
         onDictionaryClosed(context);
     }
 
-    // TODO: Support multi locale to add bigram
-    public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,
-            int frequency) {
-        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
-        if (dictionary == null) {
-            return;
-        }
-        dictionary.addToDictionary(word0, word1, isValid);
-    }
-
-    // Bulk import
-    // TODO: Support multi locale to add bigram
-    public void addBigramsToPersonalizationDictionary(
-            final ArrayList<PersonalizationLanguageModelParam> lmParams) {
-        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();
+    // TODO: Support multi locale.
+    public void addMultipleDictionaryEntriesToDictionary(
+            final ArrayList<LanguageModelParam> languageModelParams,
+            final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
+        final PersonalizationDictionary dictionary = getDictionary();
         if (dictionary == null) {
+            if (callback != null) {
+                callback.onFinished();
+            }
             return;
         }
-        for (final PersonalizationLanguageModelParam lmParam : lmParams) {
-            dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid);
-        }
+        dictionary.addMultipleDictionaryEntriesToDictionary(languageModelParams, callback);
     }
 }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
index 221ddeeba..d55cae132 100644
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
+++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java
@@ -19,11 +19,10 @@ package com.android.inputmethod.latin.personalization;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 
 import android.content.Context;
-import android.content.SharedPreferences;
-import android.preference.PreferenceManager;
 import android.util.Log;
 
 import java.lang.ref.SoftReference;
+import java.util.Locale;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class PersonalizationHelper {
@@ -31,21 +30,16 @@ public class PersonalizationHelper {
     private static final boolean DEBUG = false;
     private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
             sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap();
-
     private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>>
             sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap();
 
-    private static final ConcurrentHashMap<String,
-            SoftReference<PersonalizationPredictionDictionary>>
-                    sLangPersonalizationPredictionDictCache =
-                            CollectionUtils.newConcurrentHashMap();
-
     public static UserHistoryDictionary getUserHistoryDictionary(
-            final Context context, final String locale, final SharedPreferences sp) {
+            final Context context, final Locale locale) {
+        final String localeStr = locale.toString();
         synchronized (sLangUserHistoryDictCache) {
-            if (sLangUserHistoryDictCache.containsKey(locale)) {
+            if (sLangUserHistoryDictCache.containsKey(localeStr)) {
                 final SoftReference<UserHistoryDictionary> ref =
-                        sLangUserHistoryDictCache.get(locale);
+                        sLangUserHistoryDictCache.get(localeStr);
                 final UserHistoryDictionary dict = ref == null ? null : ref.get();
                 if (dict != null) {
                     if (DEBUG) {
@@ -55,8 +49,9 @@ public class PersonalizationHelper {
                     return dict;
                 }
             }
-            final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp);
-            sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
+            final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale);
+            sLangUserHistoryDictCache.put(localeStr,
+                    new SoftReference<UserHistoryDictionary>(dict));
             return dict;
         }
     }
@@ -74,57 +69,30 @@ public class PersonalizationHelper {
     }
 
     public static void registerPersonalizationDictionaryUpdateSession(final Context context,
-            final PersonalizationDictionaryUpdateSession session, String locale) {
-        final PersonalizationPredictionDictionary predictionDictionary =
-                getPersonalizationPredictionDictionary(context, locale,
-                        PreferenceManager.getDefaultSharedPreferences(context));
-        predictionDictionary.registerUpdateSession(session);
-        final PersonalizationDictionary dictionary =
-                getPersonalizationDictionary(context, locale,
-                        PreferenceManager.getDefaultSharedPreferences(context));
-        dictionary.registerUpdateSession(session);
+            final PersonalizationDictionaryUpdateSession session, final Locale locale) {
+        final PersonalizationDictionary personalizationDictionary =
+                getPersonalizationDictionary(context, locale);
+        personalizationDictionary.registerUpdateSession(session);
     }
 
     public static PersonalizationDictionary getPersonalizationDictionary(
-            final Context context, final String locale, final SharedPreferences sp) {
+            final Context context, final Locale locale) {
+        final String localeStr = locale.toString();
         synchronized (sLangPersonalizationDictCache) {
-            if (sLangPersonalizationDictCache.containsKey(locale)) {
+            if (sLangPersonalizationDictCache.containsKey(localeStr)) {
                 final SoftReference<PersonalizationDictionary> ref =
-                        sLangPersonalizationDictCache.get(locale);
+                        sLangPersonalizationDictCache.get(localeStr);
                 final PersonalizationDictionary dict = ref == null ? null : ref.get();
                 if (dict != null) {
                     if (DEBUG) {
-                        Log.w(TAG, "Use cached PersonalizationDictCache for " + locale);
+                        Log.w(TAG, "Use cached PersonalizationDictionary for " + locale);
                     }
                     return dict;
                 }
             }
-            final PersonalizationDictionary dict =
-                    new PersonalizationDictionary(context, locale, sp);
+            final PersonalizationDictionary dict = new PersonalizationDictionary(context, locale);
             sLangPersonalizationDictCache.put(
-                    locale, new SoftReference<PersonalizationDictionary>(dict));
-            return dict;
-        }
-    }
-
-    public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary(
-            final Context context, final String locale, final SharedPreferences sp) {
-        synchronized (sLangPersonalizationPredictionDictCache) {
-            if (sLangPersonalizationPredictionDictCache.containsKey(locale)) {
-                final SoftReference<PersonalizationPredictionDictionary> ref =
-                        sLangPersonalizationPredictionDictCache.get(locale);
-                final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get();
-                if (dict != null) {
-                    if (DEBUG) {
-                        Log.w(TAG, "Use cached PersonalizationPredictionDictionary for " + locale);
-                    }
-                    return dict;
-                }
-            }
-            final PersonalizationPredictionDictionary dict =
-                    new PersonalizationPredictionDictionary(context, locale, sp);
-            sLangPersonalizationPredictionDictCache.put(
-                    locale, new SoftReference<PersonalizationPredictionDictionary>(dict));
+                    localeStr, new SoftReference<PersonalizationDictionary>(dict));
             return dict;
         }
     }
diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
deleted file mode 100644
index 432954453..000000000
--- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2013 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.inputmethod.latin.personalization;
-
-import com.android.inputmethod.latin.Dictionary;
-import com.android.inputmethod.latin.ExpandableBinaryDictionary;
-
-import android.content.Context;
-import android.content.SharedPreferences;
-
-public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase {
-    private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
-
-    /* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
-            final SharedPreferences sp) {
-        super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA,
-                getDictionaryFileName(locale));
-    }
-
-    private static String getDictionaryFileName(final String locale) {
-        return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
-    }
-}
diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
index a60226d7e..868f21cbc 100644
--- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java
@@ -16,11 +16,13 @@
 
 package com.android.inputmethod.latin.personalization;
 
+import com.android.inputmethod.annotations.UsedForTesting;
 import com.android.inputmethod.latin.Dictionary;
-import com.android.inputmethod.latin.ExpandableBinaryDictionary;
+
+import java.io.File;
+import java.util.Locale;
 
 import android.content.Context;
-import android.content.SharedPreferences;
 
 /**
  * Locally gathers stats about the words user types and various other signals like auto-correction
@@ -29,12 +31,19 @@ import android.content.SharedPreferences;
 public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase {
     /* package for tests */ static final String NAME =
             UserHistoryDictionary.class.getSimpleName();
-    /* package */ UserHistoryDictionary(final Context context, final String locale,
-            final SharedPreferences sp) {
-        super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale));
+    /* package */ UserHistoryDictionary(final Context context, final Locale locale) {
+        super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale));
+    }
+
+    // Creates an instance that uses a given dictionary file for testing.
+    @UsedForTesting
+    public UserHistoryDictionary(final Context context, final Locale locale,
+            final File dictFile) {
+        super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale),
+                dictFile);
     }
 
-    private static String getDictionaryFileName(final String locale) {
-        return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
+    public void cancelAddingUserHistory(final String word0, final String word1) {
+        removeBigramDynamically(word0, word1);
     }
 }