Reset to 9bd6dac4708ad94fd0257c53e977df62b152e20c

The bulk merge from -bayo to klp-dev should not have been merged to master.

Change-Id: I527a03a76f5247e4939a672f27c314dc11cbb854

1 files changed, 83 insertions, 84 deletions

diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
index 1de15a333..296733fad 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java
@@ -17,18 +17,16 @@
 package com.android.inputmethod.latin.personalization;
 
 import android.content.Context;
-import android.content.SharedPreferences;
 import android.util.Log;
 
 import com.android.inputmethod.annotations.UsedForTesting;
+import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam;
 import com.android.inputmethod.latin.Constants;
 import com.android.inputmethod.latin.Dictionary;
 import com.android.inputmethod.latin.ExpandableBinaryDictionary;
-import com.android.inputmethod.latin.LatinImeLogger;
 import com.android.inputmethod.latin.makedict.DictDecoder;
 import com.android.inputmethod.latin.makedict.FormatSpec;
-import com.android.inputmethod.latin.settings.Settings;
-import com.android.inputmethod.latin.utils.CollectionUtils;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
 
@@ -36,7 +34,9 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 /**
  * This class is a base class of a dictionary that supports decaying for the personalized language
@@ -45,8 +45,7 @@ import java.util.Map;
 public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
     private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
     public static final boolean DBG_SAVE_RESTORE = false;
-    private static final boolean DBG_STRESS_TEST = false;
-    private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
+    private static final boolean DBG_DUMP_ON_CLOSE = false;
 
     /** Any pair being typed or picked */
     public static final int FREQUENCY_FOR_TYPED = 2;
@@ -54,41 +53,43 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
     public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
     public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
 
-    /** Locale for which this user history dictionary is storing words */
-    private final String mLocale;
-
-    private final String mFileName;
+    public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4;
 
-    private final SharedPreferences mPrefs;
+    /** Locale for which this user history dictionary is storing words */
+    private final Locale mLocale;
 
-    private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
-            CollectionUtils.newArrayList();
+    private final String mDictName;
 
-    // Should always be false except when we use this class for test
-    @UsedForTesting boolean mIsTest = false;
+    /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
+            final Locale locale, final String dictionaryType, final String dictName) {
+        super(context, dictName, locale, dictionaryType, true);
+        mLocale = locale;
+        mDictName = dictName;
+        if (mLocale != null && mLocale.toString().length() > 1) {
+            reloadDictionaryIfRequired();
+        }
+    }
 
+    // Creates an instance that uses a given dictionary file for testing.
+    @UsedForTesting
     /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
-            final String locale, final SharedPreferences sp, final String dictionaryType,
-            final String fileName) {
-        super(context, fileName, dictionaryType, true);
+            final Locale locale, final String dictionaryType, final String dictName,
+            final File dictFile) {
+        super(context, dictName, locale, dictionaryType, true, dictFile);
         mLocale = locale;
-        mFileName = fileName;
-        mPrefs = sp;
-        if (mLocale != null && mLocale.length() > 1) {
-            asyncLoadDictionaryToMemory();
+        mDictName = dictName;
+        if (mLocale != null && mLocale.toString().length() > 1) {
             reloadDictionaryIfRequired();
         }
     }
 
     @Override
     public void close() {
-        if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
-            closeBinaryDictionary();
+        if (DBG_DUMP_ON_CLOSE) {
+            dumpAllWordsForDebug();
         }
         // Flush pending writes.
-        // TODO: Remove after this class become to use a dynamic binary dictionary.
-        asyncFlashAllBinaryDictionary();
-        Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
+        asyncFlushBinaryDictionary();
     }
 
     @Override
@@ -98,8 +99,12 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
         attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
-        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
+        attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE,
+                FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName);
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString());
+        attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE,
+                String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
         return attributeMap;
     }
 
@@ -113,6 +118,25 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
         return false;
     }
 
+    @Override
+    protected boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) {
+        // This class is using format 4 because it's used by all version 4 dictionaries.
+        // TODO: remove this when all dynamically generated dicts use version 4.
+        return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION;
+    }
+
+    public void addMultipleDictionaryEntriesToDictionary(
+            final ArrayList<LanguageModelParam> languageModelParams,
+            final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
+        if (languageModelParams == null || languageModelParams.isEmpty()) {
+            if (callback != null) {
+                callback.onFinished();
+            }
+            return;
+        }
+        addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
+    }
+
     /**
      * Pair will be added to the decaying dictionary.
      *
@@ -121,70 +145,63 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
      * context, as in beginning of a sentence for example.
      * The second word may not be null (a NullPointerException would be thrown).
      */
-    public void addToDictionary(final String word0, final String word1, final boolean isValid) {
+    public void addToDictionary(final String word0, final String word1, final boolean isValid,
+            final int timestamp) {
         if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
                 (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
             return;
         }
-        final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
-                (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) :
-                        FREQUENCY_FOR_TYPED;
-        addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */,
-                false /* isNotAWord */);
+        final int frequency = isValid ?
+                FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS;
+        addWordDynamically(word1, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */,
+                false /* isNotAWord */, false /* isBlacklisted */, timestamp);
         // Do not insert a word as a bigram of itself
         if (word1.equals(word0)) {
             return;
         }
         if (null != word0) {
-            addBigramDynamically(word0, word1, frequency, isValid);
+            addBigramDynamically(word0, word1, frequency, timestamp);
         }
     }
 
-    public void cancelAddingUserHistory(final String word0, final String word1) {
-        removeBigramDynamically(word0, word1);
-    }
-
     @Override
     protected void loadDictionaryAsync() {
-        final int[] profTotalCount = { 0 };
-        final String locale = getLocale();
-        if (DBG_STRESS_TEST) {
-            try {
-                Log.w(TAG, "Start stress in loading: " + locale);
-                Thread.sleep(15000);
-                Log.w(TAG, "End stress in loading");
-            } catch (InterruptedException e) {
+        // Never loaded to memory in Java side.
+    }
+
+    @UsedForTesting
+    public void dumpAllWordsForDebug() {
+        runAfterGcForDebug(new Runnable() {
+            @Override
+            public void run() {
+                dumpAllWordsForDebugLocked();
             }
-        }
-        final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
-        final long now = System.currentTimeMillis();
-        final ExpandableBinaryDictionary dictionary = this;
+        });
+    }
+
+    private void dumpAllWordsForDebugLocked() {
+        Log.d(TAG, "dumpAllWordsForDebug started.");
         final OnAddWordListener listener = new OnAddWordListener() {
             @Override
             public void setUnigram(final String word, final String shortcutTarget,
                     final int frequency, final int shortcutFreq) {
-                if (DBG_SAVE_RESTORE) {
-                    Log.d(TAG, "load unigram: " + word + "," + frequency);
-                }
-                addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */);
-                ++profTotalCount[0];
+                Log.d(TAG, "load unigram: " + word + "," + frequency);
             }
 
             @Override
             public void setBigram(final String word0, final String word1, final int frequency) {
                 if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
                         && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
-                    if (DBG_SAVE_RESTORE) {
-                        Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
-                    }
-                    ++profTotalCount[0];
-                    addBigram(word0, word1, frequency, last);
+                    Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
+                } else {
+                    Log.d(TAG, "Skip inserting a too long bigram: " + word0 + "," + word1 + ","
+                            + frequency);
                 }
             }
         };
 
         // Load the dictionary from binary file
-        final File dictFile = new File(mContext.getFilesDir(), mFileName);
+        final File dictFile = new File(mContext.getFilesDir(), mDictName);
         final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
                 DictDecoder.USE_BYTEARRAY);
         if (dictDecoder == null) {
@@ -198,35 +215,17 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB
             UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
         } catch (IOException e) {
             Log.d(TAG, "IOException on opening a bytebuffer", e);
-        } finally {
-            if (PROFILE_SAVE_RESTORE) {
-                final long diff = System.currentTimeMillis() - now;
-                Log.d(TAG, "PROF: Load UserHistoryDictionary: "
-                        + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries.");
-            }
+        } catch (UnsupportedFormatException e) {
+            Log.d(TAG, "Unsupported format, can't read the dictionary", e);
         }
     }
 
-    protected String getLocale() {
-        return mLocale;
-    }
-
-    public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
-        session.setPredictionDictionary(this);
-        mSessions.add(session);
-        session.onDictionaryReady();
-    }
-
-    public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) {
-        mSessions.remove(session);
-    }
-
     @UsedForTesting
     public void clearAndFlushDictionary() {
         // Clear the node structure on memory
         clear();
         // Then flush the cleared state of the dictionary on disk.
-        asyncFlashAllBinaryDictionary();
+        asyncFlushBinaryDictionary();
     }
 
     /* package */ void decayIfNeeded() {