Quit using ExpandableDictionary.

Bug: 6669677
Change-Id: Ie90417fa9b726454fe729a665fcd549efabb9e94

4 files changed, 180 insertions, 43 deletions

diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
index 632ee0da4..61ccfcfad 100644
--- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java
@@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
 import com.android.inputmethod.latin.utils.JniUtils;
 import com.android.inputmethod.latin.utils.StringUtils;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Locale;
@@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary {
         return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
     }
 
+    private void runGCIfRequired() {
+        if (needsToRunGCNative(mNativeDict)) {
+            flushWithGC();
+        }
+    }
+
     // Add a unigram entry to binary dictionary in native code.
     public void addUnigramWord(final String word, final int probability) {
         if (TextUtils.isEmpty(word)) {
             return;
         }
+        runGCIfRequired();
         final int[] codePoints = StringUtils.toCodePointArray(word);
         addUnigramWordNative(mNativeDict, codePoints, probability);
     }
@@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary {
         if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
             return;
         }
+        runGCIfRequired();
         final int[] codePoints0 = StringUtils.toCodePointArray(word0);
         final int[] codePoints1 = StringUtils.toCodePointArray(word1);
         addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability);
@@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary {
         if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) {
             return;
         }
+        runGCIfRequired();
         final int[] codePoints0 = StringUtils.toCodePointArray(word0);
         final int[] codePoints1 = StringUtils.toCodePointArray(word1);
         removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
     }
 
-    @UsedForTesting
     public void flush() {
         if (!isValidDictionary()) return;
         flushNative(mNativeDict, mDictFilePath);
+        closeNative(mNativeDict);
+        final File dictFile = new File(mDictFilePath);
+        mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
+                dictFile.length(), true /* isUpdatable */);
     }
 
-    @UsedForTesting
     public void flushWithGC() {
         if (!isValidDictionary()) return;
         flushWithGCNative(mNativeDict, mDictFilePath);
+        closeNative(mNativeDict);
+        final File dictFile = new File(mDictFilePath);
+        mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
+                dictFile.length(), true /* isUpdatable */);
     }
 
-    @UsedForTesting
     public boolean needsToRunGC() {
         if (!isValidDictionary()) return false;
         return needsToRunGCNative(mNativeDict);
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index fcd7ede1a..0774ce203 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,14 +22,22 @@ import android.util.Log;
 
 import com.android.inputmethod.annotations.UsedForTesting;
 import com.android.inputmethod.keyboard.ProximityInfo;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
+import com.android.inputmethod.latin.makedict.DictEncoder;
+import com.android.inputmethod.latin.makedict.FormatSpec;
+import com.android.inputmethod.latin.makedict.FusionDictionary;
+import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
+import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
+import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
 import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
+import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
 import com.android.inputmethod.latin.utils.AsyncResultHolder;
 import com.android.inputmethod.latin.utils.CollectionUtils;
 import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     /** Whether to print debug output to log */
     private static boolean DEBUG = false;
 
-    // TODO: Remove and enable dynamic update in native code.
+    // TODO: Remove.
     /** Whether to call binary dictionary dynamically updating methods. */
-    private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false;
+    public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true;
 
     private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100;
 
@@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
      */
     protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;
 
+    private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
+            new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);
+
     /**
      * A static map of time recorders, each of which records the time of accesses to a single binary
      * dictionary file. The key for this map is the filename and the value is the shared dictionary
@@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     private static AbstractDictionaryWriter getDictionaryWriter(final Context context,
             final String dictType, final boolean isDynamicPersonalizationDictionary) {
         if (isDynamicPersonalizationDictionary) {
-            return new DynamicPersonalizationDictionaryWriter(context, dictType);
+            if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+                return null;
+            } else {
+                return new DynamicPersonalizationDictionaryWriter(context, dictType);
+            }
         } else {
             return new DictionaryWriter(context, dictType);
         }
@@ -198,7 +213,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
                     mBinaryDictionary.close();
                     mBinaryDictionary = null;
                 }
-                mDictionaryWriter.close();
+                if (mDictionaryWriter != null) {
+                    mDictionaryWriter.close();
+                }
             }
         });
     }
@@ -220,7 +237,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         getExecutor(mFilename).execute(new Runnable() {
             @Override
             public void run() {
-                mDictionaryWriter.clear();
+                if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
+                    mBinaryDictionary.close();
+                    final File file = new File(mContext.getFilesDir(), mFilename);
+                    final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
+                            new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
+                                    false, false));
+                    final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+                    try {
+                        dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
+                    } catch (IOException e) {
+                        Log.e(TAG, "Exception in creating new dictionary file.", e);
+                    } catch (UnsupportedFormatException e) {
+                        Log.e(TAG, "Exception in creating new dictionary file.", e);
+                    }
+                } else {
+                    mDictionaryWriter.clear();
+                }
             }
         });
     }
@@ -257,9 +290,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             public void run() {
                 if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
                     mBinaryDictionary.addUnigramWord(word, frequency);
+                } else {
+                    // TODO: Remove.
+                    mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
                 }
-                // TODO: Remove.
-                mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
             }
         });
     }
@@ -280,10 +314,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             public void run() {
                 if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
                     mBinaryDictionary.addBigramWords(word0, word1, frequency);
+                } else {
+                    // TODO: Remove.
+                    mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
+                            0 /* lastTouchedTime */);
                 }
-                // TODO: Remove.
-                mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
-                        0 /* lastTouchedTime */);
             }
         });
     }
@@ -303,9 +338,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             public void run() {
                 if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
                     mBinaryDictionary.removeBigramWords(word0, word1);
+                } else {
+                    // TODO: Remove.
+                    mDictionaryWriter.removeBigramWords(word0, word1);
                 }
-                // TODO: Remove.
-                mDictionaryWriter.removeBigramWords(word0, word1);
             }
         });
     }
@@ -322,26 +358,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         getExecutor(mFilename).executePrioritized(new Runnable() {
             @Override
             public void run() {
-                final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ?
-                        null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord,
-                                proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
-                                sessionId);
-                // TODO: Remove checking mIsUpdatable and use native suggestion.
-                if (mBinaryDictionary != null && !mIsUpdatable) {
+                if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+                    if (mBinaryDictionary == null) {
+                        holder.set(null);
+                        return;
+                    }
                     final ArrayList<SuggestedWordInfo> binarySuggestion =
                             mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
                                     proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
                                     sessionId);
-                    if (inMemDictSuggestion == null) {
-                        holder.set(binarySuggestion);
-                    } else if (binarySuggestion == null) {
-                        holder.set(inMemDictSuggestion);
+                    holder.set(binarySuggestion);
+                } else {
+                    final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
+                            composer.isBatchMode() ? null :
+                                    mDictionaryWriter.getSuggestionsWithSessionId(composer,
+                                            prevWord, proximityInfo, blockOffensiveWords,
+                                            additionalFeaturesOptions, sessionId);
+                    // TODO: Remove checking mIsUpdatable and use native suggestion.
+                    if (mBinaryDictionary != null && !mIsUpdatable) {
+                        final ArrayList<SuggestedWordInfo> binarySuggestion =
+                                mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
+                                        proximityInfo, blockOffensiveWords,
+                                        additionalFeaturesOptions, sessionId);
+                        if (inMemDictSuggestion == null) {
+                            holder.set(binarySuggestion);
+                        } else if (binarySuggestion == null) {
+                            holder.set(inMemDictSuggestion);
+                        } else {
+                            binarySuggestion.addAll(inMemDictSuggestion);
+                            holder.set(binarySuggestion);
+                        }
                     } else {
-                        binarySuggestion.addAll(inMemDictSuggestion);
-                        holder.set(binarySuggestion);
+                        holder.set(inMemDictSuggestion);
                     }
-                } else {
-                    holder.set(inMemDictSuggestion);
                 }
             }
         });
@@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length,
                 true /* useFullEditDistance */, null, mDictType, mIsUpdatable);
 
-        // Ensure all threads accessing the current dictionary have finished before swapping in
-        // the new one.
+        // Ensure all threads accessing the current dictionary have finished before
+        // swapping in the new one.
+        // TODO: Ensure multi-thread assignment of mBinaryDictionary.
         final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
         getExecutor(mFilename).executePrioritized(new Runnable() {
             @Override
@@ -443,8 +493,33 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         if (needsToReloadBeforeWriting()) {
             mDictionaryWriter.clear();
             loadDictionaryAsync();
+            mDictionaryWriter.write(mFilename);
+        } else {
+            if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+                if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {
+                    final File file = new File(mContext.getFilesDir(), mFilename);
+                    final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
+                            new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
+                                    false, false));
+                    final DictEncoder dictEncoder = new Ver3DictEncoder(file);
+                    try {
+                        dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
+                    } catch (IOException e) {
+                        Log.e(TAG, "Exception in creating new dictionary file.", e);
+                    } catch (UnsupportedFormatException e) {
+                        Log.e(TAG, "Exception in creating new dictionary file.", e);
+                    }
+                } else {
+                    if (mBinaryDictionary.needsToRunGC()) {
+                        mBinaryDictionary.flushWithGC();
+                    } else {
+                        mBinaryDictionary.flush();
+                    }
+                }
+            } else {
+                mDictionaryWriter.write(mFilename);
+            }
         }
-        mDictionaryWriter.write(mFilename);
     }
 
     /**
@@ -539,7 +614,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         getExecutor(mFilename).executePrioritized(new Runnable() {
             @Override
             public void run() {
-                loadDictionaryAsync();
+                if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+                    loadDictionaryAsync();
+                }
             }
         });
     }
@@ -547,7 +624,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
     /**
      * Generate binary dictionary using DictionaryWriter.
      */
-    protected void asyncWriteBinaryDictionary() {
+    protected void asyncFlashAllBinaryDictionary() {
         final Runnable newTask = new Runnable() {
             @Override
             public void run() {
@@ -620,8 +697,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
             @Override
             public void run() {
                 if (mDictType == Dictionary.TYPE_USER_HISTORY) {
-                    holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
-                            .isInDictionaryForTests(word));
+                    if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+                        holder.set(mBinaryDictionary.isValidWord(word));
+                    } else {
+                        holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
+                                .isInDictionaryForTests(word));
+                    }
                 }
             }
         });
diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
index 9364fb034..075d7e3c3 100644
--- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
+++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java
@@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
 
     @Override
     public void close() {
-        // Close only binary dictionary to reuse this dictionary.
-        // super.close();
-        closeBinaryDictionary();
+        if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
+            closeBinaryDictionary();
+        }
         // Flush pending writes.
         // TODO: Remove after this class become to use a dynamic binary dictionary.
-        asyncWriteBinaryDictionary();
+        asyncFlashAllBinaryDictionary();
         Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
     }
 
@@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
         // Clear the node structure on memory
         clear();
         // Then flush the cleared state of the dictionary on disk.
-        asyncWriteBinaryDictionary();
+        asyncFlashAllBinaryDictionary();
     }
 }
diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
index bf44a1424..d605cdb84 100644
--- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java
@@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
                 Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
             } catch (InterruptedException e) {
             }
-            for (int i = 0; i < 10 && i < numberOfWords; ++i) {
+            // Limit word count to check when using a Java on memory dictionary.
+            final int wordCountToCheck =
+                    ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+                            numberOfWords : 10;
+            for (int i = 0; i < wordCountToCheck; ++i) {
                 final String word = words.get(i);
                 // This may fail as long as we use tryLock on inserting the bigram words
                 assertTrue(dict.isInDictionaryForTests(word));
@@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
             }
         }
     }
+
+    public void testAddManyWords() {
+        File dictFile = null;
+        final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
+        final int numberOfWords =
+                ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
+                        10000 : 1000;
+        final Random random = new Random(123456);
+
+        UserHistoryPredictionDictionary dict =
+                PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
+                        testFilenameSuffix, mPrefs);
+        try {
+            addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
+                    true /* checksContents */);
+            dict.close();
+        } finally {
+            try {
+                Log.d(TAG, "waiting for writing ...");
+                dict.shutdownExecutorForTests();
+                while (!dict.isTerminatedForTests()) {
+                    Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS);
+                }
+            } catch (InterruptedException e) {
+                Log.d(TAG, "InterruptedException: ", e);
+            }
+            final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
+                    + ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
+            dictFile = new File(getContext().getFilesDir(), fileName);
+            if (dictFile != null) {
+                assertTrue(dictFile.exists());
+                assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
+                dictFile.delete();
+            }
+        }
+    }
+
 }