Merge "Change to a binary version of the expandable user dictionary." into jb-dev

9 files changed, 311 insertions, 14 deletions

diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
index 22787c218..4b77473d9 100644
--- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
@@ -60,7 +60,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
     private final boolean mUseFirstLastBigrams;
 
     public ContactsBinaryDictionary(final Context context, final int dicTypeId, Locale locale) {
-        super(context, getFilenameWithLocale(locale), dicTypeId);
+        super(context, getFilenameWithLocale(NAME, locale.toString()), dicTypeId);
         mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);
         registerObserver(context);
 
@@ -69,10 +69,6 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
         loadDictionary();
     }
 
-    private static String getFilenameWithLocale(Locale locale) {
-        return NAME + "." + locale.toString() + ".dict";
-    }
-
     private synchronized void registerObserver(final Context context) {
         // Perform a managed query. The Activity will handle closing and requerying the cursor
         // when needed.
@@ -175,7 +171,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
                 // capitalization of i.
                 final int wordLen = word.codePointCount(0, word.length());
                 if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {
-                    super.addWord(word, FREQUENCY_FOR_CONTACTS);
+                    super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS);
                     if (!TextUtils.isEmpty(prevWord)) {
                         if (mUseFirstLastBigrams) {
                             super.setBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM);
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
index 22d8f24f1..08f585485 100644
--- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java
@@ -22,11 +22,13 @@ import com.android.inputmethod.keyboard.ProximityInfo;
 import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
 import com.android.inputmethod.latin.makedict.FusionDictionary;
 import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
+import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
 import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -133,6 +135,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
         clearFusionDictionary();
     }
 
+    protected static String getFilenameWithLocale(final String name, final String localeStr) {
+        return name + "." + localeStr + ".dict";
+    }
+
     /**
      * Closes and cleans up the binary dictionary.
      */
@@ -166,8 +172,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
      */
     // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
     // considering performance regression.
-    protected void addWord(final String word, final int frequency) {
-        mFusionDictionary.add(word, frequency, null /* shortcutTargets */);
+    protected void addWord(final String word, final String shortcutTarget, final int frequency) {
+        if (shortcutTarget == null) {
+            mFusionDictionary.add(word, frequency, null);
+        } else {
+            // TODO: Do this in the subclass, with this class taking an arraylist.
+            final ArrayList<WeightedString> shortcutTargets = new ArrayList<WeightedString>();
+            shortcutTargets.add(new WeightedString(shortcutTarget, frequency));
+            mFusionDictionary.add(word, frequency, shortcutTargets);
+        }
     }
 
     /**
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index b632d34bf..113941de3 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -106,6 +106,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
     /** Whether to use the binary version of the contacts dictionary */
     public static final boolean USE_BINARY_CONTACTS_DICTIONARY = true;
 
+    /** Whether to use the binary version of the user dictionary */
+    public static final boolean USE_BINARY_USER_DICTIONARY = true;
+
     // TODO: migrate this to SettingsValues
     private int mSuggestionVisibility;
     private static final int SUGGESTION_VISIBILILTY_SHOW_VALUE
@@ -158,7 +161,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
     private boolean mShouldSwitchToLastSubtype = true;
 
     private boolean mIsMainDictionaryAvailable;
-    private UserDictionary mUserDictionary;
+    // TODO: revert this back to the concrete class after transition.
+    private Dictionary mUserDictionary;
     private UserHistoryDictionary mUserHistoryDictionary;
     private boolean mIsUserDictionaryAvailable;
 
@@ -476,9 +480,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
 
         mIsMainDictionaryAvailable = DictionaryFactory.isDictionaryAvailable(this, subtypeLocale);
 
-        mUserDictionary = new UserDictionary(this, localeStr);
+        if (USE_BINARY_USER_DICTIONARY) {
+            mUserDictionary = new UserBinaryDictionary(this, localeStr);
+            mIsUserDictionaryAvailable = ((UserBinaryDictionary)mUserDictionary).isEnabled();
+        } else {
+            mUserDictionary = new UserDictionary(this, localeStr);
+            mIsUserDictionaryAvailable = ((UserDictionary)mUserDictionary).isEnabled();
+        }
         mSuggest.setUserDictionary(mUserDictionary);
-        mIsUserDictionaryAvailable = mUserDictionary.isEnabled();
 
         resetContactsDictionary(oldContactsDictionary);
 
@@ -1123,7 +1132,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
 
     @Override
     public boolean addWordToDictionary(String word) {
-        mUserDictionary.addWordToUserDictionary(word, 128);
+        if (USE_BINARY_USER_DICTIONARY) {
+            ((UserBinaryDictionary)mUserDictionary).addWordToUserDictionary(word, 128);
+        } else {
+            ((UserDictionary)mUserDictionary).addWordToUserDictionary(word, 128);
+        }
         // Suggestion strip should be updated after the operation of adding word to the
         // user dictionary
         mHandler.postUpdateSuggestions();
diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java
index 188259ff8..4994e5902 100644
--- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java
+++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java
@@ -26,7 +26,6 @@ public class SynchronouslyLoadedContactsBinaryDictionary extends ContactsBinaryD
     public SynchronouslyLoadedContactsBinaryDictionary(final Context context) {
         // TODO: add locale information.
         super(context, Suggest.DIC_CONTACTS, null);
-        mClosed = false;
     }
 
     @Override
diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java
new file mode 100644
index 000000000..1606a34e0
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.content.Context;
+
+import com.android.inputmethod.keyboard.ProximityInfo;
+
+public class SynchronouslyLoadedUserBinaryDictionary extends UserBinaryDictionary {
+
+    public SynchronouslyLoadedUserBinaryDictionary(final Context context, final String locale) {
+        this(context, locale, false);
+    }
+
+    public SynchronouslyLoadedUserBinaryDictionary(final Context context, final String locale,
+            final boolean alsoUseMoreRestrictiveLocales) {
+        super(context, locale, alsoUseMoreRestrictiveLocales);
+    }
+
+    @Override
+    public synchronized void getWords(final WordComposer codes,
+            final CharSequence prevWordForBigrams, final WordCallback callback,
+            final ProximityInfo proximityInfo) {
+        syncReloadDictionaryIfRequired();
+        getWordsInner(codes, prevWordForBigrams, callback, proximityInfo);
+    }
+
+    @Override
+    public synchronized boolean isValidWord(CharSequence word) {
+        syncReloadDictionaryIfRequired();
+        return isValidWordInner(word);
+    }
+}
diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserDictionary.java
index b78be89b8..23a49c192 100644
--- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserDictionary.java
+++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserDictionary.java
@@ -21,6 +21,7 @@ import android.content.Context;
 import com.android.inputmethod.keyboard.ProximityInfo;
 
 public class SynchronouslyLoadedUserDictionary extends UserDictionary {
+    private boolean mClosed;
 
     public SynchronouslyLoadedUserDictionary(final Context context, final String locale) {
         this(context, locale, false);
@@ -44,4 +45,12 @@ public class SynchronouslyLoadedUserDictionary extends UserDictionary {
         blockingReloadDictionaryIfRequired();
         return super.isValidWord(word);
     }
+
+    // Protect against multiple closing
+    @Override
+    public synchronized void close() {
+        if (mClosed) return;
+        mClosed = true;
+        super.close();
+    }
 }
diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
new file mode 100644
index 000000000..6fa1a25a1
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.content.ContentProviderClient;
+import android.content.ContentResolver;
+import android.content.Context;
+import android.content.Intent;
+import android.database.ContentObserver;
+import android.database.Cursor;
+import android.provider.UserDictionary.Words;
+import android.text.TextUtils;
+
+import java.util.Arrays;
+
+/**
+ * An expandable dictionary that stores the words in the user unigram dictionary.
+ *
+ * Largely a copy of UserDictionary, will replace that class in the future.
+ */
+public class UserBinaryDictionary extends ExpandableBinaryDictionary {
+
+    // TODO: use Words.SHORTCUT when it's public in the SDK
+    final static String SHORTCUT = "shortcut";
+    private static final String[] PROJECTION_QUERY = {
+        Words.WORD,
+        SHORTCUT,
+        Words.FREQUENCY,
+    };
+
+    private static final String NAME = "userunigram";
+
+    // This is not exported by the framework so we pretty much have to write it here verbatim
+    private static final String ACTION_USER_DICTIONARY_INSERT =
+            "com.android.settings.USER_DICTIONARY_INSERT";
+
+    private ContentObserver mObserver;
+    final private String mLocale;
+    final private boolean mAlsoUseMoreRestrictiveLocales;
+
+    public UserBinaryDictionary(final Context context, final String locale) {
+        this(context, locale, false);
+    }
+
+    public UserBinaryDictionary(final Context context, final String locale,
+            final boolean alsoUseMoreRestrictiveLocales) {
+        super(context, getFilenameWithLocale(NAME, locale), Suggest.DIC_USER);
+        if (null == locale) throw new NullPointerException(); // Catch the error earlier
+        mLocale = locale;
+        mAlsoUseMoreRestrictiveLocales = alsoUseMoreRestrictiveLocales;
+        // Perform a managed query. The Activity will handle closing and re-querying the cursor
+        // when needed.
+        ContentResolver cres = context.getContentResolver();
+
+        mObserver = new ContentObserver(null) {
+            @Override
+            public void onChange(boolean self) {
+                setRequiresReload(true);
+            }
+        };
+        cres.registerContentObserver(Words.CONTENT_URI, true, mObserver);
+
+        loadDictionary();
+    }
+
+    @Override
+    public synchronized void close() {
+        if (mObserver != null) {
+            mContext.getContentResolver().unregisterContentObserver(mObserver);
+            mObserver = null;
+        }
+        super.close();
+    }
+
+    @Override
+    public void loadDictionaryAsync() {
+        // Split the locale. For example "en" => ["en"], "de_DE" => ["de", "DE"],
+        // "en_US_foo_bar_qux" => ["en", "US", "foo_bar_qux"] because of the limit of 3.
+        // This is correct for locale processing.
+        // For this example, we'll look at the "en_US_POSIX" case.
+        final String[] localeElements =
+                TextUtils.isEmpty(mLocale) ? new String[] {} : mLocale.split("_", 3);
+        final int length = localeElements.length;
+
+        final StringBuilder request = new StringBuilder("(locale is NULL)");
+        String localeSoFar = "";
+        // At start, localeElements = ["en", "US", "POSIX"] ; localeSoFar = "" ;
+        // and request = "(locale is NULL)"
+        for (int i = 0; i < length; ++i) {
+            // i | localeSoFar    | localeElements
+            // 0 | ""             | ["en", "US", "POSIX"]
+            // 1 | "en_"          | ["en", "US", "POSIX"]
+            // 2 | "en_US_"       | ["en", "en_US", "POSIX"]
+            localeElements[i] = localeSoFar + localeElements[i];
+            localeSoFar = localeElements[i] + "_";
+            // i | request
+            // 0 | "(locale is NULL)"
+            // 1 | "(locale is NULL) or (locale=?)"
+            // 2 | "(locale is NULL) or (locale=?) or (locale=?)"
+            request.append(" or (locale=?)");
+        }
+        // At the end, localeElements = ["en", "en_US", "en_US_POSIX"]; localeSoFar = en_US_POSIX_"
+        // and request = "(locale is NULL) or (locale=?) or (locale=?) or (locale=?)"
+
+        final String[] requestArguments;
+        // If length == 3, we already have all the arguments we need (common prefix is meaningless
+        // inside variants
+        if (mAlsoUseMoreRestrictiveLocales && length < 3) {
+            request.append(" or (locale like ?)");
+            // The following creates an array with one more (null) position
+            final String[] localeElementsWithMoreRestrictiveLocalesIncluded =
+                    Arrays.copyOf(localeElements, length + 1);
+            localeElementsWithMoreRestrictiveLocalesIncluded[length] =
+                    localeElements[length - 1] + "_%";
+            requestArguments = localeElementsWithMoreRestrictiveLocalesIncluded;
+            // If for example localeElements = ["en"]
+            // then requestArguments = ["en", "en_%"]
+            // and request = (locale is NULL) or (locale=?) or (locale like ?)
+            // If localeElements = ["en", "en_US"]
+            // then requestArguments = ["en", "en_US", "en_US_%"]
+        } else {
+            requestArguments = localeElements;
+        }
+        final Cursor cursor = mContext.getContentResolver().query(
+            Words.CONTENT_URI, PROJECTION_QUERY, request.toString(), requestArguments, null);
+        try {
+            addWords(cursor);
+        } finally {
+            if (null != cursor) cursor.close();
+        }
+    }
+
+    public boolean isEnabled() {
+        final ContentResolver cr = mContext.getContentResolver();
+        final ContentProviderClient client = cr.acquireContentProviderClient(Words.CONTENT_URI);
+        if (client != null) {
+            client.release();
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * Adds a word to the user dictionary and makes it persistent.
+     *
+     * This will call upon the system interface to do the actual work through the intent readied by
+     * the system to this effect.
+     *
+     * @param word the word to add. If the word is capitalized, then the dictionary will
+     * recognize it as a capitalized word when searched.
+     * @param frequency the frequency of occurrence of the word. A frequency of 255 is considered
+     * the highest.
+     * @TODO use a higher or float range for frequency
+     */
+    public synchronized void addWordToUserDictionary(final String word, final int frequency) {
+        // TODO: do something for the UI. With the following, any sufficiently long word will
+        // look like it will go to the user dictionary but it won't.
+        // Safeguard against adding long words. Can cause stack overflow.
+        if (word.length() >= MAX_WORD_LENGTH) return;
+
+        // TODO: Add an argument to the intent to specify the frequency.
+        Intent intent = new Intent(ACTION_USER_DICTIONARY_INSERT);
+        intent.putExtra(Words.WORD, word);
+        intent.putExtra(Words.LOCALE, mLocale);
+        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
+        mContext.startActivity(intent);
+    }
+
+    private void addWords(Cursor cursor) {
+        clearFusionDictionary();
+        if (cursor == null) return;
+        if (cursor.moveToFirst()) {
+            final int indexWord = cursor.getColumnIndex(Words.WORD);
+            final int indexShortcut = cursor.getColumnIndex(SHORTCUT);
+            final int indexFrequency = cursor.getColumnIndex(Words.FREQUENCY);
+            while (!cursor.isAfterLast()) {
+                String word = cursor.getString(indexWord);
+                String shortcut = cursor.getString(indexShortcut);
+                int frequency = cursor.getInt(indexFrequency);
+                // Safeguard against adding really long words.
+                if (word.length() < MAX_WORD_LENGTH) {
+                    super.addWord(word, null, frequency);
+                }
+                if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) {
+                    super.addWord(shortcut, word, frequency);
+                }
+                cursor.moveToNext();
+            }
+        }
+    }
+
+    @Override
+    protected boolean hasContentChanged() {
+        return true;
+    }
+}
diff --git a/java/src/com/android/inputmethod/latin/UserDictionary.java b/java/src/com/android/inputmethod/latin/UserDictionary.java
index 218bac72a..ea57db57c 100644
--- a/java/src/com/android/inputmethod/latin/UserDictionary.java
+++ b/java/src/com/android/inputmethod/latin/UserDictionary.java
@@ -29,6 +29,10 @@ import com.android.inputmethod.keyboard.ProximityInfo;
 
 import java.util.Arrays;
 
+/**
+ * An expandable dictionary that stores the words in the user unigram dictionary.
+ * To be deprecated: functionality being transferred to UserBinaryDictionary.
+*/
 public class UserDictionary extends ExpandableDictionary {
 
     // TODO: use Words.SHORTCUT when it's public in the SDK
diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
index d7c8e3850..9807d2892 100644
--- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
+++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
@@ -40,6 +40,7 @@ import com.android.inputmethod.latin.R;
 import com.android.inputmethod.latin.StringUtils;
 import com.android.inputmethod.latin.SynchronouslyLoadedContactsBinaryDictionary;
 import com.android.inputmethod.latin.SynchronouslyLoadedContactsDictionary;
+import com.android.inputmethod.latin.SynchronouslyLoadedUserBinaryDictionary;
 import com.android.inputmethod.latin.SynchronouslyLoadedUserDictionary;
 import com.android.inputmethod.latin.WhitelistDictionary;
 import com.android.inputmethod.latin.WordComposer;
@@ -403,7 +404,11 @@ public class AndroidSpellCheckerService extends SpellCheckerService
         final String localeStr = locale.toString();
         Dictionary userDictionary = mUserDictionaries.get(localeStr);
         if (null == userDictionary) {
-            userDictionary = new SynchronouslyLoadedUserDictionary(this, localeStr, true);
+            if (LatinIME.USE_BINARY_USER_DICTIONARY) {
+                userDictionary = new SynchronouslyLoadedUserBinaryDictionary(this, localeStr, true);
+            } else {
+                userDictionary = new SynchronouslyLoadedUserDictionary(this, localeStr, true);
+            }
             mUserDictionaries.put(localeStr, userDictionary);
         }
         dictionaryCollection.addDictionary(userDictionary);