Add a new binary contacts dictionary based on ExpandableBinaryDictionary and use locale for bigrams.

Bug: 6188977
Change-Id: I753422eed1effaeb5fd01124cf1ddd1e31ee9d60

5 files changed, 263 insertions, 15 deletions

diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
new file mode 100644
index 000000000..65f97e987
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.content.ContentResolver;
+import android.content.Context;
+import android.database.ContentObserver;
+import android.database.Cursor;
+import android.provider.BaseColumns;
+import android.provider.ContactsContract.Contacts;
+import android.text.TextUtils;
+import android.util.Log;
+
+import com.android.inputmethod.keyboard.Keyboard;
+
+import java.util.Locale;
+
+public class ContactsBinaryDictionary extends ExpandableBinaryDictionary {
+
+    private static final String[] PROJECTION = {BaseColumns._ID, Contacts.DISPLAY_NAME,};
+
+    private static final String TAG = ContactsBinaryDictionary.class.getSimpleName();
+    private static final String NAME = "contacts";
+
+    /**
+     * Frequency for contacts information into the dictionary
+     */
+    private static final int FREQUENCY_FOR_CONTACTS = 40;
+    private static final int FREQUENCY_FOR_CONTACTS_BIGRAM = 90;
+
+    private static final int INDEX_NAME = 1;
+
+    private ContentObserver mObserver;
+
+    /**
+     * Whether to use "firstname lastname" in bigram predictions.
+     */
+    private final boolean mUseFirstLastBigrams;
+
+    public ContactsBinaryDictionary(final Context context, final int dicTypeId, Locale locale) {
+        super(context, getFilenameWithLocale(locale), dicTypeId);
+        mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);
+        registerObserver(context);
+
+        // Load the current binary dictionary from internal storage. If no binary dictionary exists,
+        // loadDictionary will start a new thread to generate one asynchronously.
+        loadDictionary();
+    }
+
+    private static String getFilenameWithLocale(Locale locale) {
+        return NAME + "." + locale.toString() + ".dict";
+    }
+
+    private synchronized void registerObserver(final Context context) {
+        // Perform a managed query. The Activity will handle closing and requerying the cursor
+        // when needed.
+        if (mObserver != null) return;
+        ContentResolver cres = context.getContentResolver();
+        cres.registerContentObserver(Contacts.CONTENT_URI, true, mObserver =
+                new ContentObserver(null) {
+                    @Override
+                    public void onChange(boolean self) {
+                        setRequiresReload(true);
+                    }
+                });
+    }
+
+    public void reopen(final Context context) {
+        registerObserver(context);
+    }
+
+    @Override
+    public synchronized void close() {
+        if (mObserver != null) {
+            mContext.getContentResolver().unregisterContentObserver(mObserver);
+            mObserver = null;
+        }
+        super.close();
+    }
+
+    @Override
+    public void loadDictionaryAsync() {
+        try {
+            Cursor cursor = mContext.getContentResolver()
+                    .query(Contacts.CONTENT_URI, PROJECTION, null, null, null);
+            if (cursor != null) {
+                try {
+                    if (cursor.moveToFirst()) {
+                        addWords(cursor);
+                    }
+                } finally {
+                    cursor.close();
+                }
+            }
+        } catch (IllegalStateException e) {
+            Log.e(TAG, "Contacts DB is having problems");
+        }
+    }
+
+    @Override
+    public void getBigrams(final WordComposer codes, final CharSequence previousWord,
+            final WordCallback callback) {
+        super.getBigrams(codes, previousWord, callback);
+    }
+
+    private boolean useFirstLastBigramsForLocale(Locale locale) {
+        // TODO: Add firstname/lastname bigram rules for other languages.
+        if (locale != null && locale.getLanguage().equals(Locale.ENGLISH.getLanguage())) {
+            return true;
+        }
+        return false;
+    }
+
+    private void addWords(Cursor cursor) {
+        clearFusionDictionary();
+        while (!cursor.isAfterLast()) {
+            String name = cursor.getString(INDEX_NAME);
+            if (name != null && -1 == name.indexOf('@')) {
+                addName(name);
+            }
+            cursor.moveToNext();
+        }
+    }
+
+    /**
+     * Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their
+     * bigrams depending on locale.
+     */
+    private void addName(String name) {
+        int len = name.codePointCount(0, name.length());
+        String prevWord = null;
+        // TODO: Better tokenization for non-Latin writing systems
+        for (int i = 0; i < len; i++) {
+            if (Character.isLetter(name.codePointAt(i))) {
+                int j;
+                for (j = i + 1; j < len; j++) {
+                    final int codePoint = name.codePointAt(j);
+                    if (!(codePoint == Keyboard.CODE_DASH || codePoint == Keyboard.CODE_SINGLE_QUOTE
+                            || Character.isLetter(codePoint))) {
+                        break;
+                    }
+                }
+                String word = name.substring(i, j);
+                i = j - 1;
+                // Don't add single letter words, possibly confuses
+                // capitalization of i.
+                final int wordLen = word.codePointCount(0, word.length());
+                if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {
+                    super.addWord(word, FREQUENCY_FOR_CONTACTS);
+                    if (!TextUtils.isEmpty(prevWord)) {
+                        if (mUseFirstLastBigrams) {
+                            super.setBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM);
+                        }
+                    }
+                    prevWord = word;
+                }
+            }
+        }
+    }
+}
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 187252a31..245323616 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -140,6 +140,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
      */
     private static final String SCHEME_PACKAGE = "package";
 
+    /** Whether to use the binary version of the contacts dictionary */
+    public static final boolean USE_BINARY_CONTACTS_DICTIONARY = true;
+
     // TODO: migrate this to SettingsValues
     private int mSuggestionVisibility;
     private static final int SUGGESTION_VISIBILILTY_SHOW_VALUE
@@ -493,14 +496,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
         final String localeStr = mSubtypeSwitcher.getInputLocaleStr();
         final Locale keyboardLocale = mSubtypeSwitcher.getInputLocale();
 
-        final ContactsDictionary oldContactsDictionary;
+        final Dictionary oldContactsDictionary;
         if (mSuggest != null) {
             oldContactsDictionary = mSuggest.getContactsDictionary();
             mSuggest.close();
         } else {
             oldContactsDictionary = null;
         }
-
         mSuggest = new Suggest(this, keyboardLocale);
         if (mSettingsValues.mAutoCorrectEnabled) {
             mSuggest.setAutoCorrectionThreshold(mSettingsValues.mAutoCorrectionThreshold);
@@ -526,10 +528,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
      *
      * @param oldContactsDictionary an optional dictionary to use, or null
      */
-    private void resetContactsDictionary(final ContactsDictionary oldContactsDictionary) {
+    private void resetContactsDictionary(final Dictionary oldContactsDictionary) {
         final boolean shouldSetDictionary = (null != mSuggest && mSettingsValues.mUseContactsDict);
 
-        final ContactsDictionary dictionaryToUse;
+        final Dictionary dictionaryToUse;
         if (!shouldSetDictionary) {
             // Make sure the dictionary is closed. If it is already closed, this is a no-op,
             // so it's safe to call it anyways.
@@ -538,10 +540,19 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
         } else if (null != oldContactsDictionary) {
             // Make sure the old contacts dictionary is opened. If it is already open, this is a
             // no-op, so it's safe to call it anyways.
-            oldContactsDictionary.reopen(this);
+            if (USE_BINARY_CONTACTS_DICTIONARY) {
+                ((ContactsBinaryDictionary)oldContactsDictionary).reopen(this);
+            } else {
+                ((ContactsDictionary)oldContactsDictionary).reopen(this);
+            }
             dictionaryToUse = oldContactsDictionary;
         } else {
-            dictionaryToUse = new ContactsDictionary(this, Suggest.DIC_CONTACTS);
+            if (USE_BINARY_CONTACTS_DICTIONARY) {
+                dictionaryToUse = new ContactsBinaryDictionary(this, Suggest.DIC_CONTACTS,
+                        mSubtypeSwitcher.getInputLocale());
+            } else {
+                dictionaryToUse = new ContactsDictionary(this, Suggest.DIC_CONTACTS);
+            }
         }
 
         if (null != mSuggest) {
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index c3f3bd598..51ebfdad6 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -80,7 +80,7 @@ public class Suggest implements Dictionary.WordCallback {
     private static final boolean DBG = LatinImeLogger.sDBG;
 
     private Dictionary mMainDict;
-    private ContactsDictionary mContactsDict;
+    private Dictionary mContactsDict;
     private WhitelistDictionary mWhiteListDictionary;
     private final HashMap<String, Dictionary> mUnigramDictionaries =
             new HashMap<String, Dictionary>();
@@ -165,7 +165,7 @@ public class Suggest implements Dictionary.WordCallback {
         return mMainDict != null;
     }
 
-    public ContactsDictionary getContactsDictionary() {
+    public Dictionary getContactsDictionary() {
         return mContactsDict;
     }
 
@@ -190,7 +190,7 @@ public class Suggest implements Dictionary.WordCallback {
      * the contacts dictionary by passing null to this method. In this case no contacts dictionary
      * won't be used.
      */
-    public void setContactsDictionary(ContactsDictionary contactsDictionary) {
+    public void setContactsDictionary(Dictionary contactsDictionary) {
         mContactsDict = contactsDictionary;
         addOrReplaceDictionary(mUnigramDictionaries, DICT_KEY_CONTACTS, contactsDictionary);
         addOrReplaceDictionary(mBigramDictionaries, DICT_KEY_CONTACTS, contactsDictionary);
diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java
new file mode 100644
index 000000000..80825a887
--- /dev/null
+++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.inputmethod.latin;
+
+import android.content.Context;
+
+import com.android.inputmethod.keyboard.ProximityInfo;
+
+public class SynchronouslyLoadedContactsBinaryDictionary extends ContactsBinaryDictionary {
+    private boolean mClosed;
+
+    public SynchronouslyLoadedContactsBinaryDictionary(final Context context) {
+        // TODO: add locale information.
+        super(context, Suggest.DIC_CONTACTS, null);
+        mClosed = false;
+    }
+
+    @Override
+    public synchronized void getWords(final WordComposer codes, final WordCallback callback,
+            final ProximityInfo proximityInfo) {
+        syncReloadDictionaryIfRequired();
+        getWordsInner(codes, callback, proximityInfo);
+    }
+
+    @Override
+    public synchronized boolean isValidWord(CharSequence word) {
+        syncReloadDictionaryIfRequired();
+        return isValidWordInner(word);
+    }
+
+    // Protect against multiple closing
+    @Override
+    public synchronized void close() {
+        // Actually with the current implementation of ContactsDictionary it's safe to close
+        // several times, so the following protection is really only for foolproofing
+        if (mClosed) return;
+        mClosed = true;
+        super.close();
+    }
+}
\ No newline at end of file
diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
index 576fbe696..44dac30d7 100644
--- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
+++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java
@@ -33,9 +33,11 @@ import com.android.inputmethod.latin.Dictionary;
 import com.android.inputmethod.latin.Dictionary.WordCallback;
 import com.android.inputmethod.latin.DictionaryCollection;
 import com.android.inputmethod.latin.DictionaryFactory;
+import com.android.inputmethod.latin.LatinIME;
 import com.android.inputmethod.latin.LocaleUtils;
 import com.android.inputmethod.latin.R;
 import com.android.inputmethod.latin.StringUtils;
+import com.android.inputmethod.latin.SynchronouslyLoadedContactsBinaryDictionary;
 import com.android.inputmethod.latin.SynchronouslyLoadedContactsDictionary;
 import com.android.inputmethod.latin.SynchronouslyLoadedUserDictionary;
 import com.android.inputmethod.latin.WhitelistDictionary;
@@ -73,7 +75,7 @@ public class AndroidSpellCheckerService extends SpellCheckerService
             Collections.synchronizedMap(new TreeMap<String, Dictionary>());
     private Map<String, Dictionary> mWhitelistDictionaries =
             Collections.synchronizedMap(new TreeMap<String, Dictionary>());
-    private SynchronouslyLoadedContactsDictionary mContactsDictionary;
+    private Dictionary mContactsDictionary;
 
     // The threshold for a candidate to be offered as a suggestion.
     private double mSuggestionThreshold;
@@ -157,7 +159,8 @@ public class AndroidSpellCheckerService extends SpellCheckerService
 
     private void stopUsingContactsDictionaryLocked() {
         if (null == mContactsDictionary) return;
-        final SynchronouslyLoadedContactsDictionary contactsDict = mContactsDictionary;
+        final Dictionary contactsDict = mContactsDictionary;
+        // TODO: revert to the concrete type when USE_BINARY_CONTACTS_DICTIONARY is no longer needed
         mContactsDictionary = null;
         final Iterator<WeakReference<DictionaryCollection>> iterator =
                 mDictionaryCollectionsList.iterator();
@@ -360,12 +363,14 @@ public class AndroidSpellCheckerService extends SpellCheckerService
         for (Dictionary dict : oldWhitelistDictionaries.values()) {
             dict.close();
         }
-        synchronized(mUseContactsLock) {
+        synchronized (mUseContactsLock) {
             if (null != mContactsDictionary) {
                 // The synchronously loaded contacts dictionary should have been in one
                 // or several pools, but it is shielded against multiple closing and it's
                 // safe to call it several times.
-                final SynchronouslyLoadedContactsDictionary dictToClose = mContactsDictionary;
+                final Dictionary dictToClose = mContactsDictionary;
+                // TODO: revert to the concrete type when USE_BINARY_CONTACTS_DICTIONARY is no
+                // longer needed
                 mContactsDictionary = null;
                 dictToClose.close();
             }
@@ -402,10 +407,16 @@ public class AndroidSpellCheckerService extends SpellCheckerService
             mWhitelistDictionaries.put(localeStr, whitelistDictionary);
         }
         dictionaryCollection.addDictionary(whitelistDictionary);
-        synchronized(mUseContactsLock) {
+        synchronized (mUseContactsLock) {
             if (mUseContactsDictionary) {
                 if (null == mContactsDictionary) {
-                    mContactsDictionary = new SynchronouslyLoadedContactsDictionary(this);
+                    // TODO: revert to the concrete type when USE_BINARY_CONTACTS_DICTIONARY is no
+                    // longer needed
+                    if (LatinIME.USE_BINARY_CONTACTS_DICTIONARY) {
+                        mContactsDictionary = new SynchronouslyLoadedContactsBinaryDictionary(this);
+                    } else {
+                        mContactsDictionary = new SynchronouslyLoadedContactsDictionary(this);
+                    }
                 }
             }
             dictionaryCollection.addDictionary(mContactsDictionary);