Make Distracter filter use getMaxFrequencyOfExactMatches().

Bug: 13142176
Bug: 15428247

Change-Id: I5c23fbea2851f891f76f19d9da2cb70ae964569b

3 files changed, 77 insertions, 157 deletions

diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java
index 53be28139..53c78fd00 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java
@@ -89,9 +89,17 @@ public final class DictionaryCollection extends Dictionary {
         int maxFreq = -1;
         for (int i = mDictionaries.size() - 1; i >= 0; --i) {
             final int tempFreq = mDictionaries.get(i).getFrequency(word);
-            if (tempFreq >= maxFreq) {
-                maxFreq = tempFreq;
-            }
+            maxFreq = Math.max(tempFreq, maxFreq);
+        }
+        return maxFreq;
+    }
+
+    @Override
+    public int getMaxFrequencyOfExactMatches(final String word) {
+        int maxFreq = -1;
+        for (int i = mDictionaries.size() - 1; i >= 0; --i) {
+            final int tempFreq = mDictionaries.get(i).getMaxFrequencyOfExactMatches(word);
+            maxFreq = Math.max(tempFreq, maxFreq);
         }
         return maxFreq;
     }
diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
index b9c7f5671..8c3844ed8 100644
--- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
+++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java
@@ -16,33 +16,22 @@
 
 package com.android.inputmethod.latin.utils;
 
-import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import android.content.Context;
-import android.content.res.Resources;
-import android.text.InputType;
 import android.util.Log;
-import android.view.inputmethod.EditorInfo;
 import android.view.inputmethod.InputMethodSubtype;
 
-import com.android.inputmethod.keyboard.Keyboard;
-import com.android.inputmethod.keyboard.KeyboardId;
-import com.android.inputmethod.keyboard.KeyboardLayoutSet;
-import com.android.inputmethod.latin.Constants;
-import com.android.inputmethod.latin.Dictionary;
 import com.android.inputmethod.latin.DictionaryFacilitator;
 import com.android.inputmethod.latin.PrevWordsInfo;
-import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
-import com.android.inputmethod.latin.WordComposer;
 
 /**
  * This class is used to prevent distracters being added to personalization
  * or user history dictionaries
  */
+// TODO: Rename.
 public class DistracterFilterUsingSuggestion implements DistracterFilter {
     private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName();
     private static final boolean DEBUG = false;
@@ -50,10 +39,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
     private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;
 
     private final Context mContext;
-    private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;
-    private final Map<Locale, Keyboard> mLocaleToKeyboardMap;
     private final DictionaryFacilitator mDictionaryFacilitator;
-    private Keyboard mKeyboard;
     private final Object mLock = new Object();
 
     /**
@@ -63,10 +49,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
      */
     public DistracterFilterUsingSuggestion(final Context context) {
         mContext = context;
-        mLocaleToSubtypeMap = new HashMap<>();
-        mLocaleToKeyboardMap = new HashMap<>();
         mDictionaryFacilitator = new DictionaryFacilitator();
-        mKeyboard = null;
     }
 
     @Override
@@ -76,94 +59,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
 
     @Override
     public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
-        final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>();
-        if (enabledSubtypes != null) {
-            for (final InputMethodSubtype subtype : enabledSubtypes) {
-                final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
-                if (newLocaleToSubtypeMap.containsKey(locale)) {
-                    // Multiple subtypes are enabled for one locale.
-                    // TODO: Investigate what we should do for this case.
-                    continue;
-                }
-                newLocaleToSubtypeMap.put(locale, subtype);
-            }
-        }
-        if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) {
-            // Enabled subtypes have not been changed.
-            return;
-        }
-        synchronized (mLock) {
-            mLocaleToSubtypeMap.clear();
-            mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap);
-            mLocaleToKeyboardMap.clear();
-        }
-    }
-
-    private boolean isDistracter(
-            final SuggestionResults suggestionResults, final String consideredWord) {
-        int perfectMatchProbability = Dictionary.NOT_A_PROBABILITY;
-        for (final SuggestedWordInfo suggestedWordInfo : suggestionResults) {
-            if (suggestedWordInfo.mWord.equals(consideredWord)) {
-                perfectMatchProbability = mDictionaryFacilitator.getFrequency(consideredWord);
-                continue;
-            }
-            // Exact match can include case errors, accent errors, digraph conversions.
-            final boolean isExactMatch = suggestedWordInfo.isExactMatch();
-            final boolean isExactMatchWithIntentionalOmission =
-                    suggestedWordInfo.isExactMatchWithIntentionalOmission();
-
-            if (DEBUG) {
-                final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
-                        consideredWord, suggestedWordInfo.mWord, suggestedWordInfo.mScore);
-                Log.d(TAG, "consideredWord: " +  consideredWord);
-                Log.d(TAG, "top suggestion: " +  suggestedWordInfo.mWord);
-                Log.d(TAG, "suggestionScore: " +  suggestedWordInfo.mScore);
-                Log.d(TAG, "normalizedScore: " +  normalizedScore);
-                Log.d(TAG, "isExactMatch: " + isExactMatch);
-                Log.d(TAG, "isExactMatchWithIntentionalOmission: "
-                            + isExactMatchWithIntentionalOmission);
-            }
-            if (perfectMatchProbability != Dictionary.NOT_A_PROBABILITY) {
-                final int topNonPerfectProbability = mDictionaryFacilitator.getFrequency(
-                        suggestedWordInfo.mWord);
-                if (DEBUG) {
-                    Log.d(TAG, "perfectMatchProbability: " + perfectMatchProbability);
-                    Log.d(TAG, "topNonPerfectProbability: " + topNonPerfectProbability);
-                }
-                if (perfectMatchProbability > topNonPerfectProbability) {
-                    return false;
-                }
-            }
-            return isExactMatch || isExactMatchWithIntentionalOmission;
-        }
-        return false;
-    }
-
-    private void loadKeyboardForLocale(final Locale newLocale) {
-        final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale);
-        if (cachedKeyboard != null) {
-            mKeyboard = cachedKeyboard;
-            return;
-        }
-        final InputMethodSubtype subtype;
-        synchronized (mLock) {
-            subtype = mLocaleToSubtypeMap.get(newLocale);
-        }
-        if (subtype == null) {
-            return;
-        }
-        final EditorInfo editorInfo = new EditorInfo();
-        editorInfo.inputType = InputType.TYPE_CLASS_TEXT;
-        final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder(
-                mContext, editorInfo);
-        final Resources res = mContext.getResources();
-        final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res);
-        final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res);
-        builder.setKeyboardGeometry(keyboardWidth, keyboardHeight);
-        builder.setSubtype(subtype);
-        builder.setIsSpellChecker(false /* isSpellChecker */);
-        final KeyboardLayoutSet layoutSet = builder.build();
-        mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);
     }
 
     private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {
@@ -191,12 +86,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
         }
         if (!locale.equals(mDictionaryFacilitator.getLocale())) {
             synchronized (mLock) {
-                if (!mLocaleToSubtypeMap.containsKey(locale)) {
-                    Log.e(TAG, "Locale " + locale + " is not enabled.");
-                    // TODO: Investigate what we should do for disabled locales.
-                    return false;
-                }
-                loadKeyboardForLocale(locale);
                 // Reset dictionaries for the locale.
                 try {
                     loadDictionariesForLocale(locale);
@@ -207,37 +96,17 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
                 }
             }
         }
-        if (mKeyboard == null) {
-            return false;
+        // The tested word is a distracter when there is a word that is exact matched to the tested
+        // word and its probability is higher than the tested word's probability.
+        final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord);
+        final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);
+        final boolean isDistracter = perfectMatchFreq < exactMatchFreq;
+        if (DEBUG) {
+            Log.d(TAG, "testedWord: " + testedWord);
+            Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
+            Log.d(TAG, "exactMatchFreq: " + exactMatchFreq);
+            Log.d(TAG, "isDistracter: " + isDistracter);
         }
-        final WordComposer composer = new WordComposer();
-        final int[] codePoints = StringUtils.toCodePointArray(testedWord);
-        final int[] coordinates = mKeyboard.getCoordinates(codePoints);
-        composer.setComposingWord(codePoints, coordinates, PrevWordsInfo.EMPTY_PREV_WORDS_INFO);
-
-        final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
-        final String consideredWord = trailingSingleQuotesCount > 0 ?
-                testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
-                testedWord;
-        final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
-        ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
-            @Override
-            public void run() {
-                final SuggestionResults suggestionResults =
-                        mDictionaryFacilitator.getSuggestionResults(
-                                composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
-                                mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
-                                null /* additionalFeaturesOptions */, 0 /* sessionId */,
-                                null /* rawSuggestions */);
-                if (suggestionResults.isEmpty()) {
-                    holder.set(false);
-                    return;
-                }
-                holder.set(isDistracter(suggestionResults, consideredWord));
-            }
-        });
-        // It's OK to block the distracter filtering, but the dictionary lookup should be done
-        // sequentially using ExecutorUtils.
-        return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
+        return isDistracter;
     }
 }
diff --git a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
index 406e9a9b8..b7f2271be 100644
--- a/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
+++ b/tests/src/com/android/inputmethod/latin/DistracterFilterTest.java
@@ -42,11 +42,6 @@ public class DistracterFilterTest extends InputTestsBase {
         final Locale localeEnUs = new Locale("en", "US");
         String typedWord;
 
-        typedWord = "google";
-        // For this test case, we consider "google" is a distracter to "Google".
-        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
-                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
-
         typedWord = "Bill";
         // For this test case, we consider "Bill" is a distracter to "bill".
         assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
@@ -83,15 +78,20 @@ public class DistracterFilterTest extends InputTestsBase {
                 EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
 
         typedWord = "cafe";
-        // For this test case, we consider "café" is not a distracter to any word in dictionaries.
+        // For this test case, we consider "cafe" is not a distracter to any word in dictionaries.
         assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
                 EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
 
-        typedWord = "ill";
-        // For this test case, we consider "ill" is not a distracter to any word in dictionaries.
+        typedWord = "I'll";
+        // For this test case, we consider "I'll" is not a distracter to any word in dictionaries.
         assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
                 EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
 
+        typedWord = "ill";
+        // For this test case, we consider "ill" is a distracter to "I'll"
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+
         typedWord = "asdfd";
         // For this test case, we consider "asdfd" is not a distracter to any word in dictionaries.
         assertFalse(
@@ -101,8 +101,51 @@ public class DistracterFilterTest extends InputTestsBase {
         typedWord = "thank";
         // For this test case, we consider "thank" is not a distracter to any other word
         // in dictionaries.
-        assertFalse(
-                mDistracterFilter.isDistracterToWordsInDictionaries(
-                        EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+        assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));
+
+        final Locale localeDeDe = new Locale("de", "DE");
+
+        typedWord = "fuer";
+        // For this test case, we consider "fuer" is a distracter to "für".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
+
+        typedWord = "fUEr";
+        // For this test case, we consider "fUEr" is a distracter to "für".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
+
+        typedWord = "fur";
+        // For this test case, we consider "fur" is a distracter to "für".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));
+
+        final Locale localeFrFr = new Locale("fr", "FR");
+
+        typedWord = "a";
+        // For this test case, we consider "a" is a distracter to "à".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+        typedWord = "à";
+        // For this test case, we consider "à" is not a distracter to any word in dictionaries.
+        assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+        typedWord = "etre";
+        // For this test case, we consider "etre" is a distracter to "être".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+        typedWord = "États-unis";
+        // For this test case, we consider "États-unis" is a distracter to "États-Unis".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
+
+        typedWord = "ÉtatsUnis";
+        // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis".
+        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
+                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
     }
 }