Remove non-dictionary words and digit touch data.

Output to the ResearchLogger is now queued and only flushed if the word
the user was working on is a dictionary word.

multi-project commit with Ic713ec00777fbdcf4a937b3c77b995257e100fc7

Bug: 6188932
Change-Id: I9de15227ff51be23083d9096f1c1b3d83802fff7

5 files changed, 305 insertions, 162 deletions

diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java
index 7cd9bc2a8..9c3d46e70 100644
--- a/java/src/com/android/inputmethod/latin/Dictionary.java
+++ b/java/src/com/android/inputmethod/latin/Dictionary.java
@@ -115,4 +115,12 @@ public abstract class Dictionary {
     public void close() {
         // empty base implementation
     }
+
+    /**
+     * Subclasses may override to indicate that this Dictionary is not yet properly initialized.
+     */
+
+    public boolean isInitialized() {
+        return true;
+    }
 }
diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java
index 1a05fcd86..26c2e637e 100644
--- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java
+++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java
@@ -82,8 +82,9 @@ public class DictionaryCollection extends Dictionary {
         return maxFreq;
     }
 
-    public boolean isEmpty() {
-        return mDictionaries.isEmpty();
+    @Override
+    public boolean isInitialized() {
+        return !mDictionaries.isEmpty();
     }
 
     @Override
diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java
index 107d9344e..559f5ec26 100644
--- a/java/src/com/android/inputmethod/latin/LatinIME.java
+++ b/java/src/com/android/inputmethod/latin/LatinIME.java
@@ -456,6 +456,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
         }
 
         mIsMainDictionaryAvailable = DictionaryFactory.isDictionaryAvailable(this, subtypeLocale);
+        if (ProductionFlag.IS_EXPERIMENTAL) {
+            ResearchLogger.getInstance().initSuggest(mSuggest);
+        }
 
         mUserDictionary = new UserBinaryDictionary(this, localeStr);
         mIsUserDictionaryAvailable = mUserDictionary.isEnabled();
diff --git a/java/src/com/android/inputmethod/latin/ResearchLogger.java b/java/src/com/android/inputmethod/latin/ResearchLogger.java
index 7f541cfae..ee596924f 100644
--- a/java/src/com/android/inputmethod/latin/ResearchLogger.java
+++ b/java/src/com/android/inputmethod/latin/ResearchLogger.java
@@ -51,7 +51,9 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.UUID;
@@ -67,6 +69,7 @@ import java.util.UUID;
 public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChangeListener {
     private static final String TAG = ResearchLogger.class.getSimpleName();
     private static final boolean DEBUG = false;
+    private static final boolean OUTPUT_ENTIRE_BUFFER = false;  // true may disclose private info
     /* package */ static boolean sIsLogging = false;
     private static final int OUTPUT_FORMAT_VERSION = 1;
     private static final String PREF_USABILITY_STUDY_MODE = "usability_study_mode";
@@ -99,10 +102,16 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     // digits entered by the user are replaced with this codepoint.
     /* package for test */ static final int DIGIT_REPLACEMENT_CODEPOINT =
             Character.codePointAt("\uE000", 0);  // U+E000 is in the "private-use area"
+    // U+E001 is in the "private-use area"
+    /* package for test */ static final String WORD_REPLACEMENT_STRING = "\uE001";
     // set when LatinIME should ignore an onUpdateSelection() callback that
     // arises from operations in this class
     private static boolean sLatinIMEExpectingUpdateSelection = false;
 
+    // used to check whether words are not unique
+    private Suggest mSuggest;
+    private Dictionary mDictionary;
+
     private static class NullOutputStream extends OutputStream {
         /** {@inheritDoc} */
         @Override
@@ -317,6 +326,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         latinIME.showOptionDialog(builder.create());
     }
 
+    public void initSuggest(Suggest suggest) {
+        mSuggest = suggest;
+    }
+
     private void setIsPasswordView(boolean isPasswordView) {
         mIsPasswordView = isPasswordView;
     }
@@ -330,122 +343,200 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
     private static final String EVENT_TYPE_KEY = "_ty";
     private static final Object[] EVENTKEYS_NULLVALUES = {};
 
+    private LogUnit mCurrentLogUnit = new LogUnit();
+
     /**
-     * Write a description of the event out to the ResearchLog.
+     * Buffer a research log event, flagging it as privacy-sensitive.
      *
-     * Runs in the background to avoid blocking the UI thread.
+     * This event contains potentially private information.  If the word that this event is a part
+     * of is determined to be privacy-sensitive, then this event should not be included in the
+     * output log.  The system waits to output until the containing word is known.
      *
      * @param keys an array containing a descriptive name for the event, followed by the keys
      * @param values an array of values, either a String or Number.  length should be one
      * less than the keys array
      */
-    private synchronized void writeEvent(final String[] keys, final Object[] values) {
+    private synchronized void enqueuePotentiallyPrivateEvent(final String[] keys,
+            final Object[] values) {
         assert values.length + 1 == keys.length;
+        mCurrentLogUnit.addLogAtom(keys, values, true);
+    }
+
+    /**
+     * Buffer a research log event, flaggint it as not privacy-sensitive.
+     *
+     * This event contains no potentially private information.  Even if the word that this event
+     * is privacy-sensitive, this event can still safely be sent to the output log.  The system
+     * waits until the containing word is known so that this event can be written in the proper
+     * temporal order with other events that may be privacy sensitive.
+     *
+     * @param keys an array containing a descriptive name for the event, followed by the keys
+     * @param values an array of values, either a String or Number.  length should be one
+     * less than the keys array
+     */
+    private synchronized void enqueueEvent(final String[] keys, final Object[] values) {
+        assert values.length + 1 == keys.length;
+        mCurrentLogUnit.addLogAtom(keys, values, false);
+    }
+
+    private boolean isInDictionary(CharSequence word) {
+        return (mDictionary != null) && (mDictionary.isValidWord(word));
+    }
+
+    /**
+     * Write out enqueued LogEvents to the log, filtered for privacy.
+     *
+     * If word is in the dictionary, then it is not privacy-sensitive and all LogEvents related to
+     * it can be written to the log.  If the word is not in the dictionary, then it may correspond
+     * to a proper name, which might reveal private information, so neither the word nor any
+     * information related to the word (e.g. the down/motion/up coordinates) should be revealed.
+     * These LogEvents have been marked as privacy-sensitive; non privacy-sensitive events are still
+     * written out.
+     *
+     * @param word the word to be checked for inclusion in the dictionary
+     */
+    /* package for test */ synchronized void flushQueue(CharSequence word) {
         if (isAllowedToLog()) {
-            mLoggingHandler.post(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        mJsonWriter.beginObject();
-                        mJsonWriter.name(CURRENT_TIME_KEY).value(System.currentTimeMillis());
-                        mJsonWriter.name(UPTIME_KEY).value(SystemClock.uptimeMillis());
-                        mJsonWriter.name(EVENT_TYPE_KEY).value(keys[0]);
-                        final int length = values.length;
-                        for (int i = 0; i < length; i++) {
-                            mJsonWriter.name(keys[i + 1]);
-                            Object value = values[i];
-                            if (value instanceof String) {
-                                mJsonWriter.value((String) value);
-                            } else if (value instanceof Number) {
-                                mJsonWriter.value((Number) value);
-                            } else if (value instanceof Boolean) {
-                                mJsonWriter.value((Boolean) value);
-                            } else if (value instanceof CompletionInfo[]) {
-                                CompletionInfo[] ci = (CompletionInfo[]) value;
-                                mJsonWriter.beginArray();
-                                for (int j = 0; j < ci.length; j++) {
-                                    mJsonWriter.value(scrubDigitsFromString(ci[j].toString()));
-                                }
-                                mJsonWriter.endArray();
-                            } else if (value instanceof SharedPreferences) {
-                                SharedPreferences prefs = (SharedPreferences) value;
-                                mJsonWriter.beginObject();
-                                for (Map.Entry<String,?> entry : prefs.getAll().entrySet()) {
-                                    mJsonWriter.name(entry.getKey());
-                                    final Object innerValue = entry.getValue();
-                                    if (innerValue == null) {
-                                        mJsonWriter.nullValue();
-                                    } else if (innerValue instanceof Boolean) {
-                                        mJsonWriter.value((Boolean) innerValue);
-                                    } else if (innerValue instanceof Number) {
-                                        mJsonWriter.value((Number) innerValue);
-                                    } else {
-                                        mJsonWriter.value(innerValue.toString());
-                                    }
-                                }
-                                mJsonWriter.endObject();
-                            } else if (value instanceof Key[]) {
-                                Key[] keys = (Key[]) value;
-                                mJsonWriter.beginArray();
-                                for (Key key : keys) {
-                                    mJsonWriter.beginObject();
-                                    mJsonWriter.name("code").value(key.mCode);
-                                    mJsonWriter.name("altCode").value(key.mAltCode);
-                                    mJsonWriter.name("x").value(key.mX);
-                                    mJsonWriter.name("y").value(key.mY);
-                                    mJsonWriter.name("w").value(key.mWidth);
-                                    mJsonWriter.name("h").value(key.mHeight);
-                                    mJsonWriter.endObject();
-                                }
-                                mJsonWriter.endArray();
-                            } else if (value instanceof SuggestedWords) {
-                                SuggestedWords words = (SuggestedWords) value;
-                                mJsonWriter.beginObject();
-                                mJsonWriter.name("typedWordValid").value(words.mTypedWordValid);
-                                mJsonWriter.name("hasAutoCorrectionCandidate")
-                                    .value(words.mHasAutoCorrectionCandidate);
-                                mJsonWriter.name("isPunctuationSuggestions")
-                                    .value(words.mIsPunctuationSuggestions);
-                                mJsonWriter.name("allowsToBeAutoCorrected")
-                                    .value(words.mAllowsToBeAutoCorrected);
-                                mJsonWriter.name("isObsoleteSuggestions")
-                                    .value(words.mIsObsoleteSuggestions);
-                                mJsonWriter.name("isPrediction")
-                                    .value(words.mIsPrediction);
-                                mJsonWriter.name("words");
-                                mJsonWriter.beginArray();
-                                final int size = words.size();
-                                for (int j = 0; j < size; j++) {
-                                    SuggestedWordInfo wordInfo = words.getWordInfo(j);
-                                    mJsonWriter.value(scrubDigitsFromString(wordInfo.toString()));
-                                }
-                                mJsonWriter.endArray();
-                                mJsonWriter.endObject();
-                            } else if (value == null) {
-                                mJsonWriter.nullValue();
-                            } else {
-                                Log.w(TAG, "Unrecognized type to be logged: " +
-                                        (value == null ? "<null>" : value.getClass().getName()));
-                                mJsonWriter.nullValue();
-                            }
+            // check for dictionary
+            if (mDictionary == null && mSuggest != null && mSuggest.hasMainDictionary()) {
+                mDictionary = mSuggest.getMainDictionary();
+            }
+            mCurrentLogUnit.setIsPrivacySafe(word != null && isInDictionary(word));
+            mLoggingHandler.post(mCurrentLogUnit);
+            mCurrentLogUnit = new LogUnit();
+        }
+    }
+
+    private synchronized void outputEvent(final String[] keys, final Object[] values) {
+        try {
+            mJsonWriter.beginObject();
+            mJsonWriter.name(CURRENT_TIME_KEY).value(System.currentTimeMillis());
+            mJsonWriter.name(UPTIME_KEY).value(SystemClock.uptimeMillis());
+            mJsonWriter.name(EVENT_TYPE_KEY).value(keys[0]);
+            final int length = values.length;
+            for (int i = 0; i < length; i++) {
+                mJsonWriter.name(keys[i + 1]);
+                Object value = values[i];
+                if (value instanceof String) {
+                    mJsonWriter.value((String) value);
+                } else if (value instanceof Number) {
+                    mJsonWriter.value((Number) value);
+                } else if (value instanceof Boolean) {
+                    mJsonWriter.value((Boolean) value);
+                } else if (value instanceof CompletionInfo[]) {
+                    CompletionInfo[] ci = (CompletionInfo[]) value;
+                    mJsonWriter.beginArray();
+                    for (int j = 0; j < ci.length; j++) {
+                        mJsonWriter.value(ci[j].toString());
+                    }
+                    mJsonWriter.endArray();
+                } else if (value instanceof SharedPreferences) {
+                    SharedPreferences prefs = (SharedPreferences) value;
+                    mJsonWriter.beginObject();
+                    for (Map.Entry<String,?> entry : prefs.getAll().entrySet()) {
+                        mJsonWriter.name(entry.getKey());
+                        final Object innerValue = entry.getValue();
+                        if (innerValue == null) {
+                            mJsonWriter.nullValue();
+                        } else if (innerValue instanceof Boolean) {
+                            mJsonWriter.value((Boolean) innerValue);
+                        } else if (innerValue instanceof Number) {
+                            mJsonWriter.value((Number) innerValue);
+                        } else {
+                            mJsonWriter.value(innerValue.toString());
                         }
+                    }
+                    mJsonWriter.endObject();
+                } else if (value instanceof Key[]) {
+                    Key[] keyboardKeys = (Key[]) value;
+                    mJsonWriter.beginArray();
+                    for (Key keyboardKey : keyboardKeys) {
+                        mJsonWriter.beginObject();
+                        mJsonWriter.name("code").value(keyboardKey.mCode);
+                        mJsonWriter.name("altCode").value(keyboardKey.mAltCode);
+                        mJsonWriter.name("x").value(keyboardKey.mX);
+                        mJsonWriter.name("y").value(keyboardKey.mY);
+                        mJsonWriter.name("w").value(keyboardKey.mWidth);
+                        mJsonWriter.name("h").value(keyboardKey.mHeight);
                         mJsonWriter.endObject();
-                    } catch (IOException e) {
-                        e.printStackTrace();
-                        Log.w(TAG, "Error in JsonWriter; disabling logging");
-                        try {
-                            mJsonWriter.close();
-                        } catch (IllegalStateException e1) {
-                            // assume that this is just the json not being terminated properly.
-                            // ignore
-                        } catch (IOException e1) {
-                            e1.printStackTrace();
-                        } finally {
-                            mJsonWriter = NULL_JSON_WRITER;
-                        }
                     }
+                    mJsonWriter.endArray();
+                } else if (value instanceof SuggestedWords) {
+                    SuggestedWords words = (SuggestedWords) value;
+                    mJsonWriter.beginObject();
+                    mJsonWriter.name("typedWordValid").value(words.mTypedWordValid);
+                    mJsonWriter.name("hasAutoCorrectionCandidate")
+                        .value(words.mHasAutoCorrectionCandidate);
+                    mJsonWriter.name("isPunctuationSuggestions")
+                        .value(words.mIsPunctuationSuggestions);
+                    mJsonWriter.name("allowsToBeAutoCorrected")
+                        .value(words.mAllowsToBeAutoCorrected);
+                    mJsonWriter.name("isObsoleteSuggestions")
+                        .value(words.mIsObsoleteSuggestions);
+                    mJsonWriter.name("isPrediction")
+                        .value(words.mIsPrediction);
+                    mJsonWriter.name("words");
+                    mJsonWriter.beginArray();
+                    final int size = words.size();
+                    for (int j = 0; j < size; j++) {
+                        SuggestedWordInfo wordInfo = words.getWordInfo(j);
+                        mJsonWriter.value(wordInfo.toString());
+                    }
+                    mJsonWriter.endArray();
+                    mJsonWriter.endObject();
+                } else if (value == null) {
+                    mJsonWriter.nullValue();
+                } else {
+                    Log.w(TAG, "Unrecognized type to be logged: " +
+                            (value == null ? "<null>" : value.getClass().getName()));
+                    mJsonWriter.nullValue();
                 }
-            });
+            }
+            mJsonWriter.endObject();
+        } catch (IOException e) {
+            e.printStackTrace();
+            Log.w(TAG, "Error in JsonWriter; disabling logging");
+            try {
+                mJsonWriter.close();
+            } catch (IllegalStateException e1) {
+                // assume that this is just the json not being terminated properly.
+                // ignore
+            } catch (IOException e1) {
+                e1.printStackTrace();
+            } finally {
+                mJsonWriter = NULL_JSON_WRITER;
+            }
+        }
+    }
+
+    private static class LogUnit implements Runnable {
+        private final List<String[]> mKeysList = new ArrayList<String[]>();
+        private final List<Object[]> mValuesList = new ArrayList<Object[]>();
+        private final List<Boolean> mIsPotentiallyPrivate = new ArrayList<Boolean>();
+        private boolean mIsPrivacySafe = false;
+
+        private void addLogAtom(final String[] keys, final Object[] values,
+                final Boolean isPotentiallyPrivate) {
+            mKeysList.add(keys);
+            mValuesList.add(values);
+            mIsPotentiallyPrivate.add(isPotentiallyPrivate);
+        }
+
+        void setIsPrivacySafe(boolean isPrivacySafe) {
+            mIsPrivacySafe = isPrivacySafe;
+        }
+
+        @Override
+        public void run() {
+            final int numAtoms = mKeysList.size();
+            for (int atomIndex = 0; atomIndex < numAtoms; atomIndex++) {
+                if (!mIsPrivacySafe && mIsPotentiallyPrivate.get(atomIndex)) {
+                    continue;
+                }
+                final String[] keys = mKeysList.get(atomIndex);
+                final Object[] values = mValuesList.get(atomIndex);
+                ResearchLogger.getInstance().outputEvent(keys, values);
+            }
         }
     }
 
@@ -477,6 +568,16 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         }
     }
 
+    private String scrubWord(String word) {
+        if (mDictionary == null) {
+            return WORD_REPLACEMENT_STRING;
+        }
+        if (mDictionary.isValidWord(word)) {
+            return word;
+        }
+        return WORD_REPLACEMENT_STRING;
+    }
+
     private static final String[] EVENTKEYS_LATINKEYBOARDVIEW_PROCESSMOTIONEVENT = {
         "LatinKeyboardViewProcessMotionEvent", "action", "eventTime", "id", "x", "y", "size",
         "pressure"
@@ -500,7 +601,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             final Object[] values = {
                 actionString, eventTime, id, x, y, size, pressure
             };
-            getInstance().writeEvent(EVENTKEYS_LATINKEYBOARDVIEW_PROCESSMOTIONEVENT, values);
+            getInstance().enqueuePotentiallyPrivateEvent(
+                    EVENTKEYS_LATINKEYBOARDVIEW_PROCESSMOTIONEVENT, values);
         }
     }
 
@@ -511,7 +613,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             Keyboard.printableCode(scrubDigitFromCodePoint(code)), x, y
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_ONCODEINPUT, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_ONCODEINPUT, values);
     }
 
     private static final String[] EVENTKEYS_CORRECTION = {
@@ -522,7 +624,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             subgroup, scrubDigitsFromString(before), scrubDigitsFromString(after), position
         };
-        getInstance().writeEvent(EVENTKEYS_CORRECTION, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_CORRECTION, values);
     }
 
     private static final String[] EVENTKEYS_LATINIME_COMMITCURRENTAUTOCORRECTION = {
@@ -533,7 +635,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             scrubDigitsFromString(typedWord), scrubDigitsFromString(autoCorrection)
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_COMMITCURRENTAUTOCORRECTION, values);
+        final ResearchLogger researchLogger = getInstance();
+        researchLogger.enqueuePotentiallyPrivateEvent(
+                EVENTKEYS_LATINIME_COMMITCURRENTAUTOCORRECTION, values);
+        researchLogger.flushQueue(autoCorrection);
     }
 
     private static final String[] EVENTKEYS_LATINIME_COMMITTEXT = {
@@ -543,7 +648,9 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             scrubDigitsFromString(typedWord.toString())
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_COMMITTEXT, values);
+        final ResearchLogger researchLogger = getInstance();
+        researchLogger.enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_COMMITTEXT, values);
+        researchLogger.flushQueue(typedWord);
     }
 
     private static final String[] EVENTKEYS_LATINIME_DELETESURROUNDINGTEXT = {
@@ -553,14 +660,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             length
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_DELETESURROUNDINGTEXT, values);
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_DELETESURROUNDINGTEXT, values);
     }
 
     private static final String[] EVENTKEYS_LATINIME_DOUBLESPACEAUTOPERIOD = {
         "LatinIMEDoubleSpaceAutoPeriod"
     };
     public static void latinIME_doubleSpaceAutoPeriod() {
-        getInstance().writeEvent(EVENTKEYS_LATINIME_DOUBLESPACEAUTOPERIOD, EVENTKEYS_NULLVALUES);
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_DOUBLESPACEAUTOPERIOD, EVENTKEYS_NULLVALUES);
     }
 
     private static final String[] EVENTKEYS_LATINIME_ONDISPLAYCOMPLETIONS = {
@@ -571,7 +678,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             applicationSpecifiedCompletions
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_ONDISPLAYCOMPLETIONS, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_ONDISPLAYCOMPLETIONS,
+                values);
     }
 
     /* package */ static boolean getAndClearLatinIMEExpectingUpdateSelection() {
@@ -592,27 +700,35 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             ic.setSelection(savedSelectionStart, savedSelectionEnd);
             ic.endBatchEdit();
             sLatinIMEExpectingUpdateSelection = true;
-            Object[] values = new Object[2];
-            if (TextUtils.isEmpty(charSequence)) {
-                values[0] = false;
-                values[1] = "";
-            } else {
-                if (charSequence.length() > MAX_INPUTVIEW_LENGTH_TO_CAPTURE) {
-                    int length = MAX_INPUTVIEW_LENGTH_TO_CAPTURE;
-                    // do not cut in the middle of a supplementary character
-                    final char c = charSequence.charAt(length - 1);
-                    if (Character.isHighSurrogate(c)) {
-                        length--;
-                    }
-                    final CharSequence truncatedCharSequence = charSequence.subSequence(0, length);
-                    values[0] = true;
-                    values[1] = scrubDigitsFromString(truncatedCharSequence.toString());
-                } else {
+            final Object[] values = new Object[2];
+            if (OUTPUT_ENTIRE_BUFFER) {
+                if (TextUtils.isEmpty(charSequence)) {
                     values[0] = false;
-                    values[1] = scrubDigitsFromString(charSequence.toString());
+                    values[1] = "";
+                } else {
+                    if (charSequence.length() > MAX_INPUTVIEW_LENGTH_TO_CAPTURE) {
+                        int length = MAX_INPUTVIEW_LENGTH_TO_CAPTURE;
+                        // do not cut in the middle of a supplementary character
+                        final char c = charSequence.charAt(length - 1);
+                        if (Character.isHighSurrogate(c)) {
+                            length--;
+                        }
+                        final CharSequence truncatedCharSequence = charSequence.subSequence(0,
+                                length);
+                        values[0] = true;
+                        values[1] = truncatedCharSequence.toString();
+                    } else {
+                        values[0] = false;
+                        values[1] = charSequence.toString();
+                    }
                 }
+            } else {
+                values[0] = true;
+                values[1] = "";
             }
-            getInstance().writeEvent(EVENTKEYS_LATINIME_ONWINDOWHIDDEN, values);
+            final ResearchLogger researchLogger = getInstance();
+            researchLogger.enqueueEvent(EVENTKEYS_LATINIME_ONWINDOWHIDDEN, values);
+            researchLogger.flushQueue(null);
         }
     }
 
@@ -628,7 +744,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
                 Integer.toHexString(editorInfo.imeOptions), editorInfo.fieldId, Build.DISPLAY,
                 Build.MODEL, prefs, OUTPUT_FORMAT_VERSION
             };
-            getInstance().writeEvent(EVENTKEYS_LATINIME_ONSTARTINPUTVIEWINTERNAL, values);
+            getInstance().enqueueEvent(EVENTKEYS_LATINIME_ONSTARTINPUTVIEWINTERNAL, values);
         }
     }
 
@@ -662,12 +778,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
                 word = range.mWord;
             }
         }
+        final ResearchLogger researchLogger = getInstance();
+        final String scrubbedWord = researchLogger.scrubWord(word);
         final Object[] values = {
             lastSelectionStart, lastSelectionEnd, oldSelStart, oldSelEnd, newSelStart,
             newSelEnd, composingSpanStart, composingSpanEnd, expectingUpdateSelection,
-            expectingUpdateSelectionFromLogger, scrubDigitsFromString(word)
+            expectingUpdateSelectionFromLogger, scrubbedWord
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_ONUPDATESELECTION, values);
+        researchLogger.enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_ONUPDATESELECTION, values);
     }
 
     private static final String[] EVENTKEYS_LATINIME_PERFORMEDITORACTION = {
@@ -677,7 +795,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             imeActionNext
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_PERFORMEDITORACTION, values);
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_PERFORMEDITORACTION, values);
     }
 
     private static final String[] EVENTKEYS_LATINIME_PICKAPPLICATIONSPECIFIEDCOMPLETION = {
@@ -688,7 +806,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             index, cs, x, y
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_PICKAPPLICATIONSPECIFIEDCOMPLETION, values);
+        final ResearchLogger researchLogger = getInstance();
+        researchLogger.enqueuePotentiallyPrivateEvent(
+                EVENTKEYS_LATINIME_PICKAPPLICATIONSPECIFIEDCOMPLETION, values);
+        researchLogger.flushQueue(cs.toString());
     }
 
     private static final String[] EVENTKEYS_LATINIME_PICKSUGGESTIONMANUALLY = {
@@ -700,7 +821,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             scrubDigitsFromString(replacedWord), index, suggestion == null ? null :
                     scrubDigitsFromString(suggestion.toString()), x, y
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_PICKSUGGESTIONMANUALLY, values);
+        final ResearchLogger researchLogger = getInstance();
+        researchLogger.enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_PICKSUGGESTIONMANUALLY,
+                values);
+        researchLogger.flushQueue(suggestion.toString());
     }
 
     private static final String[] EVENTKEYS_LATINIME_PUNCTUATIONSUGGESTION = {
@@ -711,14 +835,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             index, suggestion, x, y
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_PUNCTUATIONSUGGESTION, values);
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_PUNCTUATIONSUGGESTION, values);
     }
 
     private static final String[] EVENTKEYS_LATINIME_REVERTDOUBLESPACEWHILEINBATCHEDIT = {
         "LatinIMERevertDoubleSpaceWhileInBatchEdit"
     };
     public static void latinIME_revertDoubleSpaceWhileInBatchEdit() {
-        getInstance().writeEvent(EVENTKEYS_LATINIME_REVERTDOUBLESPACEWHILEINBATCHEDIT,
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_REVERTDOUBLESPACEWHILEINBATCHEDIT,
                 EVENTKEYS_NULLVALUES);
     }
 
@@ -726,7 +850,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         "LatinIMERevertSwapPunctuation"
     };
     public static void latinIME_revertSwapPunctuation() {
-        getInstance().writeEvent(EVENTKEYS_LATINIME_REVERTSWAPPUNCTUATION, EVENTKEYS_NULLVALUES);
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_REVERTSWAPPUNCTUATION, EVENTKEYS_NULLVALUES);
     }
 
     private static final String[] EVENTKEYS_LATINIME_SENDKEYCODEPOINT = {
@@ -736,14 +860,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             Keyboard.printableCode(scrubDigitFromCodePoint(code))
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_SENDKEYCODEPOINT, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_SENDKEYCODEPOINT, values);
     }
 
     private static final String[] EVENTKEYS_LATINIME_SWAPSWAPPERANDSPACEWHILEINBATCHEDIT = {
         "LatinIMESwapSwapperAndSpaceWhileInBatchEdit"
     };
     public static void latinIME_swapSwapperAndSpaceWhileInBatchEdit() {
-        getInstance().writeEvent(EVENTKEYS_LATINIME_SWAPSWAPPERANDSPACEWHILEINBATCHEDIT,
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_SWAPSWAPPERANDSPACEWHILEINBATCHEDIT,
                 EVENTKEYS_NULLVALUES);
     }
 
@@ -751,14 +875,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         "LatinIMESwitchToKeyboardView"
     };
     public static void latinIME_switchToKeyboardView() {
-        getInstance().writeEvent(EVENTKEYS_LATINIME_SWITCHTOKEYBOARDVIEW, EVENTKEYS_NULLVALUES);
+        getInstance().enqueueEvent(EVENTKEYS_LATINIME_SWITCHTOKEYBOARDVIEW, EVENTKEYS_NULLVALUES);
     }
 
     private static final String[] EVENTKEYS_LATINKEYBOARDVIEW_ONLONGPRESS = {
         "LatinKeyboardViewOnLongPress"
     };
     public static void latinKeyboardView_onLongPress() {
-        getInstance().writeEvent(EVENTKEYS_LATINKEYBOARDVIEW_ONLONGPRESS, EVENTKEYS_NULLVALUES);
+        getInstance().enqueueEvent(EVENTKEYS_LATINKEYBOARDVIEW_ONLONGPRESS, EVENTKEYS_NULLVALUES);
     }
 
     private static final String[] EVENTKEYS_LATINKEYBOARDVIEW_SETKEYBOARD = {
@@ -790,7 +914,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
                     keyboard.mOccupiedHeight,
                     keyboard.mKeys
                 };
-            getInstance().writeEvent(EVENTKEYS_LATINKEYBOARDVIEW_SETKEYBOARD, values);
+            getInstance().enqueueEvent(EVENTKEYS_LATINKEYBOARDVIEW_SETKEYBOARD, values);
             getInstance().setIsPasswordView(isPasswordView);
         }
     }
@@ -802,14 +926,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             originallyTypedWord
         };
-        getInstance().writeEvent(EVENTKEYS_LATINIME_REVERTCOMMIT, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_LATINIME_REVERTCOMMIT, values);
     }
 
     private static final String[] EVENTKEYS_POINTERTRACKER_CALLLISTENERONCANCELINPUT = {
         "PointerTrackerCallListenerOnCancelInput"
     };
     public static void pointerTracker_callListenerOnCancelInput() {
-        getInstance().writeEvent(EVENTKEYS_POINTERTRACKER_CALLLISTENERONCANCELINPUT,
+        getInstance().enqueueEvent(EVENTKEYS_POINTERTRACKER_CALLLISTENERONCANCELINPUT,
                 EVENTKEYS_NULLVALUES);
     }
 
@@ -827,7 +951,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
                         : scrubDigitsFromString(outputText.toString()),
                 x, y, ignoreModifierKey, altersCode, key.isEnabled()
             };
-            getInstance().writeEvent(EVENTKEYS_POINTERTRACKER_CALLLISTENERONCODEINPUT, values);
+            getInstance().enqueuePotentiallyPrivateEvent(
+                    EVENTKEYS_POINTERTRACKER_CALLLISTENERONCODEINPUT, values);
         }
     }
 
@@ -842,7 +967,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
                 Keyboard.printableCode(scrubDigitFromCodePoint(primaryCode)), withSliding,
                 ignoreModifierKey, key.isEnabled()
             };
-            getInstance().writeEvent(EVENTKEYS_POINTERTRACKER_CALLLISTENERONRELEASE, values);
+            getInstance().enqueuePotentiallyPrivateEvent(
+                    EVENTKEYS_POINTERTRACKER_CALLLISTENERONRELEASE, values);
         }
     }
 
@@ -853,7 +979,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             deltaT, distanceSquared
         };
-        getInstance().writeEvent(EVENTKEYS_POINTERTRACKER_ONDOWNEVENT, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_POINTERTRACKER_ONDOWNEVENT, values);
     }
 
     private static final String[] EVENTKEYS_POINTERTRACKER_ONMOVEEVENT = {
@@ -864,7 +990,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         final Object[] values = {
             x, y, lastX, lastY
         };
-        getInstance().writeEvent(EVENTKEYS_POINTERTRACKER_ONMOVEEVENT, values);
+        getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_POINTERTRACKER_ONMOVEEVENT, values);
     }
 
     private static final String[] EVENTKEYS_SUDDENJUMPINGTOUCHEVENTHANDLER_ONTOUCHEVENT = {
@@ -875,8 +1001,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             final Object[] values = {
                 me.toString()
             };
-            getInstance().writeEvent(EVENTKEYS_SUDDENJUMPINGTOUCHEVENTHANDLER_ONTOUCHEVENT,
-                    values);
+            getInstance().enqueuePotentiallyPrivateEvent(
+                    EVENTKEYS_SUDDENJUMPINGTOUCHEVENTHANDLER_ONTOUCHEVENT, values);
         }
     }
 
@@ -888,7 +1014,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
             final Object[] values = {
                 suggestedWords
             };
-            getInstance().writeEvent(EVENTKEYS_SUGGESTIONSVIEW_SETSUGGESTIONS, values);
+            getInstance().enqueuePotentiallyPrivateEvent(EVENTKEYS_SUGGESTIONSVIEW_SETSUGGESTIONS,
+                    values);
         }
     }
 
@@ -896,6 +1023,6 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang
         "UserTimestamp"
     };
     public void userTimestamp() {
-        getInstance().writeEvent(EVENTKEYS_USER_TIMESTAMP, EVENTKEYS_NULLVALUES);
+        getInstance().enqueueEvent(EVENTKEYS_USER_TIMESTAMP, EVENTKEYS_NULLVALUES);
     }
 }
diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java
index 68b7b913f..7398c82fa 100644
--- a/java/src/com/android/inputmethod/latin/Suggest.java
+++ b/java/src/com/android/inputmethod/latin/Suggest.java
@@ -65,7 +65,7 @@ public class Suggest implements Dictionary.WordCallback {
 
     private static final boolean DBG = LatinImeLogger.sDBG;
 
-    private boolean mHasMainDictionary;
+    private Dictionary mMainDictionary;
     private ContactsBinaryDictionary mContactsDict;
     private WhitelistDictionary mWhiteListDictionary;
     private final ConcurrentHashMap<String, Dictionary> mUnigramDictionaries =
@@ -98,7 +98,7 @@ public class Suggest implements Dictionary.WordCallback {
             final long startOffset, final long length, final Locale locale) {
         final Dictionary mainDict = DictionaryFactory.createDictionaryForTest(context, dictionary,
                 startOffset, length /* useFullEditDistance */, false, locale);
-        mHasMainDictionary = null != mainDict;
+        mMainDictionary = mainDict;
         addOrReplaceDictionary(mUnigramDictionaries, DICT_KEY_MAIN, mainDict);
         addOrReplaceDictionary(mBigramDictionaries, DICT_KEY_MAIN, mainDict);
         initWhitelistAndAutocorrectAndPool(context, locale);
@@ -129,15 +129,15 @@ public class Suggest implements Dictionary.WordCallback {
     }
 
     public void resetMainDict(final Context context, final Locale locale) {
-        mHasMainDictionary = false;
+        mMainDictionary = null;
         new Thread("InitializeBinaryDictionary") {
             @Override
             public void run() {
                 final DictionaryCollection newMainDict =
                         DictionaryFactory.createMainDictionaryFromManager(context, locale);
-                mHasMainDictionary = null != newMainDict && !newMainDict.isEmpty();
                 addOrReplaceDictionary(mUnigramDictionaries, DICT_KEY_MAIN, newMainDict);
                 addOrReplaceDictionary(mBigramDictionaries, DICT_KEY_MAIN, newMainDict);
+                mMainDictionary = newMainDict;
             }
         }.start();
     }
@@ -145,7 +145,11 @@ public class Suggest implements Dictionary.WordCallback {
     // The main dictionary could have been loaded asynchronously.  Don't cache the return value
     // of this method.
     public boolean hasMainDictionary() {
-        return mHasMainDictionary;
+        return null != mMainDictionary && mMainDictionary.isInitialized();
+    }
+
+    public Dictionary getMainDictionary() {
+        return mMainDictionary;
     }
 
     public ContactsBinaryDictionary getContactsDictionary() {
@@ -365,7 +369,7 @@ public class Suggest implements Dictionary.WordCallback {
         // language, and it will unexpectedly auto-correct. For example, if the user types in
         // English with no dictionary and has a "Will" in their contact list, "will" would
         // always auto-correct to "Will" which is unwanted. Hence, no main dict => no auto-correct.
-                && mHasMainDictionary;
+                && hasMainDictionary();
 
         boolean autoCorrectionAvailable = hasAutoCorrection;
         if (correctionMode == CORRECTION_FULL || correctionMode == CORRECTION_FULL_BIGRAM) {
@@ -511,7 +515,7 @@ public class Suggest implements Dictionary.WordCallback {
         for (final Dictionary dictionary : dictionaries) {
             dictionary.close();
         }
-        mHasMainDictionary = false;
+        mMainDictionary = null;
     }
 
     // TODO: Resolve the inconsistencies between the native auto correction algorithms and