ResearchLogging capture full n-gram data

- Captures complete motion data for all words in an n-gram.
- Also filters n-grams properly; if any word in the n-gram is not
  in the dictionary, it is not included.
- Simplify ResearchLog to not require explicit state
- Added LogBuffer class MainLogBuffer class to allow n-gram-level decisions
  about privacy.
- Moved LogUnit out from ResearchLogger

multi-project change with Ic70ccb6c2e11eb02d887821434b44daa3eb7a3e2

Bug: 6188932
Change-Id: I731d6cff6539e997874f723b68bdb0d9b651b937

1 files changed, 96 insertions, 161 deletions

diff --git a/java/src/com/android/inputmethod/research/ResearchLog.java b/java/src/com/android/inputmethod/research/ResearchLog.java
index 18bf3c07f..71a6d6a78 100644
--- a/java/src/com/android/inputmethod/research/ResearchLog.java
+++ b/java/src/com/android/inputmethod/research/ResearchLog.java
@@ -26,7 +26,6 @@ import com.android.inputmethod.keyboard.Key;
 import com.android.inputmethod.latin.SuggestedWords;
 import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
 import com.android.inputmethod.latin.define.ProductionFlag;
-import com.android.inputmethod.research.ResearchLogger.LogUnit;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -37,6 +36,7 @@ import java.io.OutputStreamWriter;
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -51,21 +51,22 @@ import java.util.concurrent.TimeUnit;
  */
 public class ResearchLog {
     private static final String TAG = ResearchLog.class.getSimpleName();
-    private static final JsonWriter NULL_JSON_WRITER = new JsonWriter(
-            new OutputStreamWriter(new NullOutputStream()));
+    private static final boolean DEBUG = false;
+    private static final long FLUSH_DELAY_IN_MS = 1000 * 5;
+    private static final int ABORT_TIMEOUT_IN_MS = 1000 * 4;
 
-    final ScheduledExecutorService mExecutor;
+    /* package */ final ScheduledExecutorService mExecutor;
     /* package */ final File mFile;
     private JsonWriter mJsonWriter = NULL_JSON_WRITER;
+    // true if at least one byte of data has been written out to the log file.  This must be
+    // remembered because JsonWriter requires that calls matching calls to beginObject and
+    // endObject, as well as beginArray and endArray, and the file is opened lazily, only when
+    // it is certain that data will be written.  Alternatively, the matching call exceptions
+    // could be caught, but this might suppress other errors.
+    private boolean mHasWrittenData = false;
 
-    private int mLoggingState;
-    private static final int LOGGING_STATE_UNSTARTED = 0;
-    private static final int LOGGING_STATE_READY = 1;   // don't create file until necessary
-    private static final int LOGGING_STATE_RUNNING = 2;
-    private static final int LOGGING_STATE_STOPPING = 3;
-    private static final int LOGGING_STATE_STOPPED = 4;
-    private static final long FLUSH_DELAY_IN_MS = 1000 * 5;
-
+    private static final JsonWriter NULL_JSON_WRITER = new JsonWriter(
+            new OutputStreamWriter(new NullOutputStream()));
     private static class NullOutputStream extends OutputStream {
         /** {@inheritDoc} */
         @Override
@@ -84,128 +85,81 @@ public class ResearchLog {
         }
     }
 
-    public ResearchLog(File outputFile) {
-        mExecutor = Executors.newSingleThreadScheduledExecutor();
+    public ResearchLog(final File outputFile) {
         if (outputFile == null) {
             throw new IllegalArgumentException();
         }
+        mExecutor = Executors.newSingleThreadScheduledExecutor();
         mFile = outputFile;
-        mLoggingState = LOGGING_STATE_UNSTARTED;
-    }
-
-    public synchronized void start() throws IOException {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-                mLoggingState = LOGGING_STATE_READY;
-                break;
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-            case LOGGING_STATE_STOPPING:
-            case LOGGING_STATE_STOPPED:
-                break;
-        }
     }
 
-    public synchronized void stop() {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-                mLoggingState = LOGGING_STATE_STOPPED;
-                break;
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-                mExecutor.submit(new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
-                        try {
-                            mJsonWriter.endArray();
-                            mJsonWriter.flush();
-                            mJsonWriter.close();
-                        } finally {
-                            boolean success = mFile.setWritable(false, false);
-                            mLoggingState = LOGGING_STATE_STOPPED;
-                        }
-                        return null;
+    public synchronized void close() {
+        mExecutor.submit(new Callable<Object>() {
+            @Override
+            public Object call() throws Exception {
+                try {
+                    if (mHasWrittenData) {
+                        mJsonWriter.endArray();
+                        mJsonWriter.flush();
+                        mJsonWriter.close();
+                        mHasWrittenData = false;
                     }
-                });
-                removeAnyScheduledFlush();
-                mExecutor.shutdown();
-                mLoggingState = LOGGING_STATE_STOPPING;
-                break;
-            case LOGGING_STATE_STOPPING:
-            case LOGGING_STATE_STOPPED:
-        }
+                } catch (Exception e) {
+                    Log.d(TAG, "error when closing ResearchLog:");
+                    e.printStackTrace();
+                } finally {
+                    if (mFile.exists()) {
+                        mFile.setWritable(false, false);
+                    }
+                }
+                return null;
+            }
+        });
+        removeAnyScheduledFlush();
+        mExecutor.shutdown();
     }
 
-    public boolean isAlive() {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-                return true;
-        }
-        return false;
-    }
+    private boolean mIsAbortSuccessful;
 
-    public void waitUntilStopped(final int timeoutInMs) throws InterruptedException {
+    public synchronized void abort() {
+        mExecutor.submit(new Callable<Object>() {
+            @Override
+            public Object call() throws Exception {
+                try {
+                    if (mHasWrittenData) {
+                        mJsonWriter.endArray();
+                        mJsonWriter.close();
+                        mHasWrittenData = false;
+                    }
+                } finally {
+                    mIsAbortSuccessful = mFile.delete();
+                }
+                return null;
+            }
+        });
         removeAnyScheduledFlush();
         mExecutor.shutdown();
-        mExecutor.awaitTermination(timeoutInMs, TimeUnit.MILLISECONDS);
     }
 
-    public synchronized void abort() {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-                mLoggingState = LOGGING_STATE_STOPPED;
-                isAbortSuccessful = true;
-                break;
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-                mExecutor.submit(new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
-                        try {
-                            mJsonWriter.endArray();
-                            mJsonWriter.close();
-                        } finally {
-                            isAbortSuccessful = mFile.delete();
-                        }
-                        return null;
-                    }
-                });
-                removeAnyScheduledFlush();
-                mExecutor.shutdown();
-                mLoggingState = LOGGING_STATE_STOPPING;
-                break;
-            case LOGGING_STATE_STOPPING:
-            case LOGGING_STATE_STOPPED:
-        }
+    public boolean blockingAbort() throws InterruptedException {
+        abort();
+        mExecutor.awaitTermination(ABORT_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS);
+        return mIsAbortSuccessful;
     }
 
-    private boolean isAbortSuccessful;
-    public boolean isAbortSuccessful() {
-        return isAbortSuccessful;
+    public void awaitTermination(int delay, TimeUnit timeUnit) throws InterruptedException {
+        mExecutor.awaitTermination(delay, timeUnit);
     }
 
     /* package */ synchronized void flush() {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-                break;
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-                removeAnyScheduledFlush();
-                mExecutor.submit(mFlushCallable);
-                break;
-            case LOGGING_STATE_STOPPING:
-            case LOGGING_STATE_STOPPED:
-        }
+        removeAnyScheduledFlush();
+        mExecutor.submit(mFlushCallable);
     }
 
-    private Callable<Object> mFlushCallable = new Callable<Object>() {
+    private final Callable<Object> mFlushCallable = new Callable<Object>() {
         @Override
         public Object call() throws Exception {
-            if (mLoggingState == LOGGING_STATE_RUNNING) {
-                mJsonWriter.flush();
-            }
+            mJsonWriter.flush();
             return null;
         }
     };
@@ -224,56 +178,40 @@ public class ResearchLog {
         mFlushFuture = mExecutor.schedule(mFlushCallable, FLUSH_DELAY_IN_MS, TimeUnit.MILLISECONDS);
     }
 
-    public synchronized void publishPublicEvents(final LogUnit logUnit) {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-                break;
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-                mExecutor.submit(new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
-                        logUnit.publishPublicEventsTo(ResearchLog.this);
-                        scheduleFlush();
-                        return null;
-                    }
-                });
-                break;
-            case LOGGING_STATE_STOPPING:
-            case LOGGING_STATE_STOPPED:
-        }
-    }
-
-    public synchronized void publishAllEvents(final LogUnit logUnit) {
-        switch (mLoggingState) {
-            case LOGGING_STATE_UNSTARTED:
-                break;
-            case LOGGING_STATE_READY:
-            case LOGGING_STATE_RUNNING:
-                mExecutor.submit(new Callable<Object>() {
-                    @Override
-                    public Object call() throws Exception {
-                        logUnit.publishAllEventsTo(ResearchLog.this);
-                        scheduleFlush();
-                        return null;
-                    }
-                });
-                break;
-            case LOGGING_STATE_STOPPING:
-            case LOGGING_STATE_STOPPED:
+    public synchronized void publish(final LogUnit logUnit, final boolean isIncludingPrivateData) {
+        try {
+            mExecutor.submit(new Callable<Object>() {
+                @Override
+                public Object call() throws Exception {
+                    logUnit.publishTo(ResearchLog.this, isIncludingPrivateData);
+                    scheduleFlush();
+                    return null;
+                }
+            });
+        } catch (RejectedExecutionException e) {
+            // TODO: Add code to record loss of data, and report.
         }
     }
 
     private static final String CURRENT_TIME_KEY = "_ct";
     private static final String UPTIME_KEY = "_ut";
     private static final String EVENT_TYPE_KEY = "_ty";
+
     void outputEvent(final String[] keys, final Object[] values) {
-        // not thread safe.
+        // Not thread safe.
+        if (keys.length == 0) {
+            return;
+        }
+        if (DEBUG) {
+            if (keys.length != values.length + 1) {
+                Log.d(TAG, "Key and Value list sizes do not match. " + keys[0]);
+            }
+        }
         try {
             if (mJsonWriter == NULL_JSON_WRITER) {
                 mJsonWriter = new JsonWriter(new BufferedWriter(new FileWriter(mFile)));
-                mJsonWriter.setLenient(true);
                 mJsonWriter.beginArray();
+                mHasWrittenData = true;
             }
             mJsonWriter.beginObject();
             mJsonWriter.name(CURRENT_TIME_KEY).value(System.currentTimeMillis());
@@ -283,8 +221,8 @@ public class ResearchLog {
             for (int i = 0; i < length; i++) {
                 mJsonWriter.name(keys[i + 1]);
                 Object value = values[i];
-                if (value instanceof String) {
-                    mJsonWriter.value((String) value);
+                if (value instanceof CharSequence) {
+                    mJsonWriter.value(value.toString());
                 } else if (value instanceof Number) {
                     mJsonWriter.value((Number) value);
                 } else if (value instanceof Boolean) {
@@ -331,14 +269,11 @@ public class ResearchLog {
                     SuggestedWords words = (SuggestedWords) value;
                     mJsonWriter.beginObject();
                     mJsonWriter.name("typedWordValid").value(words.mTypedWordValid);
-                    mJsonWriter.name("willAutoCorrect")
-                        .value(words.mWillAutoCorrect);
+                    mJsonWriter.name("willAutoCorrect").value(words.mWillAutoCorrect);
                     mJsonWriter.name("isPunctuationSuggestions")
-                        .value(words.mIsPunctuationSuggestions);
-                    mJsonWriter.name("isObsoleteSuggestions")
-                        .value(words.mIsObsoleteSuggestions);
-                    mJsonWriter.name("isPrediction")
-                        .value(words.mIsPrediction);
+                            .value(words.mIsPunctuationSuggestions);
+                    mJsonWriter.name("isObsoleteSuggestions").value(words.mIsObsoleteSuggestions);
+                    mJsonWriter.name("isPrediction").value(words.mIsPrediction);
                     mJsonWriter.name("words");
                     mJsonWriter.beginArray();
                     final int size = words.size();
@@ -363,8 +298,8 @@ public class ResearchLog {
             try {
                 mJsonWriter.close();
             } catch (IllegalStateException e1) {
-                // assume that this is just the json not being terminated properly.
-                // ignore
+                // Assume that this is just the json not being terminated properly.
+                // Ignore
             } catch (IOException e1) {
                 e1.printStackTrace();
             } finally {