diff options
Diffstat (limited to 'java/src/com/android/inputmethod/research/ResearchLogger.java')
| -rw-r--r-- | java/src/com/android/inputmethod/research/ResearchLogger.java | 253 |
1 files changed, 182 insertions, 71 deletions
diff --git a/java/src/com/android/inputmethod/research/ResearchLogger.java b/java/src/com/android/inputmethod/research/ResearchLogger.java index 8b8ea21e9..aa4a866b8 100644 --- a/java/src/com/android/inputmethod/research/ResearchLogger.java +++ b/java/src/com/android/inputmethod/research/ResearchLogger.java @@ -83,6 +83,8 @@ import java.util.List; import java.util.Random; import java.util.regex.Pattern; +// TODO: Add a unit test for every "logging" method (i.e. that is called from the IME and calls +// enqueueEvent to record a LogStatement). /** * Logs the use of the LatinIME keyboard. * @@ -194,10 +196,17 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang // gesture, and when committing the earlier word, split the LogUnit. private long mSavedDownEventTime; private Bundle mFeedbackDialogBundle = null; + // Whether the feedback dialog is visible, and the user is typing into it. Normal logging is + // not performed on text that the user types into the feedback dialog. private boolean mInFeedbackDialog = false; private Handler mUserRecordingTimeoutHandler; private static final long USER_RECORDING_TIMEOUT_MS = 30L * DateUtils.SECOND_IN_MILLIS; + // Stores a temporary LogUnit while generating a phantom space. Needed because phantom spaces + // are issued out-of-order, immediately before the characters generated by other operations that + // have already outputted LogStatements. + private LogUnit mPhantomSpaceLogUnit = null; + private ResearchLogger() { mStatistics = Statistics.getInstance(); } @@ -253,14 +262,14 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang if (DEBUG) { final String wordsString = logUnit.getWordsAsString(); Log.d(TAG, "onPublish: '" + wordsString - + "', hc: " + logUnit.containsCorrection() + + "', hc: " + logUnit.containsUserDeletions() + ", cipd: " + canIncludePrivateData); } for (final String word : logUnit.getWordsAsStringArray()) { final Dictionary dictionary = getDictionary(); mStatistics.recordWordEntered( dictionary != null && dictionary.isValidWord(word), - logUnit.containsCorrection()); + logUnit.containsUserDeletions()); } } publishLogUnits(logUnits, mMainResearchLog, canIncludePrivateData); @@ -650,7 +659,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang feedbackLogUnit.addLogStatement(LOGSTATEMENT_FEEDBACK, SystemClock.uptimeMillis(), feedbackContents, accountName, recording); - final ResearchLog feedbackLog = new ResearchLog(mResearchLogDirectory.getLogFilePath( + final ResearchLog feedbackLog = new FeedbackLog(mResearchLogDirectory.getLogFilePath( System.currentTimeMillis(), System.nanoTime()), mLatinIME); final LogBuffer feedbackLogBuffer = new LogBuffer(); feedbackLogBuffer.shiftIn(feedbackLogUnit); @@ -713,8 +722,28 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang mIsPasswordView = isPasswordView; } - private boolean isAllowedToLog() { - return !mIsPasswordView && sIsLogging && !mInFeedbackDialog; + /** + * Returns true if logging is permitted. + * + * This method is called when adding a LogStatement to a LogUnit, and when adding a LogUnit to a + * ResearchLog. It is checked in both places in case conditions change between these times, and + * as a defensive measure in case refactoring changes the logging pipeline. + */ + private boolean isAllowedToLogTo(final ResearchLog researchLog) { + // Logging is never allowed in these circumstances + if (mIsPasswordView) return false; + if (!sIsLogging) return false; + if (mInFeedbackDialog) { + // The FeedbackDialog is up. Normal logging should not happen (the user might be trying + // out things while the dialog is up, and their reporting of an issue may not be + // representative of what they normally type). However, after the user has finished + // entering their feedback, the logger packs their comments and an encoded version of + // any demonstration of the issue into a special "FeedbackLog". So if the FeedbackLog + // is the destination, we do want to allow logging to it. + return researchLog.isFeedbackLog(); + } + // No other exclusions. Logging is permitted. + return true; } public void requestIndicatorRedraw() { @@ -747,7 +776,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang // and remove this method. // The check for MainKeyboardView ensures that the indicator only decorates the main // keyboard, not every keyboard. - if (IS_SHOWING_INDICATOR && (isAllowedToLog() || isReplaying()) + if (IS_SHOWING_INDICATOR && (isAllowedToLogTo(mMainResearchLog) || isReplaying()) && view instanceof MainKeyboardView) { final int savedColor = paint.getColor(); paint.setColor(getIndicatorColor()); @@ -782,7 +811,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang private synchronized void enqueueEvent(final LogUnit logUnit, final LogStatement logStatement, final Object... values) { assert values.length == logStatement.getKeys().length; - if (isAllowedToLog() && logUnit != null) { + if (isAllowedToLogTo(mMainResearchLog) && logUnit != null) { final long time = SystemClock.uptimeMillis(); logUnit.addLogStatement(logStatement, time, values); } @@ -792,8 +821,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang mCurrentLogUnit.setMayContainDigit(); } - private void setCurrentLogUnitContainsCorrection() { - mCurrentLogUnit.setContainsCorrection(); + private void setCurrentLogUnitContainsUserDeletions() { + mCurrentLogUnit.setContainsUserDeletions(); } private void setCurrentLogUnitCorrectionType(final int correctionType) { @@ -825,20 +854,22 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang // The user has deleted this word and returned to the previous. Check that the word in the // logUnit matches the expected word. If so, restore the last log unit committed to be the // current logUnit. I.e., pull out the last LogUnit from all the LogBuffers, and make - // restore it to mCurrentLogUnit so the new edits are captured with the word. Optionally - // dump the contents of mCurrentLogUnit (useful if they contain deletions of the next word - // that should not be reported to protect user privacy) + // it the mCurrentLogUnit so the new edits are captured with the word. Optionally dump the + // contents of mCurrentLogUnit (useful if they contain deletions of the next word that + // should not be reported to protect user privacy) // // Note that we don't use mLastLogUnit here, because it only goes one word back and is only // needed for reverts, which only happen one back. final LogUnit oldLogUnit = mMainLogBuffer.peekLastLogUnit(); - // Check that expected word matches. + // Check that expected word matches. It's ok if both strings are null, because this is the + // case where the LogUnit is storing a non-word, e.g. a separator. if (oldLogUnit != null) { + // Because the word is stored in the LogUnit with digits scrubbed, the comparison must + // be made on a scrubbed version of the expectedWord as well. + final String scrubbedExpectedWord = scrubDigitsFromString(expectedWord); final String oldLogUnitWords = oldLogUnit.getWordsAsString(); - if (oldLogUnitWords != null && !oldLogUnitWords.equals(expectedWord)) { - return; - } + if (!TextUtils.equals(scrubbedExpectedWord, oldLogUnitWords)) return; } // Uncommit, merging if necessary. @@ -881,7 +912,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang final ResearchLog researchLog, final boolean canIncludePrivateData) { final LogUnit openingLogUnit = new LogUnit(); if (logUnits.isEmpty()) return; - if (!isAllowedToLog()) return; + if (!isAllowedToLogTo(researchLog)) return; // LogUnits not containing private data, such as contextual data for the log, do not require // logSegment boundary statements. if (canIncludePrivateData) { @@ -893,7 +924,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang if (DEBUG) { Log.d(TAG, "publishLogBuffer: " + (logUnit.hasOneOrMoreWords() ? logUnit.getWordsAsString() : "<wordless>") - + ", correction?: " + logUnit.containsCorrection()); + + ", correction?: " + logUnit.containsUserDeletions()); } researchLog.publish(logUnit, canIncludePrivateData); } @@ -954,7 +985,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang return Character.isDigit(codePoint) ? DIGIT_REPLACEMENT_CODEPOINT : codePoint; } - /* package for test */ static String scrubDigitsFromString(String s) { + /* package for test */ static String scrubDigitsFromString(final String s) { + if (s == null) return null; StringBuilder sb = null; final int length = s.length(); for (int i = 0; i < length; i = s.offsetByCodePoints(i, 1)) { @@ -1247,6 +1279,16 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang } /** + * Log a revert of onTextInput() (known in the IME as "EnteredText"). + * + * SystemResponse: Remove the LogUnit recording the textInput + */ + public static void latinIME_handleBackspace_cancelTextInput(final String text) { + final ResearchLogger researchLogger = getInstance(); + researchLogger.uncommitCurrentLogUnit(text, true /* dumpCurrentLogUnit */); + } + + /** * Log a call to LatinIME.pickSuggestionManually(). * * UserAction: The user has chosen a specific word from the suggestion strip. @@ -1259,7 +1301,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang final ResearchLogger researchLogger = getInstance(); if (!replacedWord.equals(suggestion.toString())) { // The user chose something other than what was already there. - researchLogger.setCurrentLogUnitContainsCorrection(); + researchLogger.setCurrentLogUnitContainsUserDeletions(); researchLogger.setCurrentLogUnitCorrectionType(LogUnit.CORRECTIONTYPE_TYPO); } final String scrubbedWord = scrubDigitsFromString(suggestion); @@ -1291,17 +1333,32 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang /** * Log a call to LatinIME.sendKeyCodePoint(). * - * SystemResponse: The IME is inserting text into the TextView for numbers, fixed strings, or - * some other unusual mechanism. + * SystemResponse: The IME is inserting text into the TextView for non-word-constituent, + * strings (separators, numbers, other symbols). */ private static final LogStatement LOGSTATEMENT_LATINIME_SENDKEYCODEPOINT = new LogStatement("LatinIMESendKeyCodePoint", true, false, "code"); public static void latinIME_sendKeyCodePoint(final int code) { final ResearchLogger researchLogger = getInstance(); - researchLogger.enqueueEvent(LOGSTATEMENT_LATINIME_SENDKEYCODEPOINT, - Constants.printableCode(scrubDigitFromCodePoint(code))); - if (Character.isDigit(code)) { - researchLogger.setCurrentLogUnitContainsDigitFlag(); + final LogUnit phantomSpaceLogUnit = researchLogger.mPhantomSpaceLogUnit; + if (phantomSpaceLogUnit == null) { + researchLogger.enqueueEvent(LOGSTATEMENT_LATINIME_SENDKEYCODEPOINT, + Constants.printableCode(scrubDigitFromCodePoint(code))); + if (Character.isDigit(code)) { + researchLogger.setCurrentLogUnitContainsDigitFlag(); + } + researchLogger.commitCurrentLogUnit(); + } else { + researchLogger.enqueueEvent(phantomSpaceLogUnit, LOGSTATEMENT_LATINIME_SENDKEYCODEPOINT, + Constants.printableCode(scrubDigitFromCodePoint(code))); + if (Character.isDigit(code)) { + phantomSpaceLogUnit.setMayContainDigit(); + } + researchLogger.mMainLogBuffer.shiftIn(phantomSpaceLogUnit); + if (researchLogger.mUserRecordingLogBuffer != null) { + researchLogger.mUserRecordingLogBuffer.shiftIn(phantomSpaceLogUnit); + } + researchLogger.mPhantomSpaceLogUnit = null; } } @@ -1311,12 +1368,18 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang * SystemResponse: The IME is inserting a real space in place of a phantom space. */ private static final LogStatement LOGSTATEMENT_LATINIME_PROMOTEPHANTOMSPACE = - new LogStatement("LatinIMEPromotPhantomSpace", false, false); + new LogStatement("LatinIMEPromotePhantomSpace", false, false); public static void latinIME_promotePhantomSpace() { + // A phantom space is always added before the text that triggered it. The triggering text + // and the events that created it will be in mCurrentLogUnit, but the phantom space should + // be in its own LogUnit, committed before the triggering text. Although it is created + // here, it is not added to the LogBuffer until the following call to + // latinIME_sendKeyCodePoint, because SENDKEYCODEPOINT LogStatement also must go into that + // LogUnit. final ResearchLogger researchLogger = getInstance(); - final LogUnit logUnit; - logUnit = researchLogger.mMainLogBuffer.peekLastLogUnit(); - researchLogger.enqueueEvent(logUnit, LOGSTATEMENT_LATINIME_PROMOTEPHANTOMSPACE); + researchLogger.mPhantomSpaceLogUnit = new LogUnit(); + researchLogger.enqueueEvent(researchLogger.mPhantomSpaceLogUnit, + LOGSTATEMENT_LATINIME_PROMOTEPHANTOMSPACE); } /** @@ -1402,23 +1465,40 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang public static void latinIME_revertCommit(final String committedWord, final String originallyTypedWord, final boolean isBatchMode, final String separatorString) { + // TODO: Prioritize adding a unit test for this method (as it is especially complex) + // TODO: Update the UserRecording LogBuffer as well as the MainLogBuffer final ResearchLogger researchLogger = getInstance(); - // TODO: Verify that mCurrentLogUnit has been restored and contains the reverted word. - final LogUnit logUnit; - logUnit = researchLogger.mMainLogBuffer.peekLastLogUnit(); - if (originallyTypedWord.length() > 0 && hasLetters(originallyTypedWord)) { - if (logUnit != null) { - logUnit.setWords(originallyTypedWord); - } - } - researchLogger.enqueueEvent(logUnit != null ? logUnit : researchLogger.mCurrentLogUnit, - LOGSTATEMENT_LATINIME_REVERTCOMMIT, committedWord, originallyTypedWord, - separatorString); - if (logUnit != null) { - logUnit.setContainsCorrection(); + // + // 1. Remove separator LogUnit + final LogUnit lastLogUnit = researchLogger.mMainLogBuffer.peekLastLogUnit(); + // Check that we're not at the beginning of input + if (lastLogUnit == null) return; + // Check that we're after a separator + if (lastLogUnit.getWordsAsString() != null) return; + // Remove separator + final LogUnit separatorLogUnit = researchLogger.mMainLogBuffer.unshiftIn(); + + // 2. Add revert LogStatement + final LogUnit revertedLogUnit = researchLogger.mMainLogBuffer.peekLastLogUnit(); + if (revertedLogUnit == null) return; + if (!revertedLogUnit.getWordsAsString().equals(scrubDigitsFromString(committedWord))) { + // Any word associated with the reverted LogUnit has already had its digits scrubbed, so + // any digits in the committedWord argument must also be scrubbed for an accurate + // comparison. + return; } + researchLogger.enqueueEvent(revertedLogUnit, LOGSTATEMENT_LATINIME_REVERTCOMMIT, + committedWord, originallyTypedWord, separatorString); + + // 3. Update the word associated with the LogUnit + revertedLogUnit.setWords(originallyTypedWord); + revertedLogUnit.setContainsUserDeletions(); + + // 4. Re-add the separator LogUnit + researchLogger.mMainLogBuffer.shiftIn(separatorLogUnit); + + // 5. Record stats researchLogger.mStatistics.recordRevertCommit(SystemClock.uptimeMillis()); - researchLogger.commitCurrentLogUnitAsWord(originallyTypedWord, Long.MAX_VALUE, isBatchMode); } /** @@ -1528,7 +1608,12 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang private static final LogStatement LOGSTATEMENT_RICHINPUTCONNECTION_REVERTDOUBLESPACEPERIOD = new LogStatement("RichInputConnectionRevertDoubleSpacePeriod", false, false); public static void richInputConnection_revertDoubleSpacePeriod() { - getInstance().enqueueEvent(LOGSTATEMENT_RICHINPUTCONNECTION_REVERTDOUBLESPACEPERIOD); + final ResearchLogger researchLogger = getInstance(); + // An extra LogUnit is added for the period; this is removed here because of the revert. + researchLogger.uncommitCurrentLogUnit(null, true /* dumpCurrentLogUnit */); + // TODO: This will probably be lost as the user backspaces further. Figure out how to put + // it into the right logUnit. + researchLogger.enqueueEvent(LOGSTATEMENT_RICHINPUTCONNECTION_REVERTDOUBLESPACEPERIOD); } /** @@ -1571,25 +1656,6 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang } private boolean isExpectingCommitText = false; - /** - * Log a call to (UnknownClass).commitPartialText - * - * SystemResponse: The IME is committing part of a word. This happens if a space is - * automatically inserted to split a single typed string into two or more words. - */ - // TODO: This method is currently unused. Find where it should be called from in the IME and - // add invocations. - private static final LogStatement LOGSTATEMENT_COMMIT_PARTIAL_TEXT = - new LogStatement("CommitPartialText", true, false, "newCursorPosition"); - public static void commitPartialText(final String committedWord, - final long lastTimestampOfWordData, final boolean isBatchMode) { - final ResearchLogger researchLogger = getInstance(); - final String scrubbedWord = scrubDigitsFromString(committedWord); - researchLogger.enqueueEvent(LOGSTATEMENT_COMMIT_PARTIAL_TEXT); - researchLogger.mStatistics.recordAutoCorrection(SystemClock.uptimeMillis()); - researchLogger.commitCurrentLogUnitAsWord(scrubbedWord, lastTimestampOfWordData, - isBatchMode); - } /** * Log a call to RichInputConnection.commitText(). @@ -1613,12 +1679,24 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang } /** - * Shared event for logging committed text. + * Shared events for logging committed text. + * + * The "CommitTextEventHappened" LogStatement is written to the log even if privacy rules + * indicate that the word contents should not be logged. It has no contents, and only serves to + * record the event and thereby make it easier to calculate word-level statistics even when the + * word contents are unknown. */ private static final LogStatement LOGSTATEMENT_COMMITTEXT = - new LogStatement("CommitText", true, false, "committedText", "isBatchMode"); + new LogStatement("CommitText", true /* isPotentiallyPrivate */, + false /* isPotentiallyRevealing */, "committedText", "isBatchMode"); + private static final LogStatement LOGSTATEMENT_COMMITTEXT_EVENT_HAPPENED = + new LogStatement("CommitTextEventHappened", false /* isPotentiallyPrivate */, + false /* isPotentiallyRevealing */); private void enqueueCommitText(final String word, final boolean isBatchMode) { + // Event containing the word; will be published only if privacy checks pass enqueueEvent(LOGSTATEMENT_COMMITTEXT, word, isBatchMode); + // Event not containing the word; will always be published + enqueueEvent(LOGSTATEMENT_COMMITTEXT_EVENT_HAPPENED); } /** @@ -1766,17 +1844,26 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang SystemClock.uptimeMillis()); } + private static final LogStatement LOGSTATEMENT_LATINIME_HANDLEBACKSPACE = + new LogStatement("LatinIMEHandleBackspace", true, false, "numCharacters"); /** * Log a call to LatinIME.handleBackspace() that is not a batch delete. * * UserInput: The user is deleting one or more characters by hitting the backspace key once. * The covers single character deletes as well as deleting selections. + * + * @param numCharacters how many characters the backspace operation deleted + * @param shouldUncommitLogUnit whether to uncommit the last {@code LogUnit} in the + * {@code LogBuffer} */ - private static final LogStatement LOGSTATEMENT_LATINIME_HANDLEBACKSPACE = - new LogStatement("LatinIMEHandleBackspace", true, false, "numCharacters"); - public static void latinIME_handleBackspace(final int numCharacters) { + public static void latinIME_handleBackspace(final int numCharacters, + final boolean shouldUncommitLogUnit) { final ResearchLogger researchLogger = getInstance(); researchLogger.enqueueEvent(LOGSTATEMENT_LATINIME_HANDLEBACKSPACE, numCharacters); + if (shouldUncommitLogUnit) { + ResearchLogger.getInstance().uncommitCurrentLogUnit( + null, true /* dumpCurrentLogUnit */); + } } /** @@ -1794,6 +1881,8 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang numCharacters); researchLogger.mStatistics.recordGestureDelete(deletedText.length(), SystemClock.uptimeMillis()); + researchLogger.uncommitCurrentLogUnit(deletedText.toString(), + false /* dumpCurrentLogUnit */); } /** @@ -1837,6 +1926,20 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang } /** + * Call this method when the logging system has attempted publication of an n-gram. + * + * Statistics are gathered about the success or failure. + * + * @param publishabilityResultCode a result code as defined by + * {@code MainLogBuffer.PUBLISHABILITY_*} + */ + static void recordPublishabilityResultCode(final int publishabilityResultCode) { + final ResearchLogger researchLogger = getInstance(); + final Statistics statistics = researchLogger.mStatistics; + statistics.recordPublishabilityResultCode(publishabilityResultCode); + } + + /** * Log statistics. * * ContextualData, recorded at the end of a session. @@ -1848,7 +1951,11 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang "averageTimeDuringRepeatedDelete", "averageTimeAfterDelete", "dictionaryWordCount", "splitWordsCount", "gestureInputCount", "gestureCharsCount", "gesturesDeletedCount", "manualSuggestionsCount", - "revertCommitsCount", "correctedWordsCount", "autoCorrectionsCount"); + "revertCommitsCount", "correctedWordsCount", "autoCorrectionsCount", + "publishableCount", "unpublishableStoppingCount", + "unpublishableIncorrectWordCount", "unpublishableSampledTooRecentlyCount", + "unpublishableDictionaryUnavailableCount", "unpublishableMayContainDigitCount", + "unpublishableNotInDictionaryCount"); private static void logStatistics() { final ResearchLogger researchLogger = getInstance(); final Statistics statistics = researchLogger.mStatistics; @@ -1863,6 +1970,10 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang statistics.mGesturesInputCount, statistics.mGesturesCharsCount, statistics.mGesturesDeletedCount, statistics.mManualSuggestionsCount, statistics.mRevertCommitsCount, statistics.mCorrectedWordsCount, - statistics.mAutoCorrectionsCount); + statistics.mAutoCorrectionsCount, statistics.mPublishableCount, + statistics.mUnpublishableStoppingCount, statistics.mUnpublishableIncorrectWordCount, + statistics.mUnpublishableSampledTooRecently, + statistics.mUnpublishableDictionaryUnavailable, + statistics.mUnpublishableMayContainDigit, statistics.mUnpublishableNotInDictionary); } } |