diff options
Diffstat (limited to 'java/src')
39 files changed, 784 insertions, 414 deletions
diff --git a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java index 46caef625..2c87fc1e9 100644 --- a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java +++ b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java @@ -33,6 +33,7 @@ import java.util.Locale; public final class KeyCodeDescriptionMapper { private static final String TAG = KeyCodeDescriptionMapper.class.getSimpleName(); + private static final String SPOKEN_LETTER_RESOURCE_NAME_FORMAT = "spoken_accented_letter_%04X"; private static final String SPOKEN_EMOJI_RESOURCE_NAME_FORMAT = "spoken_emoji_%04X"; // The resource ID of the string spoken for obscured keys @@ -71,6 +72,15 @@ public final class KeyCodeDescriptionMapper { mKeyCodeMap.put(Constants.CODE_ACTION_PREVIOUS, R.string.spoken_description_action_previous); mKeyCodeMap.put(Constants.CODE_EMOJI, R.string.spoken_description_emoji); + // Because the upper-case and lower-case mappings of the following letters is depending on + // the locale, the upper case descriptions should be defined here. The lower case + // descriptions are handled in {@link #getSpokenLetterDescriptionId(Context,int)}. + // U+0049: "I" LATIN CAPITAL LETTER I + // U+0069: "i" LATIN SMALL LETTER I + // U+0130: "İ" LATIN CAPITAL LETTER I WITH DOT ABOVE + // U+0131: "ı" LATIN SMALL LETTER DOTLESS I + mKeyCodeMap.put(0x0049, R.string.spoken_letter_0049); + mKeyCodeMap.put(0x0130, R.string.spoken_letter_0130); } /** @@ -271,15 +281,19 @@ public final class KeyCodeDescriptionMapper { if (shouldObscure && isDefinedNonCtrl) { return context.getString(OBSCURED_KEY_RES_ID); } - if (mKeyCodeMap.indexOfKey(code) >= 0) { - return context.getString(mKeyCodeMap.get(code)); + final int index = mKeyCodeMap.indexOfKey(code); + if (index >= 0) { + return context.getString(mKeyCodeMap.valueAt(index)); } + final String accentedLetter = getSpokenAccentedLetterDescriptionId(context, code); + if (accentedLetter != null) { + return accentedLetter; + } + // Here, <code>code</code> may be a base letter. final int spokenEmojiId = getSpokenDescriptionId( context, code, SPOKEN_EMOJI_RESOURCE_NAME_FORMAT); if (spokenEmojiId != 0) { - final String spokenEmoji = context.getString(spokenEmojiId); - mKeyCodeMap.append(code, spokenEmojiId); - return spokenEmoji; + return context.getString(spokenEmojiId); } if (isDefinedNonCtrl) { return Character.toString((char) code); @@ -290,12 +304,31 @@ public final class KeyCodeDescriptionMapper { return context.getString(R.string.spoken_description_unknown, code); } - private static int getSpokenDescriptionId(final Context context, final int code, + private String getSpokenAccentedLetterDescriptionId(final Context context, final int code) { + final boolean isUpperCase = Character.isUpperCase(code); + final int baseCode = isUpperCase ? Character.toLowerCase(code) : code; + final int baseIndex = mKeyCodeMap.indexOfKey(baseCode); + final int resId = (baseIndex >= 0) ? mKeyCodeMap.valueAt(baseIndex) + : getSpokenDescriptionId(context, baseCode, SPOKEN_LETTER_RESOURCE_NAME_FORMAT); + if (resId == 0) { + return null; + } + final String spokenText = context.getString(resId); + return isUpperCase ? context.getString(R.string.spoke_description_upper_case, spokenText) + : spokenText; + } + + private int getSpokenDescriptionId(final Context context, final int code, final String resourceNameFormat) { final String resourceName = String.format(Locale.ROOT, resourceNameFormat, code); final Resources resources = context.getResources(); - final String packageName = resources.getResourcePackageName( + // Note that the resource package name may differ from the context package name. + final String resourcePackageName = resources.getResourcePackageName( R.string.spoken_description_unknown); - return resources.getIdentifier(resourceName, "string", packageName); + final int resId = resources.getIdentifier(resourceName, "string", resourcePackageName); + if (resId != 0) { + mKeyCodeMap.append(code, resId); + } + return resId; } } diff --git a/java/src/com/android/inputmethod/dictionarypack/ActionBatch.java b/java/src/com/android/inputmethod/dictionarypack/ActionBatch.java index 706bdea8e..3f69cedee 100644 --- a/java/src/com/android/inputmethod/dictionarypack/ActionBatch.java +++ b/java/src/com/android/inputmethod/dictionarypack/ActionBatch.java @@ -325,8 +325,9 @@ public final class ActionBatch { MetadataDbHelper.TYPE_BULK, MetadataDbHelper.STATUS_AVAILABLE, mWordList.mId, mWordList.mLocale, mWordList.mDescription, null == mWordList.mLocalFilename ? "" : mWordList.mLocalFilename, - mWordList.mRemoteFilename, mWordList.mLastUpdate, mWordList.mChecksum, - mWordList.mFileSize, mWordList.mVersion, mWordList.mFormatVersion); + mWordList.mRemoteFilename, mWordList.mLastUpdate, mWordList.mRawChecksum, + mWordList.mChecksum, mWordList.mFileSize, mWordList.mVersion, + mWordList.mFormatVersion); PrivateLog.log("Insert 'available' record for " + mWordList.mDescription + " and locale " + mWordList.mLocale); db.insert(MetadataDbHelper.METADATA_TABLE_NAME, null, values); @@ -374,7 +375,7 @@ public final class ActionBatch { final ContentValues values = MetadataDbHelper.makeContentValues(0, MetadataDbHelper.TYPE_BULK, MetadataDbHelper.STATUS_INSTALLED, mWordList.mId, mWordList.mLocale, mWordList.mDescription, - "", mWordList.mRemoteFilename, mWordList.mLastUpdate, + "", mWordList.mRemoteFilename, mWordList.mLastUpdate, mWordList.mRawChecksum, mWordList.mChecksum, mWordList.mFileSize, mWordList.mVersion, mWordList.mFormatVersion); PrivateLog.log("Insert 'preinstalled' record for " + mWordList.mDescription @@ -416,8 +417,9 @@ public final class ActionBatch { oldValues.getAsInteger(MetadataDbHelper.STATUS_COLUMN), mWordList.mId, mWordList.mLocale, mWordList.mDescription, oldValues.getAsString(MetadataDbHelper.LOCAL_FILENAME_COLUMN), - mWordList.mRemoteFilename, mWordList.mLastUpdate, mWordList.mChecksum, - mWordList.mFileSize, mWordList.mVersion, mWordList.mFormatVersion); + mWordList.mRemoteFilename, mWordList.mLastUpdate, mWordList.mRawChecksum, + mWordList.mChecksum, mWordList.mFileSize, mWordList.mVersion, + mWordList.mFormatVersion); PrivateLog.log("Updating record for " + mWordList.mDescription + " and locale " + mWordList.mLocale); db.update(MetadataDbHelper.METADATA_TABLE_NAME, values, diff --git a/java/src/com/android/inputmethod/dictionarypack/DictionaryProvider.java b/java/src/com/android/inputmethod/dictionarypack/DictionaryProvider.java index 80def701d..c35995b24 100644 --- a/java/src/com/android/inputmethod/dictionarypack/DictionaryProvider.java +++ b/java/src/com/android/inputmethod/dictionarypack/DictionaryProvider.java @@ -89,10 +89,13 @@ public final class DictionaryProvider extends ContentProvider { private static final class WordListInfo { public final String mId; public final String mLocale; + public final String mRawChecksum; public final int mMatchLevel; - public WordListInfo(final String id, final String locale, final int matchLevel) { + public WordListInfo(final String id, final String locale, final String rawChecksum, + final int matchLevel) { mId = id; mLocale = locale; + mRawChecksum = rawChecksum; mMatchLevel = matchLevel; } } @@ -106,7 +109,8 @@ public final class DictionaryProvider extends ContentProvider { private static final class ResourcePathCursor extends AbstractCursor { // Column names for the cursor returned by this content provider. - static private final String[] columnNames = { "id", "locale" }; + static private final String[] columnNames = { MetadataDbHelper.WORDLISTID_COLUMN, + MetadataDbHelper.LOCALE_COLUMN, MetadataDbHelper.RAW_CHECKSUM_COLUMN }; // The list of word lists served by this provider that match the client request. final WordListInfo[] mWordLists; @@ -141,6 +145,7 @@ public final class DictionaryProvider extends ContentProvider { switch (column) { case 0: return mWordLists[mPos].mId; case 1: return mWordLists[mPos].mLocale; + case 2: return mWordLists[mPos].mRawChecksum; default : return null; } } @@ -357,6 +362,8 @@ public final class DictionaryProvider extends ContentProvider { final int localeIndex = results.getColumnIndex(MetadataDbHelper.LOCALE_COLUMN); final int localFileNameIndex = results.getColumnIndex(MetadataDbHelper.LOCAL_FILENAME_COLUMN); + final int rawChecksumIndex = + results.getColumnIndex(MetadataDbHelper.RAW_CHECKSUM_COLUMN); final int statusIndex = results.getColumnIndex(MetadataDbHelper.STATUS_COLUMN); if (results.moveToFirst()) { do { @@ -379,6 +386,7 @@ public final class DictionaryProvider extends ContentProvider { } final String wordListLocale = results.getString(localeIndex); final String wordListLocalFilename = results.getString(localFileNameIndex); + final String wordListRawChecksum = results.getString(rawChecksumIndex); final int wordListStatus = results.getInt(statusIndex); // Test the requested locale against this wordlist locale. The requested locale // has to either match exactly or be more specific than the dictionary - a @@ -412,8 +420,8 @@ public final class DictionaryProvider extends ContentProvider { final WordListInfo currentBestMatch = dicts.get(wordListCategory); if (null == currentBestMatch || currentBestMatch.mMatchLevel < matchLevel) { - dicts.put(wordListCategory, - new WordListInfo(wordListId, wordListLocale, matchLevel)); + dicts.put(wordListCategory, new WordListInfo(wordListId, wordListLocale, + wordListRawChecksum, matchLevel)); } } while (results.moveToNext()); } diff --git a/java/src/com/android/inputmethod/dictionarypack/MD5Calculator.java b/java/src/com/android/inputmethod/dictionarypack/MD5Calculator.java index e47e86e4b..ccd054c84 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MD5Calculator.java +++ b/java/src/com/android/inputmethod/dictionarypack/MD5Calculator.java @@ -20,7 +20,7 @@ import java.io.InputStream; import java.io.IOException; import java.security.MessageDigest; -final class MD5Calculator { +public final class MD5Calculator { private MD5Calculator() {} // This helper class is not instantiable public static String checksum(final InputStream in) throws IOException { diff --git a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java index 4a8fa51ee..743bc8037 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java +++ b/java/src/com/android/inputmethod/dictionarypack/MetadataDbHelper.java @@ -20,6 +20,7 @@ import android.content.ContentValues; import android.content.Context; import android.database.Cursor; import android.database.sqlite.SQLiteDatabase; +import android.database.sqlite.SQLiteException; import android.database.sqlite.SQLiteOpenHelper; import android.text.TextUtils; import android.util.Log; @@ -46,7 +47,7 @@ public class MetadataDbHelper extends SQLiteOpenHelper { // used to identify the versions for upgrades. This should never change going forward. private static final int METADATA_DATABASE_VERSION_WITH_CLIENTID = 6; // The current database version. - private static final int CURRENT_METADATA_DATABASE_VERSION = 7; + private static final int CURRENT_METADATA_DATABASE_VERSION = 9; private final static long NOT_A_DOWNLOAD_ID = -1; @@ -66,7 +67,8 @@ public class MetadataDbHelper extends SQLiteOpenHelper { public static final String VERSION_COLUMN = "version"; public static final String FORMATVERSION_COLUMN = "formatversion"; public static final String FLAGS_COLUMN = "flags"; - public static final int COLUMN_COUNT = 13; + public static final String RAW_CHECKSUM_COLUMN = "rawChecksum"; + public static final int COLUMN_COUNT = 14; private static final String CLIENT_CLIENT_ID_COLUMN = "clientid"; private static final String CLIENT_METADATA_URI_COLUMN = "uri"; @@ -119,8 +121,9 @@ public class MetadataDbHelper extends SQLiteOpenHelper { + CHECKSUM_COLUMN + " TEXT, " + FILESIZE_COLUMN + " INTEGER, " + VERSION_COLUMN + " INTEGER," - + FORMATVERSION_COLUMN + " INTEGER," - + FLAGS_COLUMN + " INTEGER," + + FORMATVERSION_COLUMN + " INTEGER, " + + FLAGS_COLUMN + " INTEGER, " + + RAW_CHECKSUM_COLUMN + " TEXT," + "PRIMARY KEY (" + WORDLISTID_COLUMN + "," + VERSION_COLUMN + "));"; private static final String METADATA_CREATE_CLIENT_TABLE = "CREATE TABLE IF NOT EXISTS " + CLIENT_TABLE_NAME + " (" @@ -136,7 +139,8 @@ public class MetadataDbHelper extends SQLiteOpenHelper { static final String[] METADATA_TABLE_COLUMNS = { PENDINGID_COLUMN, TYPE_COLUMN, STATUS_COLUMN, WORDLISTID_COLUMN, LOCALE_COLUMN, DESCRIPTION_COLUMN, LOCAL_FILENAME_COLUMN, REMOTE_FILENAME_COLUMN, DATE_COLUMN, CHECKSUM_COLUMN, - FILESIZE_COLUMN, VERSION_COLUMN, FORMATVERSION_COLUMN, FLAGS_COLUMN }; + FILESIZE_COLUMN, VERSION_COLUMN, FORMATVERSION_COLUMN, FLAGS_COLUMN, + RAW_CHECKSUM_COLUMN }; // List of all client table columns. static final String[] CLIENT_TABLE_COLUMNS = { CLIENT_CLIENT_ID_COLUMN, CLIENT_METADATA_URI_COLUMN, CLIENT_PENDINGID_COLUMN, FLAGS_COLUMN }; @@ -215,6 +219,17 @@ public class MetadataDbHelper extends SQLiteOpenHelper { createClientTable(db); } + private void addRawChecksumColumnUnlessPresent(final SQLiteDatabase db, final String clientId) { + try { + db.execSQL("SELECT " + RAW_CHECKSUM_COLUMN + " FROM " + + METADATA_TABLE_NAME + " LIMIT 0;"); + } catch (SQLiteException e) { + Log.i(TAG, "No " + RAW_CHECKSUM_COLUMN + " column : creating it"); + db.execSQL("ALTER TABLE " + METADATA_TABLE_NAME + " ADD COLUMN " + + RAW_CHECKSUM_COLUMN + " TEXT;"); + } + } + /** * Upgrade the database. Upgrade from version 3 is supported. * Version 3 has a DB named METADATA_DATABASE_NAME_STEM containing a table METADATA_TABLE_NAME. @@ -260,6 +275,12 @@ public class MetadataDbHelper extends SQLiteOpenHelper { db.execSQL("DROP TABLE IF EXISTS " + CLIENT_TABLE_NAME); onCreate(db); } + // A rawChecksum column that did not exist in the previous versions was added that + // corresponds to the md5 checksum of the file after decompression/decryption. This is to + // strengthen the system against corrupted dictionary files. + // The most secure way to upgrade a database is to just test for the column presence, and + // add it if it's not there. + addRawChecksumColumnUnlessPresent(db, mClientId); } /** @@ -431,7 +452,7 @@ public class MetadataDbHelper extends SQLiteOpenHelper { public static ContentValues makeContentValues(final int pendingId, final int type, final int status, final String wordlistId, final String locale, final String description, final String filename, final String url, final long date, - final String checksum, final long filesize, final int version, + final String rawChecksum, final String checksum, final long filesize, final int version, final int formatVersion) { final ContentValues result = new ContentValues(COLUMN_COUNT); result.put(PENDINGID_COLUMN, pendingId); @@ -443,6 +464,7 @@ public class MetadataDbHelper extends SQLiteOpenHelper { result.put(LOCAL_FILENAME_COLUMN, filename); result.put(REMOTE_FILENAME_COLUMN, url); result.put(DATE_COLUMN, date); + result.put(RAW_CHECKSUM_COLUMN, rawChecksum); result.put(CHECKSUM_COLUMN, checksum); result.put(FILESIZE_COLUMN, filesize); result.put(VERSION_COLUMN, version); @@ -478,6 +500,8 @@ public class MetadataDbHelper extends SQLiteOpenHelper { if (null == result.get(REMOTE_FILENAME_COLUMN)) result.put(REMOTE_FILENAME_COLUMN, ""); // 0 for the update date : 1970/1/1. Unless specified. if (null == result.get(DATE_COLUMN)) result.put(DATE_COLUMN, 0); + // Raw checksum unknown unless specified + if (null == result.get(RAW_CHECKSUM_COLUMN)) result.put(RAW_CHECKSUM_COLUMN, ""); // Checksum unknown unless specified if (null == result.get(CHECKSUM_COLUMN)) result.put(CHECKSUM_COLUMN, ""); // No filesize unless specified @@ -525,6 +549,7 @@ public class MetadataDbHelper extends SQLiteOpenHelper { putStringResult(result, cursor, LOCAL_FILENAME_COLUMN); putStringResult(result, cursor, REMOTE_FILENAME_COLUMN); putIntResult(result, cursor, DATE_COLUMN); + putStringResult(result, cursor, RAW_CHECKSUM_COLUMN); putStringResult(result, cursor, CHECKSUM_COLUMN); putIntResult(result, cursor, FILESIZE_COLUMN); putIntResult(result, cursor, VERSION_COLUMN); diff --git a/java/src/com/android/inputmethod/dictionarypack/MetadataHandler.java b/java/src/com/android/inputmethod/dictionarypack/MetadataHandler.java index 5c2289911..63e419871 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MetadataHandler.java +++ b/java/src/com/android/inputmethod/dictionarypack/MetadataHandler.java @@ -52,6 +52,8 @@ public class MetadataHandler { final int idIndex = results.getColumnIndex(MetadataDbHelper.WORDLISTID_COLUMN); final int updateIndex = results.getColumnIndex(MetadataDbHelper.DATE_COLUMN); final int fileSizeIndex = results.getColumnIndex(MetadataDbHelper.FILESIZE_COLUMN); + final int rawChecksumIndex = + results.getColumnIndex(MetadataDbHelper.RAW_CHECKSUM_COLUMN); final int checksumIndex = results.getColumnIndex(MetadataDbHelper.CHECKSUM_COLUMN); final int localFilenameIndex = results.getColumnIndex(MetadataDbHelper.LOCAL_FILENAME_COLUMN); @@ -66,6 +68,7 @@ public class MetadataHandler { results.getString(descriptionColumn), results.getLong(updateIndex), results.getLong(fileSizeIndex), + results.getString(rawChecksumIndex), results.getString(checksumIndex), results.getString(localFilenameIndex), results.getString(remoteFilenameIndex), diff --git a/java/src/com/android/inputmethod/dictionarypack/MetadataParser.java b/java/src/com/android/inputmethod/dictionarypack/MetadataParser.java index 27670fddf..a88173e8e 100644 --- a/java/src/com/android/inputmethod/dictionarypack/MetadataParser.java +++ b/java/src/com/android/inputmethod/dictionarypack/MetadataParser.java @@ -37,6 +37,7 @@ public class MetadataParser { private static final String DESCRIPTION_FIELD_NAME = MetadataDbHelper.DESCRIPTION_COLUMN; private static final String UPDATE_FIELD_NAME = "update"; private static final String FILESIZE_FIELD_NAME = MetadataDbHelper.FILESIZE_COLUMN; + private static final String RAW_CHECKSUM_FIELD_NAME = MetadataDbHelper.RAW_CHECKSUM_COLUMN; private static final String CHECKSUM_FIELD_NAME = MetadataDbHelper.CHECKSUM_COLUMN; private static final String REMOTE_FILENAME_FIELD_NAME = MetadataDbHelper.REMOTE_FILENAME_COLUMN; @@ -80,6 +81,7 @@ public class MetadataParser { arguments.get(DESCRIPTION_FIELD_NAME), Long.parseLong(arguments.get(UPDATE_FIELD_NAME)), Long.parseLong(arguments.get(FILESIZE_FIELD_NAME)), + arguments.get(RAW_CHECKSUM_FIELD_NAME), arguments.get(CHECKSUM_FIELD_NAME), null, arguments.get(REMOTE_FILENAME_FIELD_NAME), diff --git a/java/src/com/android/inputmethod/dictionarypack/WordListMetadata.java b/java/src/com/android/inputmethod/dictionarypack/WordListMetadata.java index 69bff9597..9e510a68b 100644 --- a/java/src/com/android/inputmethod/dictionarypack/WordListMetadata.java +++ b/java/src/com/android/inputmethod/dictionarypack/WordListMetadata.java @@ -30,6 +30,7 @@ public class WordListMetadata { public final String mDescription; public final long mLastUpdate; public final long mFileSize; + public final String mRawChecksum; public final String mChecksum; public final String mLocalFilename; public final String mRemoteFilename; @@ -50,13 +51,15 @@ public class WordListMetadata { public WordListMetadata(final String id, final int type, final String description, final long lastUpdate, final long fileSize, - final String checksum, final String localFilename, final String remoteFilename, - final int version, final int formatVersion, final int flags, final String locale) { + final String rawChecksum, final String checksum, final String localFilename, + final String remoteFilename, final int version, final int formatVersion, + final int flags, final String locale) { mId = id; mType = type; mDescription = description; mLastUpdate = lastUpdate; // In milliseconds mFileSize = fileSize; + mRawChecksum = rawChecksum; mChecksum = checksum; mLocalFilename = localFilename; mRemoteFilename = remoteFilename; @@ -77,6 +80,7 @@ public class WordListMetadata { final String description = values.getAsString(MetadataDbHelper.DESCRIPTION_COLUMN); final Long lastUpdate = values.getAsLong(MetadataDbHelper.DATE_COLUMN); final Long fileSize = values.getAsLong(MetadataDbHelper.FILESIZE_COLUMN); + final String rawChecksum = values.getAsString(MetadataDbHelper.RAW_CHECKSUM_COLUMN); final String checksum = values.getAsString(MetadataDbHelper.CHECKSUM_COLUMN); final String localFilename = values.getAsString(MetadataDbHelper.LOCAL_FILENAME_COLUMN); final String remoteFilename = values.getAsString(MetadataDbHelper.REMOTE_FILENAME_COLUMN); @@ -98,8 +102,8 @@ public class WordListMetadata { || null == locale) { throw new IllegalArgumentException(); } - return new WordListMetadata(id, type, description, lastUpdate, fileSize, checksum, - localFilename, remoteFilename, version, formatVersion, flags, locale); + return new WordListMetadata(id, type, description, lastUpdate, fileSize, rawChecksum, + checksum, localFilename, remoteFilename, version, formatVersion, flags, locale); } @Override @@ -110,6 +114,7 @@ public class WordListMetadata { sb.append("\nDescription : ").append(mDescription); sb.append("\nLastUpdate : ").append(mLastUpdate); sb.append("\nFileSize : ").append(mFileSize); + sb.append("\nRawChecksum : ").append(mRawChecksum); sb.append("\nChecksum : ").append(mChecksum); sb.append("\nLocalFilename : ").append(mLocalFilename); sb.append("\nRemoteFilename : ").append(mRemoteFilename); diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java index 65242dd76..4a2b37e4c 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java @@ -130,7 +130,7 @@ public class MoreKeysKeyboardView extends KeyboardView implements MoreKeysPanel public void onUpEvent(final int x, final int y, final int pointerId, final long eventTime) { if (mCurrentKey != null && mActivePointerId == pointerId) { updateReleaseKeyGraphics(mCurrentKey); - onCodeInput(mCurrentKey.getCode(), x, y); + onKeyInput(mCurrentKey, x, y); mCurrentKey = null; } } @@ -138,7 +138,8 @@ public class MoreKeysKeyboardView extends KeyboardView implements MoreKeysPanel /** * Performs the specific action for this panel when the user presses a key on the panel. */ - protected void onCodeInput(final int code, final int x, final int y) { + protected void onKeyInput(final Key key, final int x, final int y) { + final int code = key.getCode(); if (code == Constants.CODE_OUTPUT_TEXT) { mListener.onTextInput(mCurrentKey.getOutputText()); } else if (code != Constants.CODE_UNSPECIFIED) { diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index b8cf3f89c..b77540622 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -191,7 +191,8 @@ public final class BinaryDictionary extends Dictionary { private static native void closeNative(long dict); private static native int getFormatVersionNative(long dict); private static native int getProbabilityNative(long dict, int[] word); - private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1); + private static native int getBigramProbabilityNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1); private static native void getWordPropertyNative(long dict, int[] word, int[] outCodePoints, boolean[] outFlags, int[] outProbabilityInfo, ArrayList<int[]> outBigramTargets, ArrayList<int[]> outBigramProbabilityInfo, @@ -200,15 +201,17 @@ public final class BinaryDictionary extends Dictionary { private static native void getSuggestionsNative(long dict, long proximityInfo, long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, - int[] prevWordCodePointArray, int[] outputSuggestionCount, int[] outputCodePoints, - int[] outputScores, int[] outputIndices, int[] outputTypes, - int[] outputAutoCommitFirstWordConfidence, float[] inOutLanguageWeight); + int[] prevWordCodePointArray, boolean isBeginningOfSentence, + int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores, + int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence, + float[] inOutLanguageWeight); private static native void addUnigramWordNative(long dict, int[] word, int probability, - int[] shortcutTarget, int shortcutProbability, boolean isNotAWord, - boolean isBlacklisted, int timestamp); - private static native void addBigramWordsNative(long dict, int[] word0, int[] word1, - int probability, int timestamp); - private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); + int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, + boolean isNotAWord, boolean isBlacklisted, int timestamp); + private static native void addBigramWordsNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1, int probability, int timestamp); + private static native void removeBigramWordsNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1); private static native int addMultipleDictionaryEntriesNative(long dict, LanguageModelParam[] languageModelParams, int startIndex); private static native String getPropertyNative(long dict, String query); @@ -301,7 +304,8 @@ public final class BinaryDictionary extends Dictionary { getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), inputPointers.getYCoordinates(), inputPointers.getTimes(), inputPointers.getPointerIds(), mInputCodePoints, inputSize, - mNativeSuggestOptions.getOptions(), prevWordCodePointArray, mOutputSuggestionCount, + mNativeSuggestOptions.getOptions(), prevWordCodePointArray, + prevWordsInfo.mIsBeginningOfSentence, mOutputSuggestionCount, mOutputCodePoints, mOutputScores, mSpaceIndices, mOutputTypes, mOutputAutoCommitFirstWordConfidence, mInputOutputLanguageWeight); if (inOutLanguageWeight != null) { @@ -359,15 +363,18 @@ public final class BinaryDictionary extends Dictionary { } @UsedForTesting - public boolean isValidBigram(final String word0, final String word1) { - return getBigramProbability(word0, word1) != NOT_A_PROBABILITY; + public boolean isValidNgram(final PrevWordsInfo prevWordsInfo, final String word) { + return getNgramProbability(prevWordsInfo, word) != NOT_A_PROBABILITY; } - public int getBigramProbability(final String word0, final String word1) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) return NOT_A_PROBABILITY; - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); + public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { + return NOT_A_PROBABILITY; + } + final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + final int[] codePoints1 = StringUtils.toCodePointArray(word); + return getBigramProbabilityNative(mNativeDict, codePoints0, + prevWordsInfo.mIsBeginningOfSentence, codePoints1); } public WordProperty getWordProperty(final String word) { @@ -417,40 +424,44 @@ public final class BinaryDictionary extends Dictionary { } // Add a unigram entry to binary dictionary with unigram attributes in native code. - public void addUnigramWord(final String word, final int probability, - final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord, + public void addUnigramEntry(final String word, final int probability, + final String shortcutTarget, final int shortcutProbability, + final boolean isBeginningOfSentence, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { - if (TextUtils.isEmpty(word)) { + if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { return; } final int[] codePoints = StringUtils.toCodePointArray(word); final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? StringUtils.toCodePointArray(shortcutTarget) : null; addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, - shortcutProbability, isNotAWord, isBlacklisted, timestamp); + shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp); mHasUpdated = true; } - // Add a bigram entry to binary dictionary with timestamp in native code. - public void addBigramWords(final String word0, final String word1, final int probability, + // Add an n-gram entry to the binary dictionary with timestamp in native code. + public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word, + final int probability, final int timestamp) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return; } - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp); + final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + final int[] codePoints1 = StringUtils.toCodePointArray(word); + addBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, + codePoints1, probability, timestamp); mHasUpdated = true; } - // Remove a bigram entry form binary dictionary in native code. - public void removeBigramWords(final String word0, final String word1) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { + // Remove an n-gram entry from the binary dictionary in native code. + public void removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { return; } - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); + final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + final int[] codePoints1 = StringUtils.toCodePointArray(word); + removeBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, + codePoints1); mHasUpdated = true; } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java index e428b1d54..72757e086 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java @@ -28,6 +28,7 @@ import android.text.TextUtils; import android.util.Log; import com.android.inputmethod.dictionarypack.DictionaryPackConstants; +import com.android.inputmethod.dictionarypack.MD5Calculator; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils.DictionaryInfo; @@ -38,6 +39,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.Closeable; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -167,8 +169,9 @@ public final class BinaryDictionaryFileDumper { do { final String wordListId = cursor.getString(0); final String wordListLocale = cursor.getString(1); + final String wordListRawChecksum = cursor.getString(2); if (TextUtils.isEmpty(wordListId)) continue; - list.add(new WordListInfo(wordListId, wordListLocale)); + list.add(new WordListInfo(wordListId, wordListLocale, wordListRawChecksum)); } while (cursor.moveToNext()); return list; } catch (RemoteException e) { @@ -217,7 +220,8 @@ public final class BinaryDictionaryFileDumper { * and creating it (and its containing directory) if necessary. */ private static void cacheWordList(final String wordlistId, final String locale, - final ContentProviderClient providerClient, final Context context) { + final String rawChecksum, final ContentProviderClient providerClient, + final Context context) { final int COMPRESSED_CRYPTED_COMPRESSED = 0; final int CRYPTED_COMPRESSED = 1; final int COMPRESSED_CRYPTED = 2; @@ -299,6 +303,13 @@ public final class BinaryDictionaryFileDumper { checkMagicAndCopyFileTo(bufferedInputStream, bufferedOutputStream); bufferedOutputStream.flush(); bufferedOutputStream.close(); + final String actualRawChecksum = MD5Calculator.checksum( + new BufferedInputStream(new FileInputStream(outputFile))); + Log.i(TAG, "Computed checksum for downloaded dictionary. Expected = " + rawChecksum + + " ; actual = " + actualRawChecksum); + if (!TextUtils.isEmpty(rawChecksum) && !rawChecksum.equals(actualRawChecksum)) { + throw new IOException("Could not decode the file correctly : checksum differs"); + } final File finalFile = new File(finalFileName); finalFile.delete(); if (!outputFile.renameTo(finalFile)) { @@ -408,7 +419,7 @@ public final class BinaryDictionaryFileDumper { final List<WordListInfo> idList = getWordListWordListInfos(locale, context, hasDefaultWordList); for (WordListInfo id : idList) { - cacheWordList(id.mId, id.mLocale, providerClient, context); + cacheWordList(id.mId, id.mLocale, id.mRawChecksum, providerClient, context); } } finally { providerClient.release(); diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index 67ca59540..efc5a618b 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -192,7 +192,6 @@ public final class Constants { public static final int CODE_SPACE = ' '; public static final int CODE_PERIOD = '.'; public static final int CODE_COMMA = ','; - public static final int CODE_ARMENIAN_PERIOD = 0x0589; public static final int CODE_DASH = '-'; public static final int CODE_SINGLE_QUOTE = '\''; public static final int CODE_DOUBLE_QUOTE = '"'; @@ -208,6 +207,8 @@ public final class Constants { public static final int CODE_CLOSING_SQUARE_BRACKET = ']'; public static final int CODE_CLOSING_CURLY_BRACKET = '}'; public static final int CODE_CLOSING_ANGLE_BRACKET = '>'; + public static final int CODE_INVERTED_QUESTION_MARK = 0xBF; // ¿ + public static final int CODE_INVERTED_EXCLAMATION_MARK = 0xA1; // ¡ /** * Special keys code. Must be negative. diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index e04fcda27..3fb76b142 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -142,7 +142,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { Log.d(TAG, "loadAccountVocabulary: " + word); } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, + addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } @@ -224,7 +224,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { */ private void addNameLocked(final String name) { int len = StringUtils.codePointCount(name); - String prevWord = null; + PrevWordsInfo prevWordsInfo = new PrevWordsInfo(null); // TODO: Better tokenization for non-Latin writing systems for (int i = 0; i < len; i++) { if (Character.isLetter(name.codePointAt(i))) { @@ -239,19 +239,19 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { final int wordLen = StringUtils.codePointCount(word); if (wordLen < MAX_WORD_LENGTH && wordLen > 1) { if (DEBUG) { - Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord); + Log.d(TAG, "addName " + name + ", " + word + ", " + + prevWordsInfo.mPrevWord); } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, + addUnigramLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); - if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) { + if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && mUseFirstLastBigrams) { runGCIfRequiredLocked(true /* mindsBlockByGC */); - addBigramDynamicallyLocked(prevWord, word, - FREQUENCY_FOR_CONTACTS_BIGRAM, + addNgramEntryLocked(prevWordsInfo, word, FREQUENCY_FOR_CONTACTS_BIGRAM, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } - prevWord = word; + prevWordsInfo = new PrevWordsInfo(word); } } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index 14c8bb6c3..212363895 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -19,14 +19,18 @@ package com.android.inputmethod.latin; import android.content.Context; import android.text.TextUtils; import android.util.Log; +import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.ContextualDictionary; +import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.UserHistoryDictionary; +import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.DistracterFilter; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.SuggestionResults; @@ -37,6 +41,7 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -45,8 +50,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; // TODO: Consolidate dictionaries in native code. -public class DictionaryFacilitatorForSuggest { - public static final String TAG = DictionaryFacilitatorForSuggest.class.getSimpleName(); +public class DictionaryFacilitator { + public static final String TAG = DictionaryFacilitator.class.getSimpleName(); // HACK: This threshold is being used when adding a capitalized entry in the User History // dictionary. @@ -57,6 +62,7 @@ public class DictionaryFacilitatorForSuggest { private volatile CountDownLatch mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0); // To synchronize assigning mDictionaries to ensure closing dictionaries. private final Object mLock = new Object(); + private final DistracterFilter mDistracterFilter; private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTION = new String[] { @@ -162,7 +168,17 @@ public class DictionaryFacilitatorForSuggest { public void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable); } - public DictionaryFacilitatorForSuggest() {} + public DictionaryFacilitator() { + mDistracterFilter = new DistracterFilter.EmptyDistracterFilter(); + } + + public DictionaryFacilitator(final DistracterFilter distracterFilter) { + mDistracterFilter = distracterFilter; + } + + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { + mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); + } public Locale getLocale() { return mDictionaries.mLocale; @@ -321,6 +337,7 @@ public class DictionaryFacilitatorForSuggest { for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTION) { dictionaries.closeDict(dictType); } + mDistracterFilter.close(); } // The main dictionary could have been loaded asynchronously. Don't cache the return value @@ -370,22 +387,23 @@ public class DictionaryFacilitatorForSuggest { } public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized, - final String previousWord, final int timeStampInSeconds, + final PrevWordsInfo prevWordsInfo, final int timeStampInSeconds, final boolean blockPotentiallyOffensive) { final Dictionaries dictionaries = mDictionaries; final String[] words = suggestion.split(Constants.WORD_SEPARATOR); for (int i = 0; i < words.length; i++) { final String currentWord = words[i]; - final String prevWord = (i == 0) ? previousWord : words[i - 1]; + final PrevWordsInfo prevWordsInfoForCurrentWord = + (i == 0) ? prevWordsInfo : new PrevWordsInfo(words[i - 1]); final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false; - addWordToUserHistory(dictionaries, prevWord, currentWord, + addWordToUserHistory(dictionaries, prevWordsInfoForCurrentWord, currentWord, wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive); } } - private void addWordToUserHistory(final Dictionaries dictionaries, final String prevWord, - final String word, final boolean wasAutoCapitalized, final int timeStampInSeconds, - final boolean blockPotentiallyOffensive) { + private void addWordToUserHistory(final Dictionaries dictionaries, + final PrevWordsInfo prevWordsInfo, final String word, final boolean wasAutoCapitalized, + final int timeStampInSeconds, final boolean blockPotentiallyOffensive) { final ExpandableBinaryDictionary userHistoryDictionary = dictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY); if (userHistoryDictionary == null) { @@ -430,15 +448,16 @@ public class DictionaryFacilitatorForSuggest { // We demote unrecognized words (frequency < 0, below) by specifying them as "invalid". // We don't add words with 0-frequency (assuming they would be profanity etc.). final boolean isValid = maxFreq > 0; - UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWord, secondWord, + UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord, isValid, timeStampInSeconds); } - public void cancelAddingUserHistory(final String previousWord, final String committedWord) { + public void cancelAddingUserHistory(final PrevWordsInfo prevWordsInfo, + final String committedWord) { final ExpandableBinaryDictionary userHistoryDictionary = mDictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY); if (userHistoryDictionary != null) { - userHistoryDictionary.removeBigramDynamically(previousWord, committedWord); + userHistoryDictionary.removeNgramDynamically(prevWordsInfo, committedWord); } } @@ -535,9 +554,16 @@ public class DictionaryFacilitatorForSuggest { personalizationDict.clear(); } - public void addMultipleDictionaryEntriesToPersonalizationDictionary( - final ArrayList<LanguageModelParam> languageModelParams, + public void addEntriesToPersonalizationDictionary( + final PersonalizationDataChunk personalizationDataChunk, + final SpacingAndPunctuations spacingAndPunctuations, final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { + final ArrayList<LanguageModelParam> languageModelParams = + LanguageModelParam.createLanguageModelParamsFrom( + personalizationDataChunk.mTokens, + personalizationDataChunk.mTimestampInSeconds, + this /* dictionaryFacilitator */, spacingAndPunctuations, + mDistracterFilter); final ExpandableBinaryDictionary personalizationDict = mDictionaries.getSubDict(Dictionary.TYPE_PERSONALIZATION); if (personalizationDict == null || languageModelParams == null diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 629f3fd18..2cbce045d 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -114,7 +114,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private boolean needsToMigrateDictionary(final int formatVersion) { // When we bump up the dictionary format version, the old version should be added to here // for supporting migration. Note that native code has to support reading such formats. - return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING; + return formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING + || formatVersion == FormatSpec.VERSION401; } public boolean isValidDictionaryLocked() { @@ -269,9 +270,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. + * Adds unigram information of a word to the dictionary. May overwrite an existing entry. */ - public void addWordDynamically(final String word, final int frequency, + public void addUnigramEntry(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { reloadDictionaryIfRequired(); @@ -282,23 +283,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return; } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq, + addUnigramLocked(word, frequency, shortcutTarget, shortcutFreq, isNotAWord, isBlacklisted, timestamp); } }); } - protected void addWordDynamicallyLocked(final String word, final int frequency, + protected void addUnigramLocked(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { - mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, - isNotAWord, isBlacklisted, timestamp); + mBinaryDictionary.addUnigramEntry(word, frequency, shortcutTarget, shortcutFreq, + false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, timestamp); } /** - * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. + * Adds n-gram information of a word to the dictionary. May overwrite an existing entry. */ - public void addBigramDynamically(final String word0, final String word1, + public void addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word, final int frequency, final int timestamp) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { @@ -308,20 +309,20 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return; } runGCIfRequiredLocked(true /* mindsBlockByGC */); - addBigramDynamicallyLocked(word0, word1, frequency, timestamp); + addNgramEntryLocked(prevWordsInfo, word, frequency, timestamp); } }); } - protected void addBigramDynamicallyLocked(final String word0, final String word1, + protected void addNgramEntryLocked(final PrevWordsInfo prevWordsInfo, final String word, final int frequency, final int timestamp) { - mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + mBinaryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp); } /** - * Dynamically remove a word bigram in the dictionary. + * Dynamically remove the n-gram entry in the dictionary. */ - public void removeBigramDynamically(final String word0, final String word1) { + public void removeNgramDynamically(final PrevWordsInfo prevWordsInfo, final String word1) { reloadDictionaryIfRequired(); asyncExecuteTaskWithWriteLock(new Runnable() { @Override @@ -330,7 +331,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return; } runGCIfRequiredLocked(true /* mindsBlockByGC */); - mBinaryDictionary.removeBigramWords(word0, word1); + mBinaryDictionary.removeNgramEntry(prevWordsInfo, word1); } }); } @@ -428,9 +429,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return mBinaryDictionary.isValidWord(word); } - protected boolean isValidBigramLocked(final String word1, final String word2) { + protected boolean isValidNgramLocked(final PrevWordsInfo prevWordsInfo, final String word) { if (mBinaryDictionary == null) return false; - return mBinaryDictionary.isValidBigram(word1, word2); + return mBinaryDictionary.isValidNgram(prevWordsInfo, word); } /** diff --git a/java/src/com/android/inputmethod/latin/LastComposedWord.java b/java/src/com/android/inputmethod/latin/LastComposedWord.java index 232bf7407..9caec3e01 100644 --- a/java/src/com/android/inputmethod/latin/LastComposedWord.java +++ b/java/src/com/android/inputmethod/latin/LastComposedWord.java @@ -48,7 +48,7 @@ public final class LastComposedWord { public final String mTypedWord; public final CharSequence mCommittedWord; public final String mSeparatorString; - public final String mPrevWord; + public final PrevWordsInfo mPrevWordsInfo; public final int mCapitalizedMode; public final InputPointers mInputPointers = new InputPointers(Constants.DICTIONARY_MAX_WORD_LENGTH); @@ -64,7 +64,7 @@ public final class LastComposedWord { public LastComposedWord(final ArrayList<Event> events, final InputPointers inputPointers, final String typedWord, final CharSequence committedWord, final String separatorString, - final String prevWord, final int capitalizedMode) { + final PrevWordsInfo prevWordsInfo, final int capitalizedMode) { if (inputPointers != null) { mInputPointers.copy(inputPointers); } @@ -73,7 +73,7 @@ public final class LastComposedWord { mCommittedWord = committedWord; mSeparatorString = separatorString; mActive = true; - mPrevWord = prevWord; + mPrevWordsInfo = prevWordsInfo; mCapitalizedMode = capitalizedMode; } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 5e45275f8..34d5f714c 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -81,10 +81,10 @@ import com.android.inputmethod.latin.suggestions.SuggestionStripView; import com.android.inputmethod.latin.suggestions.SuggestionStripViewAccessor; import com.android.inputmethod.latin.utils.ApplicationUtils; import com.android.inputmethod.latin.utils.CapsModeUtils; +import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CoordinateUtils; import com.android.inputmethod.latin.utils.DialogUtils; -import com.android.inputmethod.latin.utils.DistracterFilter; -import com.android.inputmethod.latin.utils.DistracterFilterUtils; +import com.android.inputmethod.latin.utils.DistracterFilterUsingSuggestion; import com.android.inputmethod.latin.utils.ImportantNoticeUtils; import com.android.inputmethod.latin.utils.IntentUtils; import com.android.inputmethod.latin.utils.JniUtils; @@ -96,6 +96,7 @@ import com.android.inputmethod.research.ResearchLogger; import java.io.FileDescriptor; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -104,7 +105,7 @@ import java.util.concurrent.TimeUnit; */ public class LatinIME extends InputMethodService implements KeyboardActionListener, SuggestionStripView.Listener, SuggestionStripViewAccessor, - DictionaryFacilitatorForSuggest.DictionaryInitializationListener, + DictionaryFacilitator.DictionaryInitializationListener, ImportantNoticeDialog.ImportantNoticeDialogListener { private static final String TAG = LatinIME.class.getSimpleName(); private static final boolean TRACE = false; @@ -123,8 +124,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private static final String SCHEME_PACKAGE = "package"; private final Settings mSettings; + private final DictionaryFacilitator mDictionaryFacilitator = + new DictionaryFacilitator(new DistracterFilterUsingSuggestion(this /* context */)); private final InputLogic mInputLogic = new InputLogic(this /* LatinIME */, - this /* SuggestionStripViewAccessor */); + this /* SuggestionStripViewAccessor */, mDictionaryFacilitator); // We expect to have only one decoder in almost all cases, hence the default capacity of 1. // If it turns out we need several, it will get grown seamlessly. final SparseArray<HardwareEventDecoder> mHardwareEventDecoders @@ -494,8 +497,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.getInstance().init(this, mKeyboardSwitcher); - ResearchLogger.getInstance().initDictionary( - mInputLogic.mSuggest.mDictionaryFacilitator); + ResearchLogger.getInstance().initDictionary(mDictionaryFacilitator); } // Register to receive ringer mode change and network state change. @@ -539,13 +541,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!mHandler.hasPendingReopenDictionaries()) { resetSuggestForLocale(locale); } + mDictionaryFacilitator.updateEnabledSubtypes(mRichImm.getMyEnabledInputMethodSubtypeList( + true /* allowsImplicitlySelectedSubtypes */)); refreshPersonalizationDictionarySession(); StatsUtils.onLoadSettings(currentSettingsValues); } private void refreshPersonalizationDictionarySession() { - final DictionaryFacilitatorForSuggest dictionaryFacilitator = - mInputLogic.mSuggest.mDictionaryFacilitator; final boolean shouldKeepUserHistoryDictionaries; final boolean shouldKeepPersonalizationDictionaries; if (mSettings.getCurrent().mUsePersonalizedDicts) { @@ -560,16 +562,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!shouldKeepUserHistoryDictionaries) { // Remove user history dictionaries. PersonalizationHelper.removeAllUserHistoryDictionaries(this); - dictionaryFacilitator.clearUserHistoryDictionary(); + mDictionaryFacilitator.clearUserHistoryDictionary(); } if (!shouldKeepPersonalizationDictionaries) { // Remove personalization dictionaries. PersonalizationHelper.removeAllPersonalizationDictionaries(this); PersonalizationDictionarySessionRegistrar.resetAll(this); } else { - final DistracterFilter distracterFilter = createDistracterFilter(); - PersonalizationDictionarySessionRegistrar.init( - this, dictionaryFacilitator, distracterFilter); + PersonalizationDictionarySessionRegistrar.init(this, mDictionaryFacilitator); } } @@ -607,10 +607,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen * @param locale the locale */ private void resetSuggestForLocale(final Locale locale) { - final DictionaryFacilitatorForSuggest dictionaryFacilitator = - mInputLogic.mSuggest.mDictionaryFacilitator; final SettingsValues settingsValues = mSettings.getCurrent(); - dictionaryFacilitator.resetDictionaries(this /* context */, locale, + mDictionaryFacilitator.resetDictionaries(this /* context */, locale, settingsValues.mUseContactsDict, settingsValues.mUsePersonalizedDicts, false /* forceReloadMainDictionary */, this); if (settingsValues.mCorrectionEnabled) { @@ -623,17 +621,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen * Reset suggest by loading the main dictionary of the current locale. */ /* package private */ void resetSuggestMainDict() { - final DictionaryFacilitatorForSuggest dictionaryFacilitator = - mInputLogic.mSuggest.mDictionaryFacilitator; final SettingsValues settingsValues = mSettings.getCurrent(); - dictionaryFacilitator.resetDictionaries(this /* context */, - dictionaryFacilitator.getLocale(), settingsValues.mUseContactsDict, + mDictionaryFacilitator.resetDictionaries(this /* context */, + mDictionaryFacilitator.getLocale(), settingsValues.mUseContactsDict, settingsValues.mUsePersonalizedDicts, true /* forceReloadMainDictionary */, this); } @Override public void onDestroy() { - mInputLogic.mSuggest.mDictionaryFacilitator.closeDictionaries(); + mDictionaryFacilitator.closeDictionaries(); mSettings.onDestroy(); unregisterReceiver(mConnectivityAndRingerModeChangeReceiver); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -667,9 +663,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mInputLogic.mConnection.finishComposingText(); mInputLogic.mConnection.endBatchEdit(); } - final DistracterFilter distracterFilter = createDistracterFilter(); PersonalizationDictionarySessionRegistrar.onConfigurationChanged(this, conf, - mInputLogic.mSuggest.mDictionaryFacilitator, distracterFilter); + mDictionaryFacilitator); super.onConfigurationChanged(conf); } @@ -842,7 +837,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen currentSettingsValues = mSettings.getCurrent(); if (currentSettingsValues.mCorrectionEnabled) { - suggest.setAutoCorrectionThreshold(currentSettingsValues.mAutoCorrectionThreshold); + suggest.setAutoCorrectionThreshold( + currentSettingsValues.mAutoCorrectionThreshold); } switcher.loadKeyboard(editorInfo, currentSettingsValues, getCurrentAutoCapsState(), @@ -871,7 +867,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mHandler.cancelUpdateSuggestionStrip(); mainKeyboardView.setMainDictionaryAvailability( - suggest.mDictionaryFacilitator.hasInitializedMainDictionary()); + mDictionaryFacilitator.hasInitializedMainDictionary()); mainKeyboardView.setKeyPreviewPopupEnabled(currentSettingsValues.mKeyPreviewPopupOn, currentSettingsValues.mKeyPreviewPopupDismissDelay); mainKeyboardView.setSlidingKeyInputPreviewEnabled( @@ -1168,8 +1164,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { wordToEdit = word; } - mInputLogic.mSuggest.mDictionaryFacilitator.addWordToUserDictionary( - this /* context */, wordToEdit); + mDictionaryFacilitator.addWordToUserDictionary(this /* context */, wordToEdit); } // Callback for the {@link SuggestionStripView}, to call when the important notice strip is @@ -1435,12 +1430,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // We're checking the previous word in the text field against the memorized previous // word. If we are composing a word we should have the second word before the cursor // memorized, otherwise we should have the first. - final CharSequence rereadPrevWord = mInputLogic.getNthPreviousWordForSuggestion( - currentSettings.mSpacingAndPunctuations, - mInputLogic.mWordComposer.isComposingWord() ? 2 : 1); - if (!TextUtils.equals(prevWordsInfo.mPrevWord, rereadPrevWord)) { + final PrevWordsInfo rereadPrevWordsInfo = + mInputLogic.getPrevWordsInfoFromNthPreviousWordForSuggestion( + currentSettings.mSpacingAndPunctuations, + mInputLogic.mWordComposer.isComposingWord() ? 2 : 1); + if (!TextUtils.equals(prevWordsInfo.mPrevWord, rereadPrevWordsInfo.mPrevWord)) { throw new RuntimeException("Unexpected previous word: " - + prevWordsInfo.mPrevWord + " <> " + rereadPrevWord); + + prevWordsInfo.mPrevWord + " <> " + rereadPrevWordsInfo.mPrevWord); } } } @@ -1725,15 +1721,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @UsedForTesting /* package for test */ void waitForLoadingDictionaries(final long timeout, final TimeUnit unit) throws InterruptedException { - mInputLogic.mSuggest.mDictionaryFacilitator.waitForLoadingDictionariesForTesting( - timeout, unit); + mDictionaryFacilitator.waitForLoadingDictionariesForTesting(timeout, unit); } // DO NOT USE THIS for any other purpose than testing. This can break the keyboard badly. @UsedForTesting /* package for test */ void replaceDictionariesForTest(final Locale locale) { final SettingsValues settingsValues = mSettings.getCurrent(); - mInputLogic.mSuggest.mDictionaryFacilitator.resetDictionaries(this, locale, + mDictionaryFacilitator.resetDictionaries(this, locale, settingsValues.mUseContactsDict, settingsValues.mUsePersonalizedDicts, false /* forceReloadMainDictionary */, this /* listener */); } @@ -1741,22 +1736,21 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // DO NOT USE THIS for any other purpose than testing. @UsedForTesting /* package for test */ void clearPersonalizedDictionariesForTest() { - mInputLogic.mSuggest.mDictionaryFacilitator.clearUserHistoryDictionary(); - mInputLogic.mSuggest.mDictionaryFacilitator.clearPersonalizationDictionary(); + mDictionaryFacilitator.clearUserHistoryDictionary(); + mDictionaryFacilitator.clearPersonalizationDictionary(); } @UsedForTesting - /* package for test */ DistracterFilter createDistracterFilter() { - return DistracterFilterUtils.createDistracterFilter(this /* Context */, mKeyboardSwitcher); + /* package for test */ List<InputMethodSubtype> getEnabledSubtypesForTest() { + return (mRichImm != null) ? mRichImm.getMyEnabledInputMethodSubtypeList( + true /* allowsImplicitlySelectedSubtypes */) : new ArrayList<InputMethodSubtype>(); } public void dumpDictionaryForDebug(final String dictName) { - final DictionaryFacilitatorForSuggest dictionaryFacilitator = - mInputLogic.mSuggest.mDictionaryFacilitator; - if (dictionaryFacilitator.getLocale() == null) { + if (mDictionaryFacilitator.getLocale() == null) { resetSuggest(); } - mInputLogic.mSuggest.mDictionaryFacilitator.dumpDictionaryForDebug(dictName); + mDictionaryFacilitator.dumpDictionaryForDebug(dictName); } public void debugDumpStateAndCrashWithException(final String context) { diff --git a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java index 9d8543183..3494d16f7 100644 --- a/java/src/com/android/inputmethod/latin/PrevWordsInfo.java +++ b/java/src/com/android/inputmethod/latin/PrevWordsInfo.java @@ -16,14 +16,34 @@ package com.android.inputmethod.latin; +import android.util.Log; + +// TODO: Support multiple previous words for n-gram. public class PrevWordsInfo { + public static final PrevWordsInfo BEGINNING_OF_SENTENCE = new PrevWordsInfo(); + // The previous word. May be null after resetting and before starting a new composing word, or // when there is no context like at the start of text for example. It can also be set to null // externally when the user enters a separator that does not let bigrams across, like a period // or a comma. public final String mPrevWord; + // TODO: Have sentence separator. + // Whether the current context is beginning of sentence or not. + public final boolean mIsBeginningOfSentence; + + // Beginning of sentence. + public PrevWordsInfo() { + mPrevWord = ""; + mIsBeginningOfSentence = true; + } + public PrevWordsInfo(final String prevWord) { mPrevWord = prevWord; + mIsBeginningOfSentence = false; + } + + public boolean isValid() { + return mPrevWord != null; } } diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 606bb775e..2c54e10aa 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -538,10 +538,12 @@ public final class RichInputConnection { } @SuppressWarnings("unused") - public String getNthPreviousWord(final SpacingAndPunctuations spacingAndPunctuations, - final int n) { + public PrevWordsInfo getPrevWordsInfoFromNthPreviousWord( + final SpacingAndPunctuations spacingAndPunctuations, final int n) { mIC = mParent.getCurrentInputConnection(); - if (null == mIC) return null; + if (null == mIC) { + return new PrevWordsInfo(null); + } final CharSequence prev = getTextBeforeCursor(LOOKBACK_CHARACTER_NUM, 0); if (DEBUG_PREVIOUS_TEXT && null != prev) { final int checkLength = LOOKBACK_CHARACTER_NUM - 1; @@ -561,46 +563,57 @@ public final class RichInputConnection { } } } - return getNthPreviousWord(prev, spacingAndPunctuations, n); + return getPrevWordsInfoFromNthPreviousWord(prev, spacingAndPunctuations, n); } private static boolean isSeparator(final int code, final int[] sortedSeparators) { return Arrays.binarySearch(sortedSeparators, code) >= 0; } - // Get the nth word before cursor. n = 1 retrieves the word immediately before the cursor, - // n = 2 retrieves the word before that, and so on. This splits on whitespace only. + // Get information of the nth word before cursor. n = 1 retrieves the word immediately before + // the cursor, n = 2 retrieves the word before that, and so on. This splits on whitespace only. // Also, it won't return words that end in a separator (if the nth word before the cursor - // ends in a separator, it returns null). + // ends in a separator, it returns information represents beginning-of-sentence). // Example : // (n = 1) "abc def|" -> def // (n = 1) "abc def |" -> def - // (n = 1) "abc def. |" -> null - // (n = 1) "abc def . |" -> null + // (n = 1) "abc def. |" -> beginning-of-sentence + // (n = 1) "abc def . |" -> beginning-of-sentence // (n = 2) "abc def|" -> abc // (n = 2) "abc def |" -> abc // (n = 2) "abc def. |" -> abc // (n = 2) "abc def . |" -> def - // (n = 2) "abc|" -> null - // (n = 2) "abc |" -> null - // (n = 2) "abc. def|" -> null - public static String getNthPreviousWord(final CharSequence prev, + // (n = 2) "abc|" -> beginning-of-sentence + // (n = 2) "abc |" -> beginning-of-sentence + // (n = 2) "abc. def|" -> beginning-of-sentence + public static PrevWordsInfo getPrevWordsInfoFromNthPreviousWord(final CharSequence prev, final SpacingAndPunctuations spacingAndPunctuations, final int n) { - if (prev == null) return null; + if (prev == null) return new PrevWordsInfo(null); final String[] w = spaceRegex.split(prev); - // If we can't find n words, or we found an empty word, return null. - if (w.length < n) return null; + // If we can't find n words, or we found an empty word, the context is + // beginning-of-sentence. + if (w.length < n) { + return new PrevWordsInfo(); + } final String nthPrevWord = w[w.length - n]; final int length = nthPrevWord.length(); - if (length <= 0) return null; + if (length <= 0) { + return new PrevWordsInfo(); + } - // If ends in a separator, return null + // If ends in a sentence separator, the context is beginning-of-sentence. final char lastChar = nthPrevWord.charAt(length - 1); + if (spacingAndPunctuations.isSentenceSeparator(lastChar)) { + new PrevWordsInfo(); + } + // If ends in a word separator or connector, the context is unclear. + // TODO: Return meaningful context for this case. if (spacingAndPunctuations.isWordSeparator(lastChar) - || spacingAndPunctuations.isWordConnector(lastChar)) return null; - - return nthPrevWord; + || spacingAndPunctuations.isWordConnector(lastChar)) { + return new PrevWordsInfo(null); + } + return new PrevWordsInfo(nthPrevWord); } /** diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index e3759a586..daa7f4b47 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -53,11 +53,14 @@ public final class Suggest { private static final int SUPPRESS_SUGGEST_THRESHOLD = -2000000000; private static final boolean DBG = LatinImeLogger.sDBG; - public final DictionaryFacilitatorForSuggest mDictionaryFacilitator = - new DictionaryFacilitatorForSuggest(); + private final DictionaryFacilitator mDictionaryFacilitator; private float mAutoCorrectionThreshold; + public Suggest(final DictionaryFacilitator dictionaryFacilitator) { + mDictionaryFacilitator = dictionaryFacilitator; + } + public Locale getLocale() { return mDictionaryFacilitator.getLocale(); } @@ -112,7 +115,10 @@ public final class Suggest { additionalFeaturesOptions, SESSION_TYPING, rawSuggestions); final boolean isFirstCharCapitalized = wordComposer.isFirstCharCapitalized(); - final boolean isAllUpperCase = wordComposer.isAllUpperCase(); + // If resumed, then we don't want to upcase everything: resuming on a fully-capitalized + // words is rarely done to switch to another fully-capitalized word, but usually to a + // normal, non-capitalized suggestion. + final boolean isAllUpperCase = wordComposer.isAllUpperCase() && !wordComposer.isResumed(); final String firstSuggestion; final String whitelistedWord; if (suggestionResults.isEmpty()) { diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index c8ffbe443..b89ab84b2 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -258,12 +258,12 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { // Safeguard against adding really long words. if (word.length() < MAX_WORD_LENGTH) { runGCIfRequiredLocked(true /* mindsBlockByGC */); - addWordDynamicallyLocked(word, adjustedFrequency, null /* shortcutTarget */, + addUnigramLocked(word, adjustedFrequency, null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) { runGCIfRequiredLocked(true /* mindsBlockByGC */); - addWordDynamicallyLocked(shortcut, adjustedFrequency, word, + addUnigramLocked(shortcut, adjustedFrequency, word, USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */, false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); } diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index 227b42bde..6ecb37346 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -294,11 +294,10 @@ public final class WordComposer { * This will register NOT_A_COORDINATE for X and Ys, and use the passed keyboard for proximity. * @param codePoints the code points to set as the composing word. * @param coordinates the x, y coordinates of the key in the CoordinateUtils format - * @param previousWord the previous word, to use as context for suggestions. Can be null if - * the context is nil (typically, at start of text). + * @param prevWordsInfo the information of previous words, to use as context for suggestions */ public void setComposingWord(final int[] codePoints, final int[] coordinates, - final CharSequence previousWord) { + final PrevWordsInfo prevWordsInfo) { reset(); final int length = codePoints.length; for (int i = 0; i < length; ++i) { @@ -307,7 +306,7 @@ public final class WordComposer { CoordinateUtils.yFromArray(coordinates, i))); } mIsResumed = true; - mPrevWordsInfo = new PrevWordsInfo(null == previousWord ? null : previousWord.toString()); + mPrevWordsInfo = prevWordsInfo; } /** @@ -372,12 +371,12 @@ public final class WordComposer { * Also, batch input needs to know about the current caps mode to display correctly * capitalized suggestions. * @param mode the mode at the time of start - * @param previousWord the previous word as context for suggestions. May be null if none. + * @param prevWordsInfo the information of previous words */ public void setCapitalizedModeAndPreviousWordAtStartComposingTime(final int mode, - final CharSequence previousWord) { + final PrevWordsInfo prevWordsInfo) { mCapitalizedMode = mode; - mPrevWordsInfo = new PrevWordsInfo(null == previousWord ? null : previousWord.toString()); + mPrevWordsInfo = prevWordsInfo; } /** @@ -413,13 +412,13 @@ public final class WordComposer { // `type' should be one of the LastComposedWord.COMMIT_TYPE_* constants above. // committedWord should contain suggestion spans if applicable. public LastComposedWord commitWord(final int type, final CharSequence committedWord, - final String separatorString, final String prevWord) { + final String separatorString, final PrevWordsInfo prevWordsInfo) { // Note: currently, we come here whenever we commit a word. If it's a MANUAL_PICK // or a DECIDED_WORD we may cancel the commit later; otherwise, we should deactivate // the last composed word to ensure this does not happen. final LastComposedWord lastComposedWord = new LastComposedWord(mEvents, mInputPointers, mTypedWordCache.toString(), committedWord, separatorString, - prevWord, mCapitalizedMode); + prevWordsInfo, mCapitalizedMode); mInputPointers.reset(); if (type != LastComposedWord.COMMIT_TYPE_DECIDED_WORD && type != LastComposedWord.COMMIT_TYPE_MANUAL_PICK) { diff --git a/java/src/com/android/inputmethod/latin/WordListInfo.java b/java/src/com/android/inputmethod/latin/WordListInfo.java index 5ac806a0c..268fe9818 100644 --- a/java/src/com/android/inputmethod/latin/WordListInfo.java +++ b/java/src/com/android/inputmethod/latin/WordListInfo.java @@ -22,8 +22,10 @@ package com.android.inputmethod.latin; public final class WordListInfo { public final String mId; public final String mLocale; - public WordListInfo(final String id, final String locale) { + public final String mRawChecksum; + public WordListInfo(final String id, final String locale, final String rawChecksum) { mId = id; mLocale = locale; + mRawChecksum = rawChecksum; } } diff --git a/java/src/com/android/inputmethod/latin/define/ProductionFlag.java b/java/src/com/android/inputmethod/latin/define/ProductionFlag.java index af899c040..761f457ea 100644 --- a/java/src/com/android/inputmethod/latin/define/ProductionFlag.java +++ b/java/src/com/android/inputmethod/latin/define/ProductionFlag.java @@ -38,4 +38,7 @@ public final class ProductionFlag { // Include all suggestions from all dictionaries in {@link SuggestedWords#mRawSuggestions}. public static final boolean INCLUDE_RAW_SUGGESTIONS = false; + + // When false, the metrics logging is not yet ready to be enabled. + public static final boolean IS_METRICS_LOGGING_SUPPORTED = false; } diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index ea58abc14..188c18b73 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -32,11 +32,12 @@ import com.android.inputmethod.event.InputTransaction; import com.android.inputmethod.keyboard.KeyboardSwitcher; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; +import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.InputPointers; import com.android.inputmethod.latin.LastComposedWord; import com.android.inputmethod.latin.LatinIME; import com.android.inputmethod.latin.LatinImeLogger; +import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.RichInputConnection; import com.android.inputmethod.latin.Suggest; import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; @@ -78,7 +79,8 @@ public final class InputLogic { private int mSpaceState; // Never null public SuggestedWords mSuggestedWords = SuggestedWords.EMPTY; - public final Suggest mSuggest = new Suggest(); + public final Suggest mSuggest; + private final DictionaryFacilitator mDictionaryFacilitator; public LastComposedWord mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; public final WordComposer mWordComposer; @@ -101,14 +103,19 @@ public final class InputLogic { * Create a new instance of the input logic. * @param latinIME the instance of the parent LatinIME. We should remove this when we can. * @param suggestionStripViewAccessor an object to access the suggestion strip view. + * @param dictionaryFacilitator facilitator for getting suggestions and updating user history + * dictionary. */ public InputLogic(final LatinIME latinIME, - final SuggestionStripViewAccessor suggestionStripViewAccessor) { + final SuggestionStripViewAccessor suggestionStripViewAccessor, + final DictionaryFacilitator dictionaryFacilitator) { mLatinIME = latinIME; mSuggestionStripViewAccessor = suggestionStripViewAccessor; mWordComposer = new WordComposer(); mConnection = new RichInputConnection(latinIME); mInputLogicHandler = InputLogicHandler.NULL_HANDLER; + mSuggest = new Suggest(dictionaryFacilitator); + mDictionaryFacilitator = dictionaryFacilitator; } /** @@ -172,7 +179,7 @@ public final class InputLogic { final InputLogicHandler inputLogicHandler = mInputLogicHandler; mInputLogicHandler = InputLogicHandler.NULL_HANDLER; inputLogicHandler.destroy(); - mSuggest.mDictionaryFacilitator.closeDictionaries(); + mDictionaryFacilitator.closeDictionaries(); } /** @@ -294,18 +301,16 @@ public final class InputLogic { // We should show the "Touch again to save" hint if the user pressed the first entry // AND it's in none of our current dictionaries (main, user or otherwise). - final DictionaryFacilitatorForSuggest dictionaryFacilitator = - mSuggest.mDictionaryFacilitator; final boolean showingAddToDictionaryHint = (SuggestedWordInfo.KIND_TYPED == suggestionInfo.mKind || SuggestedWordInfo.KIND_OOV_CORRECTION == suggestionInfo.mKind) - && !dictionaryFacilitator.isValidWord(suggestion, true /* ignoreCase */); + && !mDictionaryFacilitator.isValidWord(suggestion, true /* ignoreCase */); if (settingsValues.mIsInternal) { LatinImeLoggerUtils.onSeparator((char)Constants.CODE_SPACE, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); } - if (showingAddToDictionaryHint && dictionaryFacilitator.isUserDictionaryEnabled()) { + if (showingAddToDictionaryHint && mDictionaryFacilitator.isUserDictionaryEnabled()) { mSuggestionStripViewAccessor.showAddToDictionaryHint(suggestion); } else { // If we're not showing the "Touch again to save", then update the suggestion strip. @@ -574,7 +579,7 @@ public final class InputLogic { mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime( getActualCapsMode(settingsValues, keyboardSwitcher.getKeyboardShiftMode()), // Prev word is 1st word before cursor - getNthPreviousWordForSuggestion( + getPrevWordsInfoFromNthPreviousWordForSuggestion( settingsValues.mSpacingAndPunctuations, 1 /* nthPreviousWord */)); } @@ -613,7 +618,8 @@ public final class InputLogic { getCurrentAutoCapsState(settingsValues), getCurrentRecapitalizeState()); mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime( getActualCapsMode(settingsValues, - keyboardSwitcher.getKeyboardShiftMode()), commitParts[0]); + keyboardSwitcher.getKeyboardShiftMode()), + new PrevWordsInfo(commitParts[0])); ++mAutoCommitSequenceNumber; } } @@ -764,7 +770,8 @@ public final class InputLogic { // We pass 1 to getPreviousWordForSuggestion because we were not composing a word // yet, so the word we want is the 1st word before the cursor. mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime( - inputTransaction.mShiftState, getNthPreviousWordForSuggestion( + inputTransaction.mShiftState, + getPrevWordsInfoFromNthPreviousWordForSuggestion( settingsValues.mSpacingAndPunctuations, 1 /* nthPreviousWord */)); } mConnection.setComposingText(getTextWithUnderline( @@ -1233,7 +1240,7 @@ public final class InputLogic { } private void performAdditionToUserHistoryDictionary(final SettingsValues settingsValues, - final String suggestion, final String prevWord) { + final String suggestion, final PrevWordsInfo prevWordsInfo) { // If correction is not enabled, we don't add words to the user history dictionary. // That's to avoid unintended additions in some sensitive fields, or fields that // expect to receive non-words. @@ -1244,8 +1251,8 @@ public final class InputLogic { mWordComposer.wasAutoCapitalized() && !mWordComposer.isMostlyCaps(); final int timeStampInSeconds = (int)TimeUnit.MILLISECONDS.toSeconds( System.currentTimeMillis()); - mSuggest.mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized, prevWord, - timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive); + mDictionaryFacilitator.addToUserHistory(suggestion, wasAutoCapitalized, + prevWordsInfo, timeStampInSeconds, settingsValues.mBlockPotentiallyOffensive); } public void performUpdateSuggestionStripSync(final SettingsValues settingsValues) { @@ -1325,7 +1332,8 @@ public final class InputLogic { // Show predictions. mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime( WordComposer.CAPS_MODE_OFF, - getNthPreviousWordForSuggestion(settingsValues.mSpacingAndPunctuations, 1)); + getPrevWordsInfoFromNthPreviousWordForSuggestion( + settingsValues.mSpacingAndPunctuations, 1)); mLatinIME.mHandler.postUpdateSuggestionStrip(); return; } @@ -1370,13 +1378,14 @@ public final class InputLogic { } } final int[] codePoints = StringUtils.toCodePointArray(typedWord); + // We want the previous word for suggestion. If we have chars in the word + // before the cursor, then we want the word before that, hence 2; otherwise, + // we want the word immediately before the cursor, hence 1. + final PrevWordsInfo prevWordsInfo = getPrevWordsInfoFromNthPreviousWordForSuggestion( + settingsValues.mSpacingAndPunctuations, + 0 == numberOfCharsInWordBeforeCursor ? 1 : 2); mWordComposer.setComposingWord(codePoints, - mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), - getNthPreviousWordForSuggestion(settingsValues.mSpacingAndPunctuations, - // We want the previous word for suggestion. If we have chars in the word - // before the cursor, then we want the word before that, hence 2; otherwise, - // we want the word immediately before the cursor, hence 1. - 0 == numberOfCharsInWordBeforeCursor ? 1 : 2)); + mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), prevWordsInfo); mWordComposer.setCursorPositionWithinWord( typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor)); mConnection.setComposingRegion(expectedCursorPosition - numberOfCharsInWordBeforeCursor, @@ -1431,7 +1440,7 @@ public final class InputLogic { * @param inputTransaction The transaction in progress. */ private void revertCommit(final InputTransaction inputTransaction) { - final String previousWord = mLastComposedWord.mPrevWord; + final PrevWordsInfo prevWordsInfo = mLastComposedWord.mPrevWordsInfo; final CharSequence originallyTypedWord = mLastComposedWord.mTypedWord; final CharSequence committedWord = mLastComposedWord.mCommittedWord; final String committedWordString = committedWord.toString(); @@ -1453,9 +1462,8 @@ public final class InputLogic { } } mConnection.deleteSurroundingText(deleteLength, 0); - if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) { - mSuggest.mDictionaryFacilitator.cancelAddingUserHistory( - previousWord, committedWordString); + if (!TextUtils.isEmpty(prevWordsInfo.mPrevWord) && !TextUtils.isEmpty(committedWord)) { + mDictionaryFacilitator.cancelAddingUserHistory(prevWordsInfo, committedWordString); } final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString; final SpannableString textToCommit = new SpannableString(stringToCommit); @@ -1504,7 +1512,7 @@ public final class InputLogic { // with the typed word, so we need to resume suggestions right away. final int[] codePoints = StringUtils.toCodePointArray(stringToCommit); mWordComposer.setComposingWord(codePoints, - mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), previousWord); + mLatinIME.getCoordinatesForCurrentKeyboard(codePoints), prevWordsInfo); mConnection.setComposingText(textToCommit, 1); } if (inputTransaction.mSettingsValues.mIsInternal) { @@ -1586,21 +1594,23 @@ public final class InputLogic { } /** - * Get the nth previous word before the cursor as context for the suggestion process. + * Get information fo previous words from the nth previous word before the cursor as context + * for the suggestion process. * @param spacingAndPunctuations the current spacing and punctuations settings. * @param nthPreviousWord reverse index of the word to get (1-indexed) - * @return the nth previous word before the cursor. + * @return the information of previous words */ // TODO: Make this private - public CharSequence getNthPreviousWordForSuggestion( + public PrevWordsInfo getPrevWordsInfoFromNthPreviousWordForSuggestion( final SpacingAndPunctuations spacingAndPunctuations, final int nthPreviousWord) { if (spacingAndPunctuations.mCurrentLanguageHasSpaces) { // If we are typing in a language with spaces we can just look up the previous - // word from textview. - return mConnection.getNthPreviousWord(spacingAndPunctuations, nthPreviousWord); + // word information from textview. + return mConnection.getPrevWordsInfoFromNthPreviousWord( + spacingAndPunctuations, nthPreviousWord); } else { - return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null - : mLastComposedWord.mCommittedWord; + return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? new PrevWordsInfo() + : new PrevWordsInfo(mLastComposedWord.mCommittedWord.toString()); } } @@ -1968,17 +1978,17 @@ public final class InputLogic { suggestedWords); // Use the 2nd previous word as the previous word because the 1st previous word is the word // to be committed. - final String prevWord = mConnection.getNthPreviousWord( + final PrevWordsInfo prevWordsInfo = mConnection.getPrevWordsInfoFromNthPreviousWord( settingsValues.mSpacingAndPunctuations, 2); mConnection.commitText(chosenWordWithSuggestions, 1); // Add the word to the user history dictionary - performAdditionToUserHistoryDictionary(settingsValues, chosenWord, prevWord); + performAdditionToUserHistoryDictionary(settingsValues, chosenWord, prevWordsInfo); // TODO: figure out here if this is an auto-correct or if the best word is actually // what user typed. Note: currently this is done much later in // LastComposedWord#didCommitTypedWord by string equality of the remembered // strings. mLastComposedWord = mWordComposer.commitWord(commitType, - chosenWordWithSuggestions, separatorString, prevWord); + chosenWordWithSuggestions, separatorString, prevWordsInfo); final boolean shouldDiscardPreviousWordForSuggestion; if (0 == StringUtils.codePointCount(separatorString)) { // Separator is 0-length, we can keep the previous word for suggestion. Either this diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index f5f072b7a..a2ae74b20 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -192,8 +192,9 @@ public final class FormatSpec { public static final int VERSION2 = 2; // Dictionary version used for testing. public static final int VERSION4_ONLY_FOR_TESTING = 399; - public static final int VERSION4 = 401; - public static final int VERSION4_DEV = 402; + public static final int VERSION401 = 401; + public static final int VERSION4 = 402; + public static final int VERSION4_DEV = 403; static final int MINIMUM_SUPPORTED_VERSION = VERSION2; static final int MAXIMUM_SUPPORTED_VERSION = VERSION4_DEV; diff --git a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java index 853392200..ed832510c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java +++ b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java @@ -35,6 +35,8 @@ public final class WordProperty implements Comparable<WordProperty> { public final ProbabilityInfo mProbabilityInfo; public final ArrayList<WeightedString> mShortcutTargets; public final ArrayList<WeightedString> mBigrams; + // TODO: Support mIsBeginningOfSentence. + public final boolean mIsBeginningOfSentence; public final boolean mIsNotAWord; public final boolean mIsBlacklistEntry; public final boolean mHasShortcuts; @@ -51,6 +53,7 @@ public final class WordProperty implements Comparable<WordProperty> { mProbabilityInfo = probabilityInfo; mShortcutTargets = shortcutTargets; mBigrams = bigrams; + mIsBeginningOfSentence = false; mIsNotAWord = isNotAWord; mIsBlacklistEntry = isBlacklistEntry; mHasBigrams = bigrams != null && !bigrams.isEmpty(); @@ -77,6 +80,7 @@ public final class WordProperty implements Comparable<WordProperty> { mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo); mShortcutTargets = CollectionUtils.newArrayList(); mBigrams = CollectionUtils.newArrayList(); + mIsBeginningOfSentence = false; mIsNotAWord = isNotAWord; mIsBlacklistEntry = isBlacklisted; mHasShortcuts = hasShortcuts; diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDataChunk.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDataChunk.java new file mode 100644 index 000000000..9d72de8c5 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDataChunk.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public class PersonalizationDataChunk { + public final boolean mInputByUser; + public final List<String> mTokens; + public final int mTimestampInSeconds; + public final String mPackageName; + public final Locale mlocale = null; + + public PersonalizationDataChunk(boolean inputByUser, final List<String> tokens, + final int timestampInSeconds, final String packageName) { + mInputByUser = inputByUser; + mTokens = Collections.unmodifiableList(tokens); + mTimestampInSeconds = timestampInSeconds; + mPackageName = packageName; + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java index 9bef7a198..450644032 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegistrar.java @@ -19,18 +19,15 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; import android.content.res.Configuration; -import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; -import com.android.inputmethod.latin.utils.DistracterFilter; +import com.android.inputmethod.latin.DictionaryFacilitator; public class PersonalizationDictionarySessionRegistrar { public static void init(final Context context, - final DictionaryFacilitatorForSuggest dictionaryFacilitator, - final DistracterFilter distracterFilter) { + final DictionaryFacilitator dictionaryFacilitator) { } public static void onConfigurationChanged(final Context context, final Configuration conf, - final DictionaryFacilitatorForSuggest dictionaryFacilitator, - final DistracterFilter distracterFilter) { + final DictionaryFacilitator dictionaryFacilitator) { } public static void onUpdateData(final Context context, final String type) { diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 818cd9a5f..f89caf921 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -22,6 +22,7 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.PrevWordsInfo; import java.io.File; import java.util.Locale; @@ -52,29 +53,32 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas } /** - * Pair will be added to the user history dictionary. + * Add a word to the user history dictionary. * - * The first word may be null. That means we don't know the context, in other words, - * it's only a unigram. The first word may also be an empty string : this means start - * context, as in beginning of a sentence for example. - * The second word may not be null (a NullPointerException would be thrown). + * @param userHistoryDictionary the user history dictionary + * @param prevWordsInfo the information of previous words + * @param word the word the user inputted + * @param isValid whether the word is valid or not + * @param timestamp the timestamp when the word has been inputted */ public static void addToDictionary(final ExpandableBinaryDictionary userHistoryDictionary, - final String word0, final String word1, final boolean isValid, final int timestamp) { - if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || - (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { + final PrevWordsInfo prevWordsInfo, final String word, final boolean isValid, + final int timestamp) { + final String prevWord = prevWordsInfo.mPrevWord; + if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || + (prevWord != null && prevWord.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { return; } final int frequency = isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; - userHistoryDictionary.addWordDynamically(word1, frequency, null /* shortcutTarget */, + userHistoryDictionary.addUnigramEntry(word, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, timestamp); // Do not insert a word as a bigram of itself - if (word1.equals(word0)) { + if (word.equals(prevWord)) { return; } - if (null != word0) { - userHistoryDictionary.addBigramDynamically(word0, word1, frequency, timestamp); + if (null != prevWord) { + userHistoryDictionary.addNgramEntry(prevWordsInfo, word, frequency, timestamp); } } } diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java index 5a325ea82..e90b15ca5 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java @@ -27,14 +27,13 @@ import com.android.inputmethod.keyboard.KeyboardActionListener; import com.android.inputmethod.keyboard.internal.KeyboardBuilder; import com.android.inputmethod.keyboard.internal.KeyboardIconsSet; import com.android.inputmethod.keyboard.internal.KeyboardParams; +import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.SuggestedWords; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.TypefaceUtils; public final class MoreSuggestions extends Keyboard { - public static final int SUGGESTION_CODE_BASE = 1024; - public final SuggestedWords mSuggestedWords; public static abstract class MoreSuggestionsListener extends KeyboardActionListener.Adapter { @@ -178,7 +177,7 @@ public final class MoreSuggestions extends Keyboard { } } - private static boolean isIndexSubjectToAutoCorrection(final SuggestedWords suggestedWords, + static boolean isIndexSubjectToAutoCorrection(final SuggestedWords suggestedWords, final int index) { return suggestedWords.mWillAutoCorrect && index == SuggestedWords.INDEX_OF_AUTO_CORRECTION; } @@ -226,11 +225,7 @@ public final class MoreSuggestions extends Keyboard { word = mSuggestedWords.getLabel(index); info = mSuggestedWords.getDebugString(index); } - final int indexInMoreSuggestions = index + SUGGESTION_CODE_BASE; - final Key key = new Key(word, KeyboardIconsSet.ICON_UNDEFINED, - indexInMoreSuggestions, null /* outputText */, info, 0 /* labelFlags */, - Key.BACKGROUND_TYPE_NORMAL, x, y, width, params.mDefaultRowHeight, - params.mHorizontalGap, params.mVerticalGap); + final Key key = new MoreSuggestionKey(word, info, index, params); params.markAsEdgeKey(key, index); params.onAddKey(key); final int columnNumber = params.getColumnNumber(index); @@ -245,6 +240,19 @@ public final class MoreSuggestions extends Keyboard { } } + static final class MoreSuggestionKey extends Key { + public final int mSuggestedWordIndex; + + public MoreSuggestionKey(final String word, final String info, final int index, + final MoreSuggestionsParam params) { + super(word /* label */, KeyboardIconsSet.ICON_UNDEFINED, Constants.CODE_OUTPUT_TEXT, + word /* outputText */, info, 0 /* labelFlags */, Key.BACKGROUND_TYPE_NORMAL, + params.getX(index), params.getY(index), params.getWidth(index), + params.mDefaultRowHeight, params.mHorizontalGap, params.mVerticalGap); + mSuggestedWordIndex = index; + } + } + private static final class Divider extends Key.Spacer { private final Drawable mIcon; diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java index 549ff0d9d..7fd64c4bf 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java @@ -20,10 +20,12 @@ import android.content.Context; import android.util.AttributeSet; import android.util.Log; +import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.MoreKeysKeyboardView; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.SuggestedWords; +import com.android.inputmethod.latin.suggestions.MoreSuggestions.MoreSuggestionKey; import com.android.inputmethod.latin.suggestions.MoreSuggestions.MoreSuggestionsListener; /** @@ -59,7 +61,12 @@ public final class MoreSuggestionsView extends MoreKeysKeyboardView { } @Override - public void onCodeInput(final int code, final int x, final int y) { + protected void onKeyInput(final Key key, final int x, final int y) { + if (!(key instanceof MoreSuggestionKey)) { + Log.e(TAG, "Expected key is MoreSuggestionKey, but found " + + key.getClass().getName()); + return; + } final Keyboard keyboard = getKeyboard(); if (!(keyboard instanceof MoreSuggestions)) { Log.e(TAG, "Expected keyboard is MoreSuggestions, but found " @@ -67,7 +74,7 @@ public final class MoreSuggestionsView extends MoreKeysKeyboardView { return; } final SuggestedWords suggestedWords = ((MoreSuggestions)keyboard).mSuggestedWords; - final int index = code - MoreSuggestions.SUGGESTION_CODE_BASE; + final int index = ((MoreSuggestionKey)key).mSuggestedWordIndex; if (index < 0 || index >= suggestedWords.size()) { Log.e(TAG, "Selected suggestion has an illegal index: " + index); return; diff --git a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java index 702688f93..936219332 100644 --- a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java @@ -62,6 +62,22 @@ public final class CapsModeUtils { } /** + * Helper method to find out if a code point is starting punctuation. + * + * This include the Unicode START_PUNCTUATION category, but also some other symbols that are + * starting, like the inverted question mark or the double quote. + * + * @param codePoint the code point + * @return true if it's starting punctuation, false otherwise. + */ + private static boolean isStartPunctuation(final int codePoint) { + return (codePoint == Constants.CODE_DOUBLE_QUOTE || codePoint == Constants.CODE_SINGLE_QUOTE + || codePoint == Constants.CODE_INVERTED_QUESTION_MARK + || codePoint == Constants.CODE_INVERTED_EXCLAMATION_MARK + || Character.getType(codePoint) == Character.START_PUNCTUATION); + } + + /** * Determine what caps mode should be in effect at the current offset in * the text. Only the mode bits set in <var>reqModes</var> will be * checked. Note that the caps mode flags here are explicitly defined @@ -115,8 +131,7 @@ public final class CapsModeUtils { } else { for (i = cs.length(); i > 0; i--) { final char c = cs.charAt(i - 1); - if (c != Constants.CODE_DOUBLE_QUOTE && c != Constants.CODE_SINGLE_QUOTE - && Character.getType(c) != Character.START_PUNCTUATION) { + if (!isStartPunctuation(c)) { break; } } @@ -210,11 +225,14 @@ public final class CapsModeUtils { // We found out that we have a period. We need to determine if this is a full stop or // otherwise sentence-ending period, or an abbreviation like "e.g.". An abbreviation - // looks like (\w\.){2,} + // looks like (\w\.){2,}. Moreover, in German, you put periods after digits for dates + // and some other things, and in German specifically we need to not go into autocaps after + // a whitespace-digits-period sequence. // To find out, we will have a simple state machine with the following states : - // START, WORD, PERIOD, ABBREVIATION + // START, WORD, PERIOD, ABBREVIATION, NUMBER // On START : (just before the first period) // letter => WORD + // digit => NUMBER if German; end with caps otherwise // whitespace => end with no caps (it was a stand-alone period) // otherwise => end with caps (several periods/symbols in a row) // On WORD : (within the word just before the first period) @@ -228,6 +246,11 @@ public final class CapsModeUtils { // letter => LETTER // period => PERIOD // otherwise => end with no caps (it was an abbreviation) + // On NUMBER : (period immediately preceded by one or more digits) + // digit => NUMBER + // letter => LETTER (promote to word) + // otherwise => end with no caps (it was a whitespace-digits-period sequence, + // or a punctuation-digits-period sequence like "11.11.") // "Not an abbreviation" in the above chart essentially covers cases like "...yes.". This // should capitalize. @@ -235,6 +258,7 @@ public final class CapsModeUtils { final int WORD = 1; final int PERIOD = 2; final int LETTER = 3; + final int NUMBER = 4; final int caps = (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_WORDS | TextUtils.CAP_MODE_SENTENCES) & reqModes; final int noCaps = (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_WORDS) & reqModes; @@ -247,6 +271,8 @@ public final class CapsModeUtils { state = WORD; } else if (Character.isWhitespace(c)) { return noCaps; + } else if (Character.isDigit(c) && spacingAndPunctuations.mUsesGermanRules) { + state = NUMBER; } else { return caps; } @@ -275,6 +301,15 @@ public final class CapsModeUtils { } else { return noCaps; } + break; + case NUMBER: + if (Character.isLetter(c)) { + state = WORD; + } else if (Character.isDigit(c)) { + state = NUMBER; + } else { + return noCaps; + } } } // Here we arrived at the start of the line. This should behave exactly like whitespace. diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java index a21953259..6e0fab32a 100644 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilter.java @@ -16,129 +16,43 @@ package com.android.inputmethod.latin.utils; +import java.util.List; import java.util.Locale; -import java.util.concurrent.TimeUnit; -import android.content.Context; -import android.util.Log; +import android.view.inputmethod.InputMethodSubtype; -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.PrevWordsInfo; -import com.android.inputmethod.latin.Suggest; -import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; -import com.android.inputmethod.latin.SuggestedWords; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; - -/** - * This class is used to prevent distracters being added to personalization - * or user history dictionaries - */ -public class DistracterFilter { - private static final String TAG = DistracterFilter.class.getSimpleName(); - - private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; - - private final Context mContext; - private final Suggest mSuggest; - private final Keyboard mKeyboard; - - // If the score of the top suggestion exceeds this value, the tested word (e.g., - // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to - // words in dictionary. The greater the threshold is, the less likely the tested word would - // become a distracter, which means the tested word will be more likely to be added to - // the dictionary. - private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 2.0f; - - /** - * Create a DistracterFilter instance. - * - * @param context the context. - * @param keyboard the keyboard that is currently being used. This information is needed - * when calling mSuggest.getSuggestedWords(...) to obtain a list of suggestions. - */ - public DistracterFilter(final Context context, final Keyboard keyboard) { - mContext = context; - mSuggest = new Suggest(); - mKeyboard = keyboard; - } - - private static boolean suggestionExceedsDistracterThreshold( - final SuggestedWordInfo suggestion, final String consideredWord, - final float distracterThreshold) { - if (null != suggestion) { - final int suggestionScore = suggestion.mScore; - final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( - consideredWord, suggestion.mWord, suggestionScore); - if (normalizedScore > distracterThreshold) { - return true; - } - } - return false; - } - - private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { - mSuggest.mDictionaryFacilitator.resetDictionaries(mContext, newlocale, - false /* useContactsDict */, false /* usePersonalizedDicts */, - false /* forceReloadMainDictionary */, null /* listener */); - mSuggest.mDictionaryFacilitator.waitForLoadingMainDictionary( - TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS); - } +public interface DistracterFilter { /** * Determine whether a word is a distracter to words in dictionaries. * * @param prevWordsInfo the information of previous words. * @param testedWord the word that will be tested to see whether it is a distracter to words * in dictionaries. - * @param locale the locale of words. + * @param locale the locale of word. * @return true if testedWord is a distracter, otherwise false. */ public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, - final String testedWord, final Locale locale) { - if (mKeyboard == null || locale == null) { + final String testedWord, final Locale locale); + + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes); + + public void close(); + + public static final class EmptyDistracterFilter implements DistracterFilter { + @Override + public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo, + String testedWord, Locale locale) { return false; } - if (!locale.equals(mSuggest.mDictionaryFacilitator.getLocale())) { - // Reset dictionaries for the locale. - try { - loadDictionariesForLocale(locale); - } catch (final InterruptedException e) { - Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter", e); - return false; - } - } - - final WordComposer composer = new WordComposer(); - final int[] codePoints = StringUtils.toCodePointArray(testedWord); - final int[] coordinates; - coordinates = mKeyboard.getCoordinates(codePoints); - composer.setComposingWord(codePoints, coordinates, prevWordsInfo.mPrevWord); - final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); - final String consideredWord = trailingSingleQuotesCount > 0 ? - testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : - testedWord; - final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); - final OnGetSuggestedWordsCallback callback = new OnGetSuggestedWordsCallback() { - @Override - public void onGetSuggestedWords(final SuggestedWords suggestedWords) { - if (suggestedWords != null && suggestedWords.size() > 1) { - // The suggestedWordInfo at 0 is the typed word. The 1st suggestion from - // the decoder is at index 1. - final SuggestedWordInfo firstSuggestion = suggestedWords.getInfo(1); - final boolean hasStrongDistractor = suggestionExceedsDistracterThreshold( - firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); - holder.set(hasStrongDistractor); - } - } - }; - mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), - true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, - null /* additionalFeaturesOptions */, 0 /* sessionId */, - SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); + @Override + public void close() { + } - return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + @Override + public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) { + } } } diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java new file mode 100644 index 000000000..92033b76f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import android.content.Context; +import android.content.res.Resources; +import android.text.InputType; +import android.util.Log; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputMethodSubtype; + +import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.keyboard.KeyboardId; +import com.android.inputmethod.keyboard.KeyboardLayoutSet; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.DictionaryFacilitator; +import com.android.inputmethod.latin.PrevWordsInfo; +import com.android.inputmethod.latin.Suggest; +import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; +import com.android.inputmethod.latin.SuggestedWords; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.WordComposer; + +/** + * This class is used to prevent distracters being added to personalization + * or user history dictionaries + */ +public class DistracterFilterUsingSuggestion implements DistracterFilter { + private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName(); + + private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120; + + private final Context mContext; + private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap; + private final Map<Locale, Keyboard> mLocaleToKeyboardMap; + private final DictionaryFacilitator mDictionaryFacilitator; + private final Suggest mSuggest; + private Keyboard mKeyboard; + private final Object mLock = new Object(); + + // If the score of the top suggestion exceeds this value, the tested word (e.g., + // an OOV, a misspelling, or an in-vocabulary word) would be considered as a distracter to + // words in dictionary. The greater the threshold is, the less likely the tested word would + // become a distracter, which means the tested word will be more likely to be added to + // the dictionary. + private static final float DISTRACTER_WORD_SCORE_THRESHOLD = 2.0f; + + /** + * Create a DistracterFilter instance. + * + * @param context the context. + */ + public DistracterFilterUsingSuggestion(final Context context) { + mContext = context; + mLocaleToSubtypeMap = new HashMap<>(); + mLocaleToKeyboardMap = new HashMap<>(); + mDictionaryFacilitator = new DictionaryFacilitator(); + mSuggest = new Suggest(mDictionaryFacilitator); + mKeyboard = null; + } + + @Override + public void close() { + mDictionaryFacilitator.closeDictionaries(); + } + + @Override + public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { + final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>(); + if (enabledSubtypes != null) { + for (final InputMethodSubtype subtype : enabledSubtypes) { + final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype); + if (newLocaleToSubtypeMap.containsKey(locale)) { + // Multiple subtypes are enabled for one locale. + // TODO: Investigate what we should do for this case. + continue; + } + newLocaleToSubtypeMap.put(locale, subtype); + } + } + if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) { + // Enabled subtypes have not been changed. + return; + } + synchronized (mLock) { + mLocaleToSubtypeMap.clear(); + mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap); + mLocaleToKeyboardMap.clear(); + } + } + + private static boolean suggestionExceedsDistracterThreshold( + final SuggestedWordInfo suggestion, final String consideredWord, + final float distracterThreshold) { + if (null != suggestion) { + final int suggestionScore = suggestion.mScore; + final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore( + consideredWord, suggestion.mWord, suggestionScore); + if (normalizedScore > distracterThreshold) { + return true; + } + } + return false; + } + + private void loadKeyboardForLocale(final Locale newLocale) { + final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale); + if (cachedKeyboard != null) { + mKeyboard = cachedKeyboard; + return; + } + final InputMethodSubtype subtype; + synchronized (mLock) { + subtype = mLocaleToSubtypeMap.get(newLocale); + } + if (subtype == null) { + return; + } + final EditorInfo editorInfo = new EditorInfo(); + editorInfo.inputType = InputType.TYPE_CLASS_TEXT; + final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( + mContext, editorInfo); + final Resources res = mContext.getResources(); + final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); + final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); + builder.setKeyboardGeometry(keyboardWidth, keyboardHeight); + builder.setSubtype(subtype); + builder.setIsSpellChecker(false /* isSpellChecker */); + final KeyboardLayoutSet layoutSet = builder.build(); + mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET); + } + + private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException { + mDictionaryFacilitator.resetDictionaries(mContext, newlocale, + false /* useContactsDict */, false /* usePersonalizedDicts */, + false /* forceReloadMainDictionary */, null /* listener */); + mDictionaryFacilitator.waitForLoadingMainDictionary( + TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS); + } + + /** + * Determine whether a word is a distracter to words in dictionaries. + * + * @param prevWordsInfo the information of previous words. + * @param testedWord the word that will be tested to see whether it is a distracter to words + * in dictionaries. + * @param locale the locale of word. + * @return true if testedWord is a distracter, otherwise false. + */ + @Override + public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo, + final String testedWord, final Locale locale) { + if (locale == null) { + return false; + } + if (!locale.equals(mDictionaryFacilitator.getLocale())) { + synchronized (mLock) { + if (!mLocaleToSubtypeMap.containsKey(locale)) { + Log.e(TAG, "Locale " + locale + " is not enabled."); + // TODO: Investigate what we should do for disabled locales. + return false; + } + loadKeyboardForLocale(locale); + // Reset dictionaries for the locale. + try { + loadDictionariesForLocale(locale); + } catch (final InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter", + e); + return false; + } + } + } + if (mKeyboard == null) { + return false; + } + final WordComposer composer = new WordComposer(); + final int[] codePoints = StringUtils.toCodePointArray(testedWord); + final int[] coordinates = mKeyboard.getCoordinates(codePoints); + composer.setComposingWord(codePoints, coordinates, prevWordsInfo); + + final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord); + final String consideredWord = trailingSingleQuotesCount > 0 ? + testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) : + testedWord; + final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); + final OnGetSuggestedWordsCallback callback = new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + if (suggestedWords != null && suggestedWords.size() > 1) { + // The suggestedWordInfo at 0 is the typed word. The 1st suggestion from + // the decoder is at index 1. + final SuggestedWordInfo firstSuggestion = suggestedWords.getInfo(1); + final boolean hasStrongDistractor = suggestionExceedsDistracterThreshold( + firstSuggestion, consideredWord, DISTRACTER_WORD_SCORE_THRESHOLD); + holder.set(hasStrongDistractor); + } + } + }; + mSuggest.getSuggestedWords(composer, prevWordsInfo, mKeyboard.getProximityInfo(), + true /* blockOffensiveWords */, true /* isCorrectionEnbaled */, + null /* additionalFeaturesOptions */, 0 /* sessionId */, + SuggestedWords.NOT_A_SEQUENCE_NUMBER, callback); + + return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT); + } +} diff --git a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUtils.java b/java/src/com/android/inputmethod/latin/utils/DistracterFilterUtils.java deleted file mode 100644 index 8a711a24e..000000000 --- a/java/src/com/android/inputmethod/latin/utils/DistracterFilterUtils.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2014 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.content.Context; - -import com.android.inputmethod.keyboard.Keyboard; -import com.android.inputmethod.keyboard.KeyboardSwitcher; -import com.android.inputmethod.keyboard.MainKeyboardView; - -public class DistracterFilterUtils { - private DistracterFilterUtils() { - // This utility class is not publicly instantiable. - } - - public static final DistracterFilter createDistracterFilter(final Context context, - final KeyboardSwitcher keyboardSwitcher) { - final MainKeyboardView mainKeyboardView = keyboardSwitcher.getMainKeyboardView(); - // TODO: Create Keyboard when mainKeyboardView is null. - // TODO: Figure out the most reasonable keyboard for the filter. Refer to the - // spellchecker's logic. - final Keyboard keyboard = (mainKeyboardView != null) ? - mainKeyboardView.getKeyboard() : null; - final DistracterFilter distracterFilter = new DistracterFilter(context, keyboard); - return distracterFilter; - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java index aaf4a4064..430efdd19 100644 --- a/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java +++ b/java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java @@ -19,11 +19,12 @@ package com.android.inputmethod.latin.utils; import android.util.Log; import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; +import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.PrevWordsInfo; import com.android.inputmethod.latin.settings.SpacingAndPunctuations; import java.util.ArrayList; +import java.util.List; import java.util.Locale; // Note: this class is used as a parameter type of a native method. You should be careful when you @@ -79,8 +80,8 @@ public final class LanguageModelParam { // Process a list of words and return a list of {@link LanguageModelParam} objects. public static ArrayList<LanguageModelParam> createLanguageModelParamsFrom( - final ArrayList<String> tokens, final int timestamp, - final DictionaryFacilitatorForSuggest dictionaryFacilitator, + final List<String> tokens, final int timestamp, + final DictionaryFacilitator dictionaryFacilitator, final SpacingAndPunctuations spacingAndPunctuations, final DistracterFilter distracterFilter) { final ArrayList<LanguageModelParam> languageModelParams = @@ -124,7 +125,7 @@ public final class LanguageModelParam { private static LanguageModelParam detectWhetherVaildWordOrNotAndGetLanguageModelParam( final PrevWordsInfo prevWordsInfo, final String targetWord, final int timestamp, - final DictionaryFacilitatorForSuggest dictionaryFacilitator, + final DictionaryFacilitator dictionaryFacilitator, final DistracterFilter distracterFilter) { final Locale locale = dictionaryFacilitator.getLocale(); if (locale == null) { diff --git a/java/src/com/android/inputmethod/research/MainLogBuffer.java b/java/src/com/android/inputmethod/research/MainLogBuffer.java index ffdb43c15..3806ac755 100644 --- a/java/src/com/android/inputmethod/research/MainLogBuffer.java +++ b/java/src/com/android/inputmethod/research/MainLogBuffer.java @@ -20,7 +20,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Dictionary; -import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; +import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.define.ProductionFlag; import java.io.IOException; @@ -75,7 +75,7 @@ public abstract class MainLogBuffer extends FixedLogBuffer { // The size of the n-grams logged. E.g. N_GRAM_SIZE = 2 means to sample bigrams. public static final int N_GRAM_SIZE = 2; - private final DictionaryFacilitatorForSuggest mDictionaryFacilitator; + private final DictionaryFacilitator mDictionaryFacilitator; @UsedForTesting private Dictionary mDictionaryForTesting; private boolean mIsStopping = false; @@ -87,7 +87,7 @@ public abstract class MainLogBuffer extends FixedLogBuffer { /* package for test */ int mNumWordsUntilSafeToSample; public MainLogBuffer(final int wordsBetweenSamples, final int numInitialWordsToIgnore, - final DictionaryFacilitatorForSuggest dictionaryFacilitator) { + final DictionaryFacilitator dictionaryFacilitator) { super(N_GRAM_SIZE + wordsBetweenSamples); mNumWordsBetweenNGrams = wordsBetweenSamples; mNumWordsUntilSafeToSample = DEBUG ? 0 : numInitialWordsToIgnore; diff --git a/java/src/com/android/inputmethod/research/ResearchLogger.java b/java/src/com/android/inputmethod/research/ResearchLogger.java index d907dd1b0..d73f9c41c 100644 --- a/java/src/com/android/inputmethod/research/ResearchLogger.java +++ b/java/src/com/android/inputmethod/research/ResearchLogger.java @@ -52,7 +52,7 @@ import com.android.inputmethod.keyboard.KeyboardSwitcher; import com.android.inputmethod.keyboard.KeyboardView; import com.android.inputmethod.keyboard.MainKeyboardView; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.DictionaryFacilitatorForSuggest; +import com.android.inputmethod.latin.DictionaryFacilitator; import com.android.inputmethod.latin.LatinIME; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.RichInputConnection; @@ -167,7 +167,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang protected static final int SUSPEND_DURATION_IN_MINUTES = 1; // used to check whether words are not unique - private DictionaryFacilitatorForSuggest mDictionaryFacilitator; + private DictionaryFacilitator mDictionaryFacilitator; private MainKeyboardView mMainKeyboardView; // TODO: Check whether a superclass can be used instead of LatinIME. /* package for test */ LatinIME mLatinIME; @@ -656,7 +656,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang mInFeedbackDialog = false; } - public void initDictionary(final DictionaryFacilitatorForSuggest dictionaryFacilitator) { + public void initDictionary(final DictionaryFacilitator dictionaryFacilitator) { mDictionaryFacilitator = dictionaryFacilitator; // MainLogBuffer now has an out-of-date Suggest object. Close down MainLogBuffer and create // a new one. |