diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin')
68 files changed, 2769 insertions, 3073 deletions
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java index e6fb9807e..463d09344 100644 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -27,13 +27,15 @@ import java.io.File; import java.io.IOException; import java.util.Map; -abstract public class AbstractDictionaryWriter { +// TODO: Quit extending Dictionary after implementing dynamic binary dictionary. +abstract public class AbstractDictionaryWriter extends Dictionary { /** Used for Log actions from this class */ private static final String TAG = AbstractDictionaryWriter.class.getSimpleName(); private final Context mContext; - public AbstractDictionaryWriter(final Context context) { + public AbstractDictionaryWriter(final Context context, final String dictType) { + super(dictType); mContext = context; } @@ -53,16 +55,18 @@ abstract public class AbstractDictionaryWriter { // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve. abstract public void addBigramWords(final String word0, final String word1, - final int frequency, final boolean isValid, final long lastModifiedTime); + final int frequency, final boolean isValid, + final long lastModifiedTime); abstract public void removeBigramWords(final String word0, final String word1); abstract protected void writeDictionary(final DictEncoder dictEncoder, final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException; - public void write(final File file, final Map<String, String> attributeMap) { - final String tempFilePath = file.getAbsolutePath() + ".temp"; - final File tempFile = new File(tempFilePath); + public void write(final String fileName, final Map<String, String> attributeMap) { + final String tempFileName = fileName + ".temp"; + final File file = new File(mContext.getFilesDir(), fileName); + final File tempFile = new File(mContext.getFilesDir(), tempFileName); try { final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile); writeDictionary(dictEncoder, attributeMap); diff --git a/java/src/com/android/inputmethod/latin/AssetFileAddress.java b/java/src/com/android/inputmethod/latin/AssetFileAddress.java index fd6c24dfe..875192554 100644 --- a/java/src/com/android/inputmethod/latin/AssetFileAddress.java +++ b/java/src/com/android/inputmethod/latin/AssetFileAddress.java @@ -16,8 +16,6 @@ package com.android.inputmethod.latin; -import com.android.inputmethod.latin.utils.FileUtils; - import java.io.File; /** @@ -54,12 +52,4 @@ public final class AssetFileAddress { if (!f.isFile()) return null; return new AssetFileAddress(filename, offset, length); } - - public boolean pointsToPhysicalFile() { - return 0 == mOffset; - } - - public void deleteUnderlyingFile() { - FileUtils.deleteRecursively(new File(mFilename)); - } } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index db4234c63..fd296988e 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -26,7 +26,6 @@ import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.StringUtils; -import com.android.inputmethod.latin.utils.UnigramProperty; import java.io.File; import java.util.ArrayList; @@ -58,21 +57,6 @@ public final class BinaryDictionary extends Dictionary { @UsedForTesting public static final String MAX_BIGRAM_COUNT_QUERY = "MAX_BIGRAM_COUNT"; - public static final int NOT_A_VALID_TIMESTAMP = -1; - - // Format to get unigram flags from native side via getUnigramPropertyNative(). - private static final int FORMAT_UNIGRAM_PROPERTY_OUTPUT_FLAG_COUNT = 4; - private static final int FORMAT_UNIGRAM_PROPERTY_IS_NOT_A_WORD_INDEX = 0; - private static final int FORMAT_UNIGRAM_PROPERTY_IS_BLACKLISTED_INDEX = 1; - private static final int FORMAT_UNIGRAM_PROPERTY_HAS_BIGRAMS_INDEX = 2; - private static final int FORMAT_UNIGRAM_PROPERTY_HAS_SHORTCUTS_INDEX = 3; - - // Format to get unigram historical info from native side via getUnigramPropertyNative(). - private static final int FORMAT_UNIGRAM_PROPERTY_OUTPUT_HISTORICAL_INFO_COUNT = 3; - private static final int FORMAT_UNIGRAM_PROPERTY_TIMESTAMP_INDEX = 0; - private static final int FORMAT_UNIGRAM_PROPERTY_LEVEL_INDEX = 1; - private static final int FORMAT_UNIGRAM_PROPERTY_COUNT_INDEX = 2; - private long mNativeDict; private final Locale mLocale; private final long mDictSize; @@ -139,13 +123,8 @@ public final class BinaryDictionary extends Dictionary { private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); private static native void flushWithGCNative(long dict, String filePath); private static native void closeNative(long dict); - private static native int getFormatVersionNative(long dict); private static native int getProbabilityNative(long dict, int[] word); private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1); - private static native void getUnigramPropertyNative(long dict, int[] word, - int[] outCodePoints, boolean[] outFlags, int[] outProbability, - int[] outHistoricalInfo, ArrayList<int[]> outShortcutTargets, - ArrayList<Integer> outShortcutProbabilities); private static native int getSuggestionsNative(long dict, long proximityInfo, long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint, @@ -154,14 +133,10 @@ public final class BinaryDictionary extends Dictionary { int[] outputAutoCommitFirstWordConfidence); private static native float calcNormalizedScoreNative(int[] before, int[] after, int score); private static native int editDistanceNative(int[] before, int[] after); - private static native void addUnigramWordNative(long dict, int[] word, int probability, - int[] shortcutTarget, int shortcutProbability, boolean isNotAWord, - boolean isBlacklisted, int timestamp); + private static native void addUnigramWordNative(long dict, int[] word, int probability); private static native void addBigramWordsNative(long dict, int[] word0, int[] word1, - int probability, int timestamp); + int probability); private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); - private static native int addMultipleDictionaryEntriesNative(long dict, - LanguageModelParam[] languageModelParams, int startIndex); private static native int calculateProbabilityNative(long dict, int unigramProbability, int bigramProbability); private static native String getPropertyNative(long dict, String query); @@ -260,10 +235,6 @@ public final class BinaryDictionary extends Dictionary { return mNativeDict != 0; } - public int getFormatVersion() { - return getFormatVersionNative(mNativeDict); - } - public static float calcNormalizedScore(final String before, final String after, final int score) { return calcNormalizedScoreNative(StringUtils.toCodePointArray(before), @@ -303,55 +274,23 @@ public final class BinaryDictionary extends Dictionary { return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); } - @UsedForTesting - public UnigramProperty getUnigramProperty(final String word) { - if (TextUtils.isEmpty(word)) { - return null; - } - final int[] codePoints = StringUtils.toCodePointArray(word); - final int[] outCodePoints = new int[MAX_WORD_LENGTH]; - final boolean[] outFlags = new boolean[FORMAT_UNIGRAM_PROPERTY_OUTPUT_FLAG_COUNT]; - final int[] outProbability = new int[1]; - final int[] outHistoricalInfo = - new int[FORMAT_UNIGRAM_PROPERTY_OUTPUT_HISTORICAL_INFO_COUNT]; - final ArrayList<int[]> outShortcutTargets = CollectionUtils.newArrayList(); - final ArrayList<Integer> outShortcutProbabilities = CollectionUtils.newArrayList(); - getUnigramPropertyNative(mNativeDict, codePoints, outCodePoints, outFlags, outProbability, - outHistoricalInfo, outShortcutTargets, outShortcutProbabilities); - return new UnigramProperty(codePoints, - outFlags[FORMAT_UNIGRAM_PROPERTY_IS_NOT_A_WORD_INDEX], - outFlags[FORMAT_UNIGRAM_PROPERTY_IS_BLACKLISTED_INDEX], - outFlags[FORMAT_UNIGRAM_PROPERTY_HAS_BIGRAMS_INDEX], - outFlags[FORMAT_UNIGRAM_PROPERTY_HAS_SHORTCUTS_INDEX], outProbability[0], - outHistoricalInfo[FORMAT_UNIGRAM_PROPERTY_TIMESTAMP_INDEX], - outHistoricalInfo[FORMAT_UNIGRAM_PROPERTY_LEVEL_INDEX], - outHistoricalInfo[FORMAT_UNIGRAM_PROPERTY_COUNT_INDEX], - outShortcutTargets, outShortcutProbabilities); - } - - // Add a unigram entry to binary dictionary with unigram attributes in native code. - public void addUnigramWord(final String word, final int probability, - final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord, - final boolean isBlacklisted, final int timestamp) { + // Add a unigram entry to binary dictionary in native code. + public void addUnigramWord(final String word, final int probability) { if (TextUtils.isEmpty(word)) { return; } final int[] codePoints = StringUtils.toCodePointArray(word); - final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? - StringUtils.toCodePointArray(shortcutTarget) : null; - addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, - shortcutProbability, isNotAWord, isBlacklisted, timestamp); + addUnigramWordNative(mNativeDict, codePoints, probability); } - // Add a bigram entry to binary dictionary with timestamp in native code. - public void addBigramWords(final String word0, final String word1, final int probability, - final int timestamp) { + // Add a bigram entry to binary dictionary in native code. + public void addBigramWords(final String word0, final String word1, final int probability) { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); - addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp); + addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability); } // Remove a bigram entry form binary dictionary in native code. @@ -364,70 +303,10 @@ public final class BinaryDictionary extends Dictionary { removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); } - public static class LanguageModelParam { - public final int[] mWord0; - public final int[] mWord1; - public final int[] mShortcutTarget; - public final int mUnigramProbability; - public final int mBigramProbability; - public final int mShortcutProbability; - public final boolean mIsNotAWord; - public final boolean mIsBlacklisted; - public final int mTimestamp; - - // Constructor for unigram. - public LanguageModelParam(final String word, final int unigramProbability, - final int timestamp) { - mWord0 = null; - mWord1 = StringUtils.toCodePointArray(word); - mShortcutTarget = null; - mUnigramProbability = unigramProbability; - mBigramProbability = NOT_A_PROBABILITY; - mShortcutProbability = NOT_A_PROBABILITY; - mIsNotAWord = false; - mIsBlacklisted = false; - mTimestamp = timestamp; - } - - // Constructor for unigram and bigram. - public LanguageModelParam(final String word0, final String word1, - final int unigramProbability, final int bigramProbability, - final int timestamp) { - mWord0 = StringUtils.toCodePointArray(word0); - mWord1 = StringUtils.toCodePointArray(word1); - mShortcutTarget = null; - mUnigramProbability = unigramProbability; - mBigramProbability = bigramProbability; - mShortcutProbability = NOT_A_PROBABILITY; - mIsNotAWord = false; - mIsBlacklisted = false; - mTimestamp = timestamp; - } - } - - public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { - if (!isValidDictionary()) return; - int processedParamCount = 0; - while (processedParamCount < languageModelParams.length) { - if (needsToRunGC(true /* mindsBlockByGC */)) { - flushWithGC(); - } - processedParamCount = addMultipleDictionaryEntriesNative(mNativeDict, - languageModelParams, processedParamCount); - if (processedParamCount <= 0) { - return; - } - } - - } - private void reopen() { close(); final File dictFile = new File(mDictFilePath); - // WARNING: Because we pass 0 as the offset and file.length() as the length, this can - // only be called for actual files. Right now it's only called by the flush() family of - // functions, which require an updatable dictionary, so it's okay. But beware. - loadDictionary(dictFile.getAbsolutePath(), 0 /* startOffset */, + mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, dictFile.length(), true /* isUpdatable */); } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java index b4382bc2c..722a82961 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java @@ -98,7 +98,7 @@ public final class BinaryDictionaryFileDumper { * This creates a URI builder able to build a URI pointing to the dictionary * pack content provider for a specific dictionary id. */ - public static Uri.Builder getProviderUriBuilder(final String path) { + private static Uri.Builder getProviderUriBuilder(final String path) { return new Uri.Builder().scheme(ContentResolver.SCHEME_CONTENT) .authority(DictionaryPackConstants.AUTHORITY).appendPath(path); } @@ -339,25 +339,15 @@ public final class BinaryDictionaryFileDumper { Log.e(TAG, "Could not copy a word list. Will not be able to use it."); // If we can't copy it we should warn the dictionary provider so that it can mark it // as invalid. - reportBrokenFileToDictionaryProvider(providerClient, clientId, wordlistId); - } - - public static boolean reportBrokenFileToDictionaryProvider( - final ContentProviderClient providerClient, final String clientId, - final String wordlistId) { + wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT, + QUERY_PARAMETER_FAILURE); try { - final Uri.Builder wordListUriBuilder = getContentUriBuilderForType(clientId, - providerClient, QUERY_PATH_DATAFILE, wordlistId /* extraPath */); - wordListUriBuilder.appendQueryParameter(QUERY_PARAMETER_DELETE_RESULT, - QUERY_PARAMETER_FAILURE); if (0 >= providerClient.delete(wordListUriBuilder.build(), null, null)) { - Log.e(TAG, "Unable to delete a word list."); + Log.e(TAG, "In addition, we were unable to delete it."); } } catch (RemoteException e) { - Log.e(TAG, "Communication with the dictionary provider was cut", e); - return false; + Log.e(TAG, "In addition, communication with the dictionary provider was cut", e); } - return true; } // Ideally the two following methods should be merged, but AssetFileDescriptor does not @@ -442,9 +432,8 @@ public final class BinaryDictionaryFileDumper { // Actually copy the file final byte[] buffer = new byte[FILE_READ_BUFFER_SIZE]; - for (int readBytes = input.read(buffer); readBytes >= 0; readBytes = input.read(buffer)) { + for (int readBytes = input.read(buffer); readBytes >= 0; readBytes = input.read(buffer)) output.write(buffer, 0, readBytes); - } input.close(); } @@ -489,7 +478,8 @@ public final class BinaryDictionaryFileDumper { * @param context the context for resources and providers. * @param clientId the client ID to use. */ - public static void initializeClientRecordHelper(final Context context, final String clientId) { + public static void initializeClientRecordHelper(final Context context, + final String clientId) { try { final ContentProviderClient client = context.getContentResolver(). acquireContentProviderClient(getProviderUriBuilder("").build()); diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index c260434d5..9a9653094 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -70,47 +70,38 @@ public final class Constants { public static final class ExtraValue { /** - * The subtype extra value used to indicate that this subtype is capable of - * entering ASCII characters. + * The subtype extra value used to indicate that the subtype keyboard layout is capable + * for typing ASCII characters. */ public static final String ASCII_CAPABLE = "AsciiCapable"; /** - * The subtype extra value used to indicate that this subtype is enabled - * when the default subtype is not marked as ascii capable. - */ - public static final String ENABLED_WHEN_DEFAULT_IS_NOT_ASCII_CAPABLE = - "EnabledWhenDefaultIsNotAsciiCapable"; - - /** - * The subtype extra value used to indicate that this subtype is capable of - * entering emoji characters. + * The subtype extra value used to indicate that the subtype keyboard layout is capable + * for typing EMOJI characters. */ public static final String EMOJI_CAPABLE = "EmojiCapable"; - /** - * The subtype extra value used to indicate that this subtype requires a network - * connection to work. + * The subtype extra value used to indicate that the subtype require network connection + * to work. */ public static final String REQ_NETWORK_CONNECTIVITY = "requireNetworkConnectivity"; /** - * The subtype extra value used to indicate that the display name of this subtype - * contains a "%s" for printf-like replacement and it should be replaced by - * this extra value. + * The subtype extra value used to indicate that the subtype display name contains "%s" + * for replacement mark and it should be replaced by this extra value. * This extra value is supported on JellyBean and later. */ public static final String UNTRANSLATABLE_STRING_IN_SUBTYPE_NAME = "UntranslatableReplacementStringInSubtypeName"; /** - * The subtype extra value used to indicate this subtype keyboard layout set name. + * The subtype extra value used to indicate that the subtype keyboard layout set name. * This extra value is private to LatinIME. */ public static final String KEYBOARD_LAYOUT_SET = "KeyboardLayoutSet"; /** - * The subtype extra value used to indicate that this subtype is an additional subtype + * The subtype extra value used to indicate that the subtype is additional subtype * that the user defined. This extra value is private to LatinIME. */ public static final String IS_ADDITIONAL_SUBTYPE = "isAdditionalSubtype"; diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index b6cfcd064..47891c6b7 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -44,8 +44,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { private static final String TAG = ContactsBinaryDictionary.class.getSimpleName(); private static final String NAME = "contacts"; - private static final boolean DEBUG = false; - private static final boolean DEBUG_DUMP = false; + private static boolean DEBUG = false; /** * Frequency for contacts information into the dictionary @@ -72,8 +71,8 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { private final boolean mUseFirstLastBigrams; public ContactsBinaryDictionary(final Context context, final Locale locale) { - super(context, getDictNameWithLocale(NAME, locale), locale, - Dictionary.TYPE_CONTACTS, false /* isUpdatable */); + super(context, getFilenameWithLocale(NAME, locale.toString()), Dictionary.TYPE_CONTACTS, + false /* isUpdatable */); mLocale = locale; mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale); registerObserver(context); @@ -169,10 +168,6 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (isValidName(name)) { addName(name); ++count; - } else { - if (DEBUG_DUMP) { - Log.d(TAG, "Invalid name: " + name); - } } cursor.moveToNext(); } @@ -209,9 +204,6 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (Character.isLetter(name.codePointAt(i))) { int end = getWordEndPosition(name, len, i); String word = name.substring(i, end); - if (DEBUG_DUMP) { - Log.d(TAG, "addName word = " + word); - } i = end - 1; // Don't add single letter words, possibly confuses // capitalization of i. diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index e04524843..fa79f5af7 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -52,10 +52,13 @@ public abstract class Dictionary { public static final String TYPE_CONTACTS = "contacts"; // User dictionary, the system-managed one. public static final String TYPE_USER = "user"; - // User history dictionary internal to LatinIME. + // User history dictionary internal to LatinIME. This assumes bigram prediction for now. public static final String TYPE_USER_HISTORY = "history"; - // Personalization dictionary. + // Personalization binary dictionary internal to LatinIME. public static final String TYPE_PERSONALIZATION = "personalization"; + // Personalization prediction dictionary internal to LatinIME's Java code. + public static final String TYPE_PERSONALIZATION_PREDICTION_IN_JAVA = + "personalization_prediction_in_java"; public final String mDictType; public Dictionary(final String dictType) { diff --git a/java/src/com/android/inputmethod/latin/DictionaryFactory.java b/java/src/com/android/inputmethod/latin/DictionaryFactory.java index e09c309ea..828e54f14 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFactory.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFactory.java @@ -16,7 +16,6 @@ package com.android.inputmethod.latin; -import android.content.ContentProviderClient; import android.content.Context; import android.content.res.AssetFileDescriptor; import android.content.res.Resources; @@ -65,10 +64,6 @@ public final class DictionaryFactory { useFullEditDistance, locale, Dictionary.TYPE_MAIN); if (readOnlyBinaryDictionary.isValidDictionary()) { dictList.add(readOnlyBinaryDictionary); - } else { - readOnlyBinaryDictionary.close(); - // Prevent this dictionary to do any further harm. - killDictionary(context, f); } } } @@ -80,51 +75,6 @@ public final class DictionaryFactory { } /** - * Kills a dictionary so that it is never used again, if possible. - * @param context The context to contact the dictionary provider, if possible. - * @param f A file address to the dictionary to kill. - */ - private static void killDictionary(final Context context, final AssetFileAddress f) { - if (f.pointsToPhysicalFile()) { - f.deleteUnderlyingFile(); - // Warn the dictionary provider if the dictionary came from there. - final ContentProviderClient providerClient; - try { - providerClient = context.getContentResolver().acquireContentProviderClient( - BinaryDictionaryFileDumper.getProviderUriBuilder("").build()); - } catch (final SecurityException e) { - Log.e(TAG, "No permission to communicate with the dictionary provider", e); - return; - } - if (null == providerClient) { - Log.e(TAG, "Can't establish communication with the dictionary provider"); - return; - } - final String wordlistId = - DictionaryInfoUtils.getWordListIdFromFileName(new File(f.mFilename).getName()); - if (null != wordlistId) { - // TODO: this is a reasonable last resort, but it is suboptimal. - // The following will remove the entry for this dictionary with the dictionary - // provider. When the metadata is downloaded again, we will try downloading it - // again. - // However, in the practice that will mean the user will find themselves without - // the new dictionary. That's fine for languages where it's included in the APK, - // but for other languages it will leave the user without a dictionary at all until - // the next update, which may be a few days away. - // Ideally, we would trigger a new download right away, and use increasing retry - // delays for this particular id/version combination. - // Then again, this is expected to only ever happen in case of human mistake. If - // the wrong file is on the server, the following is still doing the right thing. - // If it's a file left over from the last version however, it's not great. - BinaryDictionaryFileDumper.reportBrokenFileToDictionaryProvider( - providerClient, - context.getString(R.string.dictionary_pack_client_id), - wordlistId); - } - } - } - - /** * Initializes a main dictionary collection from a dictionary pack, with default flags. * * This searches for a content provider providing a dictionary pack for the specified diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 89ef96d7f..3df2a2b63 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -18,6 +18,8 @@ package com.android.inputmethod.latin; import android.content.Context; +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; @@ -35,14 +37,14 @@ import java.util.Map; * An in memory dictionary for memorizing entries and writing a binary dictionary. */ public class DictionaryWriter extends AbstractDictionaryWriter { - private static final int BINARY_DICT_VERSION = 2; + private static final int BINARY_DICT_VERSION = 3; private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(BINARY_DICT_VERSION, false /* supportsDynamicUpdate */); + new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); private FusionDictionary mFusionDictionary; - public DictionaryWriter(final Context context) { - super(context); + public DictionaryWriter(final Context context, final String dictType) { + super(context, dictType); clear(); } @@ -50,7 +52,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { public void clear() { final HashMap<String, String> attributes = CollectionUtils.newHashMap(); mFusionDictionary = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(attributes)); + new FusionDictionary.DictionaryOptions(attributes, false, false)); } /** @@ -90,4 +92,18 @@ public class DictionaryWriter extends AbstractDictionaryWriter { } dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS); } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + // This class doesn't support suggestion. + return null; + } + + @Override + public boolean isValidWord(String word) { + // This class doesn't support dictionary retrieval. + return false; + } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 9f5cd162f..eb8650e6f 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -22,22 +22,18 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam; import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; import java.util.ArrayList; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -56,7 +52,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Whether to print debug output to log */ private static boolean DEBUG = false; - private static final boolean DBG_STRESS_TEST = false; + + // TODO: Remove. + /** Whether to call binary dictionary dynamically updating methods. */ + public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true; private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; @@ -65,19 +64,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; - private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4; + private static final int DICTIONARY_FORMAT_VERSION = 3; + + private static final String SUPPORTS_DYNAMIC_UPDATE = + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE; /** * A static map of update controllers, each of which records the time of accesses to a single * binary dictionary file and tracks whether the file is regenerating. The key for this map is - * the dictionary name and the value is the shared dictionary time recorder associated with - * that dictionary name. + * the filename and the value is the shared dictionary time recorder associated with that + * filename. */ private static final ConcurrentHashMap<String, DictionaryUpdateController> - sDictNameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap(); + sFilenameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap(); private static final ConcurrentHashMap<String, PrioritizedSerialExecutor> - sDictNameExecutorMap = CollectionUtils.newConcurrentHashMap(); + sFilenameExecutorMap = CollectionUtils.newConcurrentHashMap(); /** The application context. */ protected final Context mContext; @@ -93,24 +95,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected AbstractDictionaryWriter mDictionaryWriter; /** - * The name of this dictionary, used as a part of the filename for storing the binary - * dictionary. Multiple dictionary instances with the same name is supported, with access - * controlled by DictionaryUpdateController. + * The name of this dictionary, used as the filename for storing the binary dictionary. Multiple + * dictionary instances with the same filename is supported, with access controlled by + * DictionaryTimeRecorder. */ - private final String mDictName; - - /** Dictionary locale */ - private final Locale mLocale; + private final String mFilename; /** Whether to support dynamically updating the dictionary */ private final boolean mIsUpdatable; - /** Dictionary file */ - private final File mDictFile; - // TODO: remove, once dynamic operations is serialized /** Controls updating the shared binary dictionary file across multiple instances. */ - private final DictionaryUpdateController mDictNameDictionaryUpdateController; + private final DictionaryUpdateController mFilenameDictionaryUpdateController; // TODO: remove, once dynamic operations is serialized /** Controls updating the local binary dictionary for this instance. */ @@ -136,57 +132,45 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected abstract boolean hasContentChanged(); - protected boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { - // This class is using format 2 because it's used by the User and Contacts dictionary - // only, which right now use format 2 (dicts using format 4 use Decaying*, which overrides - // this method). - // TODO: Migrate these dicts to ver4 format, and remove this function. - return formatVersion == 2; - } - - public boolean isValidDictionary() { - return mBinaryDictionary.isValidDictionary(); - } - - private File getDictFile() { - return mDictFile; - } - /** - * Gets the dictionary update controller for the given dictionary name. + * Gets the dictionary update controller for the given filename. */ private static DictionaryUpdateController getDictionaryUpdateController( - final String dictName) { - DictionaryUpdateController recorder = sDictNameDictionaryUpdateControllerMap.get(dictName); + String filename) { + DictionaryUpdateController recorder = sFilenameDictionaryUpdateControllerMap.get(filename); if (recorder == null) { - synchronized(sDictNameDictionaryUpdateControllerMap) { + synchronized(sFilenameDictionaryUpdateControllerMap) { recorder = new DictionaryUpdateController(); - sDictNameDictionaryUpdateControllerMap.put(dictName, recorder); + sFilenameDictionaryUpdateControllerMap.put(filename, recorder); } } return recorder; } /** - * Gets the executor for the given dictionary name. + * Gets the executor for the given filename. */ - private static PrioritizedSerialExecutor getExecutor(final String dictName) { - PrioritizedSerialExecutor executor = sDictNameExecutorMap.get(dictName); + private static PrioritizedSerialExecutor getExecutor(final String filename) { + PrioritizedSerialExecutor executor = sFilenameExecutorMap.get(filename); if (executor == null) { - synchronized(sDictNameExecutorMap) { + synchronized(sFilenameExecutorMap) { executor = new PrioritizedSerialExecutor(); - sDictNameExecutorMap.put(dictName, executor); + sFilenameExecutorMap.put(filename, executor); } } return executor; } private static AbstractDictionaryWriter getDictionaryWriter(final Context context, - final boolean isDynamicPersonalizationDictionary) { + final String dictType, final boolean isDynamicPersonalizationDictionary) { if (isDynamicPersonalizationDictionary) { - return null; + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + return null; + } else { + return new DynamicPersonalizationDictionaryWriter(context, dictType); + } } else { - return new DictionaryWriter(context); + return new DictionaryWriter(context, dictType); } } @@ -194,37 +178,26 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Creates a new expandable binary dictionary. * * @param context The application context of the parent. - * @param dictName The name of the dictionary. Multiple instances with the same - * name is supported. - * @param locale the dictionary locale. + * @param filename The filename for this binary dictionary. Multiple dictionaries with the same + * filename is supported. * @param dictType the dictionary type, as a human-readable string * @param isUpdatable whether to support dynamically updating the dictionary. Please note that * dynamic dictionary has negative effects on memory space and computation time. */ - public ExpandableBinaryDictionary(final Context context, final String dictName, - final Locale locale, final String dictType, final boolean isUpdatable) { - this(context, dictName, locale, dictType, isUpdatable, - new File(context.getFilesDir(), dictName + DICT_FILE_EXTENSION)); - } - - // Creates an instance that uses a given dictionary file. - public ExpandableBinaryDictionary(final Context context, final String dictName, - final Locale locale, final String dictType, final boolean isUpdatable, - final File dictFile) { + public ExpandableBinaryDictionary(final Context context, final String filename, + final String dictType, final boolean isUpdatable) { super(dictType); - mDictName = dictName; + mFilename = filename; mContext = context; - mLocale = locale; mIsUpdatable = isUpdatable; - mDictFile = dictFile; mBinaryDictionary = null; - mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName); + mFilenameDictionaryUpdateController = getDictionaryUpdateController(filename); // Currently, only dynamic personalization dictionary is updatable. - mDictionaryWriter = getDictionaryWriter(context, isUpdatable); + mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); } - protected static String getDictNameWithLocale(final String name, final Locale locale) { - return name + "." + locale.toString(); + protected static String getFilenameWithLocale(final String name, final String localeStr) { + return name + "." + localeStr + DICT_FILE_EXTENSION; } /** @@ -232,20 +205,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ @Override public void close() { - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { if (mBinaryDictionary!= null) { mBinaryDictionary.close(); mBinaryDictionary = null; } + if (mDictionaryWriter != null) { + mDictionaryWriter.close(); + } } }); } protected void closeBinaryDictionary() { // Ensure that no other threads are accessing the local binary dictionary. - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { if (mBinaryDictionary != null) { @@ -258,23 +234,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString()); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE, - String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + SUPPORTS_DYNAMIC_UPDATE); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename); return attributeMap; } protected void clear() { - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - if (mDictionaryWriter == null) { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { mBinaryDictionary.close(); - final File file = getDictFile(); - if (file.exists() && !FileUtils.deleteRecursively(file)) { - Log.e(TAG, "Can't remove a file: " + file.getName()); - } + final File file = new File(mContext.getFilesDir(), mFilename); BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); mBinaryDictionary = new BinaryDictionary( @@ -314,7 +286,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Check whether GC is needed and run GC if required. */ protected void runGCIfRequired(final boolean mindsBlockByGC) { - getExecutor(mDictName).execute(new Runnable() { + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { runGCIfRequiredInternalLocked(mindsBlockByGC); @@ -323,17 +296,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; // Calls to needsToRunGC() need to be serialized. if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { - if (setProcessingLargeTaskIfNot()) { + if (setIsRegeneratingIfNotRegenerating()) { // Run GC after currently existing time sensitive operations. - getExecutor(mDictName).executePrioritized(new Runnable() { + getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { try { mBinaryDictionary.flushWithGC(); } finally { - mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); + mFilenameDictionaryUpdateController.mIsRegenerating.set(false); } } }); @@ -344,19 +318,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. */ - protected void addWordDynamically(final String word, final int frequency, - final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, - final boolean isBlacklisted, final int timestamp) { + protected void addWordDynamically(final String word, final String shortcutTarget, + final int frequency, final int shortcutFreq, final boolean isNotAWord) { if (!mIsUpdatable) { - Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mDictName); + Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename); return; } - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, - isNotAWord, isBlacklisted, timestamp); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + mBinaryDictionary.addUnigramWord(word, frequency); + } else { + // TODO: Remove. + mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, + isNotAWord); + } } }); } @@ -365,17 +343,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. */ protected void addBigramDynamically(final String word0, final String word1, - final int frequency, final int timestamp) { + final int frequency, final boolean isValid) { if (!mIsUpdatable) { Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: " - + mDictName); + + mFilename); return; } - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + mBinaryDictionary.addBigramWords(word0, word1, frequency); + } else { + // TODO: Remove. + mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, + 0 /* lastTouchedTime */); + } } }); } @@ -386,48 +370,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected void removeBigramDynamically(final String word0, final String word1) { if (!mIsUpdatable) { Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: " - + mDictName); + + mFilename); return; } - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.removeBigramWords(word0, word1); - } - }); - } - - public interface AddMultipleDictionaryEntriesCallback { - public void onFinished(); - } - - /** - * Dynamically add multiple entries to the dictionary. - */ - protected void addMultipleDictionaryEntriesDynamically( - final ArrayList<LanguageModelParam> languageModelParams, - final AddMultipleDictionaryEntriesCallback callback) { - if (!mIsUpdatable) { - Log.w(TAG, "addMultipleDictionaryEntriesDynamically is called for non-updatable " + - "dictionary: " + mDictName); - return; - } - getExecutor(mDictName).execute(new Runnable() { - @Override - public void run() { - final boolean locked = setProcessingLargeTaskIfNot(); - try { - mBinaryDictionary.addMultipleDictionaryEntries( - languageModelParams.toArray( - new LanguageModelParam[languageModelParams.size()])); - } finally { - if (callback != null) { - callback.onFinished(); - } - if (locked) { - mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); - } + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + mBinaryDictionary.removeBigramWords(word0, word1); + } else { + // TODO: Remove. + mDictionaryWriter.removeBigramWords(word0, word1); } } }); @@ -439,23 +393,49 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId) { reloadDictionaryIfRequired(); - if (processingLargeTask()) { + if (isRegenerating()) { return null; } + final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = new AsyncResultHolder<ArrayList<SuggestedWordInfo>>(); - getExecutor(mDictName).executePrioritized(new Runnable() { + getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - if (mBinaryDictionary == null) { - holder.set(null); - return; + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + if (mBinaryDictionary == null) { + holder.set(null); + return; + } + final ArrayList<SuggestedWordInfo> binarySuggestion = + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + sessionId); + holder.set(binarySuggestion); + } else { + final ArrayList<SuggestedWordInfo> inMemDictSuggestion = + composer.isBatchMode() ? null : + mDictionaryWriter.getSuggestionsWithSessionId(composer, + prevWord, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId); + // TODO: Remove checking mIsUpdatable and use native suggestion. + if (mBinaryDictionary != null && !mIsUpdatable) { + final ArrayList<SuggestedWordInfo> binarySuggestion = + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId); + if (inMemDictSuggestion == null) { + holder.set(binarySuggestion); + } else if (binarySuggestion == null) { + holder.set(inMemDictSuggestion); + } else { + binarySuggestion.addAll(inMemDictSuggestion); + holder.set(binarySuggestion); + } + } else { + holder.set(inMemDictSuggestion); + } } - final ArrayList<SuggestedWordInfo> binarySuggestion = - mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, additionalFeaturesOptions, - sessionId); - holder.set(binarySuggestion); } }); return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); @@ -476,11 +456,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } protected boolean isValidWordInner(final String word) { - if (processingLargeTask()) { + if (isRegenerating()) { return false; } final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); - getExecutor(mDictName).executePrioritized(new Runnable() { + getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { holder.set(isValidWordLocked(word)); @@ -514,22 +494,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private void loadBinaryDictionary() { if (DEBUG) { - Log.d(TAG, "Loading binary dictionary: " + mDictName + " request=" - + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" - + mDictNameDictionaryUpdateController.mLastUpdateTime); - } - if (DBG_STRESS_TEST) { - // Test if this class does not cause problems when it takes long time to load binary - // dictionary. - try { - Log.w(TAG, "Start stress in loading: " + mDictName); - Thread.sleep(15000); - Log.w(TAG, "End stress in loading"); - } catch (InterruptedException e) { - } + Log.d(TAG, "Loading binary dictionary: " + mFilename + " request=" + + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + + mFilenameDictionaryUpdateController.mLastUpdateTime); } - final File file = getDictFile(); + final File file = new File(mContext.getFilesDir(), mFilename); final String filename = file.getAbsolutePath(); final long length = file.length(); @@ -541,7 +511,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // swapping in the new one. // TODO: Ensure multi-thread assignment of mBinaryDictionary. final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; - getExecutor(mDictName).executePrioritized(new Runnable() { + getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { mBinaryDictionary = newBinaryDictionary; @@ -563,31 +533,29 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private void writeBinaryDictionary() { if (DEBUG) { - Log.d(TAG, "Generating binary dictionary: " + mDictName + " request=" - + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" - + mDictNameDictionaryUpdateController.mLastUpdateTime); + Log.d(TAG, "Generating binary dictionary: " + mFilename + " request=" + + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + + mFilenameDictionaryUpdateController.mLastUpdateTime); } if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); loadDictionaryAsync(); - mDictionaryWriter.write(getDictFile(), getHeaderAttributeMap()); + mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } else { - if (mBinaryDictionary == null || !isValidDictionary() - // TODO: remove the check below - || !matchesExpectedBinaryDictFormatVersionForThisType( - mBinaryDictionary.getFormatVersion())) { - final File file = getDictFile(); - if (file.exists() && !FileUtils.deleteRecursively(file)) { - Log.e(TAG, "Can't remove a file: " + file.getName()); - } - BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); - } else { - if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { - mBinaryDictionary.flushWithGC(); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { + final File file = new File(mContext.getFilesDir(), mFilename); + BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); } else { - mBinaryDictionary.flush(); + if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + mBinaryDictionary.flushWithGC(); + } else { + mBinaryDictionary.flush(); + } } + } else { + mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } } } @@ -602,10 +570,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected void setRequiresReload(final boolean requiresRebuild) { final long time = SystemClock.uptimeMillis(); mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = time; - mDictNameDictionaryUpdateController.mLastUpdateRequestTime = time; + mFilenameDictionaryUpdateController.mLastUpdateRequestTime = time; if (DEBUG) { - Log.d(TAG, "Reload request: " + mDictName + ": request=" + time + " update=" - + mDictNameDictionaryUpdateController.mLastUpdateTime); + Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update=" + + mFilenameDictionaryUpdateController.mLastUpdateTime); } } @@ -614,7 +582,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ public final void reloadDictionaryIfRequired() { if (!isReloadRequired()) return; - if (setProcessingLargeTaskIfNot()) { + if (setIsRegeneratingIfNotRegenerating()) { reloadDictionary(); } } @@ -626,14 +594,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.isOutOfDate(); } - private boolean processingLargeTask() { - return mDictNameDictionaryUpdateController.mProcessingLargeTask.get(); + private boolean isRegenerating() { + return mFilenameDictionaryUpdateController.mIsRegenerating.get(); } - // Returns whether the dictionary is being used for a large task. If true, we should not use - // this dictionary for latency sensitive operations. - private boolean setProcessingLargeTaskIfNot() { - return mDictNameDictionaryUpdateController.mProcessingLargeTask.compareAndSet( + // Returns whether the dictionary can be regenerated. + private boolean setIsRegeneratingIfNotRegenerating() { + return mFilenameDictionaryUpdateController.mIsRegenerating.compareAndSet( false /* expect */ , true /* update */); } @@ -644,13 +611,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private final void reloadDictionary() { // Ensure that only one thread attempts to read or write to the shared binary dictionary // file at the same time. - getExecutor(mDictName).execute(new Runnable() { + getExecutor(mFilename).execute(new Runnable() { @Override public void run() { try { final long time = SystemClock.uptimeMillis(); final boolean dictionaryFileExists = dictionaryFileExists(); - if (mDictNameDictionaryUpdateController.isOutOfDate() + if (mFilenameDictionaryUpdateController.isOutOfDate() || !dictionaryFileExists) { // If the shared dictionary file does not exist or is out of date, the // first instance that acquires the lock will generate a new one. @@ -659,44 +626,31 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // rebuild the binary dictionary. Empty dictionaries are supported (in // the case where loadDictionaryAsync() adds nothing) in order to // provide a uniform framework. - mDictNameDictionaryUpdateController.mLastUpdateTime = time; + mFilenameDictionaryUpdateController.mLastUpdateTime = time; writeBinaryDictionary(); loadBinaryDictionary(); } else { // If not, the reload request was unnecessary so revert // LastUpdateRequestTime to LastUpdateTime. - mDictNameDictionaryUpdateController.mLastUpdateRequestTime = - mDictNameDictionaryUpdateController.mLastUpdateTime; + mFilenameDictionaryUpdateController.mLastUpdateRequestTime = + mFilenameDictionaryUpdateController.mLastUpdateTime; } } else if (mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.mLastUpdateTime - < mDictNameDictionaryUpdateController.mLastUpdateTime) { + < mFilenameDictionaryUpdateController.mLastUpdateTime) { // Otherwise, if the local dictionary is older than the shared dictionary, // load the shared dictionary. loadBinaryDictionary(); } - // If we just loaded the binary dictionary, then mBinaryDictionary is not - // up-to-date yet so it's useless to test it right away. Schedule the check - // for right after it's loaded instead. - getExecutor(mDictName).executePrioritized(new Runnable() { - @Override - public void run() { - if (mBinaryDictionary != null && !(isValidDictionary() - // TODO: remove the check below - && matchesExpectedBinaryDictFormatVersionForThisType( - mBinaryDictionary.getFormatVersion()))) { - // Binary dictionary or its format version is not valid. Regenerate - // the dictionary file. writeBinaryDictionary will remove the - // existing files if appropriate. - mDictNameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); - } - mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; - } - }); + if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { + // Binary dictionary is not valid. Regenerate the dictionary file. + mFilenameDictionaryUpdateController.mLastUpdateTime = time; + writeBinaryDictionary(); + loadBinaryDictionary(); + } + mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; } finally { - mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); + mFilenameDictionaryUpdateController.mIsRegenerating.set(false); } } }); @@ -704,13 +658,28 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // TODO: cache the file's existence so that we avoid doing a disk access each time. private boolean dictionaryFileExists() { - return getDictFile().exists(); + final File file = new File(mContext.getFilesDir(), mFilename); + return file.exists(); + } + + /** + * Load the dictionary to memory. + */ + protected void asyncLoadDictionaryToMemory() { + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + loadDictionaryAsync(); + } + } + }); } /** * Generate binary dictionary using DictionaryWriter. */ - protected void asyncFlushBinaryDictionary() { + protected void asyncFlashAllBinaryDictionary() { final Runnable newTask = new Runnable() { @Override public void run() { @@ -718,32 +687,37 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } }; final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask); - getExecutor(mDictName).replaceAndExecute(oldTask, newTask); + getExecutor(mFilename).replaceAndExecute(oldTask, newTask); } /** - * For tracking whether the dictionary is out of date and the dictionary is used in a large - * task. Can be shared across multiple dictionary instances that access the same filename. + * For tracking whether the dictionary is out of date and the dictionary is regenerating. + * Can be shared across multiple dictionary instances that access the same filename. */ private static class DictionaryUpdateController { public volatile long mLastUpdateTime = 0; public volatile long mLastUpdateRequestTime = 0; - public volatile AtomicBoolean mProcessingLargeTask = new AtomicBoolean(); + public volatile AtomicBoolean mIsRegenerating = new AtomicBoolean(); public boolean isOutOfDate() { return (mLastUpdateRequestTime > mLastUpdateTime); } } - // TODO: Implement BinaryDictionary.isInDictionary(). + // TODO: Implement native binary methods once the dynamic dictionary implementation is done. @UsedForTesting public boolean isInDictionaryForTests(final String word) { final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); - getExecutor(mDictName).executePrioritized(new Runnable() { + getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { if (mDictType == Dictionary.TYPE_USER_HISTORY) { - holder.set(mBinaryDictionary.isValidWord(word)); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + holder.set(mBinaryDictionary.isValidWord(word)); + } else { + holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) + .isInBigramListForTests(word)); + } } } }); @@ -751,33 +725,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } @UsedForTesting - public void waitAllTasksForTests() { - final CountDownLatch countDownLatch = new CountDownLatch(1); - getExecutor(mDictName).execute(new Runnable() { - @Override - public void run() { - countDownLatch.countDown(); - } - }); - try { - countDownLatch.await(); - } catch (InterruptedException e) { - Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); - } + public void shutdownExecutorForTests() { + getExecutor(mFilename).shutdown(); } @UsedForTesting - protected void runAfterGcForDebug(final Runnable r) { - getExecutor(mDictName).executePrioritized(new Runnable() { - @Override - public void run() { - try { - mBinaryDictionary.flushWithGC(); - r.run(); - } finally { - mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); - } - } - }); + public boolean isTerminatedForTests() { + return getExecutor(mFilename).isTerminated(); } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java new file mode 100644 index 000000000..95c9bcab9 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -0,0 +1,894 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.text.TextUtils; +import android.util.Log; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + +import java.util.ArrayList; +import java.util.LinkedList; + +/** + * Class for an in-memory dictionary that can grow dynamically and can + * be searched for suggestions and valid words. + */ +// TODO: Remove after binary dictionary supports dynamic update. +public class ExpandableDictionary extends Dictionary { + private static final String TAG = ExpandableDictionary.class.getSimpleName(); + /** + * The weight to give to a word if it's length is the same as the number of typed characters. + */ + private static final int FULL_WORD_SCORE_MULTIPLIER = 2; + + private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH]; + private int mMaxDepth; + private int mInputLength; + + private static final class Node { + char mCode; + int mFrequency; + boolean mTerminal; + Node mParent; + NodeArray mChildren; + ArrayList<char[]> mShortcutTargets; + boolean mShortcutOnly; + LinkedList<NextWord> mNGrams; // Supports ngram + } + + private static final class NodeArray { + Node[] mData; + int mLength = 0; + private static final int INCREMENT = 2; + + NodeArray() { + mData = new Node[INCREMENT]; + } + + void add(final Node n) { + if (mLength + 1 > mData.length) { + Node[] tempData = new Node[mLength + INCREMENT]; + if (mLength > 0) { + System.arraycopy(mData, 0, tempData, 0, mLength); + } + mData = tempData; + } + mData[mLength++] = n; + } + } + + public interface NextWord { + public Node getWordNode(); + public int getFrequency(); + public ForgettingCurveParams getFcParams(); + public int notifyTypedAgainAndGetFrequency(); + } + + private static final class NextStaticWord implements NextWord { + public final Node mWord; + private final int mFrequency; + public NextStaticWord(Node word, int frequency) { + mWord = word; + mFrequency = frequency; + } + + @Override + public Node getWordNode() { + return mWord; + } + + @Override + public int getFrequency() { + return mFrequency; + } + + @Override + public ForgettingCurveParams getFcParams() { + return null; + } + + @Override + public int notifyTypedAgainAndGetFrequency() { + return mFrequency; + } + } + + private static final class NextHistoryWord implements NextWord { + public final Node mWord; + public final ForgettingCurveParams mFcp; + + public NextHistoryWord(Node word, ForgettingCurveParams fcp) { + mWord = word; + mFcp = fcp; + } + + @Override + public Node getWordNode() { + return mWord; + } + + @Override + public int getFrequency() { + return mFcp.getFrequency(); + } + + @Override + public ForgettingCurveParams getFcParams() { + return mFcp; + } + + @Override + public int notifyTypedAgainAndGetFrequency() { + return mFcp.notifyTypedAgainAndGetFrequency(); + } + } + + private NodeArray mRoots; + + private int[][] mCodes; + + public ExpandableDictionary(final String dictType) { + super(dictType); + clearDictionary(); + mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][]; + } + + public int getMaxWordLength() { + return Constants.DICTIONARY_MAX_WORD_LENGTH; + } + + /** + * Add a word with an optional shortcut to the dictionary. + * @param word The word to add. + * @param shortcutTarget A shortcut target for this word, or null if none. + * @param frequency The frequency for this unigram. + * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored + * if shortcutTarget is null. + */ + public void addWord(final String word, final String shortcutTarget, final int frequency, + final int shortcutFreq) { + if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH) { + return; + } + addWordRec(mRoots, word, 0, shortcutTarget, frequency, shortcutFreq, null); + } + + /** + * Add a word, recursively searching for its correct place in the trie tree. + * @param children The node to recursively search for addition. Initially, the root of the tree. + * @param word The word to add. + * @param depth The current depth in the tree. + * @param shortcutTarget A shortcut target for this word, or null if none. + * @param frequency The frequency for this unigram. + * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored + * if shortcutTarget is null. + * @param parentNode The parent node, for up linking. Initially null, as the root has no parent. + */ + private void addWordRec(final NodeArray children, final String word, final int depth, + final String shortcutTarget, final int frequency, final int shortcutFreq, + final Node parentNode) { + final int wordLength = word.length(); + if (wordLength <= depth) return; + final char c = word.charAt(depth); + // Does children have the current character? + final int childrenLength = children.mLength; + Node childNode = null; + for (int i = 0; i < childrenLength; i++) { + final Node node = children.mData[i]; + if (node.mCode == c) { + childNode = node; + break; + } + } + final boolean isShortcutOnly = (null != shortcutTarget); + if (childNode == null) { + childNode = new Node(); + childNode.mCode = c; + childNode.mParent = parentNode; + childNode.mShortcutOnly = isShortcutOnly; + children.add(childNode); + } + if (wordLength == depth + 1) { + // Terminate this word + childNode.mTerminal = true; + if (isShortcutOnly) { + if (null == childNode.mShortcutTargets) { + childNode.mShortcutTargets = CollectionUtils.newArrayList(); + } + childNode.mShortcutTargets.add(shortcutTarget.toCharArray()); + } else { + childNode.mShortcutOnly = false; + } + childNode.mFrequency = Math.max(frequency, childNode.mFrequency); + if (childNode.mFrequency > 255) childNode.mFrequency = 255; + return; + } + if (childNode.mChildren == null) { + childNode.mChildren = new NodeArray(); + } + addWordRec(childNode.mChildren, word, depth + 1, shortcutTarget, frequency, shortcutFreq, + childNode); + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + if (composer.size() > 1) { + if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) { + return null; + } + final ArrayList<SuggestedWordInfo> suggestions = + getWordsInner(composer, prevWord, proximityInfo); + return suggestions; + } else { + if (TextUtils.isEmpty(prevWord)) return null; + final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); + runBigramReverseLookUp(prevWord, suggestions); + return suggestions; + } + } + + private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, + final String prevWordForBigrams, final ProximityInfo proximityInfo) { + final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); + mInputLength = codes.size(); + if (mCodes.length < mInputLength) mCodes = new int[mInputLength][]; + final InputPointers ips = codes.getInputPointers(); + final int[] xCoordinates = ips.getXCoordinates(); + final int[] yCoordinates = ips.getYCoordinates(); + // Cache the codes so that we don't have to lookup an array list + for (int i = 0; i < mInputLength; i++) { + // TODO: Calculate proximity info here. + if (mCodes[i] == null || mCodes[i].length < 1) { + mCodes[i] = new int[ProximityInfo.MAX_PROXIMITY_CHARS_SIZE]; + } + final int x = xCoordinates != null && i < xCoordinates.length ? + xCoordinates[i] : Constants.NOT_A_COORDINATE; + final int y = xCoordinates != null && i < yCoordinates.length ? + yCoordinates[i] : Constants.NOT_A_COORDINATE; + proximityInfo.fillArrayWithNearestKeyCodes(x, y, codes.getCodeAt(i), mCodes[i]); + } + mMaxDepth = mInputLength * 3; + getWordsRec(mRoots, codes, mWordBuilder, 0, false, 1, 0, -1, suggestions); + for (int i = 0; i < mInputLength; i++) { + getWordsRec(mRoots, codes, mWordBuilder, 0, false, 1, 0, i, suggestions); + } + return suggestions; + } + + @Override + public synchronized boolean isValidWord(final String word) { + final Node node = searchNode(mRoots, word, 0, word.length()); + // If node is null, we didn't find the word, so it's not valid. + // If node.mShortcutOnly is true, then it exists as a shortcut but not as a word, + // so that means it's not a valid word. + // If node.mShortcutOnly is false, then it exists as a word (it may also exist as + // a shortcut, but this does not matter), so it's a valid word. + return (node == null) ? false : !node.mShortcutOnly; + } + + public boolean removeBigram(final String word0, final String word1) { + // Refer to addOrSetBigram() about word1.toLowerCase() + final Node firstWord = searchWord(mRoots, word0.toLowerCase(), 0, null); + final Node secondWord = searchWord(mRoots, word1, 0, null); + LinkedList<NextWord> bigrams = firstWord.mNGrams; + NextWord bigramNode = null; + if (bigrams == null || bigrams.size() == 0) { + return false; + } else { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + bigramNode = nw; + break; + } + } + } + if (bigramNode == null) { + return false; + } + return bigrams.remove(bigramNode); + } + + /** + * Returns the word's frequency or -1 if not found + */ + @UsedForTesting + public int getWordFrequency(final String word) { + // Case-sensitive search + final Node node = searchNode(mRoots, word, 0, word.length()); + return (node == null) ? -1 : node.mFrequency; + } + + public NextWord getBigramWord(final String word0, final String word1) { + // Refer to addOrSetBigram() about word0.toLowerCase() + final Node firstWord = searchWord(mRoots, word0.toLowerCase(), 0, null); + final Node secondWord = searchWord(mRoots, word1, 0, null); + LinkedList<NextWord> bigrams = firstWord.mNGrams; + if (bigrams == null || bigrams.size() == 0) { + return null; + } else { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + return nw; + } + } + } + return null; + } + + private static int computeSkippedWordFinalFreq(final int freq, final int snr, + final int inputLength) { + // The computation itself makes sense for >= 2, but the == 2 case returns 0 + // anyway so we may as well test against 3 instead and return the constant + if (inputLength >= 3) { + return (freq * snr * (inputLength - 2)) / (inputLength - 1); + } else { + return 0; + } + } + + /** + * Helper method to add a word and its shortcuts. + * + * @param node the terminal node + * @param word the word to insert, as an array of code points + * @param depth the depth of the node in the tree + * @param finalFreq the frequency for this word + * @param suggestions the suggestion collection to add the suggestions to + * @return whether there is still space for more words. + */ + private boolean addWordAndShortcutsFromNode(final Node node, final char[] word, final int depth, + final int finalFreq, final ArrayList<SuggestedWordInfo> suggestions) { + if (finalFreq > 0 && !node.mShortcutOnly) { + // Use KIND_CORRECTION always. This dictionary does not really have a notion of + // COMPLETION against CORRECTION; we could artificially add one by looking at + // the respective size of the typed word and the suggestion if it matters sometime + // in the future. + suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, + SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); + if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; + } + if (null != node.mShortcutTargets) { + final int length = node.mShortcutTargets.size(); + for (int shortcutIndex = 0; shortcutIndex < length; ++shortcutIndex) { + final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); + suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), + finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); + if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; + } + } + return true; + } + + /** + * Recursively traverse the tree for words that match the input. Input consists of + * a list of arrays. Each item in the list is one input character position. An input + * character is actually an array of multiple possible candidates. This function is not + * optimized for speed, assuming that the user dictionary will only be a few hundred words in + * size. + * @param roots node whose children have to be search for matches + * @param codes the input character codes + * @param word the word being composed as a possible match + * @param depth the depth of traversal - the length of the word being composed thus far + * @param completion whether the traversal is now in completion mode - meaning that we've + * exhausted the input and we're looking for all possible suffixes. + * @param snr current weight of the word being formed + * @param inputIndex position in the input characters. This can be off from the depth in + * case we skip over some punctuations such as apostrophe in the traversal. That is, if you type + * "wouldve", it could be matching "would've", so the depth will be one more than the + * inputIndex + * @param suggestions the list in which to add suggestions + */ + // TODO: Share this routine with the native code for BinaryDictionary + private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word, + final int depth, final boolean completion, final int snr, final int inputIndex, + final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) { + final int count = roots.mLength; + final int codeSize = mInputLength; + // Optimization: Prune out words that are too long compared to how much was typed. + if (depth > mMaxDepth) { + return; + } + final int[] currentChars; + if (codeSize <= inputIndex) { + currentChars = null; + } else { + currentChars = mCodes[inputIndex]; + } + + for (int i = 0; i < count; i++) { + final Node node = roots.mData[i]; + final char c = node.mCode; + final char lowerC = toLowerCase(c); + final boolean terminal = node.mTerminal; + final NodeArray children = node.mChildren; + final int freq = node.mFrequency; + if (completion || currentChars == null) { + word[depth] = c; + if (terminal) { + final int finalFreq; + if (skipPos < 0) { + finalFreq = freq * snr; + } else { + finalFreq = computeSkippedWordFinalFreq(freq, snr, mInputLength); + } + if (!addWordAndShortcutsFromNode(node, word, depth, finalFreq, suggestions)) { + // No space left in the queue, bail out + return; + } + } + if (children != null) { + getWordsRec(children, codes, word, depth + 1, true, snr, inputIndex, + skipPos, suggestions); + } + } else if ((c == Constants.CODE_SINGLE_QUOTE + && currentChars[0] != Constants.CODE_SINGLE_QUOTE) || depth == skipPos) { + // Skip the ' and continue deeper + word[depth] = c; + if (children != null) { + getWordsRec(children, codes, word, depth + 1, completion, snr, inputIndex, + skipPos, suggestions); + } + } else { + // Don't use alternatives if we're looking for missing characters + final int alternativesSize = skipPos >= 0 ? 1 : currentChars.length; + for (int j = 0; j < alternativesSize; j++) { + final int addedAttenuation = (j > 0 ? 1 : 2); + final int currentChar = currentChars[j]; + if (currentChar == Constants.NOT_A_CODE) { + break; + } + if (currentChar == lowerC || currentChar == c) { + word[depth] = c; + + if (codeSize == inputIndex + 1) { + if (terminal) { + final int finalFreq; + if (skipPos < 0) { + finalFreq = freq * snr * addedAttenuation + * FULL_WORD_SCORE_MULTIPLIER; + } else { + finalFreq = computeSkippedWordFinalFreq(freq, + snr * addedAttenuation, mInputLength); + } + if (!addWordAndShortcutsFromNode(node, word, depth, finalFreq, + suggestions)) { + // No space left in the queue, bail out + return; + } + } + if (children != null) { + getWordsRec(children, codes, word, depth + 1, + true, snr * addedAttenuation, inputIndex + 1, + skipPos, suggestions); + } + } else if (children != null) { + getWordsRec(children, codes, word, depth + 1, + false, snr * addedAttenuation, inputIndex + 1, + skipPos, suggestions); + } + } + } + } + } + } + + public int setBigramAndGetFrequency(final String word0, final String word1, + final int frequency) { + return setBigramAndGetFrequency(word0, word1, frequency, null /* unused */); + } + + public int setBigramAndGetFrequency(final String word0, final String word1, + final ForgettingCurveParams fcp) { + return setBigramAndGetFrequency(word0, word1, 0 /* unused */, fcp); + } + + /** + * Adds bigrams to the in-memory trie structure that is being used to retrieve any word + * @param word0 the first word of this bigram + * @param word1 the second word of this bigram + * @param frequency frequency for this bigram + * @param fcp an instance of ForgettingCurveParams to use for decay policy + * @return returns the final bigram frequency + */ + private int setBigramAndGetFrequency(final String word0, final String word1, + final int frequency, final ForgettingCurveParams fcp) { + if (TextUtils.isEmpty(word0)) { + Log.e(TAG, "Invalid bigram previous word: " + word0); + return frequency; + } + // We don't want results to be different according to case of the looked up left hand side + // word. We do want however to return the correct case for the right hand side. + // So we want to squash the case of the left hand side, and preserve that of the right + // hand side word. + final String word0Lower = word0.toLowerCase(); + if (TextUtils.isEmpty(word0Lower) || TextUtils.isEmpty(word1)) { + Log.e(TAG, "Invalid bigram pair: " + word0 + ", " + word0Lower + ", " + word1); + return frequency; + } + final Node firstWord = searchWord(mRoots, word0Lower, 0, null); + final Node secondWord = searchWord(mRoots, word1, 0, null); + LinkedList<NextWord> bigrams = firstWord.mNGrams; + if (bigrams == null || bigrams.size() == 0) { + firstWord.mNGrams = CollectionUtils.newLinkedList(); + bigrams = firstWord.mNGrams; + } else { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + return nw.notifyTypedAgainAndGetFrequency(); + } + } + } + if (fcp != null) { + // history + firstWord.mNGrams.add(new NextHistoryWord(secondWord, fcp)); + } else { + firstWord.mNGrams.add(new NextStaticWord(secondWord, frequency)); + } + return frequency; + } + + /** + * Searches for the word and add the word if it does not exist. + * @return Returns the terminal node of the word we are searching for. + */ + private Node searchWord(final NodeArray children, final String word, final int depth, + final Node parentNode) { + final int wordLength = word.length(); + final char c = word.charAt(depth); + // Does children have the current character? + final int childrenLength = children.mLength; + Node childNode = null; + for (int i = 0; i < childrenLength; i++) { + final Node node = children.mData[i]; + if (node.mCode == c) { + childNode = node; + break; + } + } + if (childNode == null) { + childNode = new Node(); + childNode.mCode = c; + childNode.mParent = parentNode; + children.add(childNode); + } + if (wordLength == depth + 1) { + // Terminate this word + childNode.mTerminal = true; + return childNode; + } + if (childNode.mChildren == null) { + childNode.mChildren = new NodeArray(); + } + return searchWord(childNode.mChildren, word, depth + 1, childNode); + } + + private void runBigramReverseLookUp(final String previousWord, + final ArrayList<SuggestedWordInfo> suggestions) { + // Search for the lowercase version of the word only, because that's where bigrams + // store their sons. + final Node prevWord = searchNode(mRoots, previousWord.toLowerCase(), 0, + previousWord.length()); + if (prevWord != null && prevWord.mNGrams != null) { + reverseLookUp(prevWord.mNGrams, suggestions); + } + } + + // Local to reverseLookUp, but do not allocate each time. + private final char[] mLookedUpString = new char[Constants.DICTIONARY_MAX_WORD_LENGTH]; + + /** + * reverseLookUp retrieves the full word given a list of terminal nodes and adds those words + * to the suggestions list passed as an argument. + * @param terminalNodes list of terminal nodes we want to add + * @param suggestions the suggestion collection to add the word to + */ + private void reverseLookUp(final LinkedList<NextWord> terminalNodes, + final ArrayList<SuggestedWordInfo> suggestions) { + Node node; + int freq; + for (NextWord nextWord : terminalNodes) { + node = nextWord.getWordNode(); + freq = nextWord.getFrequency(); + int index = Constants.DICTIONARY_MAX_WORD_LENGTH; + do { + --index; + mLookedUpString[index] = node.mCode; + node = node.mParent; + } while (node != null && index > 0); + + // If node is null, we have a word longer than MAX_WORD_LENGTH in the dictionary. + // It's a little unclear how this can happen, but just in case it does it's safer + // to ignore the word in this case. + if (freq >= 0 && node == null) { + suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, + Constants.DICTIONARY_MAX_WORD_LENGTH - index), + freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); + } + } + } + + /** + * Recursively search for the terminal node of the word. + * + * One iteration takes the full word to search for and the current index of the recursion. + * + * @param children the node of the trie to search under. + * @param word the word to search for. Only read [offset..length] so there may be trailing chars + * @param offset the index in {@code word} this recursion should operate on. + * @param length the length of the input word. + * @return Returns the terminal node of the word if the word exists + */ + private Node searchNode(final NodeArray children, final CharSequence word, final int offset, + final int length) { + final int count = children.mLength; + final char currentChar = word.charAt(offset); + for (int j = 0; j < count; j++) { + final Node node = children.mData[j]; + if (node.mCode == currentChar) { + if (offset == length - 1) { + if (node.mTerminal) { + return node; + } + } else { + if (node.mChildren != null) { + Node returnNode = searchNode(node.mChildren, word, offset + 1, length); + if (returnNode != null) return returnNode; + } + } + } + } + return null; + } + + public void clearDictionary() { + mRoots = new NodeArray(); + } + + private static char toLowerCase(final char c) { + char baseChar = c; + if (c < BASE_CHARS.length) { + baseChar = BASE_CHARS[c]; + } + if (baseChar >= 'A' && baseChar <= 'Z') { + return (char)(baseChar | 32); + } else if (baseChar > 127) { + return Character.toLowerCase(baseChar); + } + return baseChar; + } + + /** + * Table mapping most combined Latin, Greek, and Cyrillic characters + * to their base characters. If c is in range, BASE_CHARS[c] == c + * if c is not a combined character, or the base character if it + * is combined. + * + * cf. native/jni/src/utils/char_utils.cpp + */ + private static final char BASE_CHARS[] = { + /* U+0000 */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + /* U+0008 */ 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, + /* U+0010 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + /* U+0018 */ 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, + /* U+0020 */ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + /* U+0028 */ 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, + /* U+0030 */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + /* U+0038 */ 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, + /* U+0040 */ 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + /* U+0048 */ 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, + /* U+0050 */ 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + /* U+0058 */ 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, + /* U+0060 */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + /* U+0068 */ 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, + /* U+0070 */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + /* U+0078 */ 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, + /* U+0080 */ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + /* U+0088 */ 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, + /* U+0090 */ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + /* U+0098 */ 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, + /* U+00A0 */ 0x0020, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, + /* U+00A8 */ 0x0020, 0x00A9, 0x0061, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0020, + /* U+00B0 */ 0x00B0, 0x00B1, 0x0032, 0x0033, 0x0020, 0x03BC, 0x00B6, 0x00B7, + /* U+00B8 */ 0x0020, 0x0031, 0x006F, 0x00BB, 0x0031, 0x0031, 0x0033, 0x00BF, + /* U+00C0 */ 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x00C6, 0x0043, + /* U+00C8 */ 0x0045, 0x0045, 0x0045, 0x0045, 0x0049, 0x0049, 0x0049, 0x0049, + /* U+00D0 */ 0x00D0, 0x004E, 0x004F, 0x004F, 0x004F, 0x004F, 0x004F, 0x00D7, + /* U+00D8 */ 0x004F, 0x0055, 0x0055, 0x0055, 0x0055, 0x0059, 0x00DE, 0x0073, + // U+00D8: Manually changed from 00D8 to 004F + // TODO: Check if it's really acceptable to consider Ø a diacritical variant of O + // U+00DF: Manually changed from 00DF to 0073 + /* U+00E0 */ 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00E6, 0x0063, + /* U+00E8 */ 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + /* U+00F0 */ 0x00F0, 0x006E, 0x006F, 0x006F, 0x006F, 0x006F, 0x006F, 0x00F7, + /* U+00F8 */ 0x006F, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00FE, 0x0079, + // U+00F8: Manually changed from 00F8 to 006F + // TODO: Check if it's really acceptable to consider ø a diacritical variant of o + /* U+0100 */ 0x0041, 0x0061, 0x0041, 0x0061, 0x0041, 0x0061, 0x0043, 0x0063, + /* U+0108 */ 0x0043, 0x0063, 0x0043, 0x0063, 0x0043, 0x0063, 0x0044, 0x0064, + /* U+0110 */ 0x0110, 0x0111, 0x0045, 0x0065, 0x0045, 0x0065, 0x0045, 0x0065, + /* U+0118 */ 0x0045, 0x0065, 0x0045, 0x0065, 0x0047, 0x0067, 0x0047, 0x0067, + /* U+0120 */ 0x0047, 0x0067, 0x0047, 0x0067, 0x0048, 0x0068, 0x0126, 0x0127, + /* U+0128 */ 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, + /* U+0130 */ 0x0049, 0x0131, 0x0049, 0x0069, 0x004A, 0x006A, 0x004B, 0x006B, + /* U+0138 */ 0x0138, 0x004C, 0x006C, 0x004C, 0x006C, 0x004C, 0x006C, 0x004C, + /* U+0140 */ 0x006C, 0x004C, 0x006C, 0x004E, 0x006E, 0x004E, 0x006E, 0x004E, + // U+0141: Manually changed from 0141 to 004C + // U+0142: Manually changed from 0142 to 006C + /* U+0148 */ 0x006E, 0x02BC, 0x014A, 0x014B, 0x004F, 0x006F, 0x004F, 0x006F, + /* U+0150 */ 0x004F, 0x006F, 0x0152, 0x0153, 0x0052, 0x0072, 0x0052, 0x0072, + /* U+0158 */ 0x0052, 0x0072, 0x0053, 0x0073, 0x0053, 0x0073, 0x0053, 0x0073, + /* U+0160 */ 0x0053, 0x0073, 0x0054, 0x0074, 0x0054, 0x0074, 0x0166, 0x0167, + /* U+0168 */ 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, + /* U+0170 */ 0x0055, 0x0075, 0x0055, 0x0075, 0x0057, 0x0077, 0x0059, 0x0079, + /* U+0178 */ 0x0059, 0x005A, 0x007A, 0x005A, 0x007A, 0x005A, 0x007A, 0x0073, + /* U+0180 */ 0x0180, 0x0181, 0x0182, 0x0183, 0x0184, 0x0185, 0x0186, 0x0187, + /* U+0188 */ 0x0188, 0x0189, 0x018A, 0x018B, 0x018C, 0x018D, 0x018E, 0x018F, + /* U+0190 */ 0x0190, 0x0191, 0x0192, 0x0193, 0x0194, 0x0195, 0x0196, 0x0197, + /* U+0198 */ 0x0198, 0x0199, 0x019A, 0x019B, 0x019C, 0x019D, 0x019E, 0x019F, + /* U+01A0 */ 0x004F, 0x006F, 0x01A2, 0x01A3, 0x01A4, 0x01A5, 0x01A6, 0x01A7, + /* U+01A8 */ 0x01A8, 0x01A9, 0x01AA, 0x01AB, 0x01AC, 0x01AD, 0x01AE, 0x0055, + /* U+01B0 */ 0x0075, 0x01B1, 0x01B2, 0x01B3, 0x01B4, 0x01B5, 0x01B6, 0x01B7, + /* U+01B8 */ 0x01B8, 0x01B9, 0x01BA, 0x01BB, 0x01BC, 0x01BD, 0x01BE, 0x01BF, + /* U+01C0 */ 0x01C0, 0x01C1, 0x01C2, 0x01C3, 0x0044, 0x0044, 0x0064, 0x004C, + /* U+01C8 */ 0x004C, 0x006C, 0x004E, 0x004E, 0x006E, 0x0041, 0x0061, 0x0049, + /* U+01D0 */ 0x0069, 0x004F, 0x006F, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, + // U+01D5: Manually changed from 00DC to 0055 + // U+01D6: Manually changed from 00FC to 0075 + // U+01D7: Manually changed from 00DC to 0055 + /* U+01D8 */ 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x01DD, 0x0041, 0x0061, + // U+01D8: Manually changed from 00FC to 0075 + // U+01D9: Manually changed from 00DC to 0055 + // U+01DA: Manually changed from 00FC to 0075 + // U+01DB: Manually changed from 00DC to 0055 + // U+01DC: Manually changed from 00FC to 0075 + // U+01DE: Manually changed from 00C4 to 0041 + // U+01DF: Manually changed from 00E4 to 0061 + /* U+01E0 */ 0x0041, 0x0061, 0x00C6, 0x00E6, 0x01E4, 0x01E5, 0x0047, 0x0067, + // U+01E0: Manually changed from 0226 to 0041 + // U+01E1: Manually changed from 0227 to 0061 + /* U+01E8 */ 0x004B, 0x006B, 0x004F, 0x006F, 0x004F, 0x006F, 0x01B7, 0x0292, + // U+01EC: Manually changed from 01EA to 004F + // U+01ED: Manually changed from 01EB to 006F + /* U+01F0 */ 0x006A, 0x0044, 0x0044, 0x0064, 0x0047, 0x0067, 0x01F6, 0x01F7, + /* U+01F8 */ 0x004E, 0x006E, 0x0041, 0x0061, 0x00C6, 0x00E6, 0x004F, 0x006F, + // U+01FA: Manually changed from 00C5 to 0041 + // U+01FB: Manually changed from 00E5 to 0061 + // U+01FE: Manually changed from 00D8 to 004F + // TODO: Check if it's really acceptable to consider Ø a diacritical variant of O + // U+01FF: Manually changed from 00F8 to 006F + // TODO: Check if it's really acceptable to consider ø a diacritical variant of o + /* U+0200 */ 0x0041, 0x0061, 0x0041, 0x0061, 0x0045, 0x0065, 0x0045, 0x0065, + /* U+0208 */ 0x0049, 0x0069, 0x0049, 0x0069, 0x004F, 0x006F, 0x004F, 0x006F, + /* U+0210 */ 0x0052, 0x0072, 0x0052, 0x0072, 0x0055, 0x0075, 0x0055, 0x0075, + /* U+0218 */ 0x0053, 0x0073, 0x0054, 0x0074, 0x021C, 0x021D, 0x0048, 0x0068, + /* U+0220 */ 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0041, 0x0061, + /* U+0228 */ 0x0045, 0x0065, 0x004F, 0x006F, 0x004F, 0x006F, 0x004F, 0x006F, + // U+022A: Manually changed from 00D6 to 004F + // U+022B: Manually changed from 00F6 to 006F + // U+022C: Manually changed from 00D5 to 004F + // U+022D: Manually changed from 00F5 to 006F + /* U+0230 */ 0x004F, 0x006F, 0x0059, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, + // U+0230: Manually changed from 022E to 004F + // U+0231: Manually changed from 022F to 006F + /* U+0238 */ 0x0238, 0x0239, 0x023A, 0x023B, 0x023C, 0x023D, 0x023E, 0x023F, + /* U+0240 */ 0x0240, 0x0241, 0x0242, 0x0243, 0x0244, 0x0245, 0x0246, 0x0247, + /* U+0248 */ 0x0248, 0x0249, 0x024A, 0x024B, 0x024C, 0x024D, 0x024E, 0x024F, + /* U+0250 */ 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, + /* U+0258 */ 0x0258, 0x0259, 0x025A, 0x025B, 0x025C, 0x025D, 0x025E, 0x025F, + /* U+0260 */ 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, + /* U+0268 */ 0x0268, 0x0269, 0x026A, 0x026B, 0x026C, 0x026D, 0x026E, 0x026F, + /* U+0270 */ 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, + /* U+0278 */ 0x0278, 0x0279, 0x027A, 0x027B, 0x027C, 0x027D, 0x027E, 0x027F, + /* U+0280 */ 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, + /* U+0288 */ 0x0288, 0x0289, 0x028A, 0x028B, 0x028C, 0x028D, 0x028E, 0x028F, + /* U+0290 */ 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, + /* U+0298 */ 0x0298, 0x0299, 0x029A, 0x029B, 0x029C, 0x029D, 0x029E, 0x029F, + /* U+02A0 */ 0x02A0, 0x02A1, 0x02A2, 0x02A3, 0x02A4, 0x02A5, 0x02A6, 0x02A7, + /* U+02A8 */ 0x02A8, 0x02A9, 0x02AA, 0x02AB, 0x02AC, 0x02AD, 0x02AE, 0x02AF, + /* U+02B0 */ 0x0068, 0x0266, 0x006A, 0x0072, 0x0279, 0x027B, 0x0281, 0x0077, + /* U+02B8 */ 0x0079, 0x02B9, 0x02BA, 0x02BB, 0x02BC, 0x02BD, 0x02BE, 0x02BF, + /* U+02C0 */ 0x02C0, 0x02C1, 0x02C2, 0x02C3, 0x02C4, 0x02C5, 0x02C6, 0x02C7, + /* U+02C8 */ 0x02C8, 0x02C9, 0x02CA, 0x02CB, 0x02CC, 0x02CD, 0x02CE, 0x02CF, + /* U+02D0 */ 0x02D0, 0x02D1, 0x02D2, 0x02D3, 0x02D4, 0x02D5, 0x02D6, 0x02D7, + /* U+02D8 */ 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02DE, 0x02DF, + /* U+02E0 */ 0x0263, 0x006C, 0x0073, 0x0078, 0x0295, 0x02E5, 0x02E6, 0x02E7, + /* U+02E8 */ 0x02E8, 0x02E9, 0x02EA, 0x02EB, 0x02EC, 0x02ED, 0x02EE, 0x02EF, + /* U+02F0 */ 0x02F0, 0x02F1, 0x02F2, 0x02F3, 0x02F4, 0x02F5, 0x02F6, 0x02F7, + /* U+02F8 */ 0x02F8, 0x02F9, 0x02FA, 0x02FB, 0x02FC, 0x02FD, 0x02FE, 0x02FF, + /* U+0300 */ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, + /* U+0308 */ 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, + /* U+0310 */ 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, + /* U+0318 */ 0x0318, 0x0319, 0x031A, 0x031B, 0x031C, 0x031D, 0x031E, 0x031F, + /* U+0320 */ 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, + /* U+0328 */ 0x0328, 0x0329, 0x032A, 0x032B, 0x032C, 0x032D, 0x032E, 0x032F, + /* U+0330 */ 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, + /* U+0338 */ 0x0338, 0x0339, 0x033A, 0x033B, 0x033C, 0x033D, 0x033E, 0x033F, + /* U+0340 */ 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x0345, 0x0346, 0x0347, + /* U+0348 */ 0x0348, 0x0349, 0x034A, 0x034B, 0x034C, 0x034D, 0x034E, 0x034F, + /* U+0350 */ 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + /* U+0358 */ 0x0358, 0x0359, 0x035A, 0x035B, 0x035C, 0x035D, 0x035E, 0x035F, + /* U+0360 */ 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + /* U+0368 */ 0x0368, 0x0369, 0x036A, 0x036B, 0x036C, 0x036D, 0x036E, 0x036F, + /* U+0370 */ 0x0370, 0x0371, 0x0372, 0x0373, 0x02B9, 0x0375, 0x0376, 0x0377, + /* U+0378 */ 0x0378, 0x0379, 0x0020, 0x037B, 0x037C, 0x037D, 0x003B, 0x037F, + /* U+0380 */ 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x00A8, 0x0391, 0x00B7, + /* U+0388 */ 0x0395, 0x0397, 0x0399, 0x038B, 0x039F, 0x038D, 0x03A5, 0x03A9, + /* U+0390 */ 0x03CA, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + /* U+0398 */ 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, + /* U+03A0 */ 0x03A0, 0x03A1, 0x03A2, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, + /* U+03A8 */ 0x03A8, 0x03A9, 0x0399, 0x03A5, 0x03B1, 0x03B5, 0x03B7, 0x03B9, + /* U+03B0 */ 0x03CB, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, + /* U+03B8 */ 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, + /* U+03C0 */ 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, + /* U+03C8 */ 0x03C8, 0x03C9, 0x03B9, 0x03C5, 0x03BF, 0x03C5, 0x03C9, 0x03CF, + /* U+03D0 */ 0x03B2, 0x03B8, 0x03A5, 0x03D2, 0x03D2, 0x03C6, 0x03C0, 0x03D7, + /* U+03D8 */ 0x03D8, 0x03D9, 0x03DA, 0x03DB, 0x03DC, 0x03DD, 0x03DE, 0x03DF, + /* U+03E0 */ 0x03E0, 0x03E1, 0x03E2, 0x03E3, 0x03E4, 0x03E5, 0x03E6, 0x03E7, + /* U+03E8 */ 0x03E8, 0x03E9, 0x03EA, 0x03EB, 0x03EC, 0x03ED, 0x03EE, 0x03EF, + /* U+03F0 */ 0x03BA, 0x03C1, 0x03C2, 0x03F3, 0x0398, 0x03B5, 0x03F6, 0x03F7, + /* U+03F8 */ 0x03F8, 0x03A3, 0x03FA, 0x03FB, 0x03FC, 0x03FD, 0x03FE, 0x03FF, + /* U+0400 */ 0x0415, 0x0415, 0x0402, 0x0413, 0x0404, 0x0405, 0x0406, 0x0406, + /* U+0408 */ 0x0408, 0x0409, 0x040A, 0x040B, 0x041A, 0x0418, 0x0423, 0x040F, + /* U+0410 */ 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + /* U+0418 */ 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + // U+0419: Manually changed from 0418 to 0419 + /* U+0420 */ 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + /* U+0428 */ 0x0428, 0x0429, 0x042C, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + // U+042A: Manually changed from 042A to 042C + /* U+0430 */ 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + /* U+0438 */ 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + // U+0439: Manually changed from 0438 to 0439 + /* U+0440 */ 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + /* U+0448 */ 0x0448, 0x0449, 0x044C, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + // U+044A: Manually changed from 044A to 044C + /* U+0450 */ 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + /* U+0458 */ 0x0458, 0x0459, 0x045A, 0x045B, 0x043A, 0x0438, 0x0443, 0x045F, + /* U+0460 */ 0x0460, 0x0461, 0x0462, 0x0463, 0x0464, 0x0465, 0x0466, 0x0467, + /* U+0468 */ 0x0468, 0x0469, 0x046A, 0x046B, 0x046C, 0x046D, 0x046E, 0x046F, + /* U+0470 */ 0x0470, 0x0471, 0x0472, 0x0473, 0x0474, 0x0475, 0x0474, 0x0475, + /* U+0478 */ 0x0478, 0x0479, 0x047A, 0x047B, 0x047C, 0x047D, 0x047E, 0x047F, + /* U+0480 */ 0x0480, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + /* U+0488 */ 0x0488, 0x0489, 0x048A, 0x048B, 0x048C, 0x048D, 0x048E, 0x048F, + /* U+0490 */ 0x0490, 0x0491, 0x0492, 0x0493, 0x0494, 0x0495, 0x0496, 0x0497, + /* U+0498 */ 0x0498, 0x0499, 0x049A, 0x049B, 0x049C, 0x049D, 0x049E, 0x049F, + /* U+04A0 */ 0x04A0, 0x04A1, 0x04A2, 0x04A3, 0x04A4, 0x04A5, 0x04A6, 0x04A7, + /* U+04A8 */ 0x04A8, 0x04A9, 0x04AA, 0x04AB, 0x04AC, 0x04AD, 0x04AE, 0x04AF, + /* U+04B0 */ 0x04B0, 0x04B1, 0x04B2, 0x04B3, 0x04B4, 0x04B5, 0x04B6, 0x04B7, + /* U+04B8 */ 0x04B8, 0x04B9, 0x04BA, 0x04BB, 0x04BC, 0x04BD, 0x04BE, 0x04BF, + /* U+04C0 */ 0x04C0, 0x0416, 0x0436, 0x04C3, 0x04C4, 0x04C5, 0x04C6, 0x04C7, + /* U+04C8 */ 0x04C8, 0x04C9, 0x04CA, 0x04CB, 0x04CC, 0x04CD, 0x04CE, 0x04CF, + /* U+04D0 */ 0x0410, 0x0430, 0x0410, 0x0430, 0x04D4, 0x04D5, 0x0415, 0x0435, + /* U+04D8 */ 0x04D8, 0x04D9, 0x04D8, 0x04D9, 0x0416, 0x0436, 0x0417, 0x0437, + /* U+04E0 */ 0x04E0, 0x04E1, 0x0418, 0x0438, 0x0418, 0x0438, 0x041E, 0x043E, + /* U+04E8 */ 0x04E8, 0x04E9, 0x04E8, 0x04E9, 0x042D, 0x044D, 0x0423, 0x0443, + /* U+04F0 */ 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04F6, 0x04F7, + /* U+04F8 */ 0x042B, 0x044B, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF, + }; +} diff --git a/java/src/com/android/inputmethod/latin/InputAttributes.java b/java/src/com/android/inputmethod/latin/InputAttributes.java index fcf043031..8caf6f17f 100644 --- a/java/src/com/android/inputmethod/latin/InputAttributes.java +++ b/java/src/com/android/inputmethod/latin/InputAttributes.java @@ -52,7 +52,8 @@ public final class InputAttributes { } else if (inputClass == 0) { // TODO: is this check still necessary? Log.w(TAG, String.format("Unexpected input class: inputType=0x%08x" - + " imeOptions=0x%08x", inputType, editorInfo.imeOptions)); + + " imeOptions=0x%08x", + inputType, editorInfo.imeOptions)); } mIsSettingsSuggestionStripOn = false; mInputTypeNoAutoCorrect = false; @@ -203,7 +204,8 @@ public final class InputAttributes { public static boolean inPrivateImeOptions(String packageName, String key, EditorInfo editorInfo) { if (editorInfo == null) return false; - final String findingKey = (packageName != null) ? packageName + "." + key : key; + final String findingKey = (packageName != null) ? packageName + "." + key + : key; return StringUtils.containsInCommaSplittableText(findingKey, editorInfo.privateImeOptions); } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 6a10131b0..77d07019f 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -68,6 +68,7 @@ import com.android.inputmethod.compat.InputMethodServiceCompatUtils; import com.android.inputmethod.compat.SuggestionSpanUtils; import com.android.inputmethod.dictionarypack.DictionaryPackConstants; import com.android.inputmethod.event.EventInterpreter; +import com.android.inputmethod.keyboard.KeyDetector; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.KeyboardActionListener; import com.android.inputmethod.keyboard.KeyboardId; @@ -80,6 +81,7 @@ import com.android.inputmethod.latin.personalization.DictionaryDecayBroadcastRec import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister; import com.android.inputmethod.latin.personalization.PersonalizationHelper; +import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; import com.android.inputmethod.latin.personalization.UserHistoryDictionary; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.settings.SettingsActivity; @@ -95,8 +97,8 @@ import com.android.inputmethod.latin.utils.InputTypeUtils; import com.android.inputmethod.latin.utils.IntentUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LatinImeLoggerUtils; -import com.android.inputmethod.latin.utils.LeakGuardHandlerWrapper; import com.android.inputmethod.latin.utils.RecapitalizeStatus; +import com.android.inputmethod.latin.utils.StaticInnerHandlerWrapper; import com.android.inputmethod.latin.utils.StringUtils; import com.android.inputmethod.latin.utils.TargetPackageInfoGetterTask; import com.android.inputmethod.latin.utils.TextRange; @@ -108,7 +110,6 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Locale; import java.util.TreeSet; -import java.util.concurrent.TimeUnit; /** * Input method implementation for Qwerty'ish keyboard. @@ -181,6 +182,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean mIsMainDictionaryAvailable; private UserBinaryDictionary mUserDictionary; private UserHistoryDictionary mUserHistoryDictionary; + private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary; private PersonalizationDictionary mPersonalizationDictionary; private boolean mIsUserDictionaryAvailable; @@ -194,6 +196,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private int mLastSelectionStart = NOT_A_CURSOR_POSITION; private int mLastSelectionEnd = NOT_A_CURSOR_POSITION; + // Whether we are expecting an onUpdateSelection event to fire. If it does when we don't + // "expect" it, it means the user actually moved the cursor. + private boolean mExpectingUpdateSelection; private int mDeleteCount; private long mLastKeyTime; private final TreeSet<Long> mCurrentlyPressedHardwareKeys = CollectionUtils.newTreeSet(); @@ -222,7 +227,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen public final UIHandler mHandler = new UIHandler(this); private InputUpdater mInputUpdater; - public static final class UIHandler extends LeakGuardHandlerWrapper<LatinIME> { + public static final class UIHandler extends StaticInnerHandlerWrapper<LatinIME> { private static final int MSG_UPDATE_SHIFT_STATE = 0; private static final int MSG_PENDING_IMS_CALLBACK = 1; private static final int MSG_UPDATE_SUGGESTION_STRIP = 2; @@ -231,8 +236,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private static final int MSG_REOPEN_DICTIONARIES = 5; private static final int MSG_ON_END_BATCH_INPUT = 6; private static final int MSG_RESET_CACHES = 7; - // Update this when adding new messages - private static final int MSG_LAST = MSG_RESET_CACHES; private static final int ARG1_NOT_GESTURE_INPUT = 0; private static final int ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT = 1; @@ -245,12 +248,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private long mDoubleSpacePeriodTimeout; private long mDoubleSpacePeriodTimerStart; - public UIHandler(final LatinIME ownerInstance) { - super(ownerInstance); + public UIHandler(final LatinIME outerInstance) { + super(outerInstance); } public void onCreate() { - final Resources res = getOwnerInstance().getResources(); + final Resources res = getOuterInstance().getResources(); mDelayUpdateSuggestions = res.getInteger(R.integer.config_delay_update_suggestions); mDelayUpdateShiftState = @@ -261,7 +264,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void handleMessage(final Message msg) { - final LatinIME latinIme = getOwnerInstance(); + final LatinIME latinIme = getOuterInstance(); final KeyboardSwitcher switcher = latinIme.mKeyboardSwitcher; switch (msg.what) { case MSG_UPDATE_SUGGESTION_STRIP: @@ -344,13 +347,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen removeMessages(MSG_UPDATE_SHIFT_STATE); } - @UsedForTesting - public void removeAllMessages() { - for (int i = 0; i <= MSG_LAST; ++i) { - removeMessages(i); - } - } - public void showGesturePreviewAndSuggestionStrip(final SuggestedWords suggestedWords, final boolean dismissGestureFloatingPreviewText) { removeMessages(MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP); @@ -405,7 +401,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen removeMessages(MSG_PENDING_IMS_CALLBACK); resetPendingImsCallback(); mIsOrientationChanging = true; - final LatinIME latinIme = getOwnerInstance(); + final LatinIME latinIme = getOuterInstance(); if (latinIme.isInputViewShown()) { latinIme.mKeyboardSwitcher.saveKeyboardState(); } @@ -438,7 +434,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mIsOrientationChanging = false; mPendingSuccessiveImsCallback = true; } - final LatinIME latinIme = getOwnerInstance(); + final LatinIME latinIme = getOuterInstance(); executePendingImsCallback(latinIme, editorInfo, restarting); latinIme.onStartInputInternal(editorInfo, restarting); } @@ -457,7 +453,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen sendMessageDelayed(obtainMessage(MSG_PENDING_IMS_CALLBACK), PENDING_IMS_CALLBACK_DURATION); } - final LatinIME latinIme = getOwnerInstance(); + final LatinIME latinIme = getOuterInstance(); executePendingImsCallback(latinIme, editorInfo, restarting); latinIme.onStartInputViewInternal(editorInfo, restarting); mAppliedEditorInfo = editorInfo; @@ -469,7 +465,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Typically this is the first onFinishInputView after orientation changed. mHasPendingFinishInputView = true; } else { - final LatinIME latinIme = getOwnerInstance(); + final LatinIME latinIme = getOuterInstance(); latinIme.onFinishInputViewInternal(finishingInput); mAppliedEditorInfo = null; } @@ -480,7 +476,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Typically this is the first onFinishInput after orientation changed. mHasPendingFinishInput = true; } else { - final LatinIME latinIme = getOwnerInstance(); + final LatinIME latinIme = getOuterInstance(); executePendingImsCallback(latinIme, null, false); latinIme.onFinishInputInternal(); } @@ -613,6 +609,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Locale switcherSubtypeLocale = mSubtypeSwitcher.getCurrentSubtypeLocale(); final String switcherLocaleStr = switcherSubtypeLocale.toString(); final Locale subtypeLocale; + final String localeStr; if (TextUtils.isEmpty(switcherLocaleStr)) { // This happens in very rare corner cases - for example, immediately after a switch // to LatinIME has been requested, about a frame later another switch happens. In this @@ -622,8 +619,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // of knowing anyway. Log.e(TAG, "System is reporting no current subtype."); subtypeLocale = getResources().getConfiguration().locale; + localeStr = subtypeLocale.toString(); } else { subtypeLocale = switcherSubtypeLocale; + localeStr = switcherLocaleStr; } final Suggest newSuggest = new Suggest(this /* Context */, subtypeLocale, @@ -638,16 +637,21 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.getInstance().initSuggest(newSuggest); } - mUserDictionary = new UserBinaryDictionary(this, subtypeLocale); + mUserDictionary = new UserBinaryDictionary(this, localeStr); mIsUserDictionaryAvailable = mUserDictionary.isEnabled(); newSuggest.setUserDictionary(mUserDictionary); + final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); + mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary( - this, subtypeLocale); + this, localeStr, prefs); newSuggest.setUserHistoryDictionary(mUserHistoryDictionary); - mPersonalizationDictionary = - PersonalizationHelper.getPersonalizationDictionary(this, subtypeLocale); + mPersonalizationDictionary = PersonalizationHelper + .getPersonalizationDictionary(this, localeStr, prefs); newSuggest.setPersonalizationDictionary(mPersonalizationDictionary); + mPersonalizationPredictionDictionary = PersonalizationHelper + .getPersonalizationPredictionDictionary(this, localeStr, prefs); + newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary); final Suggest oldSuggest = mSuggest; resetContactsDictionary(null != oldSuggest ? oldSuggest.getContactsDictionary() : null); @@ -756,9 +760,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen .findViewById(android.R.id.extractArea); mKeyPreviewBackingView = view.findViewById(R.id.key_preview_backing); mSuggestionStripView = (SuggestionStripView)view.findViewById(R.id.suggestion_strip_view); - if (mSuggestionStripView != null) { + if (mSuggestionStripView != null) mSuggestionStripView.setListener(this, view); - } if (LatinImeLogger.sVISUALDEBUG) { mKeyPreviewBackingView.setBackgroundColor(0x10FF0000); } @@ -904,7 +907,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // refresh later. final boolean canReachInputConnection; if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(editorInfo.initialSelStart, - editorInfo.initialSelEnd, false /* shouldFinishComposition */)) { + false /* shouldFinishComposition */)) { // We try resetting the caches up to 5 times before giving up. mHandler.postResetCaches(isDifferentTextField, 5 /* remainingTries */); // mLastSelection{Start,End} are reset later in this method, don't need to do it here @@ -1085,9 +1088,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen + ", ce=" + composingSpanEnd); } if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { + final boolean expectingUpdateSelectionFromLogger = + ResearchLogger.getAndClearLatinIMEExpectingUpdateSelection(); ResearchLogger.latinIME_onUpdateSelection(mLastSelectionStart, mLastSelectionEnd, oldSelStart, oldSelEnd, newSelStart, newSelEnd, composingSpanStart, - composingSpanEnd, mConnection); + composingSpanEnd, mExpectingUpdateSelection, + expectingUpdateSelectionFromLogger, mConnection); + if (expectingUpdateSelectionFromLogger) { + // TODO: Investigate. Quitting now sounds wrong - we won't do the resetting work + return; + } } final boolean selectionChanged = mLastSelectionStart != newSelStart @@ -1106,8 +1116,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // TODO: revisit this when LatinIME supports hardware keyboards. // NOTE: the test harness subclasses LatinIME and overrides isInputViewShown(). // TODO: find a better way to simulate actual execution. - if (isInputViewShown() && !mConnection.isBelatedExpectedUpdate(oldSelStart, newSelStart, - oldSelEnd, newSelEnd)) { + if (isInputViewShown() && !mExpectingUpdateSelection + && !mConnection.isBelatedExpectedUpdate(oldSelStart, newSelStart)) { + // TAKE CARE: there is a race condition when we enter this test even when the user + // did not explicitly move the cursor. This happens when typing fast, where two keys + // turn this flag on in succession and both onUpdateSelection() calls arrive after + // the second one - the first call successfully avoids this test, but the second one + // enters. For the moment we rely on noComposingSpan to further reduce the impact. + // TODO: the following is probably better done in resetEntireInputState(). // it should only happen when the cursor moved, and the very purpose of the // test below is to narrow down whether this happened or not. Likewise with @@ -1133,13 +1149,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Another option would be to send suggestions each time we set the composing // text, but that is probably too expensive to do, so we decided to leave things // as is. - resetEntireInputState(newSelStart, newSelEnd); + resetEntireInputState(newSelStart); } else { - // resetEntireInputState calls resetCachesUponCursorMove, but forcing the - // composition to end. But in all cases where we don't reset the entire input - // state, we still want to tell the rich input connection about the new cursor - // position so that it can update its caches. - mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, newSelEnd, + // resetEntireInputState calls resetCachesUponCursorMove, but with the second + // argument as true. But in all cases where we don't reset the entire input state, + // we still want to tell the rich input connection about the new cursor position so + // that it can update its caches. + mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, false /* shouldFinishComposition */); } @@ -1152,6 +1168,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mRecapitalizeStatus.deactivate(); mKeyboardSwitcher.updateShiftState(); } + mExpectingUpdateSelection = false; // Make a note of the cursor position mLastSelectionStart = newSelStart; @@ -1337,7 +1354,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public boolean onEvaluateFullscreenMode() { // Reread resource value here, because this method is called by framework anytime as needed. - final boolean isFullscreenModeAllowed = Settings.readUseFullscreenMode(getResources()); + final boolean isFullscreenModeAllowed = + Settings.readUseFullscreenMode(getResources()); if (super.onEvaluateFullscreenMode() && isFullscreenModeAllowed) { // TODO: Remove this hack. Actually we should not really assume NO_EXTRACT_UI // implies NO_FULLSCREEN. However, the framework mistakenly does. i.e. NO_EXTRACT_UI @@ -1362,7 +1380,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This will reset the whole input state to the starting state. It will clear // the composing word, reset the last composed word, tell the inputconnection about it. - private void resetEntireInputState(final int newSelStart, final int newSelEnd) { + private void resetEntireInputState(final int newCursorPosition) { final boolean shouldFinishComposition = mWordComposer.isComposingWord(); resetComposingState(true /* alsoResetLastComposedWord */); final SettingsValues settingsValues = mSettings.getCurrent(); @@ -1371,15 +1389,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { setSuggestedWords(settingsValues.mSuggestPuncList, false); } - mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, newSelEnd, + mConnection.resetCachesUponCursorMoveAndReturnSuccess(newCursorPosition, shouldFinishComposition); } private void resetComposingState(final boolean alsoResetLastComposedWord) { mWordComposer.reset(); - if (alsoResetLastComposedWord) { + if (alsoResetLastComposedWord) mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; - } } private void commitTyped(final String separatorString) { @@ -1426,16 +1443,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (0 != (auto & TextUtils.CAP_MODE_CHARACTERS)) { return WordComposer.CAPS_MODE_AUTO_SHIFT_LOCKED; } - if (0 != auto) { - return WordComposer.CAPS_MODE_AUTO_SHIFTED; - } + if (0 != auto) return WordComposer.CAPS_MODE_AUTO_SHIFTED; return WordComposer.CAPS_MODE_OFF; } private void swapSwapperAndSpace() { final CharSequence lastTwo = mConnection.getTextBeforeCursor(2, 0); // It is guaranteed lastTwo.charAt(1) is a swapper - else this method is not called. - if (lastTwo != null && lastTwo.length() == 2 && lastTwo.charAt(0) == Constants.CODE_SPACE) { + if (lastTwo != null && lastTwo.length() == 2 + && lastTwo.charAt(0) == Constants.CODE_SPACE) { mConnection.deleteSurroundingText(2, 0); final String text = lastTwo.charAt(1) + " "; mConnection.commitText(text, 1); @@ -1714,7 +1730,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the character at the current cursor position. - resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); + resetEntireInputState(mLastSelectionStart); } else { commitTyped(LastComposedWord.NOT_A_SEPARATOR); } @@ -1730,6 +1746,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } handleCharacter(primaryCode, keyX, keyY, spaceState); } + mExpectingUpdateSelection = true; return didAutoCorrect; } @@ -1769,9 +1786,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mInputUpdater.onStartBatchInput(); mHandler.cancelUpdateSuggestionStrip(); mConnection.beginBatchEdit(); - final SettingsValues currentSettingsValues = mSettings.getCurrent(); + final SettingsValues settingsValues = mSettings.getCurrent(); if (mWordComposer.isComposingWord()) { - if (currentSettingsValues.mIsInternal) { + if (settingsValues.mIsInternal) { if (mWordComposer.isBatchMode()) { LatinImeLoggerUtils.onAutoCorrection( "", mWordComposer.getTypedWord(), " ", mWordComposer); @@ -1782,7 +1799,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the batch input at the current cursor position. - resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); + resetEntireInputState(mLastSelectionStart); } else if (wordComposerSize <= 1) { // We auto-correct the previous (typed, not gestured) string iff it's one character // long. The reason for this is, even in the middle of gesture typing, you'll still @@ -1795,17 +1812,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { commitTyped(LastComposedWord.NOT_A_SEPARATOR); } + mExpectingUpdateSelection = true; } final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); if (Character.isLetterOrDigit(codePointBeforeCursor) - || currentSettingsValues.isUsuallyFollowedBySpace(codePointBeforeCursor)) { + || settingsValues.isUsuallyFollowedBySpace(codePointBeforeCursor)) { mSpaceState = SPACE_STATE_PHANTOM; } mConnection.endBatchEdit(); - mKeyboardSwitcher.updateShiftState(); - mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime(getActualCapsMode(), - // Prev word is 1st word before cursor - getNthPreviousWordForSuggestion(currentSettingsValues, 1 /* nthPreviousWord */)); + mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); } static final class InputUpdater implements Handler.Callback { @@ -1978,8 +1993,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mConnection.commitText(commitParts[0], 0); mSpaceState = SPACE_STATE_PHANTOM; mKeyboardSwitcher.updateShiftState(); - mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime( - getActualCapsMode(), commitParts[0]); + mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); ++mAutoCommitSequenceNumber; } } @@ -1989,7 +2003,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This method must run in UI Thread. public void onEndBatchInputAsyncInternal(final SuggestedWords suggestedWords) { - final String batchInputText = suggestedWords.isEmpty() ? null : suggestedWords.getWord(0); + final String batchInputText = suggestedWords.isEmpty() + ? null : suggestedWords.getWord(0); if (TextUtils.isEmpty(batchInputText)) { return; } @@ -2011,6 +2026,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mWordComposer.setBatchInputWord(batchInputText); mConnection.setComposingText(batchInputText, 1); } + mExpectingUpdateSelection = true; mConnection.endBatchEdit(); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_onEndBatchInput(batchInputText, 0, suggestedWords); @@ -2064,7 +2080,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } private void handleBackspace(final int spaceState) { + // We revert these in this method if the deletion doesn't happen. mDeleteCount++; + mExpectingUpdateSelection = true; // In many cases, we may have to put the keyboard in auto-shift state again. However // we want to wait a few milliseconds before doing it to avoid the keyboard flashing @@ -2074,7 +2092,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can remove the character at the current cursor position. - resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); + resetEntireInputState(mLastSelectionStart); // When we exit this if-clause, mWordComposer.isComposingWord() will return false. } if (mWordComposer.isComposingWord()) { @@ -2145,10 +2163,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // later (typically, in a subsequent press on backspace). mLastSelectionEnd = mLastSelectionStart; mConnection.deleteSurroundingText(numCharsDeleted, 0); - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(numCharsDeleted, - false /* shouldUncommitLogUnit */); - } } else { // There is no selection, just delete one character. if (NOT_A_CURSOR_POSITION == mLastSelectionEnd) { @@ -2171,27 +2185,22 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); if (codePointBeforeCursor == Constants.NOT_A_CODE) { - // Nothing to delete before the cursor. + // Nothing to delete before the cursor. We have to revert the deletion + // states that were updated at the beginning of this method. + mDeleteCount--; + mExpectingUpdateSelection = false; return; } final int lengthToDelete = Character.isSupplementaryCodePoint(codePointBeforeCursor) ? 2 : 1; mConnection.deleteSurroundingText(lengthToDelete, 0); - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(lengthToDelete, - true /* shouldUncommitLogUnit */); - } if (mDeleteCount > DELETE_ACCELERATE_AT) { final int codePointBeforeCursorToDeleteAgain = mConnection.getCodePointBeforeCursor(); if (codePointBeforeCursorToDeleteAgain != Constants.NOT_A_CODE) { final int lengthToDeleteAgain = Character.isSupplementaryCodePoint( - codePointBeforeCursorToDeleteAgain) ? 2 : 1; + codePointBeforeCursorToDeleteAgain) ? 2 : 1; mConnection.deleteSurroundingText(lengthToDeleteAgain, 0); - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(lengthToDeleteAgain, - true /* shouldUncommitLogUnit */); - } } } } @@ -2208,8 +2217,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen /* * Strip a trailing space if necessary and returns whether it's a swap weak space situation. */ - private boolean maybeStripSpace(final int code, final int spaceState, - final boolean isFromSuggestionStrip) { + private boolean maybeStripSpace(final int code, + final int spaceState, final boolean isFromSuggestionStrip) { if (Constants.CODE_ENTER == code && SPACE_STATE_SWAP_PUNCTUATION == spaceState) { mConnection.removeTrailingSpace(); return false; @@ -2224,8 +2233,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return false; } - private void handleCharacter(final int primaryCode, final int x, final int y, - final int spaceState) { + private void handleCharacter(final int primaryCode, final int x, + final int y, final int spaceState) { // TODO: refactor this method to stop flipping isComposingWord around all the time, and // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter // which has the same name as other handle* methods but is not the same. @@ -2245,7 +2254,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the character at the current cursor position. - resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); + resetEntireInputState(mLastSelectionStart); isComposingWord = false; } // We want to find out whether to start composing a new word with this character. If so, @@ -2275,24 +2284,25 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen resetComposingState(false /* alsoResetLastComposedWord */); } if (isComposingWord) { - final MainKeyboardView mainKeyboardView = mKeyboardSwitcher.getMainKeyboardView(); - // TODO: We should reconsider which coordinate system should be used to represent - // keyboard event. - final int keyX = mainKeyboardView.getKeyX(x); - final int keyY = mainKeyboardView.getKeyY(y); + final int keyX, keyY; + if (Constants.isValidCoordinate(x) && Constants.isValidCoordinate(y)) { + final KeyDetector keyDetector = + mKeyboardSwitcher.getMainKeyboardView().getKeyDetector(); + keyX = keyDetector.getTouchX(x); + keyY = keyDetector.getTouchY(y); + } else { + keyX = x; + keyY = y; + } mWordComposer.add(primaryCode, keyX, keyY); // If it's the first letter, make note of auto-caps state if (mWordComposer.size() == 1) { - // We pass 1 to getPreviousWordForSuggestion because we were not composing a word - // yet, so the word we want is the 1st word before the cursor. - mWordComposer.setCapitalizedModeAndPreviousWordAtStartComposingTime( - getActualCapsMode(), - getNthPreviousWordForSuggestion(currentSettings, 1 /* nthPreviousWord */)); + mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); } mConnection.setComposingText(getTextWithUnderline(mWordComposer.getTypedWord()), 1); } else { - final boolean swapWeakSpace = maybeStripSpace(primaryCode, spaceState, - Constants.SUGGESTION_STRIP_COORDINATE == x); + final boolean swapWeakSpace = maybeStripSpace(primaryCode, + spaceState, Constants.SUGGESTION_STRIP_COORDINATE == x); sendKeyCodePoint(primaryCode); @@ -2356,7 +2366,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the separator at the current cursor position. - resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); + resetEntireInputState(mLastSelectionStart); } if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing if (currentSettings.mCorrectionEnabled) { @@ -2529,24 +2539,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } - /** - * Get the nth previous word before the cursor as context for the suggestion process. - * @param currentSettings the current settings values. - * @param nthPreviousWord reverse index of the word to get (1-indexed) - * @return the nth previous word before the cursor. - */ - private String getNthPreviousWordForSuggestion(final SettingsValues currentSettings, - final int nthPreviousWord) { - if (currentSettings.mCurrentLanguageHasSpaces) { - // If we are typing in a language with spaces we can just look up the previous - // word from textview. - return mConnection.getNthPreviousWord(currentSettings, nthPreviousWord); - } else { - return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null - : mLastComposedWord.mCommittedWord; - } - } - private void getSuggestedWords(final int sessionId, final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { final Keyboard keyboard = mKeyboardSwitcher.getKeyboard(); @@ -2560,31 +2552,17 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // should just skip whitespace if any, so 1. final SettingsValues currentSettings = mSettings.getCurrent(); final int[] additionalFeaturesOptions = currentSettings.mAdditionalFeaturesSettingValues; - - final String previousWord; - if (mWordComposer.isComposingWord() || mWordComposer.isBatchMode()) { - previousWord = mWordComposer.getPreviousWord(); - } else { - // Not composing: this is for prediction. - // TODO: read the previous word earlier for prediction, like we are doing for - // normal suggestions. - previousWord = getNthPreviousWordForSuggestion(currentSettings, 1 /* nthPreviousWord*/); - } - if (DEBUG) { - // TODO: this is for checking consistency with older versions. Remove this when - // we are confident this is stable. - // We're checking the previous word in the text field against the memorized previous - // word. If we are composing a word we should have the second word before the cursor - // memorized, otherwise we should have the first. - final String rereadPrevWord = getNthPreviousWordForSuggestion(currentSettings, + final String prevWord; + if (currentSettings.mCurrentLanguageHasSpaces) { + // If we are typing in a language with spaces we can just look up the previous + // word from textview. + prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, mWordComposer.isComposingWord() ? 2 : 1); - if (!TextUtils.equals(previousWord, rereadPrevWord)) { - throw new RuntimeException("Unexpected previous word: " - + previousWord + " <> " + rereadPrevWord); - } + } else { + prevWord = LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null + : mLastComposedWord.mCommittedWord; } - suggest.getSuggestedWords(mWordComposer, mWordComposer.getPreviousWord(), - keyboard.getProximityInfo(), + suggest.getSuggestedWords(mWordComposer, prevWord, keyboard.getProximityInfo(), currentSettings.mBlockPotentiallyOffensive, currentSettings.mCorrectionEnabled, additionalFeaturesOptions, sessionId, sequenceNumber, callback); } @@ -2701,6 +2679,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.latinIme_commitCurrentAutoCorrection(typedWord, autoCorrection, separator, mWordComposer.isBatchMode(), suggestedWords); } + mExpectingUpdateSelection = true; commitChosenWord(autoCorrection, LastComposedWord.COMMIT_TYPE_DECIDED_WORD, separator); if (!typedWord.equals(autoCorrection)) { @@ -2771,6 +2750,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // typed word. final String replacedWord = mWordComposer.getTypedWord(); LatinImeLogger.logOnManualSuggestion(replacedWord, suggestion, index, suggestedWords); + mExpectingUpdateSelection = true; commitChosenWord(suggestion, LastComposedWord.COMMIT_TYPE_MANUAL_PICK, LastComposedWord.NOT_A_SEPARATOR); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -2853,7 +2833,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final UserHistoryDictionary userHistoryDictionary = mUserHistoryDictionary; if (userHistoryDictionary == null) return null; - final String prevWord = mConnection.getNthPreviousWord(currentSettings, 2); + final String prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, 2); final String secondWord; if (mWordComposer.wasAutoCapitalized() && !mWordComposer.isMostlyCaps()) { secondWord = suggestion.toLowerCase(mSubtypeSwitcher.getCurrentSubtypeLocale()); @@ -2865,8 +2845,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final int maxFreq = AutoCorrectionUtils.getMaxFrequency( suggest.getUnigramDictionaries(), suggestion); if (maxFreq == 0) return null; - userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0, - (int)TimeUnit.MILLISECONDS.toSeconds((System.currentTimeMillis()))); + userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0); return prevWord; } @@ -2922,13 +2901,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } } - mWordComposer.setComposingWord(typedWord, - getNthPreviousWordForSuggestion(currentSettings, - // We want the previous word for suggestion. If we have chars in the word - // before the cursor, then we want the word before that, hence 2; otherwise, - // we want the word immediately before the cursor, hence 1. - 0 == numberOfCharsInWordBeforeCursor ? 1 : 2), - mKeyboardSwitcher.getKeyboard()); + mWordComposer.setComposingWord(typedWord, mKeyboardSwitcher.getKeyboard()); mWordComposer.setCursorPositionWithinWord( typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor)); mConnection.setComposingRegion( @@ -3006,11 +2979,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } private void restartSuggestionsOnWordBeforeCursor(final String word) { - mWordComposer.setComposingWord(word, - // Previous word is the 2nd word before cursor because we are restarting on the - // 1st word before cursor. - getNthPreviousWordForSuggestion(mSettings.getCurrent(), 2 /* nthPreviousWord */), - mKeyboardSwitcher.getKeyboard()); + mWordComposer.setComposingWord(word, mKeyboardSwitcher.getKeyboard()); final int length = word.length(); mConnection.deleteSurroundingText(length, 0); mConnection.setComposingText(word, 1); @@ -3028,8 +2997,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen * @param remainingTries How many times we may try again before giving up. */ private void retryResetCaches(final boolean tryResumeSuggestions, final int remainingTries) { - if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, - mLastSelectionEnd, false)) { + if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, false)) { if (0 < remainingTries) { mHandler.postResetCaches(tryResumeSuggestions, remainingTries - 1); return; @@ -3056,7 +3024,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen throw new RuntimeException("revertCommit, but we are composing a word"); } final CharSequence wordBeforeCursor = - mConnection.getTextBeforeCursor(deleteLength, 0).subSequence(0, cancelLength); + mConnection.getTextBeforeCursor(deleteLength, 0) + .subSequence(0, cancelLength); if (!TextUtils.equals(committedWord, wordBeforeCursor)) { throw new RuntimeException("revertCommit check failed: we thought we were " + "reverting \"" + committedWord @@ -3076,8 +3045,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { // For languages without spaces, we revert the typed string but the cursor is flush // with the typed word, so we need to resume suggestions right away. - mWordComposer.setComposingWord(stringToCommit, previousWord, - mKeyboardSwitcher.getKeyboard()); + mWordComposer.setComposingWord(stringToCommit, mKeyboardSwitcher.getKeyboard()); mConnection.setComposingText(stringToCommit, 1); } if (mSettings.isInternal()) { @@ -3127,8 +3095,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private void hapticAndAudioFeedback(final int code, final int repeatCount) { final MainKeyboardView keyboardView = mKeyboardSwitcher.getMainKeyboardView(); - if (keyboardView != null && keyboardView.isInDraggingFinger()) { - // No need to feedback while finger is dragging. + if (keyboardView != null && keyboardView.isInSlidingKeyInput()) { + // No need to feedback while sliding input. return; } if (repeatCount > 0) { @@ -3258,8 +3226,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Intent intent = IntentUtils.getInputLanguageSelectionIntent( mRichImm.getInputMethodIdOfThisIme(), Intent.FLAG_ACTIVITY_NEW_TASK - | Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED - | Intent.FLAG_ACTIVITY_CLEAR_TOP); + | Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED + | Intent.FLAG_ACTIVITY_CLEAR_TOP); startActivity(intent); break; case 1: @@ -3268,8 +3236,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } }; - final AlertDialog.Builder builder = - new AlertDialog.Builder(this).setItems(items, listener).setTitle(title); + final AlertDialog.Builder builder = new AlertDialog.Builder(this) + .setItems(items, listener) + .setTitle(title); showOptionDialog(builder.create()); } @@ -3330,13 +3299,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Printer p = new PrintWriterPrinter(fout); p.println("LatinIME state :"); - p.println(" VersionCode = " + ApplicationUtils.getVersionCode(this)); - p.println(" VersionName = " + ApplicationUtils.getVersionName(this)); final Keyboard keyboard = mKeyboardSwitcher.getKeyboard(); final int keyboardMode = keyboard != null ? keyboard.mId.mMode : -1; p.println(" Keyboard mode = " + keyboardMode); final SettingsValues settingsValues = mSettings.getCurrent(); - p.println(" mIsSuggestionsRequested = " + p.println(" mIsSuggestionsSuggestionsRequested = " + settingsValues.isSuggestionsRequested(mDisplayOrientation)); p.println(" mCorrectionEnabled=" + settingsValues.mCorrectionEnabled); p.println(" isComposingWord=" + mWordComposer.isComposingWord()); diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 37311acf2..673d1b4c2 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -57,19 +57,14 @@ public final class RichInputConnection { private static final int INVALID_CURSOR_POSITION = -1; /** - * This variable contains an expected value for the selection start position. This is where the - * cursor or selection start may end up after all the keyboard-triggered updates have passed. We - * keep this to compare it to the actual selection start to guess whether the move was caused by - * a keyboard command or not. - * It's not really the selection start position: the selection start may not be there yet, and - * in some cases, it may never arrive there. + * This variable contains an expected value for the cursor position. This is where the + * cursor may end up after all the keyboard-triggered updates have passed. We keep this to + * compare it to the actual cursor position to guess whether the move was caused by a + * keyboard command or not. + * It's not really the cursor position: the cursor may not be there yet, and it's also expected + * there be cases where it never actually comes to be there. */ - private int mExpectedSelStart = INVALID_CURSOR_POSITION; // in chars, not code points - /** - * The expected selection end. Only differs from mExpectedSelStart if a non-empty selection is - * expected. The same caveats as mExpectedSelStart apply. - */ - private int mExpectedSelEnd = INVALID_CURSOR_POSITION; // in chars, not code points + private int mExpectedCursorPosition = INVALID_CURSOR_POSITION; // in chars, not code points /** * This contains the committed text immediately preceding the cursor and the composing * text if any. It is refreshed when the cursor moves by calling upon the TextView. @@ -108,16 +103,16 @@ public final class RichInputConnection { final String reference = (beforeCursor.length() <= actualLength) ? beforeCursor.toString() : beforeCursor.subSequence(beforeCursor.length() - actualLength, beforeCursor.length()).toString(); - if (et.selectionStart != mExpectedSelStart + if (et.selectionStart != mExpectedCursorPosition || !(reference.equals(internal.toString()))) { - final String context = "Expected selection start = " + mExpectedSelStart - + "\nActual selection start = " + et.selectionStart + final String context = "Expected cursor position = " + mExpectedCursorPosition + + "\nActual cursor position = " + et.selectionStart + "\nExpected text = " + internal.length() + " " + internal + "\nActual text = " + reference.length() + " " + reference; ((LatinIME)mParent).debugDumpStateAndCrashWithException(context); } else { Log.e(TAG, DebugLogUtils.getStackTrace(2)); - Log.e(TAG, "Exp <> Actual : " + mExpectedSelStart + " <> " + et.selectionStart); + Log.e(TAG, "Exp <> Actual : " + mExpectedCursorPosition + " <> " + et.selectionStart); } } @@ -155,50 +150,16 @@ public final class RichInputConnection { * data, so we empty the cache and note that we don't know the new cursor position, and we * return false so that the caller knows about this and can retry later. * - * @param newSelStart the new position of the selection start, as received from the system. - * @param newSelEnd the new position of the selection end, as received from the system. - * @param shouldFinishComposition whether we should finish the composition in progress. + * @param newCursorPosition The new position of the cursor, as received from the system. + * @param shouldFinishComposition Whether we should finish the composition in progress. * @return true if we were able to connect to the editor successfully, false otherwise. When * this method returns false, the caches could not be correctly refreshed so they were only * reset: the caller should try again later to return to normal operation. */ - public boolean resetCachesUponCursorMoveAndReturnSuccess(final int newSelStart, - final int newSelEnd, final boolean shouldFinishComposition) { - mExpectedSelStart = newSelStart; - mExpectedSelEnd = newSelEnd; + public boolean resetCachesUponCursorMoveAndReturnSuccess(final int newCursorPosition, + final boolean shouldFinishComposition) { + mExpectedCursorPosition = newCursorPosition; mComposingText.setLength(0); - final boolean didReloadTextSuccessfully = reloadTextCache(); - if (!didReloadTextSuccessfully) { - Log.d(TAG, "Will try to retrieve text later."); - return false; - } - final int lengthOfTextBeforeCursor = mCommittedTextBeforeComposingText.length(); - if (lengthOfTextBeforeCursor > newSelStart - || (lengthOfTextBeforeCursor < Constants.EDITOR_CONTENTS_CACHE_SIZE - && newSelStart < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { - // newSelStart and newSelEnd may be lying -- when rotating the device (probably a - // framework bug). If we have less chars than we asked for, then we know how many chars - // we have, and if we got more than newSelStart says, then we know it was lying. In both - // cases the length is more reliable. Note that we only have to check newSelStart (not - // newSelEnd) since if newSelEnd is wrong, the newSelStart will be wrong as well. - mExpectedSelStart = lengthOfTextBeforeCursor; - mExpectedSelEnd = lengthOfTextBeforeCursor; - } - if (null != mIC && shouldFinishComposition) { - mIC.finishComposingText(); - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.richInputConnection_finishComposingText(); - } - } - return true; - } - - /** - * Reload the cached text from the InputConnection. - * - * @return true if successful - */ - private boolean reloadTextCache() { mCommittedTextBeforeComposingText.setLength(0); mIC = mParent.getCurrentInputConnection(); // Call upon the inputconnection directly since our own method is using the cache, and @@ -208,12 +169,27 @@ public final class RichInputConnection { if (null == textBeforeCursor) { // For some reason the app thinks we are not connected to it. This looks like a // framework bug... Fall back to ground state and return false. - mExpectedSelStart = INVALID_CURSOR_POSITION; - mExpectedSelEnd = INVALID_CURSOR_POSITION; - Log.e(TAG, "Unable to connect to the editor to retrieve text."); + mExpectedCursorPosition = INVALID_CURSOR_POSITION; + Log.e(TAG, "Unable to connect to the editor to retrieve text... will retry later"); return false; } mCommittedTextBeforeComposingText.append(textBeforeCursor); + final int lengthOfTextBeforeCursor = textBeforeCursor.length(); + if (lengthOfTextBeforeCursor > newCursorPosition + || (lengthOfTextBeforeCursor < Constants.EDITOR_CONTENTS_CACHE_SIZE + && newCursorPosition < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { + // newCursorPosition may be lying -- when rotating the device (probably a framework + // bug). If we have less chars than we asked for, then we know how many chars we have, + // and if we got more than newCursorPosition says, then we know it was lying. In both + // cases the length is more reliable + mExpectedCursorPosition = lengthOfTextBeforeCursor; + } + if (null != mIC && shouldFinishComposition) { + mIC.finishComposingText(); + if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { + ResearchLogger.richInputConnection_finishComposingText(); + } + } return true; } @@ -242,8 +218,7 @@ public final class RichInputConnection { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); mCommittedTextBeforeComposingText.append(text); - mExpectedSelStart += text.length() - mComposingText.length(); - mExpectedSelEnd = mExpectedSelStart; + mExpectedCursorPosition += text.length() - mComposingText.length(); mComposingText.setLength(0); if (null != mIC) { mIC.commitText(text, i); @@ -256,7 +231,7 @@ public final class RichInputConnection { } public boolean canDeleteCharacters() { - return mExpectedSelStart > 0; + return mExpectedCursorPosition > 0; } /** @@ -293,10 +268,11 @@ public final class RichInputConnection { // heavy pressing of delete, for example DEFAULT_TEXT_CACHE_SIZE - 5 times or so. // getCapsMode should be updated to be able to return a "not enough info" result so that // we can get more context only when needed. - if (TextUtils.isEmpty(mCommittedTextBeforeComposingText) && 0 != mExpectedSelStart) { - if (!reloadTextCache()) { - Log.w(TAG, "Unable to connect to the editor. " - + "Setting caps mode without knowing text."); + if (TextUtils.isEmpty(mCommittedTextBeforeComposingText) && 0 != mExpectedCursorPosition) { + final CharSequence textBeforeCursor = getTextBeforeCursor( + Constants.EDITOR_CONTENTS_CACHE_SIZE, 0); + if (!TextUtils.isEmpty(textBeforeCursor)) { + mCommittedTextBeforeComposingText.append(textBeforeCursor); } } // This never calls InputConnection#getCapsMode - in fact, it's a static method that @@ -319,8 +295,8 @@ public final class RichInputConnection { // However, if we don't have an expected cursor position, then we should always // go fetch the cache again (as it happens, INVALID_CURSOR_POSITION < 0, so we need to // test for this explicitly) - if (INVALID_CURSOR_POSITION != mExpectedSelStart - && (cachedLength >= n || cachedLength >= mExpectedSelStart)) { + if (INVALID_CURSOR_POSITION != mExpectedCursorPosition + && (cachedLength >= n || cachedLength >= mExpectedCursorPosition)) { final StringBuilder s = new StringBuilder(mCommittedTextBeforeComposingText); // We call #toString() here to create a temporary object. // In some situations, this method is called on a worker thread, and it's possible @@ -360,14 +336,10 @@ public final class RichInputConnection { + remainingChars, 0); mCommittedTextBeforeComposingText.setLength(len); } - if (mExpectedSelStart > beforeLength) { - mExpectedSelStart -= beforeLength; - mExpectedSelEnd -= beforeLength; + if (mExpectedCursorPosition > beforeLength) { + mExpectedCursorPosition -= beforeLength; } else { - // There are fewer characters before the cursor in the buffer than we are being asked to - // delete. Only delete what is there. - mExpectedSelStart = 0; - mExpectedSelEnd -= mExpectedSelStart; + mExpectedCursorPosition = 0; } if (null != mIC) { mIC.deleteSurroundingText(beforeLength, afterLength); @@ -401,8 +373,7 @@ public final class RichInputConnection { switch (keyEvent.getKeyCode()) { case KeyEvent.KEYCODE_ENTER: mCommittedTextBeforeComposingText.append("\n"); - mExpectedSelStart += 1; - mExpectedSelEnd = mExpectedSelStart; + mExpectedCursorPosition += 1; break; case KeyEvent.KEYCODE_DEL: if (0 == mComposingText.length()) { @@ -414,24 +385,18 @@ public final class RichInputConnection { } else { mComposingText.delete(mComposingText.length() - 1, mComposingText.length()); } - if (mExpectedSelStart > 0 && mExpectedSelStart == mExpectedSelEnd) { - // TODO: Handle surrogate pairs. - mExpectedSelStart -= 1; - } - mExpectedSelEnd = mExpectedSelStart; + if (mExpectedCursorPosition > 0) mExpectedCursorPosition -= 1; break; case KeyEvent.KEYCODE_UNKNOWN: if (null != keyEvent.getCharacters()) { mCommittedTextBeforeComposingText.append(keyEvent.getCharacters()); - mExpectedSelStart += keyEvent.getCharacters().length(); - mExpectedSelEnd = mExpectedSelStart; + mExpectedCursorPosition += keyEvent.getCharacters().length(); } break; default: - final String text = StringUtils.newSingleCodePointString(keyEvent.getUnicodeChar()); + final String text = new String(new int[] { keyEvent.getUnicodeChar() }, 0, 1); mCommittedTextBeforeComposingText.append(text); - mExpectedSelStart += text.length(); - mExpectedSelEnd = mExpectedSelStart; + mExpectedCursorPosition += text.length(); break; } } @@ -465,12 +430,10 @@ public final class RichInputConnection { public void setComposingText(final CharSequence text, final int newCursorPosition) { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); - mExpectedSelStart += text.length() - mComposingText.length(); - mExpectedSelEnd = mExpectedSelStart; + mExpectedCursorPosition += text.length() - mComposingText.length(); mComposingText.setLength(0); mComposingText.append(text); - // TODO: support values of newCursorPosition != 1. At this time, this is never called with - // newCursorPosition != 1. + // TODO: support values of i != 1. At this time, this is never called with i != 1. if (null != mIC) { mIC.setComposingText(text, newCursorPosition); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -480,31 +443,19 @@ public final class RichInputConnection { if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); } - /** - * Set the selection of the text editor. - * - * Calls through to {@link InputConnection#setSelection(int, int)}. - * - * @param start the character index where the selection should start. - * @param end the character index where the selection should end. - * @return Returns true on success, false if the input connection is no longer valid either when - * setting the selection or when retrieving the text cache at that point. - */ - public boolean setSelection(final int start, final int end) { + public void setSelection(final int start, final int end) { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); - mExpectedSelStart = start; - mExpectedSelEnd = end; if (null != mIC) { - final boolean isIcValid = mIC.setSelection(start, end); - if (!isIcValid) { - return false; - } + mIC.setSelection(start, end); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.richInputConnection_setSelection(start, end); } } - return reloadTextCache(); + mExpectedCursorPosition = start; + mCommittedTextBeforeComposingText.setLength(0); + mCommittedTextBeforeComposingText.append( + getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0)); } public void commitCorrection(final CorrectionInfo correctionInfo) { @@ -525,8 +476,7 @@ public final class RichInputConnection { // text should never be null, but just in case, it's better to insert nothing than to crash if (null == text) text = ""; mCommittedTextBeforeComposingText.append(text); - mExpectedSelStart += text.length() - mComposingText.length(); - mExpectedSelEnd = mExpectedSelStart; + mExpectedCursorPosition += text.length() - mComposingText.length(); mComposingText.setLength(0); if (null != mIC) { mIC.commitCompletion(completionInfo); @@ -538,7 +488,7 @@ public final class RichInputConnection { } @SuppressWarnings("unused") - public String getNthPreviousWord(final SettingsValues currentSettingsValues, final int n) { + public String getNthPreviousWord(final String sentenceSeperators, final int n) { mIC = mParent.getCurrentInputConnection(); if (null == mIC) return null; final CharSequence prev = getTextBeforeCursor(LOOKBACK_CHARACTER_NUM, 0); @@ -557,7 +507,7 @@ public final class RichInputConnection { } } } - return getNthPreviousWord(prev, currentSettingsValues, n); + return getNthPreviousWord(prev, sentenceSeperators, n); } private static boolean isSeparator(int code, String sep) { @@ -581,7 +531,7 @@ public final class RichInputConnection { // (n = 2) "abc |" -> null // (n = 2) "abc. def|" -> null public static String getNthPreviousWord(final CharSequence prev, - final SettingsValues currentSettingsValues, final int n) { + final String sentenceSeperators, final int n) { if (prev == null) return null; final String[] w = spaceRegex.split(prev); @@ -593,8 +543,7 @@ public final class RichInputConnection { // If ends in a separator, return null final char lastChar = nthPrevWord.charAt(length - 1); - if (currentSettingsValues.isWordSeparator(lastChar) - || currentSettingsValues.isWordConnector(lastChar)) return null; + if (sentenceSeperators.contains(String.valueOf(lastChar))) return null; return nthPrevWord; } @@ -809,30 +758,20 @@ public final class RichInputConnection { * this update and not the ones in-between. This is almost impossible to achieve even trying * very very hard. * - * @param oldSelStart The value of the old selection in the update. - * @param newSelStart The value of the new selection in the update. - * @param oldSelEnd The value of the old selection end in the update. - * @param newSelEnd The value of the new selection end in the update. + * @param oldSelStart The value of the old cursor position in the update. + * @param newSelStart The value of the new cursor position in the update. * @return whether this is a belated expected update or not. */ - public boolean isBelatedExpectedUpdate(final int oldSelStart, final int newSelStart, - final int oldSelEnd, final int newSelEnd) { - // This update is "belated" if we are expecting it. That is, mExpectedSelStart and - // mExpectedSelEnd match the new values that the TextView is updating TO. - if (mExpectedSelStart == newSelStart && mExpectedSelEnd == newSelEnd) return true; - // This update is not belated if mExpectedSelStart and mExpeectedSelend match the old - // values, and one of newSelStart or newSelEnd is updated to a different value. In this - // case, there is likely something other than the IME that has moved the selection endpoint - // to the new value. - if (mExpectedSelStart == oldSelStart && mExpectedSelEnd == oldSelEnd - && (oldSelStart != newSelStart || oldSelEnd != newSelEnd)) return false; - // If nether of the above two cases holds, then the system may be having trouble keeping up - // with updates. If 1) the selection is a cursor, 2) newSelStart is between oldSelStart - // and mExpectedSelStart, and 3) newSelEnd is between oldSelEnd and mExpectedSelEnd, then - // assume a belated update. - return (newSelStart == newSelEnd) - && (newSelStart - oldSelStart) * (mExpectedSelStart - newSelStart) >= 0 - && (newSelEnd - oldSelEnd) * (mExpectedSelEnd - newSelEnd) >= 0; + public boolean isBelatedExpectedUpdate(final int oldSelStart, final int newSelStart) { + // If this is an update that arrives at our expected position, it's a belated update. + if (newSelStart == mExpectedCursorPosition) return true; + // If this is an update that moves the cursor from our expected position, it must be + // an explicit move. + if (oldSelStart == mExpectedCursorPosition) return false; + // The following returns true if newSelStart is between oldSelStart and + // mCurrentCursorPosition. We assume that if the updated position is between the old + // position and the expected position, then it must be a belated update. + return (newSelStart - oldSelStart) * (mExpectedCursorPosition - newSelStart) >= 0; } /** diff --git a/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java b/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java index 379eaaa3d..cd9c89f04 100644 --- a/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java +++ b/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java @@ -32,9 +32,7 @@ import android.view.inputmethod.InputMethodManager; import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.compat.InputMethodSubtypeCompatUtils; import com.android.inputmethod.keyboard.KeyboardSwitcher; -import com.android.inputmethod.latin.utils.LocaleUtils; import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; import java.util.List; @@ -58,34 +56,23 @@ public final class SubtypeSwitcher { private InputMethodSubtype mEmojiSubtype; private boolean mIsNetworkConnected; - private static final String KEYBOARD_MODE = "keyboard"; // Dummy no language QWERTY subtype. See {@link R.xml.method}. - private static final int SUBTYPE_ID_OF_DUMMY_NO_LANGUAGE_SUBTYPE = 0xdde0bfd3; - private static final String EXTRA_VALUE_OF_DUMMY_NO_LANGUAGE_SUBTYPE = - "KeyboardLayoutSet=" + SubtypeLocaleUtils.QWERTY - + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE - + "," + Constants.Subtype.ExtraValue.ENABLED_WHEN_DEFAULT_IS_NOT_ASCII_CAPABLE - + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE; - private static final InputMethodSubtype DUMMY_NO_LANGUAGE_SUBTYPE = - InputMethodSubtypeCompatUtils.newInputMethodSubtype( - R.string.subtype_no_language_qwerty, R.drawable.ic_ime_switcher_dark, - SubtypeLocaleUtils.NO_LANGUAGE, KEYBOARD_MODE, - EXTRA_VALUE_OF_DUMMY_NO_LANGUAGE_SUBTYPE, - false /* isAuxiliary */, false /* overridesImplicitlyEnabledSubtype */, - SUBTYPE_ID_OF_DUMMY_NO_LANGUAGE_SUBTYPE); + private static final InputMethodSubtype DUMMY_NO_LANGUAGE_SUBTYPE = new InputMethodSubtype( + R.string.subtype_no_language_qwerty, R.drawable.ic_ime_switcher_dark, + SubtypeLocaleUtils.NO_LANGUAGE, "keyboard", "KeyboardLayoutSet=" + + SubtypeLocaleUtils.QWERTY + + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE + + ",EnabledWhenDefaultIsNotAsciiCapable," + + Constants.Subtype.ExtraValue.EMOJI_CAPABLE, + false /* isAuxiliary */, false /* overridesImplicitlyEnabledSubtype */); // Caveat: We probably should remove this when we add an Emoji subtype in {@link R.xml.method}. // Dummy Emoji subtype. See {@link R.xml.method}. - private static final int SUBTYPE_ID_OF_DUMMY_EMOJI_SUBTYPE = 0xd78b2ed0; - private static final String EXTRA_VALUE_OF_DUMMY_EMOJI_SUBTYPE = - "KeyboardLayoutSet=" + SubtypeLocaleUtils.EMOJI - + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE; - private static final InputMethodSubtype DUMMY_EMOJI_SUBTYPE = - InputMethodSubtypeCompatUtils.newInputMethodSubtype( - R.string.subtype_emoji, R.drawable.ic_ime_switcher_dark, - SubtypeLocaleUtils.NO_LANGUAGE, KEYBOARD_MODE, - EXTRA_VALUE_OF_DUMMY_EMOJI_SUBTYPE, - false /* isAuxiliary */, false /* overridesImplicitlyEnabledSubtype */, - SUBTYPE_ID_OF_DUMMY_EMOJI_SUBTYPE); + private static final InputMethodSubtype DUMMY_EMOJI_SUBTYPE = new InputMethodSubtype( + R.string.subtype_emoji, R.drawable.ic_ime_switcher_dark, + SubtypeLocaleUtils.NO_LANGUAGE, "keyboard", "KeyboardLayoutSet=" + + SubtypeLocaleUtils.EMOJI + "," + + Constants.Subtype.ExtraValue.EMOJI_CAPABLE, + false /* isAuxiliary */, false /* overridesImplicitlyEnabledSubtype */); static final class NeedsToDisplayLanguage { private int mEnabledSubtypeCount; @@ -269,23 +256,18 @@ public final class SubtypeSwitcher { return mNeedsToDisplayLanguage.getValue(); } - private static InputMethodSubtype sForcedSubtypeForTesting = null; + private static Locale sForcedLocaleForTesting = null; @UsedForTesting - void forceSubtype(final InputMethodSubtype subtype) { - sForcedSubtypeForTesting = subtype; + void forceLocale(final Locale locale) { + sForcedLocaleForTesting = locale; } public Locale getCurrentSubtypeLocale() { - if (null != sForcedSubtypeForTesting) { - return LocaleUtils.constructLocaleFromString(sForcedSubtypeForTesting.getLocale()); - } + if (null != sForcedLocaleForTesting) return sForcedLocaleForTesting; return SubtypeLocaleUtils.getSubtypeLocale(getCurrentSubtype()); } public InputMethodSubtype getCurrentSubtype() { - if (null != sForcedSubtypeForTesting) { - return sForcedSubtypeForTesting; - } return mRichImm.getCurrentInputMethodSubtype(getNoLanguageSubtype()); } @@ -297,8 +279,8 @@ public final class SubtypeSwitcher { if (mNoLanguageSubtype != null) { return mNoLanguageSubtype; } - Log.w(TAG, "Can't find any language with QWERTY subtype"); - Log.w(TAG, "No input method subtype found; returning dummy subtype: " + Log.w(TAG, "Can't find no lanugage with QWERTY subtype"); + Log.w(TAG, "No input method subtype found; return dummy subtype: " + DUMMY_NO_LANGUAGE_SUBTYPE); return DUMMY_NO_LANGUAGE_SUBTYPE; } @@ -311,9 +293,8 @@ public final class SubtypeSwitcher { if (mEmojiSubtype != null) { return mEmojiSubtype; } - Log.w(TAG, "Can't find emoji subtype"); - Log.w(TAG, "No input method subtype found; returning dummy subtype: " - + DUMMY_EMOJI_SUBTYPE); + Log.w(TAG, "Can't find Emoji subtype"); + Log.w(TAG, "No input method subtype found; return dummy subtype: " + DUMMY_EMOJI_SUBTYPE); return DUMMY_EMOJI_SUBTYPE; } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 0ecb41100..0a4c7a55d 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -25,6 +25,7 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.PersonalizationDictionary; +import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; import com.android.inputmethod.latin.personalization.UserHistoryDictionary; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.AutoCorrectionUtils; @@ -89,6 +90,7 @@ public final class Suggest { PreferenceManager.getDefaultSharedPreferences(context))) { mOnlyDictionarySetForDebug = new HashSet<String>(); mOnlyDictionarySetForDebug.add(Dictionary.TYPE_PERSONALIZATION); + mOnlyDictionarySetForDebug.add(Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA); } } @@ -192,6 +194,12 @@ public final class Suggest { addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, userHistoryDictionary); } + public void setPersonalizationPredictionDictionary( + final PersonalizationPredictionDictionary personalizationPredictionDictionary) { + addOrReplaceDictionaryInternal(Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, + personalizationPredictionDictionary); + } + public void setPersonalizationDictionary( final PersonalizationDictionary personalizationDictionary) { addOrReplaceDictionaryInternal(Dictionary.TYPE_PERSONALIZATION, @@ -424,8 +432,7 @@ public final class Suggest { final String scoreInfoString; if (normalizedScore > 0) { scoreInfoString = String.format( - Locale.ROOT, "%d (%4.2f), %s", cur.mScore, normalizedScore, - cur.mSourceDict.mDictType); + Locale.ROOT, "%d (%4.2f)", cur.mScore, normalizedScore); } else { scoreInfoString = Integer.toString(cur.mScore); } diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java index 9cb2f5bc4..6405b5e46 100644 --- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java @@ -22,15 +22,14 @@ import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.util.ArrayList; -import java.util.Locale; public final class SynchronouslyLoadedUserBinaryDictionary extends UserBinaryDictionary { - public SynchronouslyLoadedUserBinaryDictionary(final Context context, final Locale locale) { + public SynchronouslyLoadedUserBinaryDictionary(final Context context, final String locale) { this(context, locale, false); } - public SynchronouslyLoadedUserBinaryDictionary(final Context context, final Locale locale, + public SynchronouslyLoadedUserBinaryDictionary(final Context context, final String locale, final boolean alsoUseMoreRestrictiveLocales) { super(context, locale, alsoUseMoreRestrictiveLocales); } diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index 1dc3c54bb..15b3d8d02 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.content.ContentProviderClient; import android.content.ContentResolver; +import android.content.ContentUris; import android.content.Context; import android.database.ContentObserver; import android.database.Cursor; @@ -74,21 +75,20 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { final private String mLocale; final private boolean mAlsoUseMoreRestrictiveLocales; - public UserBinaryDictionary(final Context context, final Locale locale) { + public UserBinaryDictionary(final Context context, final String locale) { this(context, locale, false); } - public UserBinaryDictionary(final Context context, final Locale locale, + public UserBinaryDictionary(final Context context, final String locale, final boolean alsoUseMoreRestrictiveLocales) { - super(context, getDictNameWithLocale(NAME, locale), locale, Dictionary.TYPE_USER, + super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_USER, false /* isUpdatable */); if (null == locale) throw new NullPointerException(); // Catch the error earlier - final String localeStr = locale.toString(); - if (SubtypeLocaleUtils.NO_LANGUAGE.equals(localeStr)) { + if (SubtypeLocaleUtils.NO_LANGUAGE.equals(locale)) { // If we don't have a locale, insert into the "all locales" user dictionary. mLocale = USER_DICTIONARY_ALL_LANGUAGES; } else { - mLocale = localeStr; + mLocale = locale; } mAlsoUseMoreRestrictiveLocales = alsoUseMoreRestrictiveLocales; // Perform a managed query. The Activity will handle closing and re-querying the cursor diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index 2f81d15d5..039dadc66 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -48,10 +48,6 @@ public final class WordComposer { // at any given time. However this is not limited in size, while mPrimaryKeyCodes is limited // to MAX_WORD_LENGTH code points. private final StringBuilder mTypedWord; - // The previous word (before the composing word). Used as context for suggestions. May be null - // after resetting and before starting a new composing word, or when there is no context like - // at the start of text for example. - private String mPreviousWord; private String mAutoCorrection; private boolean mIsResumed; private boolean mIsBatchMode; @@ -89,7 +85,6 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPreviousWord = null; refreshSize(); } @@ -106,7 +101,6 @@ public final class WordComposer { mIsBatchMode = source.mIsBatchMode; mCursorPositionWithinWord = source.mCursorPositionWithinWord; mRejectedBatchModeSuggestion = source.mRejectedBatchModeSuggestion; - mPreviousWord = source.mPreviousWord; refreshSize(); } @@ -124,7 +118,6 @@ public final class WordComposer { mIsBatchMode = false; mCursorPositionWithinWord = 0; mRejectedBatchModeSuggestion = null; - mPreviousWord = null; refreshSize(); } @@ -291,13 +284,8 @@ public final class WordComposer { /** * Set the currently composing word to the one passed as an argument. * This will register NOT_A_COORDINATE for X and Ys, and use the passed keyboard for proximity. - * @param word the char sequence to set as the composing word. - * @param previousWord the previous word, to use as context for suggestions. Can be null if - * the context is nil (typically, at start of text). - * @param keyboard the keyboard this is typed on, for coordinate info/proximity. */ - public void setComposingWord(final CharSequence word, final String previousWord, - final Keyboard keyboard) { + public void setComposingWord(final CharSequence word, final Keyboard keyboard) { reset(); final int length = word.length(); for (int i = 0; i < length; i = Character.offsetByCodePoints(word, i, 1)) { @@ -305,7 +293,6 @@ public final class WordComposer { addKeyInfo(codePoint, keyboard); } mIsResumed = true; - mPreviousWord = previousWord; } /** @@ -356,10 +343,6 @@ public final class WordComposer { return mTypedWord.toString(); } - public String getPreviousWord() { - return mPreviousWord; - } - /** * Whether or not the user typed a capital letter as the first letter in the word * @return capitalization preference @@ -405,21 +388,18 @@ public final class WordComposer { } /** - * Saves the caps mode and the previous word at the start of composing. + * Saves the caps mode at the start of composing. * - * WordComposer needs to know about the caps mode for several reasons. The first is, we need - * to know after the fact what the reason was, to register the correct form into the user - * history dictionary: if the word was automatically capitalized, we should insert it in - * all-lower case but if it's a manual pressing of shift, then it should be inserted as is. + * WordComposer needs to know about this for several reasons. The first is, we need to know + * after the fact what the reason was, to register the correct form into the user history + * dictionary: if the word was automatically capitalized, we should insert it in all-lower + * case but if it's a manual pressing of shift, then it should be inserted as is. * Also, batch input needs to know about the current caps mode to display correctly * capitalized suggestions. * @param mode the mode at the time of start - * @param previousWord the previous word as context for suggestions. May be null if none. */ - public void setCapitalizedModeAndPreviousWordAtStartComposingTime(final int mode, - final String previousWord) { + public void setCapitalizedModeAtStartComposingTime(final int mode) { mCapitalizedMode = mode; - mPreviousWord = previousWord; } /** @@ -471,7 +451,6 @@ public final class WordComposer { mCapsCount = 0; mDigitsCount = 0; mIsBatchMode = false; - mPreviousWord = mTypedWord.toString(); mTypedWord.setLength(0); mCodePointSize = 0; mTrailingSingleQuotesCount = 0; @@ -485,8 +464,7 @@ public final class WordComposer { return lastComposedWord; } - public void resumeSuggestionOnLastComposedWord(final LastComposedWord lastComposedWord, - final String previousWord) { + public void resumeSuggestionOnLastComposedWord(final LastComposedWord lastComposedWord) { mPrimaryKeyCodes = lastComposedWord.mPrimaryKeyCodes; mInputPointers.set(lastComposedWord.mInputPointers); mTypedWord.setLength(0); @@ -497,7 +475,6 @@ public final class WordComposer { mCursorPositionWithinWord = mCodePointSize; mRejectedBatchModeSuggestion = null; mIsResumed = true; - mPreviousWord = previousWord; } public boolean isBatchMode() { diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index f8fa68f45..fda97dafc 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -32,35 +32,36 @@ import java.util.TreeMap; * A base class of the binary dictionary decoder. */ public abstract class AbstractDictDecoder implements DictDecoder { - private static final int SUCCESS = 0; - private static final int ERROR_CANNOT_READ = 1; - private static final int ERROR_WRONG_FORMAT = 2; - - protected FileHeader readHeader(final DictBuffer headerBuffer) + protected FileHeader readHeader(final DictBuffer dictBuffer) throws IOException, UnsupportedFormatException { - if (headerBuffer == null) { + if (dictBuffer == null) { openDictBuffer(); } - final int version = HeaderReader.readVersion(headerBuffer); + final int version = HeaderReader.readVersion(dictBuffer); if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { throw new UnsupportedFormatException("Unsupported version : " + version); } // TODO: Remove this field. - final int optionsFlags = HeaderReader.readOptionFlags(headerBuffer); - final int headerSize = HeaderReader.readHeaderSize(headerBuffer); + final int optionsFlags = HeaderReader.readOptionFlags(dictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(dictBuffer); + if (headerSize < 0) { throw new UnsupportedFormatException("header size can't be negative."); } - final HashMap<String, String> attributes = HeaderReader.readAttributes(headerBuffer, + final HashMap<String, String> attributes = HeaderReader.readAttributes(dictBuffer, headerSize); final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE), + 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); return header; } @@ -203,25 +204,4 @@ public abstract class AbstractDictDecoder implements DictDecoder { return readLength; } } - - /** - * Check whether the header contains the expected information. This is a no-error method, - * that will return an error code and never throw a checked exception. - * @return an error code, either ERROR_* or SUCCESS. - */ - private int checkHeader() { - try { - readHeader(); - } catch (IOException e) { - return ERROR_CANNOT_READ; - } catch (UnsupportedFormatException e) { - return ERROR_WRONG_FORMAT; - } - return SUCCESS; - } - - @Override - public boolean hasValidRawBinaryDictionary() { - return checkHeader() == SUCCESS; - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 7f0aa777f..216492b4d 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -24,9 +24,12 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Map; import java.util.TreeMap; @@ -166,14 +169,6 @@ public final class BinaryDictDecoderUtils { return size; } - static int getCharArraySize(final int[] chars, final int start, final int end) { - int size = 0; - for (int i = start; i < end; ++i) { - size += getCharSize(chars[i]); - } - return size; - } - /** * Writes a char array to a byte buffer. * @@ -205,7 +200,8 @@ public final class BinaryDictDecoderUtils { * @param word the string to write. * @return the size written, in bytes. */ - static int writeString(final byte[] buffer, final int origin, final String word) { + static int writeString(final byte[] buffer, final int origin, + final String word) { final int length = word.length(); int index = origin; for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { @@ -227,62 +223,22 @@ public final class BinaryDictDecoderUtils { * * This will also write the terminator byte. * - * @param stream the OutputStream to write to. + * @param buffer the OutputStream to write to. * @param word the string to write. - * @return the size written, in bytes. */ - static int writeString(final OutputStream stream, final String word) throws IOException { + static void writeString(final OutputStream buffer, final String word) throws IOException { final int length = word.length(); - int written = 0; for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); - final int charSize = getCharSize(codePoint); - if (1 == charSize) { - stream.write((byte) codePoint); - } else { - stream.write((byte) (0xFF & (codePoint >> 16))); - stream.write((byte) (0xFF & (codePoint >> 8))); - stream.write((byte) (0xFF & codePoint)); - } - written += charSize; - } - stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); - written += FormatSpec.PTNODE_TERMINATOR_SIZE; - return written; - } - - /** - * Writes an array of code points with our character format to an OutputStream. - * - * This will also write the terminator byte. - * - * @param stream the OutputStream to write to. - * @param codePoints the array of code points - * @return the size written, in bytes. - */ - // TODO: Merge this method with writeCharArray and rename the various write* methods to - // make the difference clear. - static int writeCodePoints(final OutputStream stream, final int[] codePoints, - final int startIndex, final int endIndex) - throws IOException { - int written = 0; - for (int i = startIndex; i < endIndex; ++i) { - final int codePoint = codePoints[i]; - final int charSize = getCharSize(codePoint); - if (1 == charSize) { - stream.write((byte) codePoint); + if (1 == getCharSize(codePoint)) { + buffer.write((byte) codePoint); } else { - stream.write((byte) (0xFF & (codePoint >> 16))); - stream.write((byte) (0xFF & (codePoint >> 8))); - stream.write((byte) (0xFF & codePoint)); + buffer.write((byte) (0xFF & (codePoint >> 16))); + buffer.write((byte) (0xFF & (codePoint >> 8))); + buffer.write((byte) (0xFF & codePoint)); } - written += charSize; } - if (endIndex - startIndex > 1) { - stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); - written += FormatSpec.PTNODE_TERMINATOR_SIZE; - } - return written; + buffer.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); } /** @@ -330,7 +286,7 @@ public final class BinaryDictDecoderUtils { static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { - if (options.supportsDynamicUpdate()) { + if (options.mSupportsDynamicUpdate) { final int address = dictBuffer.readUnsignedInt24(); if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; if ((address & FormatSpec.MSB24) != 0) { @@ -540,11 +496,11 @@ public final class BinaryDictDecoderUtils { } // reach the end of the array. - if (options.supportsDynamicUpdate()) { + if (options.mSupportsDynamicUpdate) { final boolean hasValidForwardLink = dictDecoder.readAndFollowForwardLink(); if (!hasValidForwardLink) break; } - } while (options.supportsDynamicUpdate() && dictDecoder.hasNextPtNodeArray()); + } while (options.mSupportsDynamicUpdate && dictDecoder.hasNextPtNodeArray()); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; @@ -600,7 +556,7 @@ public final class BinaryDictDecoderUtils { Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); Map<Integer, PtNode> reversePtNodeMapping = new TreeMap<Integer, PtNode>(); - final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mBodyOffset, + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); @@ -636,10 +592,32 @@ public final class BinaryDictDecoderUtils { /** * Basic test to find out whether the file is a binary dictionary or not. * + * Concretely this only tests the magic number. + * * @param file The file to test. * @return true if it's a binary dictionary, false otherwise */ public static boolean isBinaryDictionary(final File file) { - return FormatSpec.getDictDecoder(file).hasValidRawBinaryDictionary(); + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final ByteBuffer buffer = inStream.getChannel().map( + FileChannel.MapMode.READ_ONLY, 0, file.length()); + final int version = getFormatVersion(new ByteBufferDictBuffer(buffer)); + return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION + && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } catch (FileNotFoundException e) { + return false; + } catch (IOException e) { + return false; + } finally { + if (inStream != null) { + try { + inStream.close(); + } catch (IOException e) { + // do nothing + } + } + } } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 8ba0797de..f761829de 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -17,9 +17,9 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -160,7 +160,7 @@ public class BinaryDictEncoderUtils { node.mCachedSize = nodeSize; size += nodeSize; } - if (options.supportsDynamicUpdate()) { + if (options.mSupportsDynamicUpdate) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } ptNodeArray.mCachedSize = size; @@ -245,26 +245,6 @@ public class BinaryDictEncoderUtils { } } - static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value, - final int size) { - switch(size) { - case 4: - dictBuffer.put((byte) ((value >> 24) & 0xFF)); - /* fall through */ - case 3: - dictBuffer.put((byte) ((value >> 16) & 0xFF)); - /* fall through */ - case 2: - dictBuffer.put((byte) ((value >> 8) & 0xFF)); - /* fall through */ - case 1: - dictBuffer.put((byte) (value & 0xFF)); - break; - default: - /* nop */ - } - } - // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time @@ -397,7 +377,7 @@ public class BinaryDictEncoderUtils { nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; } } - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; } else if (null != ptNode.mChildren) { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, @@ -417,7 +397,7 @@ public class BinaryDictEncoderUtils { ptNode.mCachedSize = nodeSize; size += nodeSize; } - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } if (ptNodeArray.mCachedSize != size) { @@ -533,7 +513,7 @@ public class BinaryDictEncoderUtils { if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); } while (changesDone); - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { computeParentAddresses(flatNodes); } final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); @@ -642,7 +622,7 @@ public class BinaryDictEncoderUtils { byte flags = 0; if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { flags |= FormatSpec.FLAG_IS_NOT_MOVED; } else if (true) { switch (childrenAddressSize) { @@ -710,13 +690,6 @@ public class BinaryDictEncoderUtils { + word + " is " + unigramFrequency); bigramFrequency = unigramFrequency; } - bigramFlags += getBigramFrequencyDiff(unigramFrequency, bigramFrequency) - & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; - return bigramFlags; - } - - public static int getBigramFrequencyDiff(final int unigramFrequency, - final int bigramFrequency) { // We compute the difference between 255 (which means probability = 1) and the // unigram score. We split this into a number of discrete steps. // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 @@ -750,15 +723,22 @@ public class BinaryDictEncoderUtils { // include this bigram in the dictionary. For now, register as 0, and live with the // small over-estimation that we get in this case. TODO: actually remove this bigram // if discretizedFrequency < 0. - return discretizedFrequency > 0 ? discretizedFrequency : 0; + final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; + bigramFlags += finalBigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; + return bigramFlags; } /** - * Makes the 2-byte value for options flags. Unused at the moment, and always 0. + * Makes the 2-byte value for options flags. */ - private static final int makeOptionsValue(final FormatOptions formatOptions) { - // TODO: why doesn't this handle CONTAINS_TIMESTAMP_FLAG? - return 0; + private static final int makeOptionsValue(final FusionDictionary dictionary, + final FormatOptions formatOptions) { + final DictionaryOptions options = dictionary.mOptions; + final boolean hasBigrams = dictionary.hasBigrams(); + return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) + + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) + + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) + + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); } /** @@ -846,7 +826,7 @@ public class BinaryDictEncoderUtils { } dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); } - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); } if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate @@ -947,7 +927,7 @@ public class BinaryDictEncoderUtils { headerBuffer.write((byte) (0xFF & version)); // Options flags - final int options = makeOptionsValue(formatOptions); + final int options = makeOptionsValue(dict, formatOptions); headerBuffer.write((byte) (0xFF & (options >> 8))); headerBuffer.write((byte) (0xFF & options)); final int headerSizeOffset = headerBuffer.size(); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 640d778bb..d5516ef46 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -62,7 +62,7 @@ public final class BinaryDictIOUtils { * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder, - final int bodyOffset, final Map<Integer, String> words, + final int headerSize, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { @@ -71,7 +71,7 @@ public final class BinaryDictIOUtils { Stack<Position> stack = new Stack<Position>(); int index = 0; - Position initPos = new Position(bodyOffset, 0); + Position initPos = new Position(headerSize, 0); stack.push(initPos); while (!stack.empty()) { @@ -112,7 +112,7 @@ public final class BinaryDictIOUtils { } if (p.mPosition == p.mNumOfPtNode) { - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { final boolean hasValidForwardLinkAddress = dictDecoder.readAndFollowForwardLink(); if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) { @@ -154,7 +154,7 @@ public final class BinaryDictIOUtils { UnsupportedFormatException { // Read header final FileHeader header = dictDecoder.readHeader(); - readUnigramsAndBigramsBinaryInner(dictDecoder, header.mBodyOffset, words, + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, frequencies, bigrams, header.mFormatOptions); } @@ -228,7 +228,7 @@ public final class BinaryDictIOUtils { // a forward link address that we need to consult and possibly resume // search on the next node array in the linked list. if (foundNextPtNode) break; - if (!header.mFormatOptions.supportsDynamicUpdate()) { + if (!header.mFormatOptions.mSupportsDynamicUpdate) { return FormatSpec.NOT_VALID_WORD; } @@ -245,7 +245,8 @@ public final class BinaryDictIOUtils { /** * @return the size written, in bytes. Always 3 bytes. */ - static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) { + static int writeSInt24ToBuffer(final DictBuffer dictBuffer, + final int value) { final int absValue = Math.abs(value); dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); dictBuffer.put((byte)((absValue >> 8) & 0xFF)); @@ -300,6 +301,35 @@ public final class BinaryDictIOUtils { } /** + * Write a string to a stream. + * + * @param destination the stream to write. + * @param word the string to be written. + * @return the size written, in bytes. + * @throws IOException + */ + private static int writeString(final OutputStream destination, final String word) + throws IOException { + int size = 0; + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (CharEncoding.getCharSize(codePoint) == 1) { + destination.write((byte)codePoint); + size++; + } else { + destination.write((byte)(0xFF & (codePoint >> 16))); + destination.write((byte)(0xFF & (codePoint >> 8))); + destination.write((byte)(0xFF & codePoint)); + size += 3; + } + } + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** * Write a PtNode to an output stream from a PtNodeInfo. * A PtNode is an in-memory representation of a node in the patricia trie. * A PtNode info is a container for low-level information about how the @@ -357,7 +387,7 @@ public final class BinaryDictIOUtils { destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; - size += CharEncoding.writeString(destination, target.mWord); + size += writeString(destination, target.mWord); } } @@ -415,27 +445,6 @@ public final class BinaryDictIOUtils { } /** - * Writes a PtNodeCount to the stream. - * - * @param destination the stream to write. - * @param ptNodeCount the count. - * @return the size written in bytes. - */ - static int writePtNodeCount(final OutputStream destination, final int ptNodeCount) - throws IOException { - final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); - // the count must fit on one byte or two bytes. - // Please see comments in FormatSpec. - if (countSize != 1 && countSize != 2) { - throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); - } - final int encodedPtNodeCount = (countSize == 2) ? - (ptNodeCount | FormatSpec.LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG) : ptNodeCount; - BinaryDictEncoderUtils.writeUIntToStream(destination, encodedPtNodeCount, countSize); - return countSize; - } - - /** * Write a node array to the stream. * * @param destination the stream to write. @@ -445,7 +454,20 @@ public final class BinaryDictIOUtils { */ static int writeNodes(final OutputStream destination, final PtNodeInfo[] infos) throws IOException { - int size = writePtNodeCount(destination, infos.length); + int size = getPtNodeCountSize(infos.length); + switch (getPtNodeCountSize(infos.length)) { + case 1: + destination.write((byte)infos.length); + break; + case 2: + final int encodedPtNodeCount = + infos.length | FormatSpec.LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG; + destination.write((byte)(encodedPtNodeCount >> 8)); + destination.write((byte)(encodedPtNodeCount & 0xFF)); + break; + default: + throw new RuntimeException("Invalid node count size."); + } for (final PtNodeInfo info : infos) size += writePtNode(destination, info); writeSInt24ToStream(destination, FormatSpec.NO_FORWARD_LINK_ADDRESS); return size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; @@ -507,7 +529,7 @@ public final class BinaryDictIOUtils { * Helper method to check whether the node is moved. */ public static boolean isMovedPtNode(final int flags, final FormatOptions options) { - return options.supportsDynamicUpdate() + return options.mSupportsDynamicUpdate && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); } @@ -516,14 +538,14 @@ public final class BinaryDictIOUtils { */ public static boolean supportsDynamicUpdate(final FormatOptions options) { return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.supportsDynamicUpdate(); + && options.mSupportsDynamicUpdate; } /** * Helper method to check whether the node is deleted. */ public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { - return formatOptions.supportsDynamicUpdate() + return formatOptions.mSupportsDynamicUpdate && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); } @@ -546,7 +568,7 @@ public final class BinaryDictIOUtils { static int getChildrenAddressSize(final int optionFlags, final FormatOptions formatOptions) { - if (formatOptions.supportsDynamicUpdate()) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return 1; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index b4838f00f..3dbeee099 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -35,7 +35,6 @@ import java.util.TreeMap; /** * An interface of binary dictionary decoders. */ -// TODO: Straighten out responsibility for the buffer's file pointer. public interface DictDecoder { /** @@ -44,7 +43,7 @@ public interface DictDecoder { public FileHeader readHeader() throws IOException, UnsupportedFormatException; /** - * Reads PtNode from ptNodePos. + * Reads PtNode from nodeAddress. * @param ptNodePos the position of PtNode. * @param formatOptions the format options. * @return PtNodeInfo. @@ -128,8 +127,7 @@ public interface DictDecoder { * Opens the dictionary file and makes DictBuffer. */ @UsedForTesting - public void openDictBuffer() throws FileNotFoundException, IOException, - UnsupportedFormatException; + public void openDictBuffer() throws FileNotFoundException, IOException; @UsedForTesting public boolean isDictBufferOpen(); @@ -230,9 +228,4 @@ public interface DictDecoder { } public void skipPtNode(final FormatOptions formatOptions); - - /** - * @return whether this decoder has a valid binary dictionary that it can decode. - */ - public boolean hasValidRawBinaryDictionary(); } diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index ff03190a3..28da9ffdd 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -37,7 +37,7 @@ import java.util.Arrays; @UsedForTesting public final class DynamicBinaryDictIOUtils { private static final boolean DBG = false; - static final int MAX_JUMPS = 10000; + private static final int MAX_JUMPS = 10000; private DynamicBinaryDictIOUtils() { // This utility class is not publicly instantiable. @@ -61,7 +61,7 @@ public final class DynamicBinaryDictIOUtils { final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); final int originalPosition = dictBuffer.position(); dictBuffer.position(ptNodeOriginAddress); - if (!formatOptions.supportsDynamicUpdate()) { + if (!formatOptions.mSupportsDynamicUpdate) { throw new RuntimeException("this file format does not support parent addresses"); } final int flags = dictBuffer.readUnsignedByte(); @@ -102,7 +102,7 @@ public final class DynamicBinaryDictIOUtils { } if (!dictUpdater.readAndFollowForwardLink()) break; if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break; - } while (formatOptions.supportsDynamicUpdate()); + } while (formatOptions.mSupportsDynamicUpdate); dictUpdater.setPosition(originalPosition); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 20ddba836..b56234f6d 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -40,8 +40,12 @@ public final class FormatSpec { * p | not used 3 bits * t | each unigram and bigram entry has a time stamp? * i | 1 bit, 1 = yes, 0 = no : CONTAINS_TIMESTAMP_FLAG - * o | - * nflags + * o | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG + * n | FRENCH_LIGATURE_PROCESSING_FLAG + * f | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE + * l | GERMAN_UMLAUT_PROCESSING_FLAG + * a | + * gs * * h | * e | size of the file header, 4bytes @@ -78,36 +82,45 @@ public final class FormatSpec { * s * * f | - * o | forward link address, 3byte - * r | 1 byte = bbbbbbbb match - * w | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) - * a | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte - * r | - * dlinkaddress + * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) + * r | forward link address, 3byte + * w | 1 byte = bbbbbbbb match + * a | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) + * r | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte + * d | + * linkaddress */ /* Node (FusionDictionary.PtNode) layout is as follows: - * | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED - * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES - * | 01 = yes : FLAG_IS_MOVED - * f | the new address is stored in the same place as the parent address - * l | is deleted? 10 = yes : FLAG_IS_DELETED - * a | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS - * g | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL - * s | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS + * | IF !SUPPORTS_DYNAMIC_UPDATE + * | addressType xx : mask with MASK_CHILDREN_ADDRESS_TYPE + * | 2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS + * f | 01 = 1 byte : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE + * l | 10 = 2 bytes : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES + * a | 11 = 3 bytes : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * g | ELSE + * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED + * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 01 = yes : FLAG_IS_MOVED + * | the new address is stored in the same place as the parent address + * | is deleted? 10 = yes : FLAG_IS_DELETED + * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS + * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL + * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD * | is blacklisted ? 1 bit, 1 = yes, 0 = no : FLAG_IS_BLACKLISTED * * p | - * a | parent address, 3byte - * r | 1 byte = bbbbbbbb match - * e | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) - * n | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte - * t | This address is relative to the head of the PtNode. - * a | If the node doesn't have a parent, this field is set to 0. + * a | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) + * r | parent address, 3byte + * e | 1 byte = bbbbbbbb match + * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte + * a | This address is relative to the head of the PtNode. + * d | If the node doesn't have a parent, this field is set to 0. * d | - * dress + * ress * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers @@ -121,16 +134,23 @@ public final class FormatSpec { * e | frequency 1 byte * q | * - * c | - * h | children address, 3 bytes - * i | 1 byte = bbbbbbbb match - * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) - * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte - * r | if this node doesn't have children, this field is set to 0. - * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) - * n | This address is relative to the position of this field. - * a | - * ddress + * c | IF SUPPORTS_DYNAMIC_UPDATE + * h | children address, 3 bytes + * i | 1 byte = bbbbbbbb match + * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte + * r | if this node doesn't have children, this field is set to 0. + * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) + * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType + * a | // nothing + * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType + * d | children address, 1 byte + * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType + * e | children address, 2 bytes + * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType + * s | children address, 3 bytes + * | END + * | This address is relative to the position of this field. * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list @@ -179,22 +199,20 @@ public final class FormatSpec { */ public static final int MAGIC_NUMBER = 0x9BC13AFE; + static final int MINIMUM_SUPPORTED_VERSION = 2; + static final int MAXIMUM_SUPPORTED_VERSION = 4; static final int NOT_A_VERSION_NUMBER = -1; static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; static final int FIRST_VERSION_WITH_TERMINAL_ID = 4; - - // These MUST have the same values as the relevant constants in format_utils.h. - // From version 4 on, we use version * 100 + revision as a version number. That allows - // us to change the format during development while having testing devices remove - // older files with each upgrade, while still having a readable versioning scheme. - public static final int VERSION2 = 2; - public static final int VERSION3 = 3; - public static final int VERSION4 = 400; - static final int MINIMUM_SUPPORTED_VERSION = VERSION2; - static final int MAXIMUM_SUPPORTED_VERSION = VERSION4; + static final int VERSION3 = 3; + static final int VERSION4 = 4; // These options need to be the same numeric values as the one in the native reading code. + static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; // TODO: Make the native reading code read this variable. + static final int SUPPORTS_DYNAMIC_UPDATE = 0x2; + static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; + static final int CONTAINS_BIGRAMS_FLAG = 0x8; static final int CONTAINS_TIMESTAMP_FLAG = 0x10; // TODO: Make this value adaptative to content data, store it in the header, and @@ -245,10 +263,8 @@ public final class FormatSpec { static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; - // These values are used only by version 4 or later. They MUST match the definitions in - // ver4_dict_constants.cpp. + // These values are used only by version 4 or later. static final String TRIE_FILE_EXTENSION = ".trie"; - public static final String HEADER_FILE_EXTENSION = ".header"; static final String FREQ_FILE_EXTENSION = ".freq"; static final String UNIGRAM_TIMESTAMP_FILE_EXTENSION = ".timestamp"; // tat = Terminal Address Table @@ -262,9 +278,9 @@ public final class FormatSpec { static final int UNIGRAM_TIMESTAMP_SIZE = 4; // With the English main dictionary as of October 2013, the size of bigram address table is - // is 345KB with the block size being 16. - // This is 54% of that of full address table. - static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 16; + // is 584KB with the block size being 4. + // This is 91% of that of full address table. + static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4; static final int BIGRAM_CONTENT_COUNT = 2; static final int BIGRAM_FREQ_CONTENT_INDEX = 0; static final int BIGRAM_TIMESTAMP_CONTENT_INDEX = 1; @@ -277,7 +293,7 @@ public final class FormatSpec { static final int SHORTCUT_CONTENT_COUNT = 1; static final int SHORTCUT_CONTENT_INDEX = 0; // With the English main dictionary as of October 2013, the size of shortcut address table is - // 26KB with the block size being 64. + // 29KB with the block size being 64. // This is only 4.4% of that of full address table. static final int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64; static final String SHORTCUT_CONTENT_ID = "_shortcut"; @@ -315,36 +331,43 @@ public final class FormatSpec { */ public static final class FormatOptions { public final int mVersion; + public final boolean mSupportsDynamicUpdate; public final boolean mHasTerminalId; public final boolean mHasTimestamp; - @UsedForTesting public FormatOptions(final int version) { - this(version, false /* hasTimestamp */); + this(version, false); } - public FormatOptions(final int version, final boolean hasTimestamp) { + @UsedForTesting + public FormatOptions(final int version, final boolean supportsDynamicUpdate) { + this(version, supportsDynamicUpdate, false /* hasTimestamp */); + } + + public FormatOptions(final int version, final boolean supportsDynamicUpdate, + final boolean hasTimestamp) { mVersion = version; + if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) { + throw new RuntimeException("Dynamic updates are only supported with versions " + + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); + } + mSupportsDynamicUpdate = supportsDynamicUpdate; mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); mHasTimestamp = hasTimestamp; } - - public boolean supportsDynamicUpdate() { - return mVersion >= FIRST_VERSION_WITH_DYNAMIC_UPDATE; - } } /** * Class representing file header. */ public static final class FileHeader { - public final int mBodyOffset; + public final int mHeaderSize; public final DictionaryOptions mDictionaryOptions; public final FormatOptions mFormatOptions; // Note that these are corresponding definitions in native code in latinime::HeaderPolicy // and latinime::HeaderReadWriteUtils. + public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; - public static final String HAS_HISTORICAL_INFO_ATTRIBUTE = "HAS_HISTORICAL_INFO"; public static final String ATTRIBUTE_VALUE_TRUE = "1"; public static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; @@ -353,18 +376,9 @@ public final class FormatSpec { private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description"; public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions, final FormatOptions formatOptions) { + mHeaderSize = headerSize; mDictionaryOptions = dictionaryOptions; mFormatOptions = formatOptions; - mBodyOffset = formatOptions.mVersion < VERSION4 ? headerSize : 0; - if (null == getLocaleString()) { - throw new RuntimeException("Cannot create a FileHeader without a locale"); - } - if (null == getVersion()) { - throw new RuntimeException("Cannot create a FileHeader without a version"); - } - if (null == getId()) { - throw new RuntimeException("Cannot create a FileHeader without an ID"); - } } // Helper method to get the locale as a String diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index fdf2ae7b5..3bb218bea 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -303,9 +303,14 @@ public final class FusionDictionary implements Iterable<Word> { * Options global to the dictionary. */ public static final class DictionaryOptions { + public final boolean mGermanUmlautProcessing; + public final boolean mFrenchLigatureProcessing; public final HashMap<String, String> mAttributes; - public DictionaryOptions(final HashMap<String, String> attributes) { + public DictionaryOptions(final HashMap<String, String> attributes, + final boolean germanUmlautProcessing, final boolean frenchLigatureProcessing) { mAttributes = attributes; + mGermanUmlautProcessing = germanUmlautProcessing; + mFrenchLigatureProcessing = frenchLigatureProcessing; } @Override public String toString() { // Convenience method @@ -334,6 +339,14 @@ public final class FusionDictionary implements Iterable<Word> { } s.append("\n"); } + if (mGermanUmlautProcessing) { + s.append(indent); + s.append("Needs German umlaut processing\n"); + } + if (mFrenchLigatureProcessing) { + s.append(indent); + s.append("Needs French ligature processing\n"); + } return s.toString(); } } @@ -688,6 +701,138 @@ public final class FusionDictionary implements Iterable<Word> { } /** + * Recursively count the number of nodes in a given branch of the trie. + * + * @param nodeArray the node array to count. + * @return the number of nodes in this branch. + */ + public static int countNodeArrays(final PtNodeArray nodeArray) { + int size = 1; + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countNodeArrays(ptNode.mChildren); + } + return size; + } + + // Recursively find out whether there are any bigrams. + // This can be pretty expensive especially if there aren't any (we return as soon + // as we find one, so it's much cheaper if there are bigrams) + private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { + if (null == nodeArray) return false; + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mBigrams) return true; + if (hasBigramsInternal(ptNode.mChildren)) return true; + } + return false; + } + + /** + * Finds out whether there are any bigrams in this dictionary. + * + * @return true if there is any bigram, false otherwise. + */ + // TODO: this is expensive especially for large dictionaries without any bigram. + // The up side is, this is always accurate and correct and uses no memory. We should + // find a more efficient way of doing this, without compromising too much on memory + // and ease of use. + public boolean hasBigrams() { + return hasBigramsInternal(mRootNodeArray); + } + + // Historically, the tails of the words were going to be merged to save space. + // However, that would prevent the code to search for a specific address in log(n) + // time so this was abandoned. + // The code is still of interest as it does add some compression to any dictionary + // that has no need for attributes. Implementations that does not read attributes should be + // able to read a dictionary with merged tails. + // Also, the following code does support frequencies, as in, it will only merges + // tails that share the same frequency. Though it would result in the above loss of + // performance while searching by address, it is still technically possible to merge + // tails that contain attributes, but this code does not take that into account - it does + // not compare attributes and will merge terminals with different attributes regardless. + public void mergeTails() { + MakedictLog.i("Do not merge tails"); + return; + +// MakedictLog.i("Merging PtNodes. Number of PtNodes : " + countPtNodes(root)); +// MakedictLog.i("Number of PtNodes : " + countPtNodes(root)); +// +// final HashMap<String, ArrayList<PtNodeArray>> repository = +// new HashMap<String, ArrayList<PtNodeArray>>(); +// mergeTailsInner(repository, root); +// +// MakedictLog.i("Number of different pseudohashes : " + repository.size()); +// int size = 0; +// for (ArrayList<PtNodeArray> a : repository.values()) { +// size += a.size(); +// } +// MakedictLog.i("Number of nodes after merge : " + (1 + size)); +// MakedictLog.i("Recursively seen nodes : " + countNodes(root)); + } + + // The following methods are used by the deactivated mergeTails() +// private static boolean isEqual(PtNodeArray a, PtNodeArray b) { +// if (null == a && null == b) return true; +// if (null == a || null == b) return false; +// if (a.data.size() != b.data.size()) return false; +// final int size = a.data.size(); +// for (int i = size - 1; i >= 0; --i) { +// PtNode aPtNode = a.data.get(i); +// PtNode bPtNode = b.data.get(i); +// if (aPtNode.frequency != bPtNode.frequency) return false; +// if (aPtNode.alternates == null && bPtNode.alternates != null) return false; +// if (aPtNode.alternates != null && !aPtNode.equals(bPtNode.alternates)) return false; +// if (!Arrays.equals(aPtNode.chars, bPtNode.chars)) return false; +// if (!isEqual(aPtNode.children, bPtNode.children)) return false; +// } +// return true; +// } + +// static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner( +// final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) { +// final ArrayList<PtNode> branches = nodeArray.data; +// final int nodeSize = branches.size(); +// for (int i = 0; i < nodeSize; ++i) { +// PtNode ptNode = branches.get(i); +// if (null != ptNode.children) { +// String pseudoHash = getPseudoHash(ptNode.children); +// ArrayList<PtNodeArray> similarList = map.get(pseudoHash); +// if (null == similarList) { +// similarList = new ArrayList<PtNodeArray>(); +// map.put(pseudoHash, similarList); +// } +// boolean merged = false; +// for (PtNodeArray similar : similarList) { +// if (isEqual(ptNode.children, similar)) { +// ptNode.children = similar; +// merged = true; +// break; +// } +// } +// if (!merged) { +// similarList.add(ptNode.children); +// } +// mergeTailsInner(map, ptNode.children); +// } +// } +// return map; +// } + +// private static String getPseudoHash(final PtNodeArray nodeArray) { +// StringBuilder s = new StringBuilder(); +// for (PtNode ptNode : nodeArray.data) { +// s.append(ptNode.frequency); +// for (int ch : ptNode.chars) { +// s.append(Character.toChars(ch)); +// } +// } +// return s.toString(); +// } + + /** * Iterator to walk through a dictionary. * * This is purely for convenience. diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java deleted file mode 100644 index 06088b651..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; - -/** - * An auxiliary class for reading SparseTable and data written by SparseTableContentWriter. - */ -public class SparseTableContentReader { - - /** - * An interface of a function which is passed to SparseTableContentReader.read. - */ - public interface SparseTableContentReaderInterface { - /** - * Reads data. - * - * @param buffer the DictBuffer. The position of the buffer is set to the head of data. - */ - public void read(final DictBuffer buffer); - } - - protected final int mContentCount; - protected final int mBlockSize; - protected final File mBaseDir; - protected final File mLookupTableFile; - protected final File[] mAddressTableFiles; - protected final File[] mContentFiles; - protected DictBuffer mLookupTableBuffer; - protected final DictBuffer[] mAddressTableBuffers; - private final DictBuffer[] mContentBuffers; - protected final DictionaryBufferFactory mFactory; - - /** - * Sole constructor of SparseTableContentReader. - * - * @param name the name of SparseTable. - * @param blockSize the block size of the content table. - * @param baseDir the directory which contains the files of the content table. - * @param contentFilenames the file names of content files. - * @param contentIds the ids of contents. These ids are used for a suffix of a name of - * address files and content files. - * @param factory the DictionaryBufferFactory which is used for opening the files. - */ - public SparseTableContentReader(final String name, final int blockSize, final File baseDir, - final String[] contentFilenames, final String[] contentIds, - final DictionaryBufferFactory factory) { - if (contentFilenames.length != contentIds.length) { - throw new RuntimeException("The length of contentFilenames and the length of" - + " contentIds are different " + contentFilenames.length + ", " - + contentIds.length); - } - mBlockSize = blockSize; - mBaseDir = baseDir; - mFactory = factory; - mContentCount = contentFilenames.length; - mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - mAddressTableFiles = new File[mContentCount]; - mContentFiles = new File[mContentCount]; - for (int i = 0; i < mContentCount; ++i) { - mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); - } - mAddressTableBuffers = new DictBuffer[mContentCount]; - mContentBuffers = new DictBuffer[mContentCount]; - } - - public void openBuffers() throws FileNotFoundException, IOException { - mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); - for (int i = 0; i < mContentCount; ++i) { - mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); - mContentBuffers[i] = mFactory.getDictionaryBuffer(mContentFiles[i]); - } - } - - protected void read(final int contentIndex, final int index, - final SparseTableContentReaderInterface reader) { - if (index < 0 || (index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES - >= mLookupTableBuffer.limit()) { - return; - } - - mLookupTableBuffer.position((index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - final int posInAddressTable = mLookupTableBuffer.readInt(); - if (posInAddressTable == SparseTable.NOT_EXIST) { - return; - } - - mAddressTableBuffers[contentIndex].position( - (posInAddressTable + index % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - final int address = mAddressTableBuffers[contentIndex].readInt(); - if (address == SparseTable.NOT_EXIST) { - return; - } - - mContentBuffers[contentIndex].position(address); - reader.read(mContentBuffers[contentIndex]); - } -}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java deleted file mode 100644 index 4518f21b9..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * An auxiliary class for updating data associated with SparseTable. - */ -public class SparseTableContentUpdater extends SparseTableContentReader { - protected OutputStream mLookupTableOutStream; - protected OutputStream[] mAddressTableOutStreams; - protected OutputStream[] mContentOutStreams; - - public SparseTableContentUpdater(final String name, final int blockSize, - final File baseDir, final String[] contentFilenames, final String[] contentIds, - final DictionaryBufferFactory factory) { - super(name, blockSize, baseDir, contentFilenames, contentIds, factory); - mAddressTableOutStreams = new OutputStream[mContentCount]; - mContentOutStreams = new OutputStream[mContentCount]; - } - - protected void openStreamsAndBuffers() throws IOException { - openBuffers(); - mLookupTableOutStream = new FileOutputStream(mLookupTableFile, true /* append */); - for (int i = 0; i < mContentCount; ++i) { - mAddressTableOutStreams[i] = new FileOutputStream(mAddressTableFiles[i], - true /* append */); - mContentOutStreams[i] = new FileOutputStream(mContentFiles[i], true /* append */); - } - } - - /** - * Set the contentIndex-th elements of contentId-th table. - * - * @param contentId the id of the content table. - * @param contentIndex the index where to set the valie. - * @param value the value to set. - */ - protected void setContentValue(final int contentId, final int contentIndex, final int value) - throws IOException { - if ((contentIndex / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES - >= mLookupTableBuffer.limit()) { - // Need to extend the lookup table - final int currentSize = mLookupTableBuffer.limit() - / SparseTable.SIZE_OF_INT_IN_BYTES; - final int target = contentIndex / mBlockSize + 1; - for (int i = currentSize; i < target; ++i) { - BinaryDictEncoderUtils.writeUIntToStream(mLookupTableOutStream, - SparseTable.NOT_EXIST, SparseTable.SIZE_OF_INT_IN_BYTES); - } - // We need to reopen the byte buffer of the lookup table because a MappedByteBuffer in - // Java isn't expanded automatically when the underlying file is expanded. - reopenLookupTable(); - } - - mLookupTableBuffer.position((contentIndex / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); - int posInAddressTable = mLookupTableBuffer.readInt(); - if (posInAddressTable == SparseTable.NOT_EXIST) { - // Need to extend the address table - mLookupTableBuffer.position(mLookupTableBuffer.position() - - SparseTable.SIZE_OF_INT_IN_BYTES); - posInAddressTable = mAddressTableBuffers[0].limit() / mBlockSize; - BinaryDictEncoderUtils.writeUIntToDictBuffer(mLookupTableBuffer, - posInAddressTable, SparseTable.SIZE_OF_INT_IN_BYTES); - for (int i = 0; i < mContentCount; ++i) { - for (int j = 0; j < mBlockSize; ++j) { - BinaryDictEncoderUtils.writeUIntToStream(mAddressTableOutStreams[i], - SparseTable.NOT_EXIST, SparseTable.SIZE_OF_INT_IN_BYTES); - } - } - // We need to reopen the byte buffers of the address tables because a MappedByteBuffer - // in Java isn't expanded automatically when the underlying file is expanded. - reopenAddressTables(); - } - posInAddressTable += (contentIndex % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES; - - mAddressTableBuffers[contentId].position(posInAddressTable); - BinaryDictEncoderUtils.writeUIntToDictBuffer(mAddressTableBuffers[contentId], - value, SparseTable.SIZE_OF_INT_IN_BYTES); - } - - private void reopenLookupTable() throws IOException { - mLookupTableOutStream.flush(); - mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); - } - - private void reopenAddressTables() throws IOException { - for (int i = 0; i < mContentCount; ++i) { - mAddressTableOutStreams[i].flush(); - mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); - } - } - - protected void close() throws IOException { - mLookupTableOutStream.close(); - for (final OutputStream stream : mAddressTableOutStreams) { - stream.close(); - } - for (final OutputStream stream : mContentOutStreams) { - stream.close(); - } - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java deleted file mode 100644 index 49f0fd624..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * An auxiliary class for writing data associated with SparseTable to files. - */ -public class SparseTableContentWriter { - public interface SparseTableContentWriterInterface { - public void write(final OutputStream outStream) throws IOException; - } - - private final int mContentCount; - private final SparseTable mSparseTable; - private final File mLookupTableFile; - protected final File mBaseDir; - private final File[] mAddressTableFiles; - private final File[] mContentFiles; - protected final OutputStream[] mContentOutStreams; - - /** - * Sole constructor of SparseTableContentWriter. - * - * @param name the name of SparseTable. - * @param initialCapacity the initial capacity of SparseTable. - * @param blockSize the block size of the content table. - * @param baseDir the directory which contains the files of the content table. - * @param contentFilenames the file names of content files. - * @param contentIds the ids of contents. These ids are used for a suffix of a name of address - * files and content files. - */ - public SparseTableContentWriter(final String name, final int initialCapacity, - final int blockSize, final File baseDir, final String[] contentFilenames, - final String[] contentIds) { - if (contentFilenames.length != contentIds.length) { - throw new RuntimeException("The length of contentFilenames and the length of" - + " contentIds are different " + contentFilenames.length + ", " - + contentIds.length); - } - mContentCount = contentFilenames.length; - mSparseTable = new SparseTable(initialCapacity, blockSize, mContentCount); - mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - mAddressTableFiles = new File[mContentCount]; - mContentFiles = new File[mContentCount]; - mBaseDir = baseDir; - for (int i = 0; i < mContentCount; ++i) { - mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); - } - mContentOutStreams = new OutputStream[mContentCount]; - } - - public void openStreams() throws FileNotFoundException { - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); - } - } - - protected void write(final int contentIndex, final int index, - final SparseTableContentWriterInterface writer) throws IOException { - mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); - writer.write(mContentOutStreams[contentIndex]); - mContentOutStreams[contentIndex].flush(); - } - - public void closeStreams() throws IOException { - mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i].close(); - } - } -}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 92eb861d6..5da34534e 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -169,7 +169,7 @@ public class Ver3DictEncoder implements DictEncoder { private void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition, childrenPos); } else { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 3be62f066..734223ec2 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -40,52 +40,26 @@ import java.util.Arrays; public class Ver4DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver4DictDecoder.class.getSimpleName(); - protected static final int FILETYPE_TRIE = 1; - protected static final int FILETYPE_FREQUENCY = 2; - protected static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; - protected static final int FILETYPE_BIGRAM_FREQ = 4; - protected static final int FILETYPE_SHORTCUT = 5; - protected static final int FILETYPE_HEADER = 6; - - protected final File mDictDirectory; - protected final DictionaryBufferFactory mBufferFactory; + private static final int FILETYPE_TRIE = 1; + private static final int FILETYPE_FREQUENCY = 2; + private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; + private static final int FILETYPE_BIGRAM_FREQ = 4; + private static final int FILETYPE_SHORTCUT = 5; + + private final File mDictDirectory; + private final DictionaryBufferFactory mBufferFactory; protected DictBuffer mDictBuffer; - protected DictBuffer mHeaderBuffer; - protected DictBuffer mFrequencyBuffer; - protected DictBuffer mTerminalAddressTableBuffer; - private BigramContentReader mBigramReader; - private ShortcutContentReader mShortcutReader; - - /** - * Raw PtNode info straight out of a trie file in version 4 dictionary. - */ - protected static final class Ver4PtNodeInfo { - public final int mFlags; - public final int[] mCharacters; - public final int mTerminalId; - public final int mChildrenPos; - public final int mParentPos; - public final int mNodeSize; - public int mStartIndexOfCharacters; - public int mEndIndexOfCharacters; // exclusive - - public Ver4PtNodeInfo(final int flags, final int[] characters, final int terminalId, - final int childrenPos, final int parentPos, final int nodeSize) { - mFlags = flags; - mCharacters = characters; - mTerminalId = terminalId; - mChildrenPos = childrenPos; - mParentPos = parentPos; - mNodeSize = nodeSize; - mStartIndexOfCharacters = 0; - mEndIndexOfCharacters = characters.length; - } - } + private DictBuffer mFrequencyBuffer; + private DictBuffer mTerminalAddressTableBuffer; + private DictBuffer mBigramBuffer; + private DictBuffer mShortcutBuffer; + private SparseTable mBigramAddressTable; + private SparseTable mShortcutAddressTable; @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { mDictDirectory = dictDirectory; - mDictBuffer = mHeaderBuffer = mFrequencyBuffer = null; + mDictBuffer = mFrequencyBuffer = null; if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); @@ -102,16 +76,13 @@ public class Ver4DictDecoder extends AbstractDictDecoder { /* package */ Ver4DictDecoder(final File dictDirectory, final DictionaryBufferFactory factory) { mDictDirectory = dictDirectory; mBufferFactory = factory; - mDictBuffer = mHeaderBuffer = mFrequencyBuffer = null; + mDictBuffer = mFrequencyBuffer = null; } - protected File getFile(final int fileType) throws UnsupportedFormatException { + private File getFile(final int fileType) { if (fileType == FILETYPE_TRIE) { return new File(mDictDirectory, mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION); - } else if (fileType == FILETYPE_HEADER) { - return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.HEADER_FILE_EXTENSION); } else if (fileType == FILETYPE_FREQUENCY) { return new File(mDictDirectory, mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION); @@ -127,27 +98,20 @@ public class Ver4DictDecoder extends AbstractDictDecoder { mDictDirectory.getName() + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.SHORTCUT_CONTENT_ID); } else { - throw new UnsupportedFormatException("Unsupported kind of file : " + fileType); + throw new RuntimeException("Unsupported kind of file : " + fileType); } } @Override - public void openDictBuffer() throws FileNotFoundException, IOException, - UnsupportedFormatException { - if (!mDictDirectory.isDirectory()) { - throw new UnsupportedFormatException("Format 4 dictionary needs a directory"); - } - mHeaderBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_HEADER)); + public void openDictBuffer() throws FileNotFoundException, IOException { mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE)); mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); - mBigramReader = new BigramContentReader(mDictDirectory.getName(), - mDictDirectory, mBufferFactory, false); - mBigramReader.openBuffers(); - mShortcutReader = new ShortcutContentReader(mDictDirectory.getName(), mDictDirectory, - mBufferFactory); - mShortcutReader.openBuffers(); + mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM_FREQ)); + loadBigramAddressSparseTable(); + mShortcutBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_SHORTCUT)); + loadShortcutAddressSparseTable(); } @Override @@ -155,134 +119,46 @@ public class Ver4DictDecoder extends AbstractDictDecoder { return mDictBuffer != null; } - @UsedForTesting - /* package */ DictBuffer getHeaderBuffer() { - return mHeaderBuffer; - } - - @UsedForTesting /* package */ DictBuffer getDictBuffer() { return mDictBuffer; } @Override public FileHeader readHeader() throws IOException, UnsupportedFormatException { - if (mHeaderBuffer == null) { + if (mDictBuffer == null) { openDictBuffer(); } - mHeaderBuffer.position(0); - final FileHeader header = super.readHeader(mHeaderBuffer); + final FileHeader header = super.readHeader(mDictBuffer); final int version = header.mFormatOptions.mVersion; - if (version != FormatSpec.VERSION4) { + if (version != 4) { throw new UnsupportedFormatException("File header has a wrong version : " + version); } return header; } - /** - * An auxiliary class for reading bigrams. - */ - protected static class BigramContentReader extends SparseTableContentReader { - public BigramContentReader(final String name, final File baseDir, - final DictionaryBufferFactory factory, final boolean hasTimestamp) { - super(name + FormatSpec.BIGRAM_FILE_EXTENSION, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - getContentFilenames(name, hasTimestamp), getContentIds(hasTimestamp), factory); - } - - // TODO: Consolidate this method and BigramContentWriter.getContentFilenames. - protected static String[] getContentFilenames(final String name, - final boolean hasTimestamp) { - final String[] contentFilenames; - if (hasTimestamp) { - contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION, - name + FormatSpec.BIGRAM_FILE_EXTENSION }; - } else { - contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }; - } - return contentFilenames; - } - - // TODO: Consolidate this method and BigramContentWriter.getContentIds. - protected static String[] getContentIds(final boolean hasTimestamp) { - final String[] contentIds; - if (hasTimestamp) { - contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID, - FormatSpec.BIGRAM_TIMESTAMP_CONTENT_ID }; - } else { - contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }; - } - return contentIds; - } - - public ArrayList<PendingAttribute> readTargetsAndFrequencies(final int terminalId, - final DictBuffer terminalAddressTableBuffer) { - final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); - read(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, - new SparseTableContentReaderInterface() { - @Override - public void read(final DictBuffer buffer) { - while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, - // remaining bigram entries are ignored. - final int bigramFlags = buffer.readUnsignedByte(); - final int targetTerminalId = buffer.readUnsignedInt24(); - terminalAddressTableBuffer.position(targetTerminalId - * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - final int targetAddress = - terminalAddressTableBuffer.readUnsignedInt24(); - bigrams.add(new PendingAttribute(bigramFlags - & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, - targetAddress)); - if (0 == (bigramFlags - & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { - break; - } - } - if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode (" - + bigrams.size() + " but max is " - + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); - } - } - }); - if (bigrams.isEmpty()) return null; - return bigrams; - } + private void loadBigramAddressSparseTable() throws IOException { + final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + final File freqsFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + + FormatSpec.BIGRAM_FREQ_CONTENT_ID); + mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { freqsFile }, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); } - /** - * An auxiliary class for reading shortcuts. - */ - protected static class ShortcutContentReader extends SparseTableContentReader { - public ShortcutContentReader(final String name, final File baseDir, - final DictionaryBufferFactory factory) { - super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, - FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, - new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, factory); - } - - public ArrayList<WeightedString> readShortcuts(final int terminalId) { - final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList(); - read(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, - new SparseTableContentReaderInterface() { - @Override - public void read(final DictBuffer buffer) { - while (true) { - final int flags = buffer.readUnsignedByte(); - final String word = CharEncoding.readString(buffer); - shortcuts.add(new WeightedString(word, - flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { - break; - } - } - } - }); - if (shortcuts.isEmpty()) return null; - return shortcuts; - } + // TODO: Let's have something like SparseTableContentsReader in this class. + private void loadShortcutAddressSparseTable() throws IOException { + final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + final File contentFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + + FormatSpec.SHORTCUT_CONTENT_ID); + final File timestampsFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + + FormatSpec.SHORTCUT_CONTENT_ID); + mShortcutAddressTable = SparseTable.readFromFiles(lookupIndexFile, + new File[] { contentFile, timestampsFile }, + FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE); } protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { @@ -296,82 +172,102 @@ public class Ver4DictDecoder extends AbstractDictDecoder { } } - private final int[] mCharacterBufferForReadingVer4PtNodeInfo - = new int[FormatSpec.MAX_WORD_LENGTH]; + private ArrayList<WeightedString> readShortcuts(final int terminalId) { + if (mShortcutAddressTable.get(0, terminalId) == SparseTable.NOT_EXIST) return null; + + final ArrayList<WeightedString> ret = CollectionUtils.newArrayList(); + final int posOfShortcuts = mShortcutAddressTable.get(FormatSpec.SHORTCUT_CONTENT_INDEX, + terminalId); + mShortcutBuffer.position(posOfShortcuts); + while (true) { + final int flags = mShortcutBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(mShortcutBuffer); + ret.add(new WeightedString(word, + flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return ret; + } - /** - * Reads PtNode from ptNodePos in the trie file and returns Ver4PtNodeInfo. - * - * @param ptNodePos the position of PtNode. - * @param options the format options. - * @return Ver4PtNodeInfo. - */ // TODO: Make this buffer thread safe. // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH. - protected Ver4PtNodeInfo readVer4PtNodeInfo(final int ptNodePos, final FormatOptions options) { - int readingPos = ptNodePos; + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { + int addressPointer = ptNodePos; final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - readingPos += FormatSpec.PTNODE_FLAGS_SIZE; + addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; - final int parentPos = PtNodeReader.readParentAddress(mDictBuffer, options); + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - readingPos += FormatSpec.PARENT_ADDRESS_SIZE; + addressPointer += FormatSpec.PARENT_ADDRESS_SIZE; } final int characters[]; if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { int index = 0; int character = CharEncoding.readChar(mDictBuffer); - readingPos += CharEncoding.getCharSize(character); + addressPointer += CharEncoding.getCharSize(character); while (FormatSpec.INVALID_CHARACTER != character && index < FormatSpec.MAX_WORD_LENGTH) { - mCharacterBufferForReadingVer4PtNodeInfo[index++] = character; + mCharacterBuffer[index++] = character; character = CharEncoding.readChar(mDictBuffer); - readingPos += CharEncoding.getCharSize(character); + addressPointer += CharEncoding.getCharSize(character); } - characters = Arrays.copyOfRange(mCharacterBufferForReadingVer4PtNodeInfo, 0, index); + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); } else { final int character = CharEncoding.readChar(mDictBuffer); - readingPos += CharEncoding.getCharSize(character); + addressPointer += CharEncoding.getCharSize(character); characters = new int[] { character }; } final int terminalId; if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { terminalId = PtNodeReader.readTerminalId(mDictBuffer); - readingPos += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + addressPointer += FormatSpec.PTNODE_TERMINAL_ID_SIZE; } else { terminalId = PtNode.NOT_A_TERMINAL; } - int childrenPos = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); - if (childrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenPos += readingPos; - } - readingPos += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - - return new Ver4PtNodeInfo(flags, characters, terminalId, childrenPos, parentPos, - readingPos - ptNodePos); - } - - @Override - public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { - final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(ptNodePos, options); - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & nodeInfo.mFlags)) { - frequency = PtNodeReader.readFrequency(mFrequencyBuffer, nodeInfo.mTerminalId); + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + frequency = PtNodeReader.readFrequency(mFrequencyBuffer, terminalId); } else { frequency = PtNode.NOT_A_TERMINAL; } - - final ArrayList<WeightedString> shortcutTargets = mShortcutReader.readShortcuts( - nodeInfo.mTerminalId); - final ArrayList<PendingAttribute> bigrams = mBigramReader.readTargetsAndFrequencies( - nodeInfo.mTerminalId, mTerminalAddressTableBuffer); - - return new PtNodeInfo(ptNodePos, ptNodePos + nodeInfo.mNodeSize, nodeInfo.mFlags, - nodeInfo.mCharacters, frequency, nodeInfo.mParentPos, nodeInfo.mChildrenPos, - shortcutTargets, bigrams); + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + final ArrayList<WeightedString> shortcutTargets = readShortcuts(terminalId); + + final ArrayList<PendingAttribute> bigrams; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId); + mBigramBuffer.position(posOfBigrams); + while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, + // remaining bigram entries are ignored. + final int bigramFlags = mBigramBuffer.readUnsignedByte(); + final int targetTerminalId = mBigramBuffer.readUnsignedInt24(); + mTerminalAddressTableBuffer.position( + targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + final int targetAddress = mTerminalAddressTableBuffer.readUnsignedInt24(); + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + targetAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() + + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); + } + } else { + bigrams = null; + } + return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); } private void deleteDictFiles() { @@ -422,14 +318,10 @@ public class Ver4DictDecoder extends AbstractDictDecoder { @Override public boolean readAndFollowForwardLink() { - final int forwardLinkPos = mDictBuffer.position(); - int nextRelativePos = BinaryDictDecoderUtils.readSInt24(mDictBuffer); - if (nextRelativePos != FormatSpec.NO_FORWARD_LINK_ADDRESS) { - final int nextPos = forwardLinkPos + nextRelativePos; - if (nextPos >= 0 && nextPos < mDictBuffer.limit()) { - mDictBuffer.position(nextPos); - return true; - } + final int nextAddress = mDictBuffer.readUnsignedInt24(); + if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { + mDictBuffer.position(nextAddress); + return true; } return false; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 8b80ebe63..8d5b48a9b 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -25,8 +25,6 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.FileUtils; import java.io.File; import java.io.FileNotFoundException; @@ -34,8 +32,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.Iterator; /** @@ -46,8 +42,8 @@ public class Ver4DictEncoder implements DictEncoder { private final File mDictPlacedDir; private byte[] mTrieBuf; private int mTriePos; + private int mHeaderSize; private OutputStream mTrieOutStream; - private OutputStream mHeaderOutStream; private OutputStream mFreqOutStream; private OutputStream mUnigramTimestampOutStream; private OutputStream mTerminalAddressTableOutStream; @@ -61,6 +57,62 @@ public class Ver4DictEncoder implements DictEncoder { mDictPlacedDir = dictPlacedDir; } + private interface SparseTableContentWriterInterface { + public void write(final OutputStream outStream) throws IOException; + } + + private static class SparseTableContentWriter { + private final int mContentCount; + private final SparseTable mSparseTable; + private final File mLookupTableFile; + protected final File mBaseDir; + private final File[] mAddressTableFiles; + private final File[] mContentFiles; + protected final OutputStream[] mContentOutStreams; + + public SparseTableContentWriter(final String name, final int initialCapacity, + final int blockSize, final File baseDir, final String[] contentFilenames, + final String[] contentIds) { + if (contentFilenames.length != contentIds.length) { + throw new RuntimeException("The length of contentFilenames and the length of" + + " contentIds are different " + contentFilenames.length + ", " + + contentIds.length); + } + mContentCount = contentFilenames.length; + mSparseTable = new SparseTable(initialCapacity, blockSize, mContentCount); + mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + mAddressTableFiles = new File[mContentCount]; + mContentFiles = new File[mContentCount]; + mBaseDir = baseDir; + for (int i = 0; i < mContentCount; ++i) { + mAddressTableFiles[i] = new File(mBaseDir, + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + } + mContentOutStreams = new OutputStream[mContentCount]; + } + + public void openStreams() throws FileNotFoundException { + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); + } + } + + protected void write(final int contentIndex, final int index, + final SparseTableContentWriterInterface writer) throws IOException { + mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); + writer.write(mContentOutStreams[contentIndex]); + mContentOutStreams[contentIndex].flush(); + } + + public void closeStreams() throws IOException { + mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i].close(); + } + } + } + private static class BigramContentWriter extends SparseTableContentWriter { private final boolean mWriteTimestamp; @@ -186,21 +238,16 @@ public class Ver4DictEncoder implements DictEncoder { mBaseFilename = header.getId() + "." + header.getVersion(); mDictDir = new File(mDictPlacedDir, mBaseFilename); final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION); - final File headerFile = new File(mDictDir, - mBaseFilename + FormatSpec.HEADER_FILE_EXTENSION); final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION); final File timestampFile = new File(mDictDir, mBaseFilename + FormatSpec.UNIGRAM_TIMESTAMP_FILE_EXTENSION); final File terminalAddressTableFile = new File(mDictDir, mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); if (!mDictDir.isDirectory()) { - if (mDictDir.exists()) { - FileUtils.deleteRecursively(mDictDir); - } + if (mDictDir.exists()) mDictDir.delete(); mDictDir.mkdirs(); } mTrieOutStream = new FileOutputStream(trieFile); - mHeaderOutStream = new FileOutputStream(headerFile); mFreqOutStream = new FileOutputStream(freqFile); mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile); if (formatOptions.mHasTimestamp) { @@ -213,9 +260,6 @@ public class Ver4DictEncoder implements DictEncoder { if (mTrieOutStream != null) { mTrieOutStream.close(); } - if (mHeaderOutStream != null) { - mHeaderOutStream.close(); - } if (mFreqOutStream != null) { mFreqOutStream.close(); } @@ -227,7 +271,6 @@ public class Ver4DictEncoder implements DictEncoder { } } finally { mTrieOutStream = null; - mHeaderOutStream = null; mFreqOutStream = null; mTerminalAddressTableOutStream = null; } @@ -248,34 +291,16 @@ public class Ver4DictEncoder implements DictEncoder { openStreams(formatOptions, dict.mOptions); } - BinaryDictEncoderUtils.writeDictionaryHeader(mHeaderOutStream, dict, formatOptions); + mHeaderSize = BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, + formatOptions); MakedictLog.i("Flattening the tree..."); ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); int terminalCount = 0; - final ArrayList<PtNode> nodes = CollectionUtils.newArrayList(); for (final PtNodeArray array : flatNodes) { for (final PtNode node : array.mData) { - if (node.isTerminal()) { - nodes.add(node); - node.mTerminalId = terminalCount++; - } - } - } - Collections.sort(nodes, new Comparator<PtNode>() { - @Override - public int compare(final PtNode lhs, final PtNode rhs) { - if (lhs.mFrequency != rhs.mFrequency) { - return lhs.mFrequency < rhs.mFrequency ? -1 : 1; - } - if (lhs.mTerminalId < rhs.mTerminalId) return -1; - if (lhs.mTerminalId > rhs.mTerminalId) return 1; - return 0; + if (node.isTerminal()) node.mTerminalId = terminalCount++; } - }); - int count = 0; - for (final PtNode node : nodes) { - node.mTerminalId = count++; } MakedictLog.i("Computing addresses..."); @@ -312,7 +337,7 @@ public class Ver4DictEncoder implements DictEncoder { @Override public void setPosition(int position) { - if (mTrieBuf == null || position < 0 || position > mTrieBuf.length) return; + if (mTrieBuf == null || position < 0 || position >- mTrieBuf.length) return; mTriePos = position; } @@ -365,7 +390,7 @@ public class Ver4DictEncoder implements DictEncoder { private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - if (formatOptions.supportsDynamicUpdate()) { + if (formatOptions.mSupportsDynamicUpdate) { mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, mTriePos, childrenPos); } else { @@ -432,7 +457,7 @@ public class Ver4DictEncoder implements DictEncoder { ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE); BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf, ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, - ptNode.mCachedAddressAfterUpdate, + ptNode.mCachedAddressAfterUpdate + mHeaderSize, FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java index c46bc36bb..3d8f186ba 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -17,130 +17,29 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import android.util.Log; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; /** * An implementation of DictUpdater for version 4 binary dictionary. */ @UsedForTesting public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { - private static final String TAG = Ver4DictUpdater.class.getSimpleName(); - - private OutputStream mDictStream; - private final File mFrequencyFile; @UsedForTesting - public Ver4DictUpdater(final File dictDirectory, final int factoryType) - throws UnsupportedFormatException { + public Ver4DictUpdater(final File dictDirectory, final int factoryType) { // DictUpdater must have an updatable DictBuffer. super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); - mFrequencyFile = getFile(FILETYPE_FREQUENCY); - } - - private static class BigramContentUpdater extends SparseTableContentUpdater { - public BigramContentUpdater(final String name, final File baseDir, - final boolean hasTimestamp) { - super(name + FormatSpec.BIGRAM_FILE_EXTENSION, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - BigramContentReader.getContentFilenames(name, hasTimestamp), - BigramContentReader.getContentIds(hasTimestamp), - new DictionaryBufferFromWritableByteBufferFactory()); - } - - public void insertBigramEntries(final int terminalId, final int frequency, - final ArrayList<PendingAttribute> entries) throws IOException { - if (terminalId < 0) { - throw new RuntimeException("Invalid terminal id : " + terminalId); - } - openStreamsAndBuffers(); - - if (entries == null || entries.isEmpty()) { - setContentValue(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, - SparseTable.NOT_EXIST); - return; - } - final int positionOfEntries = - (int) mContentFiles[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX].length(); - setContentValue(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, positionOfEntries); - - final Iterator<PendingAttribute> bigramIterator = entries.iterator(); - while (bigramIterator.hasNext()) { - final PendingAttribute entry = bigramIterator.next(); - final int flags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), - 0 /* offset */, entry.mFrequency, frequency, "" /* word */); - BinaryDictEncoderUtils.writeUIntToStream( - mContentOutStreams[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX], flags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - BinaryDictEncoderUtils.writeUIntToStream( - mContentOutStreams[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX], entry.mAddress, - FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); - } - close(); - } - } - - private static class ShortcutContentUpdater extends SparseTableContentUpdater { - public ShortcutContentUpdater(final String name, final File baseDir) { - super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, - FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, - new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, - new DictionaryBufferFromWritableByteBufferFactory()); - } - - public void insertShortcuts(final int terminalId, - final ArrayList<WeightedString> shortcuts) throws IOException { - if (terminalId < 0) { - throw new RuntimeException("Invalid terminal id : " + terminalId); - } - openStreamsAndBuffers(); - if (shortcuts == null || shortcuts.isEmpty()) { - setContentValue(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, - SparseTable.NOT_EXIST); - return; - } - - final int positionOfShortcuts = - (int) mContentFiles[FormatSpec.SHORTCUT_CONTENT_INDEX].length(); - setContentValue(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, positionOfShortcuts); - - final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), target.mFrequency); - BinaryDictEncoderUtils.writeUIntToStream( - mContentOutStreams[FormatSpec.SHORTCUT_CONTENT_INDEX], shortcutFlags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - CharEncoding.writeString(mContentOutStreams[FormatSpec.SHORTCUT_CONTENT_INDEX], - target.mWord); - } - close(); - } } @Override public void deleteWord(final String word) throws IOException, UnsupportedFormatException { - if (mDictBuffer == null) { - openDictBuffer(); - readHeader(); - } + if (mDictBuffer == null) openDictBuffer(); + readHeader(); final int wordPos = getTerminalPosition(word); if (wordPos != FormatSpec.NOT_VALID_WORD) { mDictBuffer.position(wordPos); @@ -150,623 +49,11 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { } } - private int getNewTerminalId() { - // The size of frequency file is FormatSpec.FREQUENCY_AND_FLAGS_SIZE * number of terminals - // because each terminal always has a frequency. - // So we can get a fresh terminal id by this logic. - // CAVEAT: we are reading the file size from the disk each time: beware of race conditions, - // even on one thread. - return (int) (mFrequencyFile.length() / FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - } - - private void updateParentPosIfNotMoved(final int nodePos, final int newParentPos, - final FormatOptions formatOptions) { - final int originalPos = getPosition(); - setPosition(nodePos); - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - if (!BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { - final int parentOffset = newParentPos - nodePos; - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, parentOffset); - } - setPosition(originalPos); - } - - private void updateParentPositions(final int nodeArrayPos, final int newParentPos, - final FormatOptions formatOptions) { - final int originalPos = mDictBuffer.position(); - mDictBuffer.position(nodeArrayPos); - int jumpCount = 0; - do { - final int count = readPtNodeCount(); - for (int i = 0; i < count; ++i) { - updateParentPosIfNotMoved(getPosition(), newParentPos, formatOptions); - skipPtNode(formatOptions); - } - if (!readAndFollowForwardLink()) break; - } while (jumpCount++ < DynamicBinaryDictIOUtils.MAX_JUMPS); - setPosition(originalPos); - } - - private void updateChildrenPos(final int nodePos, final int newChildrenPos, - final FormatOptions options) { - final int originalPos = getPosition(); - setPosition(nodePos); - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - PtNodeReader.readParentAddress(mDictBuffer, options); - BinaryDictIOUtils.skipString(mDictBuffer, - (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer); - final int basePos = getPosition(); - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, newChildrenPos - basePos); - setPosition(originalPos); - } - - private void updateTerminalPosition(final int terminalId, final int position) { - if (terminalId == PtNode.NOT_A_TERMINAL - || terminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE - >= mTerminalAddressTableBuffer.limit()) return; - mTerminalAddressTableBuffer.position(terminalId - * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - BinaryDictEncoderUtils.writeUIntToDictBuffer(mTerminalAddressTableBuffer, position, - FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - } - - private void updateForwardLink(final int nodeArrayPos, final int newForwardLink, - final FormatOptions formatOptions) { - final int originalPos = getPosition(); - setPosition(nodeArrayPos); - int jumpCount = 0; - while (jumpCount++ < DynamicBinaryDictIOUtils.MAX_JUMPS) { - final int ptNodeCount = readPtNodeCount(); - for (int i = 0; i < ptNodeCount; ++i) { - skipPtNode(formatOptions); - } - final int forwardLinkPos = getPosition(); - if (!readAndFollowForwardLink()) { - setPosition(forwardLinkPos); - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, newForwardLink - forwardLinkPos); - break; - } - } - setPosition(originalPos); - } - - private void markPtNodeAsMoved(final int nodePos, final int newNodePos, - final FormatOptions options) { - final int originalPos = getPosition(); - updateParentPosIfNotMoved(nodePos, newNodePos, options); - setPosition(nodePos); - final int currentFlags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - setPosition(nodePos); - mDictBuffer.put((byte) (FormatSpec.FLAG_IS_MOVED - | (currentFlags & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); - final int offset = newNodePos - nodePos; - BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, offset); - setPosition(originalPos); - } - - /** - * Writes a PtNode to an output stream from a Ver4PtNodeInfo. - * - * @param nodePos the position of the head of the PtNode. - * @param info the PtNode info to be written. - * @return the size written, in bytes. - */ - private int writePtNode(final int nodePos, final Ver4PtNodeInfo info) throws IOException { - int written = 0; - - // Write flags. - mDictStream.write((byte) (info.mFlags & 0xFF)); - written += FormatSpec.PTNODE_FLAGS_SIZE; - - // Write the parent position. - final int parentOffset = info.mParentPos == FormatSpec.NO_PARENT_ADDRESS ? - FormatSpec.NO_PARENT_ADDRESS : info.mParentPos - nodePos; - BinaryDictIOUtils.writeSInt24ToStream(mDictStream, parentOffset); - written += FormatSpec.PARENT_ADDRESS_SIZE; - - // Write a string. - if (((info.mFlags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0) - != (info.mEndIndexOfCharacters - info.mStartIndexOfCharacters > 1)) { - throw new RuntimeException("Inconsistent flags : hasMultipleChars = " - + ((info.mFlags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0) + ", length = " - + (info.mEndIndexOfCharacters - info.mStartIndexOfCharacters)); - } - written += CharEncoding.writeCodePoints(mDictStream, info.mCharacters, - info.mStartIndexOfCharacters, info.mEndIndexOfCharacters); - - // Write the terminal id. - if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - BinaryDictEncoderUtils.writeUIntToStream(mDictStream, info.mTerminalId, - FormatSpec.PTNODE_TERMINAL_ID_SIZE); - written += FormatSpec.PTNODE_TERMINAL_ID_SIZE; - } - - // Write the children position. - final int childrenOffset = info.mChildrenPos == FormatSpec.NO_CHILDREN_ADDRESS - ? 0 : info.mChildrenPos - (nodePos + written); - BinaryDictIOUtils.writeSInt24ToStream(mDictStream, childrenOffset); - written += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - - return written; - } - - /** - * Helper method to split and move PtNode. - * - * @param ptNodeArrayPos the position of PtNodeArray which contains the split and moved PtNode. - * @param splittedPtNodeToMovePos the position of the split and moved PtNode. - * @param newParent the parent PtNode after splitting. - * @param newChildren the children PtNodes after splitting. - * @param newParentStartPos where to write the new parent. - * @param formatOptions the format options. - */ - private void writeSplittedPtNodes(final int ptNodeArrayPos, final int splittedPtNodeToMovePos, - final Ver4PtNodeInfo newParent, final Ver4PtNodeInfo[] newChildren, - final int newParentStartPos, - final FormatOptions formatOptions) throws IOException { - updateTerminalPosition(newParent.mTerminalId, - newParentStartPos + 1 /* size of PtNodeCount */); - int written = writePtNodeArray(newParentStartPos, new Ver4PtNodeInfo[] { newParent }, - FormatSpec.NO_FORWARD_LINK_ADDRESS); - final int childrenStartPos = newParentStartPos + written; - writePtNodeArray(childrenStartPos, newChildren, FormatSpec.NO_FORWARD_LINK_ADDRESS); - int childrenNodePos = childrenStartPos + 1 /* size of PtNodeCount */; - for (final Ver4PtNodeInfo info : newChildren) { - updateTerminalPosition(info.mTerminalId, childrenNodePos); - childrenNodePos += computePtNodeSize(info.mCharacters, info.mStartIndexOfCharacters, - info.mEndIndexOfCharacters, - (info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0); - } - - // Mark as moved. - markPtNodeAsMoved(splittedPtNodeToMovePos, newParentStartPos + 1 /* size of PtNodeCount */, - formatOptions); - updateForwardLink(ptNodeArrayPos, newParentStartPos, formatOptions); - } - - /** - * Writes a node array to the stream. - * - * @param nodeArrayPos the position of the head of the node array. - * @param infos an array of Ver4PtNodeInfo to be written. - * @return the written length in bytes. - */ - private int writePtNodeArray(final int nodeArrayPos, final Ver4PtNodeInfo[] infos, - final int forwardLink) throws IOException { - int written = BinaryDictIOUtils.writePtNodeCount(mDictStream, infos.length); - for (int i = 0; i < infos.length; ++i) { - written += writePtNode(nodeArrayPos + written, infos[i]); - } - BinaryDictIOUtils.writeSInt24ToStream(mDictStream, forwardLink); - written += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - return written; - } - - private int computePtNodeSize(final int[] codePoints, final int startIndex, final int endIndex, - final boolean isTerminal) { - return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + CharEncoding.getCharArraySize(codePoints, startIndex, endIndex) - + (endIndex - startIndex > 1 ? FormatSpec.PTNODE_TERMINATOR_SIZE : 0) - + (isTerminal ? FormatSpec.PTNODE_TERMINAL_ID_SIZE : 0) - + FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } - - private void writeNewSinglePtNodeWithAttributes(final int[] codePoints, - final boolean hasShortcuts, final int terminalId, final boolean hasBigrams, - final boolean isNotAWord, final boolean isBlackListEntry, final int parentPos, - final FormatOptions formatOptions) throws IOException { - final int newNodeArrayPos = mDictBuffer.limit(); - final int newNodeFlags = BinaryDictEncoderUtils.makePtNodeFlags(codePoints.length > 1, - terminalId != PtNode.NOT_A_TERMINAL, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, - hasBigrams, isNotAWord, isBlackListEntry, formatOptions); - final Ver4PtNodeInfo info = new Ver4PtNodeInfo(newNodeFlags, codePoints, terminalId, - FormatSpec.NO_CHILDREN_ADDRESS, parentPos, 0 /* nodeSize */); - writePtNodeArray(newNodeArrayPos, new Ver4PtNodeInfo[] { info }, - FormatSpec.NO_FORWARD_LINK_ADDRESS); - } - - private int setMultipleCharsInFlags(final int currentFlags, final boolean hasMultipleChars) { - final int flags; - if (hasMultipleChars) { - flags = currentFlags | FormatSpec.FLAG_HAS_MULTIPLE_CHARS; - } else { - flags = currentFlags & (~FormatSpec.FLAG_HAS_MULTIPLE_CHARS); - } - return flags; - } - - private int setIsNotAWordInFlags(final int currentFlags, final boolean isNotAWord) { - final int flags; - if (isNotAWord) { - flags = currentFlags | FormatSpec.FLAG_IS_NOT_A_WORD; - } else { - flags = currentFlags & (~FormatSpec.FLAG_IS_NOT_A_WORD); - } - return flags; - } - - private int setIsBlackListEntryInFlags(final int currentFlags, final boolean isBlackListEntry) { - final int flags; - if (isBlackListEntry) { - flags = currentFlags | FormatSpec.FLAG_IS_BLACKLISTED; - } else { - flags = currentFlags & (~FormatSpec.FLAG_IS_BLACKLISTED); - } - return flags; - } - - /** - * Splits a PtNode. - * - * abcd - ef - * - * -> inserting "abc" - * - * abc - d - ef - * - * @param nodeArrayToSplitPos the position of PtNodeArray which contains the PtNode to split. - * @param nodeToSplitPos the position of the PtNode to split. - * @param nodeToSplitInfo the information of the PtNode to split. - * @param indexToSplit the index where to split in the code points array. - * @param parentOfNodeToSplitPos the absolute position of a parent of the node to split. - * @param newTerminalId the terminal id of the inserted node (corresponds to "d"). - * @param hasShortcuts whether the inserted word should have shortcuts. - * @param hasBigrams whether the inserted word should have bigrams. - * @param isNotAWord whether the inserted word should be not a word. - * @param isBlackListEntry whether the inserted word should be a black list entry. - * @param formatOptions the format options. - */ - private void splitOnly(final int nodeArrayToSplitPos, final int nodeToSplitPos, - final Ver4PtNodeInfo nodeToSplitInfo, final int indexToSplit, - final int parentOfNodeToSplitPos, final int newTerminalId, final boolean hasShortcuts, - final boolean hasBigrams, final boolean isNotAWord, final boolean isBlackListEntry, - final FormatOptions formatOptions) throws IOException { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - final int parentNodeStartPos = parentNodeArrayStartPos + 1 /* size of PtNodeCount */; - final int parentFlags = BinaryDictEncoderUtils.makePtNodeFlags(indexToSplit > 1, - true /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, formatOptions); - final Ver4PtNodeInfo parentInfo = new Ver4PtNodeInfo(parentFlags, - nodeToSplitInfo.mCharacters, newTerminalId, parentNodeStartPos - + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, true) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, - parentOfNodeToSplitPos, 0 /* nodeSize */); - parentInfo.mStartIndexOfCharacters = 0; - parentInfo.mEndIndexOfCharacters = indexToSplit; - - // Write the child. - final int childrenFlags = setMultipleCharsInFlags(nodeToSplitInfo.mFlags, - nodeToSplitInfo.mCharacters.length - indexToSplit > 1); - final Ver4PtNodeInfo childrenInfo = new Ver4PtNodeInfo(childrenFlags, - nodeToSplitInfo.mCharacters, nodeToSplitInfo.mTerminalId, - nodeToSplitInfo.mChildrenPos, parentNodeStartPos, 0 /* nodeSize */); - childrenInfo.mStartIndexOfCharacters = indexToSplit; - childrenInfo.mEndIndexOfCharacters = nodeToSplitInfo.mCharacters.length; - if (nodeToSplitInfo.mChildrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentPositions(nodeToSplitInfo.mChildrenPos, - parentInfo.mChildrenPos + 1 /* size of PtNodeCount */, formatOptions); - } - - writeSplittedPtNodes(nodeArrayToSplitPos, nodeToSplitPos, parentInfo, - new Ver4PtNodeInfo[] { childrenInfo }, parentNodeArrayStartPos, formatOptions); - } - - /** - * Split and branch a PtNode. - * - * ab - cd - * - * -> inserting "ac" - * - * a - b - cd - * | - * - c - * - * @param nodeArrayToSplitPos the position of PtNodeArray which contains the PtNode to split. - * @param nodeToSplitPos the position of the PtNode to split. - * @param nodeToSplitInfo the information of the PtNode to split. - * @param indexToSplit the index where to split in the code points array. - * @param parentOfNodeToSplitPos the absolute position of parent of the node to split. - * @param newWordSuffixCodePoints the suffix of the newly inserted word (corresponds to "c"). - * @param startIndexOfNewWordSuffixCodePoints the start index in newWordSuffixCodePoints where - * the suffix starts. - * @param newTerminalId the terminal id of the inserted node (correspond to "c"). - * @param hasShortcuts whether the inserted word should have shortcuts. - * @param hasBigrams whether the inserted word should have bigrams. - * @param isNotAWord whether the inserted word should be not a word. - * @param isBlackListEntry whether the inserted word should be a black list entry. - * @param formatOptions the format options. - */ - private void splitAndBranch(final int nodeArrayToSplitPos, final int nodeToSplitPos, - final Ver4PtNodeInfo nodeToSplitInfo, final int indexToSplit, - final int parentOfNodeToSplitPos, final int[] newWordSuffixCodePoints, - final int startIndexOfNewWordSuffixCodePoints, - final int newTerminalId, - final boolean hasShortcuts, final boolean hasBigrams, final boolean isNotAWord, - final boolean isBlackListEntry, final FormatOptions formatOptions) throws IOException { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - final int parentNodeStartPos = parentNodeArrayStartPos + 1 /* size of PtNodeCount */; - final int parentFlags = BinaryDictEncoderUtils.makePtNodeFlags( - indexToSplit > 1, - false /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, - false /* hasShortcut */, false /* hasBigrams */, - false /* isNotAWord */, false /* isBlackListEntry */, formatOptions); - final Ver4PtNodeInfo parentInfo = new Ver4PtNodeInfo(parentFlags, - nodeToSplitInfo.mCharacters, PtNode.NOT_A_TERMINAL, - parentNodeStartPos - + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, false) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, - parentOfNodeToSplitPos, 0 /* nodeSize */); - parentInfo.mStartIndexOfCharacters = 0; - parentInfo.mEndIndexOfCharacters = indexToSplit; - - final int childrenNodeArrayStartPos = parentNodeStartPos - + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, false) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - final int firstChildrenFlags = BinaryDictEncoderUtils.makePtNodeFlags( - newWordSuffixCodePoints.length - startIndexOfNewWordSuffixCodePoints > 1, - true /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, formatOptions); - final Ver4PtNodeInfo firstChildrenInfo = new Ver4PtNodeInfo(firstChildrenFlags, - newWordSuffixCodePoints, newTerminalId, - FormatSpec.NO_CHILDREN_ADDRESS, parentNodeStartPos, - 0 /* nodeSize */); - firstChildrenInfo.mStartIndexOfCharacters = startIndexOfNewWordSuffixCodePoints; - firstChildrenInfo.mEndIndexOfCharacters = newWordSuffixCodePoints.length; - - final int secondChildrenStartPos = childrenNodeArrayStartPos + 1 /* size of ptNodeCount */ - + computePtNodeSize(newWordSuffixCodePoints, startIndexOfNewWordSuffixCodePoints, - newWordSuffixCodePoints.length, true /* isTerminal */); - final int secondChildrenFlags = setMultipleCharsInFlags(nodeToSplitInfo.mFlags, - nodeToSplitInfo.mCharacters.length - indexToSplit > 1); - final Ver4PtNodeInfo secondChildrenInfo = new Ver4PtNodeInfo(secondChildrenFlags, - nodeToSplitInfo.mCharacters, nodeToSplitInfo.mTerminalId, - nodeToSplitInfo.mChildrenPos, parentNodeStartPos, 0 /* nodeSize */); - secondChildrenInfo.mStartIndexOfCharacters = indexToSplit; - secondChildrenInfo.mEndIndexOfCharacters = nodeToSplitInfo.mCharacters.length; - if (nodeToSplitInfo.mChildrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentPositions(nodeToSplitInfo.mChildrenPos, secondChildrenStartPos, - formatOptions); - } - - writeSplittedPtNodes(nodeArrayToSplitPos, nodeToSplitPos, parentInfo, - new Ver4PtNodeInfo[] { firstChildrenInfo, secondChildrenInfo }, - parentNodeArrayStartPos, formatOptions); - } - - /** - * Inserts a word into the trie file and returns the position of inserted terminal node. - * If the insertion is failed, returns FormatSpec.NOT_VALID_WORD. - */ - @UsedForTesting - private int insertWordToTrie(final String word, final int newTerminalId, - final boolean isNotAWord, final boolean isBlackListEntry, final boolean hasBigrams, - final boolean hasShortcuts) throws IOException, UnsupportedFormatException { - setPosition(0); - final FileHeader header = readHeader(); - - final int[] codePoints = FusionDictionary.getCodePoints(word); - final int wordLen = codePoints.length; - - int wordPos = 0; - for (int depth = 0; depth < FormatSpec.MAX_WORD_LENGTH; /* nop */) { - final int nodeArrayPos = getPosition(); - final int ptNodeCount = readPtNodeCount(); - boolean goToChildren = false; - int parentPos = FormatSpec.NO_PARENT_ADDRESS; - for (int i = 0; i < ptNodeCount; ++i) { - final int nodePos = getPosition(); - final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(nodePos, header.mFormatOptions); - if (BinaryDictIOUtils.isMovedPtNode(nodeInfo.mFlags, header.mFormatOptions)) { - continue; - } - if (nodeInfo.mParentPos != FormatSpec.NO_PARENT_ADDRESS) { - parentPos = nodePos + nodeInfo.mParentPos; - } - - final boolean firstCharacterMatched = - codePoints[wordPos] == nodeInfo.mCharacters[0]; - boolean allCharactersMatched = true; - int firstDifferentCharacterIndex = -1; - for (int p = 0; p < nodeInfo.mCharacters.length; ++p) { - if (wordPos + p >= codePoints.length) break; - if (codePoints[wordPos + p] != nodeInfo.mCharacters[p]) { - if (firstDifferentCharacterIndex == -1) { - firstDifferentCharacterIndex = p; - } - allCharactersMatched = false; - } - } - - if (!firstCharacterMatched) { - // Go to the next sibling node. - continue; - } - - if (!allCharactersMatched) { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - splitAndBranch(nodeArrayPos, nodePos, nodeInfo, firstDifferentCharacterIndex, - parentPos, codePoints, wordPos + firstDifferentCharacterIndex, - newTerminalId, hasShortcuts, hasBigrams, isNotAWord, - isBlackListEntry, header.mFormatOptions); - - return parentNodeArrayStartPos + computePtNodeSize(codePoints, wordPos, - wordPos + firstDifferentCharacterIndex, false) - + FormatSpec.FORWARD_LINK_ADDRESS_SIZE + 1 /* size of PtNodeCount */; - } - - if (wordLen - wordPos < nodeInfo.mCharacters.length) { - final int parentNodeArrayStartPos = mDictBuffer.limit(); - splitOnly(nodeArrayPos, nodePos, nodeInfo, wordLen - wordPos, parentPos, - newTerminalId, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, - header.mFormatOptions); - - // Return the position of the inserted word. - return parentNodeArrayStartPos + 1 /* size of PtNodeCount */; - } - - wordPos += nodeInfo.mCharacters.length; - if (wordPos == wordLen) { - // This dictionary already contains the word. - Log.e(TAG, "Something went wrong. If the word is already contained, " - + " there is no need to insert new PtNode."); - return FormatSpec.NOT_VALID_WORD; - } - if (nodeInfo.mChildrenPos == FormatSpec.NO_CHILDREN_ADDRESS) { - // There are no children. - // We need to add a new node as a child of this node. - final int newNodeArrayPos = mDictBuffer.limit(); - final int[] newNodeCodePoints = Arrays.copyOfRange(codePoints, wordPos, - codePoints.length); - writeNewSinglePtNodeWithAttributes(newNodeCodePoints, hasShortcuts, - newTerminalId, hasBigrams, isNotAWord, isBlackListEntry, nodePos, - header.mFormatOptions); - updateChildrenPos(nodePos, newNodeArrayPos, header.mFormatOptions); - return newNodeArrayPos + 1 /* size of PtNodeCount */; - } else { - // Found the matched node. - // Go to the children of this node. - setPosition(nodeInfo.mChildrenPos); - goToChildren = true; - depth++; - break; - } - } - - if (goToChildren) continue; - if (!readAndFollowForwardLink()) { - // Add a new node that contains [wordPos, word.length()-1]. - // and update the forward link. - final int newNodeArrayPos = mDictBuffer.limit(); - final int[] newCodePoints = Arrays.copyOfRange(codePoints, wordPos, - codePoints.length); - writeNewSinglePtNodeWithAttributes(newCodePoints, hasShortcuts, newTerminalId, - hasBigrams, isNotAWord, isBlackListEntry, parentPos, header.mFormatOptions); - updateForwardLink(nodeArrayPos, newNodeArrayPos, header.mFormatOptions); - return newNodeArrayPos + 1 /* size of PtNodeCount */; - } - } - return FormatSpec.NOT_VALID_WORD; - } - - private void updateFrequency(final int terminalId, final int frequency) { - mFrequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - BinaryDictEncoderUtils.writeUIntToDictBuffer(mFrequencyBuffer, frequency, - FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - } - - private void insertFrequency(final int frequency) throws IOException { - final OutputStream frequencyStream = new FileOutputStream(mFrequencyFile, - true /* append */); - BinaryDictEncoderUtils.writeUIntToStream(frequencyStream, frequency, - FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - frequencyStream.close(); - } - - private void insertTerminalPosition(final int posOfTerminal) throws IOException, - UnsupportedFormatException { - final OutputStream terminalPosStream = new FileOutputStream( - getFile(FILETYPE_TERMINAL_ADDRESS_TABLE), true /* append */); - BinaryDictEncoderUtils.writeUIntToStream(terminalPosStream, posOfTerminal, - FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - terminalPosStream.close(); - } - - private void insertBigrams(final int terminalId, final int frequency, - final ArrayList<PendingAttribute> bigramAddresses) - throws IOException, UnsupportedFormatException { - openDictBuffer(); - final BigramContentUpdater updater = new BigramContentUpdater(mDictDirectory.getName(), - mDictDirectory, false); - - // Convert addresses to terminal ids. - final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); - mDictBuffer.position(0); - final FileHeader header = readHeader(); - for (PendingAttribute attr : bigramAddresses) { - mDictBuffer.position(attr.mAddress); - final Ver4PtNodeInfo info = readVer4PtNodeInfo(attr.mAddress, header.mFormatOptions); - if (info.mTerminalId == PtNode.NOT_A_TERMINAL) { - throw new RuntimeException("We can't have a bigram target that's not a terminal."); - } - bigrams.add(new PendingAttribute(frequency, info.mTerminalId)); - } - updater.insertBigramEntries(terminalId, frequency, bigrams); - close(); - } - - private void insertShortcuts(final int terminalId, final ArrayList<WeightedString> shortcuts) - throws IOException { - final ShortcutContentUpdater updater = new ShortcutContentUpdater(mDictDirectory.getName(), - mDictDirectory); - updater.insertShortcuts(terminalId, shortcuts); - } - - private void openBuffersAndStream() throws IOException, UnsupportedFormatException { - openDictBuffer(); - mDictStream = new FileOutputStream(getFile(FILETYPE_TRIE), true /* append */); - } - - private void close() throws IOException { - if (mDictStream != null) { - mDictStream.close(); - mDictStream = null; - } - mDictBuffer = null; - mFrequencyBuffer = null; - mTerminalAddressTableBuffer = null; - } - - private void updateAttributes(final int posOfWord, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, - final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { - mDictBuffer.position(0); - final FileHeader header = readHeader(); - mDictBuffer.position(posOfWord); - final Ver4PtNodeInfo info = readVer4PtNodeInfo(posOfWord, header.mFormatOptions); - final int terminalId = info.mTerminalId; - - // Update the flags. - final int newFlags = setIsNotAWordInFlags( - setIsBlackListEntryInFlags(info.mFlags, isBlackListEntry), isNotAWord); - mDictBuffer.position(posOfWord); - mDictBuffer.put((byte) newFlags); - - updateFrequency(terminalId, frequency); - insertBigrams(terminalId, frequency, - DynamicBinaryDictIOUtils.resolveBigramPositions(this, bigramStrings)); - insertShortcuts(terminalId, shortcuts); - } - - @Override @UsedForTesting + @Override public void insertWord(final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { - final int newTerminalId = getNewTerminalId(); - - openBuffersAndStream(); - final int posOfWord = getTerminalPosition(word); - if (posOfWord != FormatSpec.NOT_VALID_WORD) { - // The word is already contained in the dictionary. - updateAttributes(posOfWord, frequency, bigramStrings, shortcuts, isNotAWord, - isBlackListEntry); - close(); - return; - } - - // Insert new PtNode into trie. - final int posOfTerminal = insertWordToTrie(word, newTerminalId, isNotAWord, - isBlackListEntry, bigramStrings != null && !bigramStrings.isEmpty(), - shortcuts != null && !shortcuts.isEmpty()); - insertFrequency(frequency); - insertTerminalPosition(posOfTerminal); - close(); - - insertBigrams(newTerminalId, frequency, - DynamicBinaryDictIOUtils.resolveBigramPositions(this, bigramStrings)); - insertShortcuts(newTerminalId, shortcuts); + // TODO: Implement this method. } } diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 9b573b4b8..1de15a333 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -17,16 +17,18 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; +import android.content.SharedPreferences; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.makedict.DictDecoder; import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.settings.Settings; +import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; @@ -34,9 +36,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; /** * This class is a base class of a dictionary that supports decaying for the personalized language @@ -45,7 +45,8 @@ import java.util.concurrent.TimeUnit; public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary { private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName(); public static final boolean DBG_SAVE_RESTORE = false; - private static final boolean DBG_DUMP_ON_CLOSE = false; + private static final boolean DBG_STRESS_TEST = false; + private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; /** Any pair being typed or picked */ public static final int FREQUENCY_FOR_TYPED = 2; @@ -53,56 +54,52 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED; public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY; - public static final int REQUIRED_BINARY_DICTIONARY_VERSION = 4; - /** Locale for which this user history dictionary is storing words */ - private final Locale mLocale; + private final String mLocale; - private final String mDictName; + private final String mFileName; - /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, - final Locale locale, final String dictionaryType, final String dictName) { - super(context, dictName, locale, dictionaryType, true); - mLocale = locale; - mDictName = dictName; - if (mLocale != null && mLocale.toString().length() > 1) { - reloadDictionaryIfRequired(); - } - } + private final SharedPreferences mPrefs; + + private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = + CollectionUtils.newArrayList(); + + // Should always be false except when we use this class for test + @UsedForTesting boolean mIsTest = false; - // Creates an instance that uses a given dictionary file for testing. - @UsedForTesting /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, - final Locale locale, final String dictionaryType, final String dictName, - final File dictFile) { - super(context, dictName, locale, dictionaryType, true, dictFile); + final String locale, final SharedPreferences sp, final String dictionaryType, + final String fileName) { + super(context, fileName, dictionaryType, true); mLocale = locale; - mDictName = dictName; - if (mLocale != null && mLocale.toString().length() > 1) { + mFileName = fileName; + mPrefs = sp; + if (mLocale != null && mLocale.length() > 1) { + asyncLoadDictionaryToMemory(); reloadDictionaryIfRequired(); } } @Override public void close() { - if (DBG_DUMP_ON_CLOSE) { - dumpAllWordsForDebug(); + if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + closeBinaryDictionary(); } // Flush pending writes. - asyncFlushBinaryDictionary(); + // TODO: Remove after this class become to use a dynamic binary dictionary. + asyncFlashAllBinaryDictionary(); + Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); } @Override protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(FormatSpec.FileHeader.HAS_HISTORICAL_INFO_ATTRIBUTE, + attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mDictName); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale.toString()); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_VERSION_ATTRIBUTE, - String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale); return attributeMap; } @@ -116,25 +113,6 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB return false; } - @Override - protected boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { - // This class is using format 4 because it's used by all version 4 dictionaries. - // TODO: remove this when all dynamically generated dicts use version 4. - return formatVersion == REQUIRED_BINARY_DICTIONARY_VERSION; - } - - public void addMultipleDictionaryEntriesToDictionary( - final ArrayList<LanguageModelParam> languageModelParams, - final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { - if (languageModelParams == null || languageModelParams.isEmpty()) { - if (callback != null) { - callback.onFinished(); - } - return; - } - addMultipleDictionaryEntriesDynamically(languageModelParams, callback); - } - /** * Pair will be added to the decaying dictionary. * @@ -143,63 +121,70 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public void addToDictionary(final String word0, final String word1, final boolean isValid, - final int timestamp) { + public void addToDictionary(final String word0, final String word1, final boolean isValid) { if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { return; } - final int frequency = isValid ? - FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; - addWordDynamically(word1, frequency, null /* shortcutTarget */, 0 /* shortcutFreq */, - false /* isNotAWord */, false /* isBlacklisted */, timestamp); + final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) : + FREQUENCY_FOR_TYPED; + addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */, + false /* isNotAWord */); // Do not insert a word as a bigram of itself if (word1.equals(word0)) { return; } if (null != word0) { - addBigramDynamically(word0, word1, frequency, timestamp); + addBigramDynamically(word0, word1, frequency, isValid); } } - @Override - protected void loadDictionaryAsync() { - // Never loaded to memory in Java side. + public void cancelAddingUserHistory(final String word0, final String word1) { + removeBigramDynamically(word0, word1); } - @UsedForTesting - public void dumpAllWordsForDebug() { - runAfterGcForDebug(new Runnable() { - @Override - public void run() { - dumpAllWordsForDebugLocked(); + @Override + protected void loadDictionaryAsync() { + final int[] profTotalCount = { 0 }; + final String locale = getLocale(); + if (DBG_STRESS_TEST) { + try { + Log.w(TAG, "Start stress in loading: " + locale); + Thread.sleep(15000); + Log.w(TAG, "End stress in loading"); + } catch (InterruptedException e) { } - }); - } - - private void dumpAllWordsForDebugLocked() { - Log.d(TAG, "dumpAllWordsForDebug started."); + } + final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale); + final long now = System.currentTimeMillis(); + final ExpandableBinaryDictionary dictionary = this; final OnAddWordListener listener = new OnAddWordListener() { @Override public void setUnigram(final String word, final String shortcutTarget, final int frequency, final int shortcutFreq) { - Log.d(TAG, "load unigram: " + word + "," + frequency); + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "load unigram: " + word + "," + frequency); + } + addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */); + ++profTotalCount[0]; } @Override public void setBigram(final String word0, final String word1, final int frequency) { if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { - Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); - } else { - Log.d(TAG, "Skip inserting a too long bigram: " + word0 + "," + word1 + "," - + frequency); + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); + } + ++profTotalCount[0]; + addBigram(word0, word1, frequency, last); } } }; // Load the dictionary from binary file - final File dictFile = new File(mContext.getFilesDir(), mDictName); + final File dictFile = new File(mContext.getFilesDir(), mFileName); final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, DictDecoder.USE_BYTEARRAY); if (dictDecoder == null) { @@ -213,17 +198,35 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); } catch (IOException e) { Log.d(TAG, "IOException on opening a bytebuffer", e); - } catch (UnsupportedFormatException e) { - Log.d(TAG, "Unsupported format, can't read the dictionary", e); + } finally { + if (PROFILE_SAVE_RESTORE) { + final long diff = System.currentTimeMillis() - now; + Log.d(TAG, "PROF: Load UserHistoryDictionary: " + + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries."); + } } } + protected String getLocale() { + return mLocale; + } + + public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { + session.setPredictionDictionary(this); + mSessions.add(session); + session.onDictionaryReady(); + } + + public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { + mSessions.remove(session); + } + @UsedForTesting public void clearAndFlushDictionary() { // Clear the node structure on memory clear(); // Then flush the cleared state of the dictionary on disk. - asyncFlushBinaryDictionary(); + asyncFlashAllBinaryDictionary(); } /* package */ void decayIfNeeded() { diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java new file mode 100644 index 000000000..6f152bb91 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.compat.ActivityManagerCompatUtils; +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.AbstractDictionaryWriter; +import com.android.inputmethod.latin.ExpandableDictionary; +import com.android.inputmethod.latin.WordComposer; +import com.android.inputmethod.latin.ExpandableDictionary.NextWord; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; + +// Currently this class is used to implement dynamic prodiction dictionary. +// TODO: Move to native code. +public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { + private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); + /** Maximum number of pairs. Pruning will start when databases goes above this number. */ + public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000; + public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000; + + /** Any pair being typed or picked */ + private static final int FREQUENCY_FOR_TYPED = 2; + + private static final int BINARY_DICT_VERSION = 3; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); + + private final UserHistoryDictionaryBigramList mBigramList = + new UserHistoryDictionaryBigramList(); + private final ExpandableDictionary mExpandableDictionary; + private final int mMaxHistoryBigrams; + + public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { + super(context, dictType); + mExpandableDictionary = new ExpandableDictionary(dictType); + final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context); + mMaxHistoryBigrams = isLowRamDevice ? + LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS; + } + + @Override + public void clear() { + mBigramList.evictAll(); + mExpandableDictionary.clearDictionary(); + } + + /** + * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes + * are done to update the binary dictionary. + * @param word The word to add. + * @param shortcutTarget A shortcut target for this word, or null if none. + * @param frequency The frequency for this unigram. + * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored + * if shortcutTarget is null. + * @param isNotAWord true if this is not a word, i.e. shortcut only. + */ + @Override + public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, + final int shortcutFreq, final boolean isNotAWord) { + if (mBigramList.size() > mMaxHistoryBigrams * 2) { + // Too many entries: just stop adding new vocabulary and wait next refresh. + return; + } + mExpandableDictionary.addWord(word, shortcutTarget, frequency, shortcutFreq); + mBigramList.addBigram(null, word, (byte)frequency); + } + + @Override + public void addBigramWords(final String word0, final String word1, final int frequency, + final boolean isValid, final long lastModifiedTime) { + if (mBigramList.size() > mMaxHistoryBigrams * 2) { + // Too many entries: just stop adding new vocabulary and wait next refresh. + return; + } + if (lastModifiedTime > 0) { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(frequency, System.currentTimeMillis(), + lastModifiedTime)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } else { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(isValid)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } + } + + @Override + public void removeBigramWords(final String word0, final String word1) { + if (mBigramList.removeBigram(word0, word1)) { + mExpandableDictionary.removeBigram(word0, word1); + } + } + + @Override + protected void writeDictionary(final DictEncoder dictEncoder, + final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException { + UserHistoryDictIOUtils.writeDictionary(dictEncoder, + new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams), + mBigramList, FORMAT_OPTIONS); + } + + private static class FrequencyProvider implements BigramDictionaryInterface { + private final UserHistoryDictionaryBigramList mBigramList; + private final ExpandableDictionary mExpandableDictionary; + private final int mMaxHistoryBigrams; + + public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, + final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) { + mBigramList = bigramList; + mExpandableDictionary = expandableDictionary; + mMaxHistoryBigrams = maxHistoryBigrams; + } + + @Override + public int getFrequency(final String word0, final String word1) { + final int freq; + if (word0 == null) { // unigram + freq = FREQUENCY_FOR_TYPED; + } else { // bigram + final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1); + if (nw != null) { + final ForgettingCurveParams forgettingCurveParams = nw.getFcParams(); + final byte prevFc = mBigramList.getBigrams(word0).get(word1); + final byte fc = forgettingCurveParams.getFc(); + final boolean isValid = forgettingCurveParams.isValid(); + if (prevFc > 0 && prevFc == fc) { + freq = fc & 0xFF; + } else if (UserHistoryForgettingCurveUtils. + needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) { + freq = fc & 0xFF; + } else { + // Delete this entry + freq = -1; + } + } else { + // Delete this entry + freq = -1; + } + } + return freq; + } + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo, + blockOffensiveWords, additionalFeaturesOptions); + } + + @Override + public boolean isValidWord(final String word) { + return mExpandableDictionary.isValidWord(word); + } + + @UsedForTesting + public boolean isInBigramListForTests(final String word) { + // TODO: Use native method to determine whether the word is in dictionary or not + return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java index 596562f1d..f257165cb 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -16,37 +16,53 @@ package com.android.inputmethod.latin.personalization; -import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.utils.CollectionUtils; -import java.io.File; -import java.util.ArrayList; -import java.util.Locale; - import android.content.Context; +import android.content.SharedPreferences; -public class PersonalizationDictionary extends DecayingExpandableBinaryDictionaryBase { - private static final String NAME = PersonalizationDictionary.class.getSimpleName(); +import java.util.ArrayList; +/** + * This class is a dictionary for the personalized language model that uses binary dictionary. + */ +public class PersonalizationDictionary extends ExpandableBinaryDictionary { + private static final String NAME = "personalization"; private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = CollectionUtils.newArrayList(); - /* package */ PersonalizationDictionary(final Context context, final Locale locale) { - super(context, locale, Dictionary.TYPE_PERSONALIZATION, - getDictNameWithLocale(NAME, locale)); + /** Locale for which this user history dictionary is storing words */ + private final String mLocale; + + public PersonalizationDictionary(final Context context, final String locale, + final SharedPreferences prefs) { + // TODO: Make isUpdatable true. + super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION, + false /* isUpdatable */); + mLocale = locale; + // TODO: Restore last updated time + loadDictionary(); + } + + @Override + protected void loadDictionaryAsync() { + // TODO: Implement + } + + @Override + protected boolean hasContentChanged() { + return false; } - // Creates an instance that uses a given dictionary file for testing. - @UsedForTesting - public PersonalizationDictionary(final Context context, final Locale locale, - final File dictFile) { - super(context, locale, Dictionary.TYPE_PERSONALIZATION, getDictNameWithLocale(NAME, locale), - dictFile); + @Override + protected boolean needsToReloadBeforeWriting() { + return false; } public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { - session.setPredictionDictionary(this); + session.setDictionary(this); mSessions.add(session); session.onDictionaryReady(); } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java index 542bda621..c1833ff14 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java @@ -20,18 +20,18 @@ import android.content.Context; import android.content.res.Configuration; public class PersonalizationDictionarySessionRegister { - public static void init(final Context context) { + public static void init(Context context) { } public static void onConfigurationChanged(final Context context, final Configuration conf) { } - public static void onUpdateData(final Context context, final String type) { + public static void onUpdateData(Context context, String type) { } - public static void onRemoveData(final Context context, final String type) { + public static void onRemoveData(Context context, String type) { } - public static void onDestroy(final Context context) { + public static void onDestroy(Context context) { } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index 61354762b..a86f6e584 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -18,37 +18,61 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; -import com.android.inputmethod.latin.BinaryDictionary.LanguageModelParam; -import com.android.inputmethod.latin.ExpandableBinaryDictionary; - import java.lang.ref.WeakReference; import java.util.ArrayList; -import java.util.Locale; /** * This class is a session where a data provider can communicate with a personalization * dictionary. */ public abstract class PersonalizationDictionaryUpdateSession { - public WeakReference<PersonalizationDictionary> mDictionary; - public final Locale mSystemLocale; + /** + * This class is a parameter for a new unigram or bigram word which will be added + * to the personalization dictionary. + */ + public static class PersonalizationLanguageModelParam { + public final String mWord0; + public final String mWord1; + public final boolean mIsValid; + public final int mFrequency; + public PersonalizationLanguageModelParam(String word0, String word1, boolean isValid, + int frequency) { + mWord0 = word0; + mWord1 = word1; + mIsValid = isValid; + mFrequency = frequency; + } + } - public PersonalizationDictionaryUpdateSession(final Locale locale) { + // TODO: Use a dynamic binary dictionary instead + public WeakReference<PersonalizationDictionary> mDictionary; + public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary; + public final String mSystemLocale; + public PersonalizationDictionaryUpdateSession(String locale) { mSystemLocale = locale; } public abstract void onDictionaryReady(); - public abstract void onDictionaryClosed(final Context context); + public abstract void onDictionaryClosed(Context context); - public void setPredictionDictionary(final PersonalizationDictionary dictionary) { + public void setDictionary(PersonalizationDictionary dictionary) { mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); } + public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) { + mPredictionDictionary = + new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary); + } + protected PersonalizationDictionary getDictionary() { return mDictionary == null ? null : mDictionary.get(); } + protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() { + return mPredictionDictionary == null ? null : mPredictionDictionary.get(); + } + private void unsetDictionary() { final PersonalizationDictionary dictionary = getDictionary(); if (dictionary == null) { @@ -57,30 +81,48 @@ public abstract class PersonalizationDictionaryUpdateSession { dictionary.unRegisterUpdateSession(this); } - public void clearAndFlushDictionary(final Context context) { - final PersonalizationDictionary dictionary = getDictionary(); + private void unsetPredictionDictionary() { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.unRegisterUpdateSession(this); + } + + public void clearAndFlushPredictionDictionary(Context context) { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } dictionary.clearAndFlushDictionary(); } - public void closeSession(final Context context) { + public void closeSession(Context context) { unsetDictionary(); + unsetPredictionDictionary(); onDictionaryClosed(context); } - // TODO: Support multi locale. - public void addMultipleDictionaryEntriesToDictionary( - final ArrayList<LanguageModelParam> languageModelParams, - final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { - final PersonalizationDictionary dictionary = getDictionary(); + // TODO: Support multi locale to add bigram + public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, + int frequency) { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { - if (callback != null) { - callback.onFinished(); - } return; } - dictionary.addMultipleDictionaryEntriesToDictionary(languageModelParams, callback); + dictionary.addToDictionary(word0, word1, isValid); + } + + // Bulk import + // TODO: Support multi locale to add bigram + public void addBigramsToPersonalizationDictionary( + final ArrayList<PersonalizationLanguageModelParam> lmParams) { + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + for (final PersonalizationLanguageModelParam lmParam : lmParams) { + dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid); + } } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index d55cae132..221ddeeba 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -19,10 +19,11 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.utils.CollectionUtils; import android.content.Context; +import android.content.SharedPreferences; +import android.preference.PreferenceManager; import android.util.Log; import java.lang.ref.SoftReference; -import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; public class PersonalizationHelper { @@ -30,16 +31,21 @@ public class PersonalizationHelper { private static final boolean DEBUG = false; private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>> sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, + SoftReference<PersonalizationPredictionDictionary>> + sLangPersonalizationPredictionDictCache = + CollectionUtils.newConcurrentHashMap(); + public static UserHistoryDictionary getUserHistoryDictionary( - final Context context, final Locale locale) { - final String localeStr = locale.toString(); + final Context context, final String locale, final SharedPreferences sp) { synchronized (sLangUserHistoryDictCache) { - if (sLangUserHistoryDictCache.containsKey(localeStr)) { + if (sLangUserHistoryDictCache.containsKey(locale)) { final SoftReference<UserHistoryDictionary> ref = - sLangUserHistoryDictCache.get(localeStr); + sLangUserHistoryDictCache.get(locale); final UserHistoryDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -49,9 +55,8 @@ public class PersonalizationHelper { return dict; } } - final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale); - sLangUserHistoryDictCache.put(localeStr, - new SoftReference<UserHistoryDictionary>(dict)); + final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp); + sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict)); return dict; } } @@ -69,30 +74,57 @@ public class PersonalizationHelper { } public static void registerPersonalizationDictionaryUpdateSession(final Context context, - final PersonalizationDictionaryUpdateSession session, final Locale locale) { - final PersonalizationDictionary personalizationDictionary = - getPersonalizationDictionary(context, locale); - personalizationDictionary.registerUpdateSession(session); + final PersonalizationDictionaryUpdateSession session, String locale) { + final PersonalizationPredictionDictionary predictionDictionary = + getPersonalizationPredictionDictionary(context, locale, + PreferenceManager.getDefaultSharedPreferences(context)); + predictionDictionary.registerUpdateSession(session); + final PersonalizationDictionary dictionary = + getPersonalizationDictionary(context, locale, + PreferenceManager.getDefaultSharedPreferences(context)); + dictionary.registerUpdateSession(session); } public static PersonalizationDictionary getPersonalizationDictionary( - final Context context, final Locale locale) { - final String localeStr = locale.toString(); + final Context context, final String locale, final SharedPreferences sp) { synchronized (sLangPersonalizationDictCache) { - if (sLangPersonalizationDictCache.containsKey(localeStr)) { + if (sLangPersonalizationDictCache.containsKey(locale)) { final SoftReference<PersonalizationDictionary> ref = - sLangPersonalizationDictCache.get(localeStr); + sLangPersonalizationDictCache.get(locale); final PersonalizationDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { - Log.w(TAG, "Use cached PersonalizationDictionary for " + locale); + Log.w(TAG, "Use cached PersonalizationDictCache for " + locale); } return dict; } } - final PersonalizationDictionary dict = new PersonalizationDictionary(context, locale); + final PersonalizationDictionary dict = + new PersonalizationDictionary(context, locale, sp); sLangPersonalizationDictCache.put( - localeStr, new SoftReference<PersonalizationDictionary>(dict)); + locale, new SoftReference<PersonalizationDictionary>(dict)); + return dict; + } + } + + public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( + final Context context, final String locale, final SharedPreferences sp) { + synchronized (sLangPersonalizationPredictionDictCache) { + if (sLangPersonalizationPredictionDictCache.containsKey(locale)) { + final SoftReference<PersonalizationPredictionDictionary> ref = + sLangPersonalizationPredictionDictCache.get(locale); + final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get(); + if (dict != null) { + if (DEBUG) { + Log.w(TAG, "Use cached PersonalizationPredictionDictionary for " + locale); + } + return dict; + } + } + final PersonalizationPredictionDictionary dict = + new PersonalizationPredictionDictionary(context, locale, sp); + sLangPersonalizationPredictionDictCache.put( + locale, new SoftReference<PersonalizationPredictionDictionary>(dict)); return dict; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java new file mode 100644 index 000000000..432954453 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; + +import android.content.Context; +import android.content.SharedPreferences; + +public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { + private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); + + /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, + final SharedPreferences sp) { + super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, + getDictionaryFileName(locale)); + } + + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index 868f21cbc..a60226d7e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -16,13 +16,11 @@ package com.android.inputmethod.latin.personalization; -import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Dictionary; - -import java.io.File; -import java.util.Locale; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; +import android.content.SharedPreferences; /** * Locally gathers stats about the words user types and various other signals like auto-correction @@ -31,19 +29,12 @@ import android.content.Context; public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase { /* package for tests */ static final String NAME = UserHistoryDictionary.class.getSimpleName(); - /* package */ UserHistoryDictionary(final Context context, final Locale locale) { - super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale)); - } - - // Creates an instance that uses a given dictionary file for testing. - @UsedForTesting - public UserHistoryDictionary(final Context context, final Locale locale, - final File dictFile) { - super(context, locale, Dictionary.TYPE_USER_HISTORY, getDictNameWithLocale(NAME, locale), - dictFile); + /* package */ UserHistoryDictionary(final Context context, final String locale, + final SharedPreferences sp) { + super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); } - public void cancelAddingUserHistory(final String word0, final String word1) { - removeBigramDynamically(word0, word1); + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java index d060485bd..da1fb73fe 100644 --- a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java +++ b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java @@ -112,7 +112,8 @@ public final class DebugSettings extends PreferenceFragment updateDebugMode(); mServiceNeedsRestart = true; } - } else if (key.equals(PREF_FORCE_NON_DISTINCT_MULTITOUCH)) { + } else if (key.equals(PREF_FORCE_NON_DISTINCT_MULTITOUCH) + || key.equals(PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG)) { mServiceNeedsRestart = true; } } diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java index 714c3a97a..df2c6907f 100644 --- a/java/src/com/android/inputmethod/latin/settings/Settings.java +++ b/java/src/com/android/inputmethod/latin/settings/Settings.java @@ -27,10 +27,12 @@ import com.android.inputmethod.latin.AudioAndHapticFeedbackManager; import com.android.inputmethod.latin.InputAttributes; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils; +import com.android.inputmethod.latin.utils.LocaleUtils; import com.android.inputmethod.latin.utils.ResourceUtils; import com.android.inputmethod.latin.utils.RunInLocale; import com.android.inputmethod.latin.utils.StringUtils; +import java.util.HashMap; import java.util.Locale; import java.util.concurrent.locks.ReentrantLock; @@ -51,6 +53,8 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang public static final String PREF_AUTO_CORRECTION_THRESHOLD = "auto_correction_threshold"; public static final String PREF_SHOW_SUGGESTIONS_SETTING = "show_suggestions_setting"; public static final String PREF_MISC_SETTINGS = "misc_settings"; + public static final String PREF_LAST_USER_DICTIONARY_WRITE_TIME = + "last_user_dictionary_write_time"; public static final String PREF_ADVANCED_SETTINGS = "pref_advanced_settings"; public static final String PREF_KEY_USE_CONTACTS_DICT = "pref_key_use_contacts_dict"; public static final String PREF_KEY_USE_DOUBLE_SPACE_PERIOD = @@ -225,15 +229,16 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang res.getBoolean(R.bool.config_default_phrase_gesture_enabled)); } - public static boolean readFromBuildConfigIfToShowKeyPreviewPopupOption(final Resources res) { - return res.getBoolean(R.bool.config_enable_show_key_preview_popup_option); + public static boolean readFromBuildConfigIfToShowKeyPreviewPopupSettingsOption( + final Resources res) { + return res.getBoolean(R.bool.config_enable_show_option_of_key_preview_popup); } public static boolean readKeyPreviewPopupEnabled(final SharedPreferences prefs, final Resources res) { final boolean defaultKeyPreviewPopup = res.getBoolean( R.bool.config_default_key_preview_popup); - if (!readFromBuildConfigIfToShowKeyPreviewPopupOption(res)) { + if (!readFromBuildConfigIfToShowKeyPreviewPopupSettingsOption(res)) { return defaultKeyPreviewPopup; } return prefs.getBoolean(PREF_POPUP_ON, defaultKeyPreviewPopup); @@ -328,6 +333,25 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang return prefs.getBoolean(DebugSettings.PREF_USABILITY_STUDY_MODE, true); } + public static long readLastUserHistoryWriteTime(final SharedPreferences prefs, + final String locale) { + final String str = prefs.getString(PREF_LAST_USER_DICTIONARY_WRITE_TIME, ""); + final HashMap<String, Long> map = LocaleUtils.localeAndTimeStrToHashMap(str); + if (map.containsKey(locale)) { + return map.get(locale); + } + return 0; + } + + public static void writeLastUserHistoryWriteTime(final SharedPreferences prefs, + final String locale) { + final String oldStr = prefs.getString(PREF_LAST_USER_DICTIONARY_WRITE_TIME, ""); + final HashMap<String, Long> map = LocaleUtils.localeAndTimeStrToHashMap(oldStr); + map.put(locale, System.currentTimeMillis()); + final String newStr = LocaleUtils.localeAndTimeHashMapToStr(map); + prefs.edit().putString(PREF_LAST_USER_DICTIONARY_WRITE_TIME, newStr).apply(); + } + public static boolean readUseFullscreenMode(final Resources res) { return res.getBoolean(R.bool.config_use_fullscreen_mode); } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java b/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java index d7a3e95b3..5c60a7350 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java @@ -48,6 +48,7 @@ import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils; import com.android.inputmethod.latin.utils.ApplicationUtils; import com.android.inputmethod.latin.utils.FeedbackUtils; import com.android.inputmethod.latin.utils.SubtypeLocaleUtils; +import com.android.inputmethod.research.ResearchLogger; import com.android.inputmethodcommon.InputMethodSettingsFragment; import java.util.TreeSet; @@ -60,6 +61,13 @@ public final class SettingsFragment extends InputMethodSettingsFragment DBG_USE_INTERNAL_PERSONAL_DICTIONARY_SETTINGS || Build.VERSION.SDK_INT <= 18 /* Build.VERSION.JELLY_BEAN_MR2 */; + private CheckBoxPreference mVoiceInputKeyPreference; + private ListPreference mShowCorrectionSuggestionsPreference; + private ListPreference mAutoCorrectionThresholdPreference; + private ListPreference mKeyPreviewPopupDismissDelay; + // Use bigrams to predict the next word when there is no input for it yet + private CheckBoxPreference mBigramPrediction; + private void setPreferenceEnabled(final String preferenceKey, final boolean enabled) { final Preference preference = findPreference(preferenceKey); if (preference != null) { @@ -67,18 +75,6 @@ public final class SettingsFragment extends InputMethodSettingsFragment } } - private void updateListPreferenceSummaryToCurrentValue(final String prefKey) { - // Because the "%s" summary trick of {@link ListPreference} doesn't work properly before - // KitKat, we need to update the summary programmatically. - final ListPreference listPreference = (ListPreference)findPreference(prefKey); - if (listPreference == null) { - return; - } - final CharSequence entries[] = listPreference.getEntries(); - final int entryIndex = listPreference.findIndexOfValue(listPreference.getValue()); - listPreference.setSummary(entryIndex < 0 ? null : entries[entryIndex]); - } - private static void removePreference(final String preferenceKey, final PreferenceGroup parent) { if (parent == null) { return; @@ -111,9 +107,16 @@ public final class SettingsFragment extends InputMethodSettingsFragment SubtypeLocaleUtils.init(context); AudioAndHapticFeedbackManager.init(context); + mVoiceInputKeyPreference = + (CheckBoxPreference) findPreference(Settings.PREF_VOICE_INPUT_KEY); + mShowCorrectionSuggestionsPreference = + (ListPreference) findPreference(Settings.PREF_SHOW_SUGGESTIONS_SETTING); final SharedPreferences prefs = getPreferenceManager().getSharedPreferences(); prefs.registerOnSharedPreferenceChangeListener(this); + mAutoCorrectionThresholdPreference = + (ListPreference) findPreference(Settings.PREF_AUTO_CORRECTION_THRESHOLD); + mBigramPrediction = (CheckBoxPreference) findPreference(Settings.PREF_BIGRAM_PREDICTIONS); ensureConsistencyOfAutoCorrectionSettings(); final PreferenceGroup generalSettings = @@ -140,7 +143,12 @@ public final class SettingsFragment extends InputMethodSettingsFragment feedbackSettings.setOnPreferenceClickListener(new OnPreferenceClickListener() { @Override public boolean onPreferenceClick(final Preference pref) { - FeedbackUtils.showFeedbackForm(getActivity()); + if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { + // Use development-only feedback mechanism + ResearchLogger.getInstance().presentFeedbackDialogFromSettings(); + } else { + FeedbackUtils.showFeedbackForm(getActivity()); + } return true; } }); @@ -159,7 +167,7 @@ public final class SettingsFragment extends InputMethodSettingsFragment final boolean showVoiceKeyOption = res.getBoolean( R.bool.config_enable_show_voice_key_option); if (!showVoiceKeyOption) { - removePreference(Settings.PREF_VOICE_INPUT_KEY, generalSettings); + generalSettings.removePreference(mVoiceInputKeyPreference); } final PreferenceGroup advancedSettings = @@ -169,27 +177,26 @@ public final class SettingsFragment extends InputMethodSettingsFragment removePreference(Settings.PREF_VIBRATION_DURATION_SETTINGS, advancedSettings); } - if (!Settings.readFromBuildConfigIfToShowKeyPreviewPopupOption(res)) { + mKeyPreviewPopupDismissDelay = + (ListPreference) findPreference(Settings.PREF_KEY_PREVIEW_POPUP_DISMISS_DELAY); + if (!Settings.readFromBuildConfigIfToShowKeyPreviewPopupSettingsOption(res)) { removePreference(Settings.PREF_POPUP_ON, generalSettings); removePreference(Settings.PREF_KEY_PREVIEW_POPUP_DISMISS_DELAY, advancedSettings); } else { - // TODO: Cleanup this setup. - final ListPreference keyPreviewPopupDismissDelay = - (ListPreference) findPreference(Settings.PREF_KEY_PREVIEW_POPUP_DISMISS_DELAY); final String popupDismissDelayDefaultValue = Integer.toString(res.getInteger( R.integer.config_key_preview_linger_timeout)); - keyPreviewPopupDismissDelay.setEntries(new String[] { + mKeyPreviewPopupDismissDelay.setEntries(new String[] { res.getString(R.string.key_preview_popup_dismiss_no_delay), res.getString(R.string.key_preview_popup_dismiss_default_delay), }); - keyPreviewPopupDismissDelay.setEntryValues(new String[] { + mKeyPreviewPopupDismissDelay.setEntryValues(new String[] { "0", popupDismissDelayDefaultValue }); - if (null == keyPreviewPopupDismissDelay.getValue()) { - keyPreviewPopupDismissDelay.setValue(popupDismissDelayDefaultValue); + if (null == mKeyPreviewPopupDismissDelay.getValue()) { + mKeyPreviewPopupDismissDelay.setValue(popupDismissDelayDefaultValue); } - keyPreviewPopupDismissDelay.setEnabled( + mKeyPreviewPopupDismissDelay.setEnabled( Settings.readKeyPreviewPopupEnabled(prefs, res)); } @@ -236,19 +243,20 @@ public final class SettingsFragment extends InputMethodSettingsFragment @Override public void onResume() { super.onResume(); - final SharedPreferences prefs = getPreferenceManager().getSharedPreferences(); - final Resources res = getResources(); final boolean isShortcutImeEnabled = SubtypeSwitcher.getInstance().isShortcutImeEnabled(); - setPreferenceEnabled(Settings.PREF_VOICE_INPUT_KEY, isShortcutImeEnabled); + if (!isShortcutImeEnabled) { + getPreferenceScreen().removePreference(mVoiceInputKeyPreference); + } + final SharedPreferences prefs = getPreferenceManager().getSharedPreferences(); final CheckBoxPreference showSetupWizardIcon = (CheckBoxPreference)findPreference(Settings.PREF_SHOW_SETUP_WIZARD_ICON); if (showSetupWizardIcon != null) { showSetupWizardIcon.setChecked(Settings.readShowSetupWizardIcon(prefs, getActivity())); } - updateListPreferenceSummaryToCurrentValue(Settings.PREF_SHOW_SUGGESTIONS_SETTING); - updateListPreferenceSummaryToCurrentValue(Settings.PREF_KEY_PREVIEW_POPUP_DISMISS_DELAY); - updateListPreferenceSummaryToCurrentValue(Settings.PREF_KEYBOARD_LAYOUT); - updateCustomInputStylesSummary(prefs, res); + updateShowCorrectionSuggestionsSummary(); + updateKeyPreviewPopupDelaySummary(); + updateColorSchemeSummary(prefs, getResources()); + updateCustomInputStylesSummary(); } @Override @@ -279,26 +287,50 @@ public final class SettingsFragment extends InputMethodSettingsFragment LauncherIconVisibilityManager.updateSetupWizardIconVisibility(getActivity()); } ensureConsistencyOfAutoCorrectionSettings(); - updateListPreferenceSummaryToCurrentValue(Settings.PREF_SHOW_SUGGESTIONS_SETTING); - updateListPreferenceSummaryToCurrentValue(Settings.PREF_KEY_PREVIEW_POPUP_DISMISS_DELAY); - updateListPreferenceSummaryToCurrentValue(Settings.PREF_KEYBOARD_LAYOUT); + updateShowCorrectionSuggestionsSummary(); + updateKeyPreviewPopupDelaySummary(); + updateColorSchemeSummary(prefs, res); refreshEnablingsOfKeypressSoundAndVibrationSettings(prefs, getResources()); } private void ensureConsistencyOfAutoCorrectionSettings() { final String autoCorrectionOff = getResources().getString( R.string.auto_correction_threshold_mode_index_off); - final ListPreference autoCorrectionThresholdPref = (ListPreference)findPreference( - Settings.PREF_AUTO_CORRECTION_THRESHOLD); - final String currentSetting = autoCorrectionThresholdPref.getValue(); - setPreferenceEnabled( - Settings.PREF_BIGRAM_PREDICTIONS, !currentSetting.equals(autoCorrectionOff)); + final String currentSetting = mAutoCorrectionThresholdPreference.getValue(); + mBigramPrediction.setEnabled(!currentSetting.equals(autoCorrectionOff)); } - private void updateCustomInputStylesSummary(final SharedPreferences prefs, - final Resources res) { + private void updateShowCorrectionSuggestionsSummary() { + mShowCorrectionSuggestionsPreference.setSummary( + getResources().getStringArray(R.array.prefs_suggestion_visibilities) + [mShowCorrectionSuggestionsPreference.findIndexOfValue( + mShowCorrectionSuggestionsPreference.getValue())]); + } + + private void updateColorSchemeSummary(final SharedPreferences prefs, final Resources res) { + // Because the "%s" summary trick of {@link ListPreference} doesn't work properly before + // KitKat, we need to update the summary by code. + final Preference preference = findPreference(Settings.PREF_KEYBOARD_LAYOUT); + if (!(preference instanceof ListPreference)) { + Log.w(TAG, "Can't find Keyboard Color Scheme preference"); + return; + } + final ListPreference colorSchemePreference = (ListPreference)preference; + final int themeIndex = Settings.readKeyboardThemeIndex(prefs, res); + int entryIndex = colorSchemePreference.findIndexOfValue(Integer.toString(themeIndex)); + if (entryIndex < 0) { + final int defaultThemeIndex = Settings.resetAndGetDefaultKeyboardThemeIndex(prefs, res); + entryIndex = colorSchemePreference.findIndexOfValue( + Integer.toString(defaultThemeIndex)); + } + colorSchemePreference.setSummary(colorSchemePreference.getEntries()[entryIndex]); + } + + private void updateCustomInputStylesSummary() { final PreferenceScreen customInputStyles = (PreferenceScreen)findPreference(Settings.PREF_CUSTOM_INPUT_STYLES); + final SharedPreferences prefs = getPreferenceManager().getSharedPreferences(); + final Resources res = getResources(); final String prefSubtype = Settings.readPrefAdditionalSubtypes(prefs, res); final InputMethodSubtype[] subtypes = AdditionalSubtypeUtils.createAdditionalSubtypesArray(prefSubtype); @@ -310,6 +342,13 @@ public final class SettingsFragment extends InputMethodSettingsFragment customInputStyles.setSummary(styles); } + private void updateKeyPreviewPopupDelaySummary() { + final ListPreference lp = mKeyPreviewPopupDismissDelay; + final CharSequence[] entries = lp.getEntries(); + if (entries == null || entries.length <= 0) return; + lp.setSummary(entries[lp.findIndexOfValue(lp.getValue())]); + } + private void refreshEnablingsOfKeypressSoundAndVibrationSettings( final SharedPreferences sp, final Resources res) { setPreferenceEnabled(Settings.PREF_VIBRATION_DURATION_SETTINGS, diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 06406c19b..f331c78e5 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -67,7 +67,7 @@ public final class SettingsValues { public final boolean mVibrateOn; public final boolean mSoundOn; public final boolean mKeyPreviewPopupOn; - public final boolean mShowsVoiceInputKey; + private final boolean mShowsVoiceInputKey; public final boolean mIncludesOtherImesInLanguageSwitchList; public final boolean mShowsLanguageSwitchKey; public final boolean mUseContactsDict; @@ -271,6 +271,13 @@ public final class SettingsValues { return mInputAttributes.mShouldInsertSpacesAutomatically; } + public boolean isVoiceKeyEnabled(final EditorInfo editorInfo) { + final boolean shortcutImeEnabled = SubtypeSwitcher.getInstance().isShortcutImeEnabled(); + final int inputType = (editorInfo != null) ? editorInfo.inputType : 0; + return shortcutImeEnabled && mShowsVoiceInputKey + && !InputTypeUtils.isPasswordInputType(inputType); + } + public boolean isLanguageSwitchKeyEnabled() { if (!mShowsLanguageSwitchKey) { return false; @@ -367,20 +374,16 @@ public final class SettingsValues { return autoCorrectionThreshold; } - private static boolean needsToShowVoiceInputKey(final SharedPreferences prefs, - final Resources res) { - if (!prefs.contains(Settings.PREF_VOICE_INPUT_KEY)) { - // Migrate preference from {@link Settings#PREF_VOICE_MODE_OBSOLETE} to - // {@link Settings#PREF_VOICE_INPUT_KEY}. - final String voiceModeMain = res.getString(R.string.voice_mode_main); - final String voiceMode = prefs.getString( - Settings.PREF_VOICE_MODE_OBSOLETE, voiceModeMain); - final boolean shouldShowVoiceInputKey = voiceModeMain.equals(voiceMode); - prefs.edit().putBoolean(Settings.PREF_VOICE_INPUT_KEY, shouldShowVoiceInputKey).apply(); - } - // Remove the obsolete preference if exists. - if (prefs.contains(Settings.PREF_VOICE_MODE_OBSOLETE)) { - prefs.edit().remove(Settings.PREF_VOICE_MODE_OBSOLETE).apply(); + private static boolean needsToShowVoiceInputKey(SharedPreferences prefs, Resources res) { + final String voiceModeMain = res.getString(R.string.voice_mode_main); + final String voiceMode = prefs.getString(Settings.PREF_VOICE_MODE_OBSOLETE, voiceModeMain); + final boolean showsVoiceInputKey = voiceMode == null || voiceMode.equals(voiceModeMain); + if (!showsVoiceInputKey) { + // Migrate settings from PREF_VOICE_MODE_OBSOLETE to PREF_VOICE_INPUT_KEY + // Set voiceModeMain as a value of obsolete voice mode settings. + prefs.edit().putString(Settings.PREF_VOICE_MODE_OBSOLETE, voiceModeMain).apply(); + // Disable voice input key. + prefs.edit().putBoolean(Settings.PREF_VOICE_INPUT_KEY, false).apply(); } return prefs.getBoolean(Settings.PREF_VOICE_INPUT_KEY, true); } diff --git a/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java b/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java index 5072fabd6..c4a813c24 100644 --- a/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java +++ b/java/src/com/android/inputmethod/latin/setup/SetupWizardActivity.java @@ -38,7 +38,7 @@ import com.android.inputmethod.compat.ViewCompatUtils; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.settings.SettingsActivity; import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.LeakGuardHandlerWrapper; +import com.android.inputmethod.latin.utils.StaticInnerHandlerWrapper; import java.util.ArrayList; @@ -74,21 +74,21 @@ public final class SetupWizardActivity extends Activity implements View.OnClickL private SettingsPoolingHandler mHandler; private static final class SettingsPoolingHandler - extends LeakGuardHandlerWrapper<SetupWizardActivity> { + extends StaticInnerHandlerWrapper<SetupWizardActivity> { private static final int MSG_POLLING_IME_SETTINGS = 0; private static final long IME_SETTINGS_POLLING_INTERVAL = 200; private final InputMethodManager mImmInHandler; - public SettingsPoolingHandler(final SetupWizardActivity ownerInstance, + public SettingsPoolingHandler(final SetupWizardActivity outerInstance, final InputMethodManager imm) { - super(ownerInstance); + super(outerInstance); mImmInHandler = imm; } @Override public void handleMessage(final Message msg) { - final SetupWizardActivity setupWizardActivity = getOwnerInstance(); + final SetupWizardActivity setupWizardActivity = getOuterInstance(); if (setupWizardActivity == null) { return; } diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java index c108b20ed..503b18b1b 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java @@ -428,7 +428,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService final String localeStr = locale.toString(); UserBinaryDictionary userDictionary = mUserDictionaries.get(localeStr); if (null == userDictionary) { - userDictionary = new SynchronouslyLoadedUserBinaryDictionary(this, locale, true); + userDictionary = new SynchronouslyLoadedUserBinaryDictionary(this, localeStr, true); mUserDictionaries.put(localeStr, userDictionary); } dictionaryCollection.addDictionary(userDictionary); diff --git a/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java b/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java index b7a5a4026..a0aed2829 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java @@ -49,7 +49,6 @@ public final class DictionaryPool extends LinkedBlockingQueue<DictAndKeyboard> { final static ArrayList<SuggestedWordInfo> noSuggestions = CollectionUtils.newArrayList(); private final static DictAndKeyboard dummyDict = new DictAndKeyboard( new Dictionary(Dictionary.TYPE_MAIN) { - // TODO: this dummy dictionary should be a singleton in the Dictionary class. @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java index 52012c846..acd47450b 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java @@ -66,8 +66,7 @@ public final class MoreSuggestions extends Keyboard { clearKeys(); mDivider = res.getDrawable(R.drawable.more_suggestions_divider); mDividerWidth = mDivider.getIntrinsicWidth(); - final float padding = res.getDimension( - R.dimen.config_more_suggestions_key_horizontal_padding); + final float padding = res.getDimension(R.dimen.more_suggestions_key_horizontal_padding); int row = 0; int index = fromIndex; @@ -76,7 +75,7 @@ public final class MoreSuggestions extends Keyboard { while (index < size) { final String word = suggestedWords.getWord(index); // TODO: Should take care of text x-scaling. - mWidths[index] = (int)(TypefaceUtils.getStringWidth(word, paint) + padding); + mWidths[index] = (int)(TypefaceUtils.getLabelWidth(word, paint) + padding); final int numColumn = index - rowStartIndex + 1; final int columnWidth = (maxWidth - mDividerWidth * (numColumn - 1)) / numColumn; diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java index 549ff0d9d..0ebe37782 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java @@ -54,7 +54,7 @@ public final class MoreSuggestionsView extends MoreKeysKeyboardView { public void adjustVerticalCorrectionForModalMode() { // Set vertical correction to zero (Reset more keys keyboard sliding allowance - // {@link R#dimen.config_more_keys_keyboard_slide_allowance}). + // {@link R#dimen.more_keys_keyboard_slide_allowance}). mKeyDetector.setKeyboard(getKeyboard(), -getPaddingLeft(), -getPaddingTop()); } diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index 72281e62c..faa5560e4 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -119,8 +119,7 @@ final class SuggestionStripLayoutHelper { mDividerWidth = dividerView.getMeasuredWidth(); final Resources res = wordView.getResources(); - mSuggestionsStripHeight = res.getDimensionPixelSize( - R.dimen.config_suggestions_strip_height); + mSuggestionsStripHeight = res.getDimensionPixelSize(R.dimen.suggestions_strip_height); final TypedArray a = context.obtainStyledAttributes(attrs, R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripView); @@ -146,17 +145,15 @@ final class SuggestionStripLayoutHelper { a.recycle(); mMoreSuggestionsHint = getMoreSuggestionsHint(res, - res.getDimension(R.dimen.config_more_suggestions_hint_text_size), - mColorAutoCorrect); + res.getDimension(R.dimen.more_suggestions_hint_text_size), mColorAutoCorrect); mCenterPositionInStrip = mSuggestionsCountInStrip / 2; // Assuming there are at least three suggestions. Also, note that the suggestions are // laid out according to script direction, so this is left of the center for LTR scripts // and right of the center for RTL scripts. mTypedWordPositionWhenAutocorrect = mCenterPositionInStrip - 1; mMoreSuggestionsBottomGap = res.getDimensionPixelOffset( - R.dimen.config_more_suggestions_bottom_gap); - mMoreSuggestionsRowHeight = res.getDimensionPixelSize( - R.dimen.config_more_suggestions_row_height); + R.dimen.more_suggestions_bottom_gap); + mMoreSuggestionsRowHeight = res.getDimensionPixelSize(R.dimen.more_suggestions_row_height); final LayoutInflater inflater = LayoutInflater.from(context); mWordToSaveView = (TextView)inflater.inflate(R.layout.suggestion_word, null); diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java index aa87affa2..75f17c559 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java @@ -112,7 +112,7 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick final Resources res = context.getResources(); mMoreSuggestionsModalTolerance = res.getDimensionPixelOffset( - R.dimen.config_more_suggestions_modal_tolerance); + R.dimen.more_suggestions_modal_tolerance); mMoreSuggestionsSlidingDetector = new GestureDetector( context, mMoreSuggestionsSlidingListener); } diff --git a/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java b/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java index ef1d0f42c..d87f6f3c4 100644 --- a/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java @@ -17,25 +17,21 @@ package com.android.inputmethod.latin.utils; import static com.android.inputmethod.latin.Constants.Subtype.KEYBOARD_MODE; -import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.EMOJI_CAPABLE; import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.IS_ADDITIONAL_SUBTYPE; import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.KEYBOARD_LAYOUT_SET; import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.UNTRANSLATABLE_STRING_IN_SUBTYPE_NAME; import android.os.Build; import android.text.TextUtils; -import android.util.Log; import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.compat.InputMethodSubtypeCompatUtils; +import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; import java.util.ArrayList; -import java.util.Arrays; public final class AdditionalSubtypeUtils { - private static final String TAG = AdditionalSubtypeUtils.class.getSimpleName(); - private static final InputMethodSubtype[] EMPTY_SUBTYPE_ARRAY = new InputMethodSubtype[0]; private AdditionalSubtypeUtils() { @@ -47,11 +43,6 @@ public final class AdditionalSubtypeUtils { } private static final String LOCALE_AND_LAYOUT_SEPARATOR = ":"; - private static final int INDEX_OF_LOCALE = 0; - private static final int INDEX_OF_KEYBOARD_LAYOUT = 1; - private static final int INDEX_OF_EXTRA_VALUE = 2; - private static final int LENGTH_WITHOUT_EXTRA_VALUE = (INDEX_OF_KEYBOARD_LAYOUT + 1); - private static final int LENGTH_WITH_EXTRA_VALUE = (INDEX_OF_EXTRA_VALUE + 1); private static final String PREF_SUBTYPE_SEPARATOR = ";"; public static InputMethodSubtype createAdditionalSubtype(final String localeString, @@ -88,6 +79,17 @@ public final class AdditionalSubtypeUtils { : basePrefSubtype + LOCALE_AND_LAYOUT_SEPARATOR + extraValue; } + public static InputMethodSubtype createAdditionalSubtype(final String prefSubtype) { + final String elems[] = prefSubtype.split(LOCALE_AND_LAYOUT_SEPARATOR); + if (elems.length < 2 || elems.length > 3) { + throw new RuntimeException("Unknown additional subtype specified: " + prefSubtype); + } + final String localeString = elems[0]; + final String keyboardLayoutSetName = elems[1]; + final String extraValue = (elems.length == 3) ? elems[2] : null; + return createAdditionalSubtype(localeString, keyboardLayoutSetName, extraValue); + } + public static InputMethodSubtype[] createAdditionalSubtypesArray(final String prefSubtypes) { if (TextUtils.isEmpty(prefSubtypes)) { return EMPTY_SUBTYPE_ARRAY; @@ -96,19 +98,7 @@ public final class AdditionalSubtypeUtils { final ArrayList<InputMethodSubtype> subtypesList = CollectionUtils.newArrayList(prefSubtypeArray.length); for (final String prefSubtype : prefSubtypeArray) { - final String elems[] = prefSubtype.split(LOCALE_AND_LAYOUT_SEPARATOR); - if (elems.length != LENGTH_WITHOUT_EXTRA_VALUE - && elems.length != LENGTH_WITH_EXTRA_VALUE) { - Log.w(TAG, "Unknown additional subtype specified: " + prefSubtype + " in " - + prefSubtypes); - continue; - } - final String localeString = elems[INDEX_OF_LOCALE]; - final String keyboardLayoutSetName = elems[INDEX_OF_KEYBOARD_LAYOUT]; - final String extraValue = (elems.length == LENGTH_WITH_EXTRA_VALUE) - ? elems[INDEX_OF_EXTRA_VALUE] : null; - final InputMethodSubtype subtype = createAdditionalSubtype( - localeString, keyboardLayoutSetName, extraValue); + final InputMethodSubtype subtype = createAdditionalSubtype(prefSubtype); if (subtype.getNameResId() == SubtypeLocaleUtils.UNKNOWN_KEYBOARD_LAYOUT) { // Skip unknown keyboard layout subtype. This may happen when predefined keyboard // layout has been removed. @@ -147,36 +137,31 @@ public final class AdditionalSubtypeUtils { return sb.toString(); } - private static InputMethodSubtype buildInputMethodSubtype(final int nameId, - final String localeString, final String layoutExtraValue, - final String additionalSubtypeExtraValue) { - // To preserve additional subtype settings and user's selection across OS updates, subtype - // id shouldn't be changed. New attributes, such as emojiCapable, are carefully excluded - // from the calculation of subtype id. - final String compatibleExtraValue = StringUtils.joinCommaSplittableText( - layoutExtraValue, additionalSubtypeExtraValue); - final int compatibleSubtypeId = getInputMethodSubtypeId(localeString, compatibleExtraValue); + private static InputMethodSubtype buildInputMethodSubtype(int nameId, String localeString, + String layoutExtraValue, String additionalSubtypeExtraValue) { + // CAVEAT! If you want to change subtypeId after changing the extra values, + // you must change "getInputMethodSubtypeId". But it will remove the additional keyboard + // from the current users. So, you should be really careful to change it. + final int subtypeId = getInputMethodSubtypeId(nameId, localeString, layoutExtraValue, + additionalSubtypeExtraValue); final String extraValue; - // Color Emoji is supported from KitKat. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { - extraValue = StringUtils.appendToCommaSplittableTextIfNotExists( - EMOJI_CAPABLE, compatibleExtraValue); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { + extraValue = layoutExtraValue + "," + additionalSubtypeExtraValue + + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE + + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE; } else { - extraValue = compatibleExtraValue; + extraValue = layoutExtraValue + "," + additionalSubtypeExtraValue; } return InputMethodSubtypeCompatUtils.newInputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, localeString, KEYBOARD_MODE, extraValue, - false, false, compatibleSubtypeId); + false, false, subtypeId); } - private static int getInputMethodSubtypeId(final String localeString, final String extraValue) { - // From the compatibility point of view, the calculation of subtype id has been copied from - // {@link InputMethodSubtype} of JellyBean MR2. - return Arrays.hashCode(new Object[] { - localeString, - KEYBOARD_MODE, - extraValue, - false /* isAuxiliary */, - false /* overrideImplicitlyEnabledSubtype */ }); + private static int getInputMethodSubtypeId(int nameId, String localeString, + String layoutExtraValue, String additionalSubtypeExtraValue) { + // TODO: Use InputMethodSubtypeBuilder once we use SDK version 19. + return (new InputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, + localeString, KEYBOARD_MODE, layoutExtraValue + "," + additionalSubtypeExtraValue, + false, false)).hashCode(); } } diff --git a/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java b/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java index e521ec807..08a2a8c5a 100644 --- a/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java @@ -62,22 +62,4 @@ public final class ApplicationUtils { } return ""; } - - /** - * A utility method to get the application's PackageInfo.versionCode - * @return the application's PackageInfo.versionCode - */ - public static int getVersionCode(final Context context) { - try { - if (context == null) { - return 0; - } - final String packageName = context.getPackageName(); - final PackageInfo info = context.getPackageManager().getPackageInfo(packageName, 0); - return info.versionCode; - } catch (final NameNotFoundException e) { - Log.e(TAG, "Could not find version info.", e); - } - return 0; - } } diff --git a/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java b/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java index d12aad639..c2e97a36f 100644 --- a/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java +++ b/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java @@ -20,7 +20,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; /** - * This class is a holder of the result of an asynchronous computation. + * This class is a holder of a result of asynchronous computation. * * @param <E> the type of the result. */ @@ -36,9 +36,9 @@ public class AsyncResultHolder<E> { } /** - * Sets the result value of this holder. + * Sets the result value to this holder. * - * @param result the value to set. + * @param result the value which is set. */ public void set(final E result) { synchronized(mLock) { @@ -54,12 +54,12 @@ public class AsyncResultHolder<E> { * Causes the current thread to wait unless the value is set or the specified time is elapsed. * * @param defaultValue the default value. - * @param timeOut the maximum time to wait. - * @return if the result is set before the time limit then the result, otherwise defaultValue. + * @param timeOut the time to wait. + * @return if the result is set until the time limit then the result, otherwise defaultValue. */ public E get(final E defaultValue, final long timeOut) { try { - if (mLatch.await(timeOut, TimeUnit.MILLISECONDS)) { + if(mLatch.await(timeOut, TimeUnit.MILLISECONDS)) { return mResult; } else { return defaultValue; diff --git a/java/src/com/android/inputmethod/latin/utils/FileUtils.java b/java/src/com/android/inputmethod/latin/utils/FileUtils.java deleted file mode 100644 index 83c1e7c4d..000000000 --- a/java/src/com/android/inputmethod/latin/utils/FileUtils.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.android.inputmethod.latin.utils; - -import java.io.File; - -/** - * A simple class to help with removing directories recursively. - */ -public class FileUtils { - public static boolean deleteRecursively(final File path) { - if (path.isDirectory()) { - for (final File child : path.listFiles()) { - deleteRecursively(child); - } - } - return path.delete(); - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/JsonUtils.java b/java/src/com/android/inputmethod/latin/utils/JsonUtils.java deleted file mode 100644 index 764ef72ce..000000000 --- a/java/src/com/android/inputmethod/latin/utils/JsonUtils.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.util.JsonReader; -import android.util.JsonWriter; -import android.util.Log; - -import java.io.Closeable; -import java.io.IOException; -import java.io.StringReader; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public final class JsonUtils { - private static final String TAG = JsonUtils.class.getSimpleName(); - - private static final String INTEGER_CLASS_NAME = Integer.class.getSimpleName(); - private static final String STRING_CLASS_NAME = String.class.getSimpleName(); - - private static final String EMPTY_STRING = ""; - - public static List<Object> jsonStrToList(final String s) { - final ArrayList<Object> list = CollectionUtils.newArrayList(); - final JsonReader reader = new JsonReader(new StringReader(s)); - try { - reader.beginArray(); - while (reader.hasNext()) { - reader.beginObject(); - while (reader.hasNext()) { - final String name = reader.nextName(); - if (name.equals(INTEGER_CLASS_NAME)) { - list.add(reader.nextInt()); - } else if (name.equals(STRING_CLASS_NAME)) { - list.add(reader.nextString()); - } else { - Log.w(TAG, "Invalid name: " + name); - reader.skipValue(); - } - } - reader.endObject(); - } - reader.endArray(); - return list; - } catch (final IOException e) { - } finally { - close(reader); - } - return Collections.<Object>emptyList(); - } - - public static String listToJsonStr(final List<Object> list) { - if (list == null || list.isEmpty()) { - return EMPTY_STRING; - } - final StringWriter sw = new StringWriter(); - final JsonWriter writer = new JsonWriter(sw); - try { - writer.beginArray(); - for (final Object o : list) { - writer.beginObject(); - if (o instanceof Integer) { - writer.name(INTEGER_CLASS_NAME).value((Integer)o); - } else if (o instanceof String) { - writer.name(STRING_CLASS_NAME).value((String)o); - } - writer.endObject(); - } - writer.endArray(); - return sw.toString(); - } catch (final IOException e) { - } finally { - close(writer); - } - return EMPTY_STRING; - } - - private static void close(final Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (final IOException e) { - // Ignore - } - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java b/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java index d14ba508b..e958a7e71 100644 --- a/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/LatinImeLoggerUtils.java @@ -35,7 +35,7 @@ public final class LatinImeLoggerUtils { public static void onSeparator(final int code, final int x, final int y) { // Helper method to log a single code point separator // TODO: cache this mapping of a code point to a string in a sparse array in StringUtils - onSeparator(StringUtils.newSingleCodePointString(code), x, y); + onSeparator(new String(new int[]{code}, 0, 1), x, y); } public static void onSeparator(final String separator, final int x, final int y) { diff --git a/java/src/com/android/inputmethod/latin/utils/LocaleUtils.java b/java/src/com/android/inputmethod/latin/utils/LocaleUtils.java index 0c55484b4..22045aa38 100644 --- a/java/src/com/android/inputmethod/latin/utils/LocaleUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/LocaleUtils.java @@ -30,6 +30,9 @@ import java.util.Locale; * dictionary pack. */ public final class LocaleUtils { + private static final HashMap<String, Long> EMPTY_LT_HASH_MAP = CollectionUtils.newHashMap(); + private static final String LOCALE_AND_TIME_STR_SEPARATER = ","; + private LocaleUtils() { // Intentional empty constructor for utility class. } @@ -165,14 +168,12 @@ public final class LocaleUtils { * Creates a locale from a string specification. */ public static Locale constructLocaleFromString(final String localeStr) { - if (localeStr == null) { + if (localeStr == null) return null; - } synchronized (sLocaleCache) { - Locale retval = sLocaleCache.get(localeStr); - if (retval != null) { - return retval; - } + if (sLocaleCache.containsKey(localeStr)) + return sLocaleCache.get(localeStr); + Locale retval = null; String[] localeParams = localeStr.split("_", 3); if (localeParams.length == 1) { retval = new Locale(localeParams[0]); @@ -187,4 +188,38 @@ public final class LocaleUtils { return retval; } } + + public static HashMap<String, Long> localeAndTimeStrToHashMap(String str) { + if (TextUtils.isEmpty(str)) { + return EMPTY_LT_HASH_MAP; + } + final String[] ss = str.split(LOCALE_AND_TIME_STR_SEPARATER); + final int N = ss.length; + if (N < 2 || N % 2 != 0) { + return EMPTY_LT_HASH_MAP; + } + final HashMap<String, Long> retval = CollectionUtils.newHashMap(); + for (int i = 0; i < N / 2; ++i) { + final String localeStr = ss[i * 2]; + final long time = Long.valueOf(ss[i * 2 + 1]); + retval.put(localeStr, time); + } + return retval; + } + + public static String localeAndTimeHashMapToStr(HashMap<String, Long> map) { + if (map == null || map.isEmpty()) { + return ""; + } + final StringBuilder builder = new StringBuilder(); + for (String localeStr : map.keySet()) { + if (builder.length() > 0) { + builder.append(LOCALE_AND_TIME_STR_SEPARATER); + } + final Long time = map.get(localeStr); + builder.append(localeStr).append(LOCALE_AND_TIME_STR_SEPARATER); + builder.append(String.valueOf(time)); + } + return builder.toString(); + } } diff --git a/java/src/com/android/inputmethod/latin/utils/ResourceUtils.java b/java/src/com/android/inputmethod/latin/utils/ResourceUtils.java index deb28a08d..22c92446a 100644 --- a/java/src/com/android/inputmethod/latin/utils/ResourceUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/ResourceUtils.java @@ -227,19 +227,19 @@ public final class ResourceUtils { final String keyboardHeightString = getDeviceOverrideValue(res, R.array.keyboard_heights); final float keyboardHeight; if (TextUtils.isEmpty(keyboardHeightString)) { - keyboardHeight = res.getDimension(R.dimen.config_default_keyboard_height); + keyboardHeight = res.getDimension(R.dimen.keyboardHeight); } else { keyboardHeight = Float.parseFloat(keyboardHeightString) * dm.density; } final float maxKeyboardHeight = res.getFraction( - R.fraction.config_max_keyboard_height, dm.heightPixels, dm.heightPixels); + R.fraction.maxKeyboardHeight, dm.heightPixels, dm.heightPixels); float minKeyboardHeight = res.getFraction( - R.fraction.config_min_keyboard_height, dm.heightPixels, dm.heightPixels); + R.fraction.minKeyboardHeight, dm.heightPixels, dm.heightPixels); if (minKeyboardHeight < 0.0f) { // Specified fraction was negative, so it should be calculated against display // width. minKeyboardHeight = -res.getFraction( - R.fraction.config_min_keyboard_height, dm.widthPixels, dm.widthPixels); + R.fraction.minKeyboardHeight, dm.widthPixels, dm.widthPixels); } // Keyboard height will not exceed maxKeyboardHeight and will not be less than // minKeyboardHeight. diff --git a/java/src/com/android/inputmethod/latin/utils/LeakGuardHandlerWrapper.java b/java/src/com/android/inputmethod/latin/utils/StaticInnerHandlerWrapper.java index 8469c87b0..44e5d17b4 100644 --- a/java/src/com/android/inputmethod/latin/utils/LeakGuardHandlerWrapper.java +++ b/java/src/com/android/inputmethod/latin/utils/StaticInnerHandlerWrapper.java @@ -21,22 +21,22 @@ import android.os.Looper; import java.lang.ref.WeakReference; -public class LeakGuardHandlerWrapper<T> extends Handler { - private final WeakReference<T> mOwnerInstanceRef; +public class StaticInnerHandlerWrapper<T> extends Handler { + private final WeakReference<T> mOuterInstanceRef; - public LeakGuardHandlerWrapper(final T ownerInstance) { - this(ownerInstance, Looper.myLooper()); + public StaticInnerHandlerWrapper(final T outerInstance) { + this(outerInstance, Looper.myLooper()); } - public LeakGuardHandlerWrapper(final T ownerInstance, final Looper looper) { + public StaticInnerHandlerWrapper(final T outerInstance, final Looper looper) { super(looper); - if (ownerInstance == null) { - throw new NullPointerException("ownerInstance is null"); + if (outerInstance == null) { + throw new NullPointerException("outerInstance is null"); } - mOwnerInstanceRef = new WeakReference<T>(ownerInstance); + mOuterInstanceRef = new WeakReference<T>(outerInstance); } - public T getOwnerInstance() { - return mOwnerInstanceRef.get(); + public T getOuterInstance() { + return mOuterInstanceRef.get(); } } diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index df420417d..a36548392 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -16,15 +16,20 @@ package com.android.inputmethod.latin.utils; -import android.text.TextUtils; -import android.util.Log; - import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.settings.SettingsValues; +import android.text.TextUtils; +import android.util.JsonReader; +import android.util.JsonWriter; +import android.util.Log; + import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; @@ -34,8 +39,6 @@ public final class StringUtils { public static final int CAPITALIZE_FIRST = 1; // First only public static final int CAPITALIZE_ALL = 2; // All caps - private static final String EMPTY_STRING = ""; - private StringUtils() { // This utility class is not publicly instantiable. } @@ -77,20 +80,6 @@ public final class StringUtils { return containsInArray(text, extraValues.split(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT)); } - public static String joinCommaSplittableText(final String head, final String tail) { - if (TextUtils.isEmpty(head) && TextUtils.isEmpty(tail)) { - return EMPTY_STRING; - } - // Here either head or tail is not null. - if (TextUtils.isEmpty(head)) { - return tail; - } - if (TextUtils.isEmpty(tail)) { - return head; - } - return head + SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT + tail; - } - public static String appendToCommaSplittableTextIfNotExists(final String text, final String extraValues) { if (TextUtils.isEmpty(extraValues)) { @@ -105,7 +94,7 @@ public final class StringUtils { public static String removeFromCommaSplittableTextIfExists(final String text, final String extraValues) { if (TextUtils.isEmpty(extraValues)) { - return EMPTY_STRING; + return ""; } final String[] elements = extraValues.split(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT); if (!containsInArray(text, elements)) { @@ -378,7 +367,7 @@ public final class StringUtils { return false; } - public static boolean isEmptyStringOrWhiteSpaces(final String s) { + public static boolean isEmptyStringOrWhiteSpaces(String s) { final int N = codePointCount(s); for (int i = 0; i < N; ++i) { if (!Character.isWhitespace(s.codePointAt(i))) { @@ -389,9 +378,9 @@ public final class StringUtils { } @UsedForTesting - public static String byteArrayToHexString(final byte[] bytes) { + public static String byteArrayToHexString(byte[] bytes) { if (bytes == null || bytes.length == 0) { - return EMPTY_STRING; + return ""; } final StringBuilder sb = new StringBuilder(); for (byte b : bytes) { @@ -404,7 +393,7 @@ public final class StringUtils { * Convert hex string to byte array. The string length must be an even number. */ @UsedForTesting - public static byte[] hexStringToByteArray(final String hexString) { + public static byte[] hexStringToByteArray(String hexString) { if (TextUtils.isEmpty(hexString)) { return null; } @@ -420,4 +409,67 @@ public final class StringUtils { } return bytes; } + + public static List<Object> jsonStrToList(String s) { + final ArrayList<Object> retval = CollectionUtils.newArrayList(); + final JsonReader reader = new JsonReader(new StringReader(s)); + try { + reader.beginArray(); + while(reader.hasNext()) { + reader.beginObject(); + while (reader.hasNext()) { + final String name = reader.nextName(); + if (name.equals(Integer.class.getSimpleName())) { + retval.add(reader.nextInt()); + } else if (name.equals(String.class.getSimpleName())) { + retval.add(reader.nextString()); + } else { + Log.w(TAG, "Invalid name: " + name); + reader.skipValue(); + } + } + reader.endObject(); + } + reader.endArray(); + return retval; + } catch (IOException e) { + } finally { + try { + reader.close(); + } catch (IOException e) { + } + } + return Collections.<Object>emptyList(); + } + + public static String listToJsonStr(List<Object> list) { + if (list == null || list.isEmpty()) { + return ""; + } + final StringWriter sw = new StringWriter(); + final JsonWriter writer = new JsonWriter(sw); + try { + writer.beginArray(); + for (final Object o : list) { + writer.beginObject(); + if (o instanceof Integer) { + writer.name(Integer.class.getSimpleName()).value((Integer)o); + } else if (o instanceof String) { + writer.name(String.class.getSimpleName()).value((String)o); + } + writer.endObject(); + } + writer.endArray(); + return sw.toString(); + } catch (IOException e) { + } finally { + try { + if (writer != null) { + writer.close(); + } + } catch (IOException e) { + } + } + return ""; + } } diff --git a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java index fdbe81ab6..102a41b4e 100644 --- a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java @@ -197,9 +197,7 @@ public final class SubtypeLocaleUtils { // es_US spanish F Español (EE.UU.) exception // fr azerty F Français // fr_CA qwerty F Français (Canada) - // fr_CH swiss F Français (Suisse) // de qwertz F Deutsch - // de_CH swiss T Deutsch (Schweiz) // zz qwerty F No language (QWERTY) in system locale // fr qwertz T Français (QWERTZ) // de qwerty T Deutsch (QWERTY) @@ -300,9 +298,7 @@ public final class SubtypeLocaleUtils { // es_US spanish F Es Español Español (EE.UU.) exception // fr azerty F Fr Français Français // fr_CA qwerty F Fr Français Français (Canada) - // fr_CH swiss F Fr Français Français (Suisse) // de qwertz F De Deutsch Deutsch - // de_CH swiss T De Deutsch Deutsch (Schweiz) // zz qwerty F QWERTY QWERTY // fr qwertz T Fr Français Français // de qwerty T De Deutsch Deutsch diff --git a/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java b/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java index 087a7f255..47ea1ea75 100644 --- a/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java @@ -22,9 +22,6 @@ import android.graphics.Typeface; import android.util.SparseArray; public final class TypefaceUtils { - private static final char[] KEY_LABEL_REFERENCE_CHAR = { 'M' }; - private static final char[] KEY_NUMERIC_HINT_LABEL_REFERENCE_CHAR = { '8' }; - private TypefaceUtils() { // This utility class is not publicly instantiable. } @@ -34,7 +31,7 @@ public final class TypefaceUtils { // Working variable for the following method. private static final Rect sTextHeightBounds = new Rect(); - private static float getCharHeight(final char[] referenceChar, final Paint paint) { + public static float getCharHeight(final char[] referenceChar, final Paint paint) { final int key = getCharGeometryCacheKey(referenceChar[0], paint); synchronized (sTextHeightCache) { final Float cachedValue = sTextHeightCache.get(key); @@ -54,7 +51,7 @@ public final class TypefaceUtils { // Working variable for the following method. private static final Rect sTextWidthBounds = new Rect(); - private static float getCharWidth(final char[] referenceChar, final Paint paint) { + public static float getCharWidth(final char[] referenceChar, final Paint paint) { final int key = getCharGeometryCacheKey(referenceChar[0], paint); synchronized (sTextWidthCache) { final Float cachedValue = sTextWidthCache.get(key); @@ -69,6 +66,11 @@ public final class TypefaceUtils { } } + public static float getStringWidth(final String string, final Paint paint) { + paint.getTextBounds(string, 0, string.length(), sTextWidthBounds); + return sTextWidthBounds.width(); + } + private static int getCharGeometryCacheKey(final char referenceChar, final Paint paint) { final int labelSize = (int)paint.getTextSize(); final Typeface face = paint.getTypeface(); @@ -84,25 +86,9 @@ public final class TypefaceUtils { } } - public static float getReferenceCharHeight(final Paint paint) { - return getCharHeight(KEY_LABEL_REFERENCE_CHAR, paint); - } - - public static float getReferenceCharWidth(final Paint paint) { - return getCharWidth(KEY_LABEL_REFERENCE_CHAR, paint); - } - - public static float getReferenceDigitWidth(final Paint paint) { - return getCharWidth(KEY_NUMERIC_HINT_LABEL_REFERENCE_CHAR, paint); - } - - // Working variable for the following method. - private static final Rect sStringWidthBounds = new Rect(); - - public static float getStringWidth(final String string, final Paint paint) { - synchronized (sStringWidthBounds) { - paint.getTextBounds(string, 0, string.length(), sStringWidthBounds); - return sStringWidthBounds.width(); - } + public static float getLabelWidth(final String label, final Paint paint) { + final Rect textBounds = new Rect(); + paint.getTextBounds(label, 0, label.length(), textBounds); + return textBounds.width(); } } diff --git a/java/src/com/android/inputmethod/latin/utils/UnigramProperty.java b/java/src/com/android/inputmethod/latin/utils/UnigramProperty.java deleted file mode 100644 index 4feee4393..000000000 --- a/java/src/com/android/inputmethod/latin/utils/UnigramProperty.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.android.inputmethod.latin.utils; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.BinaryDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; - -import java.util.ArrayList; - -// This has information that belong to a unigram. This class has some detailed attributes such as -// historical information but they have to be checked only for testing purpose. -@UsedForTesting -public class UnigramProperty { - public final String mCodePoints; - public final boolean mIsNotAWord; - public final boolean mIsBlacklisted; - public final boolean mHasBigrams; - public final boolean mHasShortcuts; - public final int mProbability; - // mTimestamp, mLevel and mCount are historical info. These values are depend on the - // implementation in native code; thus, we must not use them and have any assumptions about - // them except for tests. - public final int mTimestamp; - public final int mLevel; - public final int mCount; - public final ArrayList<WeightedString> mShortcutTargets = CollectionUtils.newArrayList(); - - private static int getCodePointCount(final int[] codePoints) { - for (int i = 0; i < codePoints.length; i++) { - if (codePoints[i] == 0) { - return i; - } - } - return codePoints.length; - } - - // This represents invalid unigram when the probability is BinaryDictionary.NOT_A_PROBABILITY. - public UnigramProperty(final int[] codePoints, final boolean isNotAWord, - final boolean isBlacklisted, final boolean hasBigram, - final boolean hasShortcuts, final int probability, final int timestamp, - final int level, final int count, final ArrayList<int[]> shortcutTargets, - final ArrayList<Integer> shortcutProbabilities) { - mCodePoints = new String(codePoints, 0 /* offset */, getCodePointCount(codePoints)); - mIsNotAWord = isNotAWord; - mIsBlacklisted = isBlacklisted; - mHasBigrams = hasBigram; - mHasShortcuts = hasShortcuts; - mProbability = probability; - mTimestamp = timestamp; - mLevel = level; - mCount = count; - final int shortcutTargetCount = shortcutTargets.size(); - for (int i = 0; i < shortcutTargetCount; i++) { - final int[] shortcutTargetCodePointArray = shortcutTargets.get(i); - final String shortcutTargetString = new String(shortcutTargetCodePointArray, - 0 /* offset */, getCodePointCount(shortcutTargetCodePointArray)); - mShortcutTargets.add( - new WeightedString(shortcutTargetString, shortcutProbabilities.get(i))); - } - } - - @UsedForTesting - public boolean isValid() { - return mProbability != BinaryDictionary.NOT_A_PROBABILITY; - } -}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index db628fe18..635afe7cc 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -70,11 +70,10 @@ public final class UserHistoryDictIOUtils { /** * Writes dictionary to file. */ - @UsedForTesting public static void writeDictionary(final DictEncoder dictEncoder, final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams, - final FormatOptions formatOptions, final HashMap<String, String> options) { - final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams, options); + final FormatOptions formatOptions) { + final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); fusionDict.addOptionAttribute(USES_FORGETTING_CURVE_KEY, USES_FORGETTING_CURVE_VALUE); fusionDict.addOptionAttribute(LAST_UPDATED_TIME_KEY, String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); @@ -92,10 +91,11 @@ public final class UserHistoryDictIOUtils { * Constructs a new FusionDictionary from BigramDictionaryInterface. */ @UsedForTesting - static FusionDictionary constructFusionDictionary(final BigramDictionaryInterface dict, - final UserHistoryDictionaryBigramList bigrams, final HashMap<String, String> options) { + static FusionDictionary constructFusionDictionary( + final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams) { final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(options)); + new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, + false)); int profTotal = 0; for (final String word1 : bigrams.keySet()) { final HashMap<String, Byte> word1Bigrams = bigrams.getBigrams(word1); diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java index 677035ed6..1992b2f5d 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java @@ -20,9 +20,6 @@ import android.util.Log; import java.util.concurrent.TimeUnit; -import com.android.inputmethod.annotations.UsedForTesting; - -@UsedForTesting public final class UserHistoryForgettingCurveUtils { private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); private static final boolean DEBUG = false; @@ -121,22 +118,18 @@ public final class UserHistoryForgettingCurveUtils { } } - @UsedForTesting /* package */ static int fcToElapsedTime(byte fc) { return fc & 0x0F; } - @UsedForTesting /* package */ static int fcToCount(byte fc) { return (fc >> 4) & 0x03; } - @UsedForTesting /* package */ static int fcToLevel(byte fc) { return (fc >> 6) & 0x03; } - @UsedForTesting private static int calcFreq(int elapsedTime, int count, int level) { if (level <= 0) { // Reserved words, just return -1 @@ -165,7 +158,6 @@ public final class UserHistoryForgettingCurveUtils { return calcFreq(elapsedTime, count, level); } - @UsedForTesting public static byte pushElapsedTime(byte fc) { int elapsedTime = fcToElapsedTime(fc); int count = fcToCount(fc); @@ -181,7 +173,6 @@ public final class UserHistoryForgettingCurveUtils { return calcFc(elapsedTime, count, level); } - @UsedForTesting public static byte pushCount(byte fc, boolean isValid) { final int elapsedTime = fcToElapsedTime(fc); int count = fcToCount(fc); |