diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin')
25 files changed, 920 insertions, 363 deletions
diff --git a/java/src/com/android/inputmethod/latin/AdditionalSubtype.java b/java/src/com/android/inputmethod/latin/AdditionalSubtype.java index ffdbfbb67..f8f1395b3 100644 --- a/java/src/com/android/inputmethod/latin/AdditionalSubtype.java +++ b/java/src/com/android/inputmethod/latin/AdditionalSubtype.java @@ -28,6 +28,7 @@ import android.view.inputmethod.InputMethodSubtype; import java.util.ArrayList; public class AdditionalSubtype { + private static final InputMethodSubtype[] EMPTY_SUBTYPE_ARRAY = new InputMethodSubtype[0]; private AdditionalSubtype() { // This utility class is not publicly instantiable. @@ -86,7 +87,7 @@ public class AdditionalSubtype { public static InputMethodSubtype[] createAdditionalSubtypesArray(String prefSubtypes) { if (TextUtils.isEmpty(prefSubtypes)) { - return null; + return EMPTY_SUBTYPE_ARRAY; } final String[] prefSubtypeArray = prefSubtypes.split(PREF_SUBTYPE_SEPARATOR); final ArrayList<InputMethodSubtype> subtypesList = diff --git a/java/src/com/android/inputmethod/latin/AutoCorrection.java b/java/src/com/android/inputmethod/latin/AutoCorrection.java index da1936aef..e0452483c 100644 --- a/java/src/com/android/inputmethod/latin/AutoCorrection.java +++ b/java/src/com/android/inputmethod/latin/AutoCorrection.java @@ -22,7 +22,7 @@ import android.text.TextUtils; import android.util.Log; import java.util.ArrayList; -import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; public class AutoCorrection { private static final boolean DBG = LatinImeLogger.sDBG; @@ -33,10 +33,10 @@ public class AutoCorrection { } public static CharSequence computeAutoCorrectionWord( - HashMap<String, Dictionary> dictionaries, - WordComposer wordComposer, ArrayList<SuggestedWordInfo> suggestions, - CharSequence consideredWord, float autoCorrectionThreshold, - CharSequence whitelistedWord) { + final ConcurrentHashMap<String, Dictionary> dictionaries, + final WordComposer wordComposer, final ArrayList<SuggestedWordInfo> suggestions, + final CharSequence consideredWord, final float autoCorrectionThreshold, + final CharSequence whitelistedWord) { if (hasAutoCorrectionForWhitelistedWord(whitelistedWord)) { return whitelistedWord; } else if (hasAutoCorrectionForConsideredWord( @@ -49,8 +49,8 @@ public class AutoCorrection { return null; } - public static boolean isValidWord( - HashMap<String, Dictionary> dictionaries, CharSequence word, boolean ignoreCase) { + public static boolean isValidWord(final ConcurrentHashMap<String, Dictionary> dictionaries, + CharSequence word, boolean ignoreCase) { if (TextUtils.isEmpty(word)) { return false; } @@ -74,8 +74,27 @@ public class AutoCorrection { return false; } + public static int getMaxFrequency(final ConcurrentHashMap<String, Dictionary> dictionaries, + CharSequence word) { + if (TextUtils.isEmpty(word)) { + return Dictionary.NOT_A_PROBABILITY; + } + int maxFreq = -1; + for (final String key : dictionaries.keySet()) { + if (key.equals(Suggest.DICT_KEY_WHITELIST)) continue; + final Dictionary dictionary = dictionaries.get(key); + if (null == dictionary) continue; + final int tempFreq = dictionary.getFrequency(word); + if (tempFreq >= maxFreq) { + maxFreq = tempFreq; + } + } + return maxFreq; + } + public static boolean allowsToBeAutoCorrected( - HashMap<String, Dictionary> dictionaries, CharSequence word, boolean ignoreCase) { + final ConcurrentHashMap<String, Dictionary> dictionaries, + final CharSequence word, final boolean ignoreCase) { final WhitelistDictionary whitelistDictionary = (WhitelistDictionary)dictionaries.get(Suggest.DICT_KEY_WHITELIST); // If "word" is in the whitelist dictionary, it should not be auto corrected. @@ -91,8 +110,9 @@ public class AutoCorrection { } private static boolean hasAutoCorrectionForConsideredWord( - HashMap<String, Dictionary> dictionaries, WordComposer wordComposer, - ArrayList<SuggestedWordInfo> suggestions, CharSequence consideredWord) { + final ConcurrentHashMap<String, Dictionary> dictionaries, + final WordComposer wordComposer, final ArrayList<SuggestedWordInfo> suggestions, + final CharSequence consideredWord) { if (TextUtils.isEmpty(consideredWord)) return false; return wordComposer.size() > 1 && suggestions.size() > 0 && !allowsToBeAutoCorrected(dictionaries, consideredWord, false); diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index e18aee6ff..d0613bd72 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -84,7 +84,7 @@ public class BinaryDictionary extends Dictionary { private native long openNative(String sourceDir, long dictOffset, long dictSize, int typedLetterMultiplier, int fullWordMultiplier, int maxWordLength, int maxWords); private native void closeNative(long dict); - private native boolean isValidWordNative(long dict, int[] word, int wordLength); + private native int getFrequencyNative(long dict, int[] word, int wordLength); private native boolean isValidBigramNative(long dict, int[] word1, int[] word2); private native int getSuggestionsNative(long dict, long proximityInfo, int[] xCoordinates, int[] yCoordinates, int[] inputCodes, int codesSize, int[] prevWordForBigrams, @@ -201,9 +201,14 @@ public class BinaryDictionary extends Dictionary { @Override public boolean isValidWord(CharSequence word) { - if (word == null) return false; + return getFrequency(word) >= 0; + } + + @Override + public int getFrequency(CharSequence word) { + if (word == null) return -1; int[] chars = StringUtils.toCodePointArray(word.toString()); - return isValidWordNative(mNativeDict, chars, chars.length); + return getFrequencyNative(mNativeDict, chars, chars.length); } // TODO: Add a batch process version (isValidBigramMultiple?) to avoid excessive numbers of jni diff --git a/java/src/com/android/inputmethod/latin/ContactsDictionary.java b/java/src/com/android/inputmethod/latin/ContactsDictionary.java index 2f3395245..cbfbd0ec8 100644 --- a/java/src/com/android/inputmethod/latin/ContactsDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsDictionary.java @@ -34,6 +34,7 @@ import com.android.inputmethod.keyboard.Keyboard; * * @deprecated Use {@link ContactsBinaryDictionary}. */ +@Deprecated public class ContactsDictionary extends ExpandableDictionary { private static final String[] PROJECTION = { @@ -158,7 +159,7 @@ public class ContactsDictionary extends ExpandableDictionary { super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS); if (!TextUtils.isEmpty(prevWord)) { - super.setBigram(prevWord, word, + super.setBigramAndGetFrequency(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM); } prevWord = word; diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 1ec678f7f..7cd9bc2a8 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -31,9 +31,10 @@ public abstract class Dictionary { public static final int UNIGRAM = 0; public static final int BIGRAM = 1; + public static final int NOT_A_PROBABILITY = -1; /** * Interface to be implemented by classes requesting words to be fetched from the dictionary. - * @see #getWords(WordComposer, WordCallback, ProximityInfo) + * @see #getWords(WordComposer, CharSequence, WordCallback, ProximityInfo) */ public interface WordCallback { /** @@ -84,6 +85,10 @@ public abstract class Dictionary { */ abstract public boolean isValidWord(CharSequence word); + public int getFrequency(CharSequence word) { + return NOT_A_PROBABILITY; + } + /** * Compares the contents of the character array with the typed word and returns true if they * are the same. diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java index f3aa27a22..1a05fcd86 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java +++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java @@ -70,6 +70,18 @@ public class DictionaryCollection extends Dictionary { return false; } + @Override + public int getFrequency(CharSequence word) { + int maxFreq = -1; + for (int i = mDictionaries.size() - 1; i >= 0; --i) { + final int tempFreq = mDictionaries.get(i).getFrequency(word); + if (tempFreq >= maxFreq) { + maxFreq = tempFreq; + } + } + return maxFreq; + } + public boolean isEmpty() { return mDictionaries.isEmpty(); } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFactory.java b/java/src/com/android/inputmethod/latin/DictionaryFactory.java index 4cd1b3883..a22d73af7 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFactory.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFactory.java @@ -89,7 +89,6 @@ public class DictionaryFactory { /** * Initializes a dictionary from a raw resource file * @param context application context for reading resources - * @param resId the resource containing the raw binary dictionary * @param locale the locale to use for the resource * @return an initialized instance of BinaryDictionary */ diff --git a/java/src/com/android/inputmethod/latin/DictionaryPackInstallBroadcastReceiver.java b/java/src/com/android/inputmethod/latin/DictionaryPackInstallBroadcastReceiver.java index 9d30af84b..9c37d7673 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryPackInstallBroadcastReceiver.java +++ b/java/src/com/android/inputmethod/latin/DictionaryPackInstallBroadcastReceiver.java @@ -51,6 +51,8 @@ public class DictionaryPackInstallBroadcastReceiver extends BroadcastReceiver { if (null == packageUri) return; // No package name : we can't do anything final String packageName = packageUri.getSchemeSpecificPart(); if (null == packageName) return; + // TODO: do this in a more appropriate place + TargetApplicationGetter.removeApplicationInfoCache(packageName); final PackageInfo packageInfo; try { packageInfo = manager.getPackageInfo(packageName, PackageManager.GET_PROVIDERS); diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index 7a740b3f1..34a92fd30 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -21,6 +21,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.KeyDetector; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import java.util.ArrayList; import java.util.LinkedList; @@ -80,31 +81,72 @@ public class ExpandableDictionary extends Dictionary { } } - private static class NextWord { - public final Node mWord; - private int mFrequency; + protected interface NextWord { + public Node getWordNode(); + public int getFrequency(); + public ForgettingCurveParams getFcParams(); + public int notifyTypedAgainAndGetFrequency(); + } - public NextWord(Node word, int frequency) { + private static class NextStaticWord implements NextWord { + public final Node mWord; + private final int mFrequency; + public NextStaticWord(Node word, int frequency) { mWord = word; mFrequency = frequency; } + @Override + public Node getWordNode() { + return mWord; + } + + @Override public int getFrequency() { return mFrequency; } - public int setFrequency(int freq) { - mFrequency = freq; - return mFrequency; + @Override + public ForgettingCurveParams getFcParams() { + return null; } - public int addFrequency(int add) { - mFrequency += add; - if (mFrequency > BIGRAM_MAX_FREQUENCY) mFrequency = BIGRAM_MAX_FREQUENCY; + @Override + public int notifyTypedAgainAndGetFrequency() { return mFrequency; } } + private static class NextHistoryWord implements NextWord { + public final Node mWord; + public final ForgettingCurveParams mFcp; + + public NextHistoryWord(Node word, ForgettingCurveParams fcp) { + mWord = word; + mFcp = fcp; + } + + @Override + public Node getWordNode() { + return mWord; + } + + @Override + public int getFrequency() { + return mFcp.getFrequency(); + } + + @Override + public ForgettingCurveParams getFcParams() { + return mFcp; + } + + @Override + public int notifyTypedAgainAndGetFrequency() { + return mFcp.notifyTypedAgainAndGetFrequency(); + } + } + private NodeArray mRoots; private int[][] mCodes; @@ -183,7 +225,7 @@ public class ExpandableDictionary extends Dictionary { childNode.mShortcutOnly = isShortcutOnly; children.add(childNode); } - if (wordLength == depth + 1) { + if (wordLength == depth + 1 && shortcutTarget != null) { // Terminate this word childNode.mTerminal = true; if (isShortcutOnly) { @@ -221,7 +263,7 @@ public class ExpandableDictionary extends Dictionary { protected final void getWordsInner(final WordComposer codes, final CharSequence prevWordForBigrams, final WordCallback callback, - @SuppressWarnings("unused") final ProximityInfo proximityInfo) { + final ProximityInfo proximityInfo) { mInputLength = codes.size(); if (mCodes.length < mInputLength) mCodes = new int[mInputLength][]; final int[] xCoordinates = codes.getXCoordinates(); @@ -261,6 +303,28 @@ public class ExpandableDictionary extends Dictionary { return (node == null) ? false : !node.mShortcutOnly; } + protected boolean removeBigram(String word1, String word2) { + // Refer to addOrSetBigram() about word1.toLowerCase() + final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); + final Node secondWord = searchWord(mRoots, word2, 0, null); + LinkedList<NextWord> bigrams = firstWord.mNGrams; + NextWord bigramNode = null; + if (bigrams == null || bigrams.size() == 0) { + return false; + } else { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + bigramNode = nw; + break; + } + } + } + if (bigramNode == null) { + return false; + } + return bigrams.remove(bigramNode); + } + /** * Returns the word's frequency or -1 if not found */ @@ -270,6 +334,23 @@ public class ExpandableDictionary extends Dictionary { return (node == null) ? -1 : node.mFrequency; } + protected NextWord getBigramWord(String word1, String word2) { + // Refer to addOrSetBigram() about word1.toLowerCase() + final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); + final Node secondWord = searchWord(mRoots, word2, 0, null); + LinkedList<NextWord> bigrams = firstWord.mNGrams; + if (bigrams == null || bigrams.size() == 0) { + return null; + } else { + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + return nw; + } + } + } + return null; + } + private static int computeSkippedWordFinalFreq(int freq, int snr, int inputLength) { // The computation itself makes sense for >= 2, but the == 2 case returns 0 // anyway so we may as well test against 3 instead and return the constant @@ -287,7 +368,8 @@ public class ExpandableDictionary extends Dictionary { * @param word the word to insert, as an array of code points * @param depth the depth of the node in the tree * @param finalFreq the frequency for this word - * @return whether there is still space for more words. {@see Dictionary.WordCallback#addWord}. + * @return whether there is still space for more words. + * @see Dictionary.WordCallback#addWord(char[], int, int, int, int, int) */ private boolean addWordAndShortcutsFromNode(final Node node, final char[] word, final int depth, final int finalFreq, final WordCallback callback) { @@ -422,43 +504,45 @@ public class ExpandableDictionary extends Dictionary { } } - protected int setBigram(String word1, String word2, int frequency) { - return addOrSetBigram(word1, word2, frequency, false); + public int setBigramAndGetFrequency(String word1, String word2, int frequency) { + return setBigramAndGetFrequency(word1, word2, frequency, null /* unused */); } - protected int addBigram(String word1, String word2, int frequency) { - return addOrSetBigram(word1, word2, frequency, true); + public int setBigramAndGetFrequency(String word1, String word2, ForgettingCurveParams fcp) { + return setBigramAndGetFrequency(word1, word2, 0 /* unused */, fcp); } /** * Adds bigrams to the in-memory trie structure that is being used to retrieve any word * @param frequency frequency for this bigram * @param addFrequency if true, it adds to current frequency, else it overwrites the old value - * @return returns the final frequency + * @return returns the final bigram frequency */ - private int addOrSetBigram(String word1, String word2, int frequency, boolean addFrequency) { + private int setBigramAndGetFrequency( + String word1, String word2, int frequency, ForgettingCurveParams fcp) { // We don't want results to be different according to case of the looked up left hand side // word. We do want however to return the correct case for the right hand side. // So we want to squash the case of the left hand side, and preserve that of the right // hand side word. Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); Node secondWord = searchWord(mRoots, word2, 0, null); - LinkedList<NextWord> bigram = firstWord.mNGrams; - if (bigram == null || bigram.size() == 0) { + LinkedList<NextWord> bigrams = firstWord.mNGrams; + if (bigrams == null || bigrams.size() == 0) { firstWord.mNGrams = new LinkedList<NextWord>(); - bigram = firstWord.mNGrams; + bigrams = firstWord.mNGrams; } else { - for (NextWord nw : bigram) { - if (nw.mWord == secondWord) { - if (addFrequency) { - return nw.addFrequency(frequency); - } else { - return nw.setFrequency(frequency); - } + for (NextWord nw : bigrams) { + if (nw.getWordNode() == secondWord) { + return nw.notifyTypedAgainAndGetFrequency(); } } } - firstWord.mNGrams.add(new NextWord(secondWord, frequency)); + if (fcp != null) { + // history + firstWord.mNGrams.add(new NextHistoryWord(secondWord, fcp)); + } else { + firstWord.mNGrams.add(new NextStaticWord(secondWord, frequency)); + } return frequency; } @@ -557,7 +641,7 @@ public class ExpandableDictionary extends Dictionary { Node node; int freq; for (NextWord nextWord : terminalNodes) { - node = nextWord.mWord; + node = nextWord.getWordNode(); freq = nextWord.getFrequency(); int index = BinaryDictionary.MAX_WORD_LENGTH; do { @@ -566,8 +650,10 @@ public class ExpandableDictionary extends Dictionary { node = node.mParent; } while (node != null); - callback.addWord(mLookedUpString, index, BinaryDictionary.MAX_WORD_LENGTH - index, - freq, mDicTypeId, Dictionary.BIGRAM); + if (freq >= 0) { + callback.addWord(mLookedUpString, index, BinaryDictionary.MAX_WORD_LENGTH - index, + freq, mDicTypeId, Dictionary.BIGRAM); + } } } @@ -639,167 +725,167 @@ public class ExpandableDictionary extends Dictionary { * is combined. */ private static final char BASE_CHARS[] = { - 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, - 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, - 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, - 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, - 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, - 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, - 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, - 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, - 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, - 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, - 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, - 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, - 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, - 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, - 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, - 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, - 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, - 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, - 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, - 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0020, - 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7, - 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf, - 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x00c6, 0x0043, - 0x0045, 0x0045, 0x0045, 0x0045, 0x0049, 0x0049, 0x0049, 0x0049, - 0x00d0, 0x004e, 0x004f, 0x004f, 0x004f, 0x004f, 0x004f, 0x00d7, + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x0020, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x0020, 0x00a9, 0x0061, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x0020, + 0x00b0, 0x00b1, 0x0032, 0x0033, 0x0020, 0x03bc, 0x00b6, 0x00b7, + 0x0020, 0x0031, 0x006f, 0x00bb, 0x0031, 0x0031, 0x0033, 0x00bf, + 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x00c6, 0x0043, + 0x0045, 0x0045, 0x0045, 0x0045, 0x0049, 0x0049, 0x0049, 0x0049, + 0x00d0, 0x004e, 0x004f, 0x004f, 0x004f, 0x004f, 0x004f, 0x00d7, 0x004f, 0x0055, 0x0055, 0x0055, 0x0055, 0x0059, 0x00de, 0x0073, // Manually changed d8 to 4f // Manually changed df to 73 - 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, - 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, - 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7, + 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00e6, 0x0063, + 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, + 0x00f0, 0x006e, 0x006f, 0x006f, 0x006f, 0x006f, 0x006f, 0x00f7, 0x006f, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00fe, 0x0079, // Manually changed f8 to 6f - 0x0041, 0x0061, 0x0041, 0x0061, 0x0041, 0x0061, 0x0043, 0x0063, - 0x0043, 0x0063, 0x0043, 0x0063, 0x0043, 0x0063, 0x0044, 0x0064, - 0x0110, 0x0111, 0x0045, 0x0065, 0x0045, 0x0065, 0x0045, 0x0065, - 0x0045, 0x0065, 0x0045, 0x0065, 0x0047, 0x0067, 0x0047, 0x0067, - 0x0047, 0x0067, 0x0047, 0x0067, 0x0048, 0x0068, 0x0126, 0x0127, - 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, - 0x0049, 0x0131, 0x0049, 0x0069, 0x004a, 0x006a, 0x004b, 0x006b, - 0x0138, 0x004c, 0x006c, 0x004c, 0x006c, 0x004c, 0x006c, 0x004c, - 0x006c, 0x0141, 0x0142, 0x004e, 0x006e, 0x004e, 0x006e, 0x004e, - 0x006e, 0x02bc, 0x014a, 0x014b, 0x004f, 0x006f, 0x004f, 0x006f, - 0x004f, 0x006f, 0x0152, 0x0153, 0x0052, 0x0072, 0x0052, 0x0072, - 0x0052, 0x0072, 0x0053, 0x0073, 0x0053, 0x0073, 0x0053, 0x0073, - 0x0053, 0x0073, 0x0054, 0x0074, 0x0054, 0x0074, 0x0166, 0x0167, - 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, - 0x0055, 0x0075, 0x0055, 0x0075, 0x0057, 0x0077, 0x0059, 0x0079, - 0x0059, 0x005a, 0x007a, 0x005a, 0x007a, 0x005a, 0x007a, 0x0073, - 0x0180, 0x0181, 0x0182, 0x0183, 0x0184, 0x0185, 0x0186, 0x0187, - 0x0188, 0x0189, 0x018a, 0x018b, 0x018c, 0x018d, 0x018e, 0x018f, - 0x0190, 0x0191, 0x0192, 0x0193, 0x0194, 0x0195, 0x0196, 0x0197, - 0x0198, 0x0199, 0x019a, 0x019b, 0x019c, 0x019d, 0x019e, 0x019f, - 0x004f, 0x006f, 0x01a2, 0x01a3, 0x01a4, 0x01a5, 0x01a6, 0x01a7, - 0x01a8, 0x01a9, 0x01aa, 0x01ab, 0x01ac, 0x01ad, 0x01ae, 0x0055, - 0x0075, 0x01b1, 0x01b2, 0x01b3, 0x01b4, 0x01b5, 0x01b6, 0x01b7, - 0x01b8, 0x01b9, 0x01ba, 0x01bb, 0x01bc, 0x01bd, 0x01be, 0x01bf, - 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0044, 0x0044, 0x0064, 0x004c, - 0x004c, 0x006c, 0x004e, 0x004e, 0x006e, 0x0041, 0x0061, 0x0049, - 0x0069, 0x004f, 0x006f, 0x0055, 0x0075, 0x00dc, 0x00fc, 0x00dc, - 0x00fc, 0x00dc, 0x00fc, 0x00dc, 0x00fc, 0x01dd, 0x00c4, 0x00e4, - 0x0226, 0x0227, 0x00c6, 0x00e6, 0x01e4, 0x01e5, 0x0047, 0x0067, - 0x004b, 0x006b, 0x004f, 0x006f, 0x01ea, 0x01eb, 0x01b7, 0x0292, - 0x006a, 0x0044, 0x0044, 0x0064, 0x0047, 0x0067, 0x01f6, 0x01f7, - 0x004e, 0x006e, 0x00c5, 0x00e5, 0x00c6, 0x00e6, 0x00d8, 0x00f8, - 0x0041, 0x0061, 0x0041, 0x0061, 0x0045, 0x0065, 0x0045, 0x0065, - 0x0049, 0x0069, 0x0049, 0x0069, 0x004f, 0x006f, 0x004f, 0x006f, - 0x0052, 0x0072, 0x0052, 0x0072, 0x0055, 0x0075, 0x0055, 0x0075, - 0x0053, 0x0073, 0x0054, 0x0074, 0x021c, 0x021d, 0x0048, 0x0068, - 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0041, 0x0061, - 0x0045, 0x0065, 0x00d6, 0x00f6, 0x00d5, 0x00f5, 0x004f, 0x006f, - 0x022e, 0x022f, 0x0059, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, - 0x0238, 0x0239, 0x023a, 0x023b, 0x023c, 0x023d, 0x023e, 0x023f, - 0x0240, 0x0241, 0x0242, 0x0243, 0x0244, 0x0245, 0x0246, 0x0247, - 0x0248, 0x0249, 0x024a, 0x024b, 0x024c, 0x024d, 0x024e, 0x024f, - 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, - 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f, - 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, - 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f, - 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, - 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f, - 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, - 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f, - 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, - 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, - 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, - 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, - 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077, - 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, - 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, - 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, - 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, - 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df, - 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7, - 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, - 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, - 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, - 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, - 0x0308, 0x0309, 0x030a, 0x030b, 0x030c, 0x030d, 0x030e, 0x030f, - 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, - 0x0318, 0x0319, 0x031a, 0x031b, 0x031c, 0x031d, 0x031e, 0x031f, - 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, - 0x0328, 0x0329, 0x032a, 0x032b, 0x032c, 0x032d, 0x032e, 0x032f, - 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, - 0x0338, 0x0339, 0x033a, 0x033b, 0x033c, 0x033d, 0x033e, 0x033f, - 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x0345, 0x0346, 0x0347, - 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x034f, - 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, - 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, - 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, - 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, - 0x0370, 0x0371, 0x0372, 0x0373, 0x02b9, 0x0375, 0x0376, 0x0377, - 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f, - 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x00a8, 0x0391, 0x00b7, - 0x0395, 0x0397, 0x0399, 0x038b, 0x039f, 0x038d, 0x03a5, 0x03a9, - 0x03ca, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, - 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, - 0x03a0, 0x03a1, 0x03a2, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, - 0x03a8, 0x03a9, 0x0399, 0x03a5, 0x03b1, 0x03b5, 0x03b7, 0x03b9, - 0x03cb, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, - 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, - 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, - 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03cf, - 0x03b2, 0x03b8, 0x03a5, 0x03d2, 0x03d2, 0x03c6, 0x03c0, 0x03d7, - 0x03d8, 0x03d9, 0x03da, 0x03db, 0x03dc, 0x03dd, 0x03de, 0x03df, - 0x03e0, 0x03e1, 0x03e2, 0x03e3, 0x03e4, 0x03e5, 0x03e6, 0x03e7, - 0x03e8, 0x03e9, 0x03ea, 0x03eb, 0x03ec, 0x03ed, 0x03ee, 0x03ef, - 0x03ba, 0x03c1, 0x03c2, 0x03f3, 0x0398, 0x03b5, 0x03f6, 0x03f7, - 0x03f8, 0x03a3, 0x03fa, 0x03fb, 0x03fc, 0x03fd, 0x03fe, 0x03ff, - 0x0415, 0x0415, 0x0402, 0x0413, 0x0404, 0x0405, 0x0406, 0x0406, - 0x0408, 0x0409, 0x040a, 0x040b, 0x041a, 0x0418, 0x0423, 0x040f, - 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, - 0x0418, 0x0418, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, - 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, - 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, - 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, - 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, - 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, - 0x0460, 0x0461, 0x0462, 0x0463, 0x0464, 0x0465, 0x0466, 0x0467, - 0x0468, 0x0469, 0x046a, 0x046b, 0x046c, 0x046d, 0x046e, 0x046f, - 0x0470, 0x0471, 0x0472, 0x0473, 0x0474, 0x0475, 0x0474, 0x0475, - 0x0478, 0x0479, 0x047a, 0x047b, 0x047c, 0x047d, 0x047e, 0x047f, - 0x0480, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, - 0x0488, 0x0489, 0x048a, 0x048b, 0x048c, 0x048d, 0x048e, 0x048f, - 0x0490, 0x0491, 0x0492, 0x0493, 0x0494, 0x0495, 0x0496, 0x0497, - 0x0498, 0x0499, 0x049a, 0x049b, 0x049c, 0x049d, 0x049e, 0x049f, - 0x04a0, 0x04a1, 0x04a2, 0x04a3, 0x04a4, 0x04a5, 0x04a6, 0x04a7, - 0x04a8, 0x04a9, 0x04aa, 0x04ab, 0x04ac, 0x04ad, 0x04ae, 0x04af, - 0x04b0, 0x04b1, 0x04b2, 0x04b3, 0x04b4, 0x04b5, 0x04b6, 0x04b7, - 0x04b8, 0x04b9, 0x04ba, 0x04bb, 0x04bc, 0x04bd, 0x04be, 0x04bf, - 0x04c0, 0x0416, 0x0436, 0x04c3, 0x04c4, 0x04c5, 0x04c6, 0x04c7, - 0x04c8, 0x04c9, 0x04ca, 0x04cb, 0x04cc, 0x04cd, 0x04ce, 0x04cf, - 0x0410, 0x0430, 0x0410, 0x0430, 0x04d4, 0x04d5, 0x0415, 0x0435, - 0x04d8, 0x04d9, 0x04d8, 0x04d9, 0x0416, 0x0436, 0x0417, 0x0437, - 0x04e0, 0x04e1, 0x0418, 0x0438, 0x0418, 0x0438, 0x041e, 0x043e, - 0x04e8, 0x04e9, 0x04e8, 0x04e9, 0x042d, 0x044d, 0x0423, 0x0443, - 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04f6, 0x04f7, - 0x042b, 0x044b, 0x04fa, 0x04fb, 0x04fc, 0x04fd, 0x04fe, 0x04ff, + 0x0041, 0x0061, 0x0041, 0x0061, 0x0041, 0x0061, 0x0043, 0x0063, + 0x0043, 0x0063, 0x0043, 0x0063, 0x0043, 0x0063, 0x0044, 0x0064, + 0x0110, 0x0111, 0x0045, 0x0065, 0x0045, 0x0065, 0x0045, 0x0065, + 0x0045, 0x0065, 0x0045, 0x0065, 0x0047, 0x0067, 0x0047, 0x0067, + 0x0047, 0x0067, 0x0047, 0x0067, 0x0048, 0x0068, 0x0126, 0x0127, + 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, + 0x0049, 0x0131, 0x0049, 0x0069, 0x004a, 0x006a, 0x004b, 0x006b, + 0x0138, 0x004c, 0x006c, 0x004c, 0x006c, 0x004c, 0x006c, 0x004c, + 0x006c, 0x0141, 0x0142, 0x004e, 0x006e, 0x004e, 0x006e, 0x004e, + 0x006e, 0x02bc, 0x014a, 0x014b, 0x004f, 0x006f, 0x004f, 0x006f, + 0x004f, 0x006f, 0x0152, 0x0153, 0x0052, 0x0072, 0x0052, 0x0072, + 0x0052, 0x0072, 0x0053, 0x0073, 0x0053, 0x0073, 0x0053, 0x0073, + 0x0053, 0x0073, 0x0054, 0x0074, 0x0054, 0x0074, 0x0166, 0x0167, + 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, + 0x0055, 0x0075, 0x0055, 0x0075, 0x0057, 0x0077, 0x0059, 0x0079, + 0x0059, 0x005a, 0x007a, 0x005a, 0x007a, 0x005a, 0x007a, 0x0073, + 0x0180, 0x0181, 0x0182, 0x0183, 0x0184, 0x0185, 0x0186, 0x0187, + 0x0188, 0x0189, 0x018a, 0x018b, 0x018c, 0x018d, 0x018e, 0x018f, + 0x0190, 0x0191, 0x0192, 0x0193, 0x0194, 0x0195, 0x0196, 0x0197, + 0x0198, 0x0199, 0x019a, 0x019b, 0x019c, 0x019d, 0x019e, 0x019f, + 0x004f, 0x006f, 0x01a2, 0x01a3, 0x01a4, 0x01a5, 0x01a6, 0x01a7, + 0x01a8, 0x01a9, 0x01aa, 0x01ab, 0x01ac, 0x01ad, 0x01ae, 0x0055, + 0x0075, 0x01b1, 0x01b2, 0x01b3, 0x01b4, 0x01b5, 0x01b6, 0x01b7, + 0x01b8, 0x01b9, 0x01ba, 0x01bb, 0x01bc, 0x01bd, 0x01be, 0x01bf, + 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x0044, 0x0044, 0x0064, 0x004c, + 0x004c, 0x006c, 0x004e, 0x004e, 0x006e, 0x0041, 0x0061, 0x0049, + 0x0069, 0x004f, 0x006f, 0x0055, 0x0075, 0x00dc, 0x00fc, 0x00dc, + 0x00fc, 0x00dc, 0x00fc, 0x00dc, 0x00fc, 0x01dd, 0x00c4, 0x00e4, + 0x0226, 0x0227, 0x00c6, 0x00e6, 0x01e4, 0x01e5, 0x0047, 0x0067, + 0x004b, 0x006b, 0x004f, 0x006f, 0x01ea, 0x01eb, 0x01b7, 0x0292, + 0x006a, 0x0044, 0x0044, 0x0064, 0x0047, 0x0067, 0x01f6, 0x01f7, + 0x004e, 0x006e, 0x00c5, 0x00e5, 0x00c6, 0x00e6, 0x00d8, 0x00f8, + 0x0041, 0x0061, 0x0041, 0x0061, 0x0045, 0x0065, 0x0045, 0x0065, + 0x0049, 0x0069, 0x0049, 0x0069, 0x004f, 0x006f, 0x004f, 0x006f, + 0x0052, 0x0072, 0x0052, 0x0072, 0x0055, 0x0075, 0x0055, 0x0075, + 0x0053, 0x0073, 0x0054, 0x0074, 0x021c, 0x021d, 0x0048, 0x0068, + 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0041, 0x0061, + 0x0045, 0x0065, 0x00d6, 0x00f6, 0x00d5, 0x00f5, 0x004f, 0x006f, + 0x022e, 0x022f, 0x0059, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, + 0x0238, 0x0239, 0x023a, 0x023b, 0x023c, 0x023d, 0x023e, 0x023f, + 0x0240, 0x0241, 0x0242, 0x0243, 0x0244, 0x0245, 0x0246, 0x0247, + 0x0248, 0x0249, 0x024a, 0x024b, 0x024c, 0x024d, 0x024e, 0x024f, + 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, + 0x0258, 0x0259, 0x025a, 0x025b, 0x025c, 0x025d, 0x025e, 0x025f, + 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, + 0x0268, 0x0269, 0x026a, 0x026b, 0x026c, 0x026d, 0x026e, 0x026f, + 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, + 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x027d, 0x027e, 0x027f, + 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, + 0x0288, 0x0289, 0x028a, 0x028b, 0x028c, 0x028d, 0x028e, 0x028f, + 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, + 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, + 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, + 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, + 0x0068, 0x0266, 0x006a, 0x0072, 0x0279, 0x027b, 0x0281, 0x0077, + 0x0079, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, + 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, + 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, + 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, + 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02de, 0x02df, + 0x0263, 0x006c, 0x0073, 0x0078, 0x0295, 0x02e5, 0x02e6, 0x02e7, + 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, + 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, + 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, + 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, + 0x0308, 0x0309, 0x030a, 0x030b, 0x030c, 0x030d, 0x030e, 0x030f, + 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, + 0x0318, 0x0319, 0x031a, 0x031b, 0x031c, 0x031d, 0x031e, 0x031f, + 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, + 0x0328, 0x0329, 0x032a, 0x032b, 0x032c, 0x032d, 0x032e, 0x032f, + 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, + 0x0338, 0x0339, 0x033a, 0x033b, 0x033c, 0x033d, 0x033e, 0x033f, + 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x0345, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x034f, + 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, + 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, + 0x0370, 0x0371, 0x0372, 0x0373, 0x02b9, 0x0375, 0x0376, 0x0377, + 0x0378, 0x0379, 0x0020, 0x037b, 0x037c, 0x037d, 0x003b, 0x037f, + 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x00a8, 0x0391, 0x00b7, + 0x0395, 0x0397, 0x0399, 0x038b, 0x039f, 0x038d, 0x03a5, 0x03a9, + 0x03ca, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0x03a2, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x0399, 0x03a5, 0x03b1, 0x03b5, 0x03b7, 0x03b9, + 0x03cb, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03b9, 0x03c5, 0x03bf, 0x03c5, 0x03c9, 0x03cf, + 0x03b2, 0x03b8, 0x03a5, 0x03d2, 0x03d2, 0x03c6, 0x03c0, 0x03d7, + 0x03d8, 0x03d9, 0x03da, 0x03db, 0x03dc, 0x03dd, 0x03de, 0x03df, + 0x03e0, 0x03e1, 0x03e2, 0x03e3, 0x03e4, 0x03e5, 0x03e6, 0x03e7, + 0x03e8, 0x03e9, 0x03ea, 0x03eb, 0x03ec, 0x03ed, 0x03ee, 0x03ef, + 0x03ba, 0x03c1, 0x03c2, 0x03f3, 0x0398, 0x03b5, 0x03f6, 0x03f7, + 0x03f8, 0x03a3, 0x03fa, 0x03fb, 0x03fc, 0x03fd, 0x03fe, 0x03ff, + 0x0415, 0x0415, 0x0402, 0x0413, 0x0404, 0x0405, 0x0406, 0x0406, + 0x0408, 0x0409, 0x040a, 0x040b, 0x041a, 0x0418, 0x0423, 0x040f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0418, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0438, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, + 0x0458, 0x0459, 0x045a, 0x045b, 0x043a, 0x0438, 0x0443, 0x045f, + 0x0460, 0x0461, 0x0462, 0x0463, 0x0464, 0x0465, 0x0466, 0x0467, + 0x0468, 0x0469, 0x046a, 0x046b, 0x046c, 0x046d, 0x046e, 0x046f, + 0x0470, 0x0471, 0x0472, 0x0473, 0x0474, 0x0475, 0x0474, 0x0475, + 0x0478, 0x0479, 0x047a, 0x047b, 0x047c, 0x047d, 0x047e, 0x047f, + 0x0480, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + 0x0488, 0x0489, 0x048a, 0x048b, 0x048c, 0x048d, 0x048e, 0x048f, + 0x0490, 0x0491, 0x0492, 0x0493, 0x0494, 0x0495, 0x0496, 0x0497, + 0x0498, 0x0499, 0x049a, 0x049b, 0x049c, 0x049d, 0x049e, 0x049f, + 0x04a0, 0x04a1, 0x04a2, 0x04a3, 0x04a4, 0x04a5, 0x04a6, 0x04a7, + 0x04a8, 0x04a9, 0x04aa, 0x04ab, 0x04ac, 0x04ad, 0x04ae, 0x04af, + 0x04b0, 0x04b1, 0x04b2, 0x04b3, 0x04b4, 0x04b5, 0x04b6, 0x04b7, + 0x04b8, 0x04b9, 0x04ba, 0x04bb, 0x04bc, 0x04bd, 0x04be, 0x04bf, + 0x04c0, 0x0416, 0x0436, 0x04c3, 0x04c4, 0x04c5, 0x04c6, 0x04c7, + 0x04c8, 0x04c9, 0x04ca, 0x04cb, 0x04cc, 0x04cd, 0x04ce, 0x04cf, + 0x0410, 0x0430, 0x0410, 0x0430, 0x04d4, 0x04d5, 0x0415, 0x0435, + 0x04d8, 0x04d9, 0x04d8, 0x04d9, 0x0416, 0x0436, 0x0417, 0x0437, + 0x04e0, 0x04e1, 0x0418, 0x0438, 0x0418, 0x0438, 0x041e, 0x043e, + 0x04e8, 0x04e9, 0x04e8, 0x04e9, 0x042d, 0x044d, 0x0423, 0x0443, + 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04f6, 0x04f7, + 0x042b, 0x044b, 0x04fa, 0x04fb, 0x04fc, 0x04fd, 0x04fe, 0x04ff, }; // generated with: diff --git a/java/src/com/android/inputmethod/latin/LastComposedWord.java b/java/src/com/android/inputmethod/latin/LastComposedWord.java index af0ef4b37..4e1f5fe92 100644 --- a/java/src/com/android/inputmethod/latin/LastComposedWord.java +++ b/java/src/com/android/inputmethod/latin/LastComposedWord.java @@ -46,17 +46,18 @@ public class LastComposedWord { public final String mTypedWord; public final String mCommittedWord; public final int mSeparatorCode; + public final CharSequence mPrevWord; private boolean mActive; public static final LastComposedWord NOT_A_COMPOSED_WORD = - new LastComposedWord(null, null, null, "", "", NOT_A_SEPARATOR); + new LastComposedWord(null, null, null, "", "", NOT_A_SEPARATOR, null); // Warning: this is using the passed objects as is and fully expects them to be // immutable. Do not fiddle with their contents after you passed them to this constructor. public LastComposedWord(final int[] primaryKeyCodes, final int[] xCoordinates, final int[] yCoordinates, final String typedWord, final String committedWord, - final int separatorCode) { + final int separatorCode, final CharSequence prevWord) { mPrimaryKeyCodes = primaryKeyCodes; mXCoordinates = xCoordinates; mYCoordinates = yCoordinates; @@ -64,6 +65,7 @@ public class LastComposedWord { mCommittedWord = committedWord; mSeparatorCode = separatorCode; mActive = true; + mPrevWord = prevWord; } public void deactivate() { diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 83658f77e..38549436b 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -27,6 +27,7 @@ import android.content.DialogInterface; import android.content.Intent; import android.content.IntentFilter; import android.content.SharedPreferences; +import android.content.pm.ApplicationInfo; import android.content.res.Configuration; import android.content.res.Resources; import android.graphics.Rect; @@ -44,6 +45,8 @@ import android.text.TextUtils; import android.util.Log; import android.util.PrintWriterPrinter; import android.util.Printer; +import android.view.KeyCharacterMap; +import android.view.KeyEvent; import android.view.View; import android.view.ViewGroup; import android.view.ViewGroup.LayoutParams; @@ -80,7 +83,7 @@ import java.util.Locale; * Input method implementation for Qwerty'ish keyboard. */ public class LatinIME extends InputMethodService implements KeyboardActionListener, - SuggestionsView.Listener { + SuggestionsView.Listener, TargetApplicationGetter.OnTargetApplicationKnownListener { private static final String TAG = LatinIME.class.getSimpleName(); private static final boolean TRACE = false; private static boolean DEBUG; @@ -152,6 +155,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private SuggestionsView mSuggestionsView; /* package for tests */ Suggest mSuggest; private CompletionInfo[] mApplicationSpecifiedCompletions; + private ApplicationInfo mTargetApplicationInfo; private InputMethodManagerCompatWrapper mImm; private Resources mResources; @@ -492,7 +496,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen resetContactsDictionary(oldContactsDictionary); mUserHistoryDictionary = new UserHistoryDictionary( - this, localeStr, Suggest.DIC_USER_HISTORY); + this, localeStr, Suggest.DIC_USER_HISTORY, mPrefs); mSuggest.setUserHistoryDictionary(mUserHistoryDictionary); } @@ -665,6 +669,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen Log.w(TAG, "Use EditorInfo.IME_FLAG_FORCE_ASCII flag instead"); } + mTargetApplicationInfo = + TargetApplicationGetter.getCachedApplicationInfo(editorInfo.packageName); + if (null == mTargetApplicationInfo) { + new TargetApplicationGetter(this /* context */, this /* listener */) + .execute(editorInfo.packageName); + } + LatinImeLogger.onStartInputView(editorInfo); // In landscape mode, this method gets called without the input view being created. if (inputView == null) { @@ -718,6 +729,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (TRACE) Debug.startMethodTracing("/data/trace/latinime"); } + public void onTargetApplicationKnown(final ApplicationInfo info) { + mTargetApplicationInfo = info; + } + @Override public void onWindowHidden() { super.onWindowHidden(); @@ -732,7 +747,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen KeyboardView inputView = mKeyboardSwitcher.getKeyboardView(); if (inputView != null) inputView.closing(); - if (mUserHistoryDictionary != null) mUserHistoryDictionary.flushPendingWrites(); } private void onFinishInputViewInternal(boolean finishingInput) { @@ -1027,16 +1041,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!mWordComposer.isComposingWord()) return; final CharSequence typedWord = mWordComposer.getTypedWord(); if (typedWord.length() > 0) { - mLastComposedWord = mWordComposer.commitWord( - LastComposedWord.COMMIT_TYPE_USER_TYPED_WORD, typedWord.toString(), - separatorCode); if (ic != null) { ic.commitText(typedWord, 1); if (ProductionFlag.IS_EXPERIMENTAL) { ResearchLogger.latinIME_commitText(typedWord); } } - addToUserHistoryDictionary(typedWord); + final CharSequence prevWord = addToUserHistoryDictionary(typedWord); + mLastComposedWord = mWordComposer.commitWord( + LastComposedWord.COMMIT_TYPE_USER_TYPED_WORD, typedWord.toString(), + separatorCode, prevWord); } updateSuggestions(); } @@ -1211,6 +1225,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } + static private void sendUpDownEnterOrBackspace(final int code, final InputConnection ic) { + final long eventTime = SystemClock.uptimeMillis(); + ic.sendKeyEvent(new KeyEvent(eventTime, eventTime, + KeyEvent.ACTION_DOWN, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, + KeyEvent.FLAG_SOFT_KEYBOARD | KeyEvent.FLAG_KEEP_TOUCH_MODE)); + ic.sendKeyEvent(new KeyEvent(SystemClock.uptimeMillis(), eventTime, + KeyEvent.ACTION_UP, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, + KeyEvent.FLAG_SOFT_KEYBOARD | KeyEvent.FLAG_KEEP_TOUCH_MODE)); + } + private void sendKeyCodePoint(int code) { // TODO: Remove this special handling of digit letters. // For backward compatibility. See {@link InputMethodService#sendKeyChar(char)}. @@ -1221,8 +1245,19 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final InputConnection ic = getCurrentInputConnection(); if (ic != null) { - final String text = new String(new int[] { code }, 0, 1); - ic.commitText(text, text.length()); + // 16 is android.os.Build.VERSION_CODES.JELLY_BEAN but we can't write it because + // we want to be able to compile against the Ice Cream Sandwich SDK. + if (Keyboard.CODE_ENTER == code && mTargetApplicationInfo != null + && mTargetApplicationInfo.targetSdkVersion < 16) { + // Backward compatibility mode. Before Jelly bean, the keyboard would simulate + // a hardware keyboard event on pressing enter or delete. This is bad for many + // reasons (there are race conditions with commits) but some applications are + // relying on this behavior so we continue to support it for older apps. + sendUpDownEnterOrBackspace(KeyEvent.KEYCODE_ENTER, ic); + } else { + final String text = new String(new int[] { code }, 0, 1); + ic.commitText(text, text.length()); + } if (ProductionFlag.IS_EXPERIMENTAL) { ResearchLogger.latinIME_sendKeyCodePoint(code); } @@ -1451,7 +1486,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This should never happen. Log.e(TAG, "Backspace when we don't know the selection position"); } - ic.deleteSurroundingText(1, 0); + // 16 is android.os.Build.VERSION_CODES.JELLY_BEAN but we can't write it because + // we want to be able to compile against the Ice Cream Sandwich SDK. + if (mTargetApplicationInfo != null + && mTargetApplicationInfo.targetSdkVersion < 16) { + // Backward compatibility mode. Before Jelly bean, the keyboard would simulate + // a hardware keyboard event on pressing enter or delete. This is bad for many + // reasons (there are race conditions with commits) but some applications are + // relying on this behavior so we continue to support it for older apps. + sendUpDownEnterOrBackspace(KeyEvent.KEYCODE_DEL, ic); + } else { + ic.deleteSurroundingText(1, 0); + } if (ProductionFlag.IS_EXPERIMENTAL) { ResearchLogger.latinIME_deleteSurroundingText(1); } @@ -1836,8 +1882,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mExpectingUpdateSelection = true; commitChosenWord(autoCorrection, LastComposedWord.COMMIT_TYPE_DECIDED_WORD, separatorCodePoint); - // Add the word to the user history dictionary - addToUserHistoryDictionary(autoCorrection); if (!typedWord.equals(autoCorrection) && null != ic) { // This will make the correction flash for a short while as a visual clue // to the user that auto-correction happened. @@ -1915,8 +1959,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen LastComposedWord.NOT_A_SEPARATOR); // Don't allow cancellation of manual pick mLastComposedWord.deactivate(); - // Add the word to the user history dictionary - addToUserHistoryDictionary(suggestion); mSpaceState = SPACE_STATE_PHANTOM; // TODO: is this necessary? mKeyboardSwitcher.updateShiftState(); @@ -1959,31 +2001,33 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen /** * Commits the chosen word to the text field and saves it for later retrieval. */ - private void commitChosenWord(final CharSequence bestWord, final int commitType, + private void commitChosenWord(final CharSequence chosenWord, final int commitType, final int separatorCode) { final InputConnection ic = getCurrentInputConnection(); if (ic != null) { if (mSettingsValues.mEnableSuggestionSpanInsertion) { final SuggestedWords suggestedWords = mSuggestionsView.getSuggestions(); ic.commitText(SuggestionSpanUtils.getTextWithSuggestionSpan( - this, bestWord, suggestedWords, mIsMainDictionaryAvailable), + this, chosenWord, suggestedWords, mIsMainDictionaryAvailable), 1); if (ProductionFlag.IS_EXPERIMENTAL) { - ResearchLogger.latinIME_commitText(bestWord); + ResearchLogger.latinIME_commitText(chosenWord); } } else { - ic.commitText(bestWord, 1); + ic.commitText(chosenWord, 1); if (ProductionFlag.IS_EXPERIMENTAL) { - ResearchLogger.latinIME_commitText(bestWord); + ResearchLogger.latinIME_commitText(chosenWord); } } } + // Add the word to the user history dictionary + final CharSequence prevWord = addToUserHistoryDictionary(chosenWord); // TODO: figure out here if this is an auto-correct or if the best word is actually // what user typed. Note: currently this is done much later in // LastComposedWord#didCommitTypedWord by string equality of the remembered // strings. - mLastComposedWord = mWordComposer.commitWord(commitType, bestWord.toString(), - separatorCode); + mLastComposedWord = mWordComposer.commitWord(commitType, chosenWord.toString(), + separatorCode, prevWord); } public void updateBigramPredictions() { @@ -2023,15 +2067,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen setSuggestionStripShown(isSuggestionsStripVisible()); } - private void addToUserHistoryDictionary(final CharSequence suggestion) { - if (TextUtils.isEmpty(suggestion)) return; + private CharSequence addToUserHistoryDictionary(final CharSequence suggestion) { + if (TextUtils.isEmpty(suggestion)) return null; // Only auto-add to dictionary if auto-correct is ON. Otherwise we'll be // adding words in situations where the user or application really didn't // want corrections enabled or learned. if (!(mCorrectionMode == Suggest.CORRECTION_FULL || mCorrectionMode == Suggest.CORRECTION_FULL_BIGRAM)) { - return; + return null; } if (mUserHistoryDictionary != null) { @@ -2049,9 +2093,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { secondWord = suggestion.toString(); } + // We demote unrecognized word and words with 0-frequency (assuming they would be + // profanity etc.) by specifying them as "invalid". + final int maxFreq = AutoCorrection.getMaxFrequency( + mSuggest.getUnigramDictionaries(), suggestion); mUserHistoryDictionary.addToUserHistory(null == prevWord ? null : prevWord.toString(), - secondWord); + secondWord, maxFreq > 0); + return prevWord; } + return null; } public boolean isCursorTouchingWord() { @@ -2136,6 +2186,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // "ic" must not be null private void revertCommit(final InputConnection ic) { + final CharSequence previousWord = mLastComposedWord.mPrevWord; final String originallyTypedWord = mLastComposedWord.mTypedWord; final CharSequence committedWord = mLastComposedWord.mCommittedWord; final int cancelLength = committedWord.length(); @@ -2160,6 +2211,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.IS_EXPERIMENTAL) { ResearchLogger.latinIME_deleteSurroundingText(deleteLength); } + if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) { + mUserHistoryDictionary.cancelAddingUserHistory( + previousWord.toString(), committedWord.toString()); + } if (0 == separatorLength || mLastComposedWord.didCommitTypedWord()) { // This is the case when we cancel a manual pick. // We should restart suggestion on the word right away. diff --git a/java/src/com/android/inputmethod/latin/ResearchLogger.java b/java/src/com/android/inputmethod/latin/ResearchLogger.java index aa979a66f..66d6d58b1 100644 --- a/java/src/com/android/inputmethod/latin/ResearchLogger.java +++ b/java/src/com/android/inputmethod/latin/ResearchLogger.java @@ -54,7 +54,7 @@ import java.util.Map; * This class logs operations on the IME keyboard, including what the user has typed. * Data is stored locally in a file in app-specific storage. * - * This functionality is off by default. See {@link ProductionFlag.IS_EXPERIMENTAL}. + * This functionality is off by default. See {@link ProductionFlag#IS_EXPERIMENTAL}. */ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChangeListener { private static final String TAG = ResearchLogger.class.getSimpleName(); diff --git a/java/src/com/android/inputmethod/latin/Settings.java b/java/src/com/android/inputmethod/latin/Settings.java index 74c4aea0c..08f3e8456 100644 --- a/java/src/com/android/inputmethod/latin/Settings.java +++ b/java/src/com/android/inputmethod/latin/Settings.java @@ -58,6 +58,8 @@ public class Settings extends InputMethodSettingsFragment public static final String PREF_SHOW_SUGGESTIONS_SETTING = "show_suggestions_setting"; public static final String PREF_MISC_SETTINGS = "misc_settings"; public static final String PREF_USABILITY_STUDY_MODE = "usability_study_mode"; + public static final String PREF_LAST_USER_DICTIONARY_WRITE_TIME = + "last_user_dictionary_write_time"; public static final String PREF_ADVANCED_SETTINGS = "pref_advanced_settings"; public static final String PREF_SUPPRESS_LANGUAGE_SWITCH_KEY = "pref_suppress_language_switch_key"; @@ -244,7 +246,6 @@ public class Settings extends InputMethodSettingsFragment refreshEnablingsOfKeypressSoundAndVibrationSettings(prefs, res); } - @SuppressWarnings("unused") @Override public void onResume() { super.onResume(); diff --git a/java/src/com/android/inputmethod/latin/SettingsValues.java b/java/src/com/android/inputmethod/latin/SettingsValues.java index 932920a60..4aae6a85e 100644 --- a/java/src/com/android/inputmethod/latin/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/SettingsValues.java @@ -28,6 +28,8 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; /** * When you call the constructor of this class, you may want to change the current system locale by @@ -351,4 +353,23 @@ public class SettingsValues { // TODO: use mUsabilityStudyMode instead of reading it again here return prefs.getBoolean(Settings.PREF_USABILITY_STUDY_MODE, true); } + + public static long getLastUserHistoryWriteTime( + final SharedPreferences prefs, final String locale) { + final String str = prefs.getString(Settings.PREF_LAST_USER_DICTIONARY_WRITE_TIME, ""); + final HashMap<String, Long> map = Utils.localeAndTimeStrToHashMap(str); + if (map.containsKey(locale)) { + return map.get(locale); + } + return 0; + } + + public static void setLastUserHistoryWriteTime( + final SharedPreferences prefs, final String locale) { + final String oldStr = prefs.getString(Settings.PREF_LAST_USER_DICTIONARY_WRITE_TIME, ""); + final HashMap<String, Long> map = Utils.localeAndTimeStrToHashMap(oldStr); + map.put(locale, System.currentTimeMillis()); + final String newStr = Utils.localeAndTimeHashMapToStr(map); + prefs.edit().putString(Settings.PREF_LAST_USER_DICTIONARY_WRITE_TIME, newStr).apply(); + } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index c98a27b64..336a76f4b 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -26,9 +26,9 @@ import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import java.io.File; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; /** * This class loads a dictionary and provides a list of suggestions for a given sequence of @@ -68,10 +68,10 @@ public class Suggest implements Dictionary.WordCallback { private boolean mHasMainDictionary; private Dictionary mContactsDict; private WhitelistDictionary mWhiteListDictionary; - private final HashMap<String, Dictionary> mUnigramDictionaries = - new HashMap<String, Dictionary>(); - private final HashMap<String, Dictionary> mBigramDictionaries = - new HashMap<String, Dictionary>(); + private final ConcurrentHashMap<String, Dictionary> mUnigramDictionaries = + new ConcurrentHashMap<String, Dictionary>(); + private final ConcurrentHashMap<String, Dictionary> mBigramDictionaries = + new ConcurrentHashMap<String, Dictionary>(); private int mPrefMaxSuggestions = 18; @@ -117,8 +117,9 @@ public class Suggest implements Dictionary.WordCallback { initWhitelistAndAutocorrectAndPool(context, locale); } - private static void addOrReplaceDictionary(HashMap<String, Dictionary> dictionaries, String key, - Dictionary dict) { + private static void addOrReplaceDictionary( + final ConcurrentHashMap<String, Dictionary> dictionaries, + final String key, final Dictionary dict) { final Dictionary oldDict = (dict == null) ? dictionaries.remove(key) : dictionaries.put(key, dict); @@ -151,7 +152,7 @@ public class Suggest implements Dictionary.WordCallback { return mContactsDict; } - public HashMap<String, Dictionary> getUnigramDictionaries() { + public ConcurrentHashMap<String, Dictionary> getUnigramDictionaries() { return mUnigramDictionaries; } diff --git a/java/src/com/android/inputmethod/latin/TargetApplicationGetter.java b/java/src/com/android/inputmethod/latin/TargetApplicationGetter.java new file mode 100644 index 000000000..4fb2e9704 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/TargetApplicationGetter.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.latin; + +import android.content.Context; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageManager; +import android.os.AsyncTask; +import android.util.LruCache; + +public class TargetApplicationGetter extends AsyncTask<String, Void, ApplicationInfo> { + + private static final int MAX_CACHE_ENTRIES = 64; // arbitrary + private static LruCache<String, ApplicationInfo> sCache = + new LruCache<String, ApplicationInfo>(MAX_CACHE_ENTRIES); + + public static ApplicationInfo getCachedApplicationInfo(final String packageName) { + return sCache.get(packageName); + } + public static void removeApplicationInfoCache(final String packageName) { + sCache.remove(packageName); + } + + public interface OnTargetApplicationKnownListener { + public void onTargetApplicationKnown(final ApplicationInfo info); + } + + private Context mContext; + private final OnTargetApplicationKnownListener mListener; + + public TargetApplicationGetter(final Context context, + final OnTargetApplicationKnownListener listener) { + mContext = context; + mListener = listener; + } + + @Override + protected ApplicationInfo doInBackground(final String... packageName) { + final PackageManager pm = mContext.getPackageManager(); + mContext = null; // Bazooka-powered anti-leak device + try { + final ApplicationInfo targetAppInfo = + pm.getApplicationInfo(packageName[0], 0 /* flags */); + sCache.put(packageName[0], targetAppInfo); + return targetAppInfo; + } catch (android.content.pm.PackageManager.NameNotFoundException e) { + return null; + } + } + + @Override + protected void onPostExecute(final ApplicationInfo info) { + mListener.onTargetApplicationKnown(info); + } +} diff --git a/java/src/com/android/inputmethod/latin/UserDictionary.java b/java/src/com/android/inputmethod/latin/UserDictionary.java index 81e2fdce4..c1efadd44 100644 --- a/java/src/com/android/inputmethod/latin/UserDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserDictionary.java @@ -35,6 +35,7 @@ import java.util.Arrays; * * @deprecated Use {@link UserBinaryDictionary}. */ +@Deprecated public class UserDictionary extends ExpandableDictionary { // TODO: use Words.SHORTCUT when it's public in the SDK diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java index e13602e50..c8ad40b12 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionary.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.content.ContentValues; import android.content.Context; +import android.content.SharedPreferences; import android.database.Cursor; import android.database.sqlite.SQLiteDatabase; import android.database.sqlite.SQLiteOpenHelper; @@ -26,9 +27,9 @@ import android.os.AsyncTask; import android.provider.BaseColumns; import android.util.Log; +import com.android.inputmethod.latin.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; /** * Locally gathers stats about the words user types and various other signals like auto-correction @@ -36,13 +37,11 @@ import java.util.Iterator; */ public class UserHistoryDictionary extends ExpandableDictionary { private static final String TAG = "UserHistoryDictionary"; + public static final boolean DBG_SAVE_RESTORE = false; /** Any pair being typed or picked */ private static final int FREQUENCY_FOR_TYPED = 2; - /** Maximum frequency for all pairs */ - private static final int FREQUENCY_MAX = 127; - /** Maximum number of pairs. Pruning will start when databases goes above this number. */ private static int sMaxHistoryBigrams = 10000; @@ -78,9 +77,11 @@ public class UserHistoryDictionary extends ExpandableDictionary { /** Locale for which this auto dictionary is storing words */ private String mLocale; - private HashSet<Bigram> mPendingWrites = new HashSet<Bigram>(); + private UserHistoryDictionaryBigramList mBigramList = + new UserHistoryDictionaryBigramList(); private final Object mPendingWritesLock = new Object(); private static volatile boolean sUpdatingDB = false; + private final SharedPreferences mPrefs; private final static HashMap<String, String> sDictProjectionMap; @@ -98,37 +99,6 @@ public class UserHistoryDictionary extends ExpandableDictionary { private static DatabaseHelper sOpenHelper = null; - private static class Bigram { - public final String mWord1; - public final String mWord2; - public final int mFrequency; - - Bigram(String word1, String word2, int frequency) { - this.mWord1 = word1; - this.mWord2 = word2; - this.mFrequency = frequency; - } - - @Override - public boolean equals(Object bigram) { - if (!(bigram instanceof Bigram)) { - return false; - } - final Bigram bigram2 = (Bigram) bigram; - final boolean eq1 = - mWord1 == null ? bigram2.mWord1 == null : mWord1.equals(bigram2.mWord1); - if (!eq1) { - return false; - } - return mWord2 == null ? bigram2.mWord2 == null : mWord2.equals(bigram2.mWord2); - } - - @Override - public int hashCode() { - return (mWord1 + " " + mWord2).hashCode(); - } - } - public void setDatabaseMax(int maxHistoryBigram) { sMaxHistoryBigrams = maxHistoryBigram; } @@ -137,7 +107,8 @@ public class UserHistoryDictionary extends ExpandableDictionary { sDeleteHistoryBigrams = deleteHistoryBigram; } - public UserHistoryDictionary(final Context context, final String locale, final int dicTypeId) { + public UserHistoryDictionary(final Context context, final String locale, final int dicTypeId, + SharedPreferences sp) { super(context, dicTypeId); mLocale = locale; if (sOpenHelper == null) { @@ -146,11 +117,13 @@ public class UserHistoryDictionary extends ExpandableDictionary { if (mLocale != null && mLocale.length() > 1) { loadDictionary(); } + mPrefs = sp; } @Override public void close() { flushPendingWrites(); + SettingsValues.setLastUserHistoryWriteTime(mPrefs, mLocale); // Don't close the database as locale changes will require it to be reopened anyway // Also, the database is written to somewhat frequently, so it needs to be kept alive // throughout the life of the process. @@ -175,44 +148,45 @@ public class UserHistoryDictionary extends ExpandableDictionary { * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public int addToUserHistory(final String word1, String word2) { - super.addWord(word2, null /* shortcut */, FREQUENCY_FOR_TYPED); + public int addToUserHistory(final String word1, String word2, boolean isValid) { + super.addWord(word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED); // Do not insert a word as a bigram of itself if (word2.equals(word1)) { return 0; } - - int freq; + final int freq; if (null == word1) { freq = FREQUENCY_FOR_TYPED; } else { - freq = super.addBigram(word1, word2, FREQUENCY_FOR_TYPED); + freq = super.setBigramAndGetFrequency(word1, word2, new ForgettingCurveParams(isValid)); } - if (freq > FREQUENCY_MAX) freq = FREQUENCY_MAX; synchronized (mPendingWritesLock) { - if (freq == FREQUENCY_FOR_TYPED || mPendingWrites.isEmpty()) { - mPendingWrites.add(new Bigram(word1, word2, freq)); - } else { - Bigram bi = new Bigram(word1, word2, freq); - mPendingWrites.remove(bi); - mPendingWrites.add(bi); - } + mBigramList.addBigram(word1, word2); } return freq; } + public boolean cancelAddingUserHistory(String word1, String word2) { + synchronized (mPendingWritesLock) { + if (mBigramList.removeBigram(word1, word2)) { + return super.removeBigram(word1, word2); + } + } + return false; + } + /** * Schedules a background thread to write any pending words to the database. */ - public void flushPendingWrites() { + private void flushPendingWrites() { synchronized (mPendingWritesLock) { // Nothing pending? Return - if (mPendingWrites.isEmpty()) return; + if (mBigramList.isEmpty()) return; // Create a background thread to write the pending entries - new UpdateDbTask(sOpenHelper, mPendingWrites, mLocale).execute(); + new UpdateDbTask(sOpenHelper, mBigramList, mLocale, this).execute(); // Create a new map for writing new entries into while the old one is written to db - mPendingWrites = new HashSet<Bigram>(); + mBigramList = new UserHistoryDictionaryBigramList(); } } @@ -231,25 +205,34 @@ public class UserHistoryDictionary extends ExpandableDictionary { @Override public void loadDictionaryAsync() { + final long last = SettingsValues.getLastUserHistoryWriteTime(mPrefs, mLocale); + final long now = System.currentTimeMillis(); // Load the words that correspond to the current input locale final Cursor cursor = query(MAIN_COLUMN_LOCALE + "=?", new String[] { mLocale }); if (null == cursor) return; try { if (cursor.moveToFirst()) { - int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1); - int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2); - int frequencyIndex = cursor.getColumnIndex(FREQ_COLUMN_FREQUENCY); + final int word1Index = cursor.getColumnIndex(MAIN_COLUMN_WORD1); + final int word2Index = cursor.getColumnIndex(MAIN_COLUMN_WORD2); + final int frequencyIndex = cursor.getColumnIndex(FREQ_COLUMN_FREQUENCY); while (!cursor.isAfterLast()) { - String word1 = cursor.getString(word1Index); - String word2 = cursor.getString(word2Index); - int frequency = cursor.getInt(frequencyIndex); + final String word1 = cursor.getString(word1Index); + final String word2 = cursor.getString(word2Index); + final int frequency = cursor.getInt(frequencyIndex); + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "--- Load user history: " + word1 + ", " + word2); + } // Safeguard against adding really long words. Stack may overflow due // to recursive lookup if (null == word1) { super.addWord(word2, null /* shortcut */, frequency); } else if (word1.length() < BinaryDictionary.MAX_WORD_LENGTH && word2.length() < BinaryDictionary.MAX_WORD_LENGTH) { - super.setBigram(word1, word2, frequency); + super.setBigramAndGetFrequency( + word1, word2, new ForgettingCurveParams(frequency, now, last)); + } + synchronized(mPendingWritesLock) { + mBigramList.addBigram(word1, word2); } cursor.moveToNext(); } @@ -328,15 +311,18 @@ public class UserHistoryDictionary extends ExpandableDictionary { * the in-memory trie. */ private static class UpdateDbTask extends AsyncTask<Void, Void, Void> { - private final HashSet<Bigram> mMap; + private final UserHistoryDictionaryBigramList mBigramList; private final DatabaseHelper mDbHelper; private final String mLocale; + private final UserHistoryDictionary mUserHistoryDictionary; - public UpdateDbTask(DatabaseHelper openHelper, HashSet<Bigram> pendingWrites, - String locale) { - mMap = pendingWrites; + public UpdateDbTask( + DatabaseHelper openHelper, UserHistoryDictionaryBigramList pendingWrites, + String locale, UserHistoryDictionary dict) { + mBigramList = pendingWrites; mLocale = locale; mDbHelper = openHelper; + mUserHistoryDictionary = dict; } /** Prune any old data if the database is getting too big. */ @@ -348,7 +334,8 @@ public class UserHistoryDictionary extends ExpandableDictionary { int totalRowCount = c.getCount(); // prune out old data if we have too much data if (totalRowCount > sMaxHistoryBigrams) { - int numDeleteRows = (totalRowCount - sMaxHistoryBigrams) + sDeleteHistoryBigrams; + int numDeleteRows = (totalRowCount - sMaxHistoryBigrams) + + sDeleteHistoryBigrams; int pairIdColumnId = c.getColumnIndex(FREQ_COLUMN_PAIR_ID); c.moveToFirst(); int count = 0; @@ -387,44 +374,76 @@ public class UserHistoryDictionary extends ExpandableDictionary { return null; } db.execSQL("PRAGMA foreign_keys = ON;"); - // Write all the entries to the db - Iterator<Bigram> iterator = mMap.iterator(); - while (iterator.hasNext()) { - // TODO: this process of making a text search for each pair each time - // is terribly inefficient. Optimize this. - Bigram bi = iterator.next(); - - // find pair id - final Cursor c; - if (null != bi.mWord1) { - c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, - MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " - + MAIN_COLUMN_LOCALE + "=?", - new String[] { bi.mWord1, bi.mWord2, mLocale }, null, null, null); - } else { - c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, - MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + "=? AND " - + MAIN_COLUMN_LOCALE + "=?", - new String[] { bi.mWord2, mLocale }, null, null, null); - } + final boolean addLevel0Bigram = mBigramList.size() <= sMaxHistoryBigrams; - int pairId; - if (c.moveToFirst()) { - // existing pair - pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); - db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", - new String[] { Integer.toString(pairId) }); - } else { - // new pair - Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, - getContentValues(bi.mWord1, bi.mWord2, mLocale)); - pairId = pairIdLong.intValue(); + // Write all the entries to the db + for (String word1 : mBigramList.keySet()) { + for (String word2 : mBigramList.getBigrams(word1)) { + // TODO: this process of making a text search for each pair each time + // is terribly inefficient. Optimize this. + // find pair id + Cursor c = null; + try { + if (null != word1) { + c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, + MAIN_COLUMN_WORD1 + "=? AND " + MAIN_COLUMN_WORD2 + "=? AND " + + MAIN_COLUMN_LOCALE + "=?", + new String[] { word1, word2, mLocale }, null, null, + null); + } else { + c = db.query(MAIN_TABLE_NAME, new String[] { MAIN_COLUMN_ID }, + MAIN_COLUMN_WORD1 + " IS NULL AND " + MAIN_COLUMN_WORD2 + + "=? AND " + MAIN_COLUMN_LOCALE + "=?", + new String[] { word2, mLocale }, null, null, null); + } + + final int pairId; + if (c.moveToFirst()) { + // existing pair + pairId = c.getInt(c.getColumnIndex(MAIN_COLUMN_ID)); + db.delete(FREQ_TABLE_NAME, FREQ_COLUMN_PAIR_ID + "=?", + new String[] { Integer.toString(pairId) }); + } else { + // new pair + Long pairIdLong = db.insert(MAIN_TABLE_NAME, null, + getContentValues(word1, word2, mLocale)); + pairId = pairIdLong.intValue(); + } + // insert new frequency + final int freq; + if (word1 == null) { + freq = FREQUENCY_FOR_TYPED; + } else { + final NextWord nw = mUserHistoryDictionary.getBigramWord(word1, word2); + if (nw != null) { + final ForgettingCurveParams fcp = nw.getFcParams(); + final int tempFreq = fcp.getFc(); + final boolean isValid = fcp.isValid(); + if (UserHistoryForgettingCurveUtils.needsToSave( + (byte)tempFreq, isValid, addLevel0Bigram)) { + freq = tempFreq; + } else { + freq = -1; + } + } else { + freq = -1; + } + } + if (freq > 0) { + if (DBG_SAVE_RESTORE) { + Log.d(TAG, "--- Save user history: " + word1 + ", " + word2); + } + db.insert(FREQ_TABLE_NAME, null, + getFrequencyContentValues(pairId, freq)); + } + } finally { + if (c != null) { + c.close(); + } + } } - c.close(); - - // insert new frequency - db.insert(FREQ_TABLE_NAME, null, getFrequencyContentValues(pairId, bi.mFrequency)); } + checkPruneData(db); sUpdatingDB = false; diff --git a/java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java new file mode 100644 index 000000000..409f921ff --- /dev/null +++ b/java/src/com/android/inputmethod/latin/UserHistoryDictionaryBigramList.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.util.Log; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; + +/** + * A store of bigrams which will be updated when the user history dictionary is closed + * All bigrams including stale ones in SQL DB should be stored in this class to avoid adding stale + * bigrams when we write to the SQL DB. + */ +public class UserHistoryDictionaryBigramList { + private static final String TAG = UserHistoryDictionaryBigramList.class.getSimpleName(); + private static final HashSet<String> EMPTY_STRING_SET = new HashSet<String>(); + private final HashMap<String, HashSet<String>> mBigramMap = + new HashMap<String, HashSet<String>>(); + private int mSize = 0; + + public void evictAll() { + mSize = 0; + mBigramMap.clear(); + } + + public void addBigram(String word1, String word2) { + if (UserHistoryDictionary.DBG_SAVE_RESTORE) { + Log.d(TAG, "--- add bigram: " + word1 + ", " + word2); + } + final HashSet<String> set; + if (mBigramMap.containsKey(word1)) { + set = mBigramMap.get(word1); + } else { + set = new HashSet<String>(); + mBigramMap.put(word1, set); + } + if (!set.contains(word2)) { + ++mSize; + set.add(word2); + } + } + + public int size() { + return mSize; + } + + public boolean isEmpty() { + return mBigramMap.isEmpty(); + } + + public Set<String> keySet() { + return mBigramMap.keySet(); + } + + public HashSet<String> getBigrams(String word1) { + if (!mBigramMap.containsKey(word1)) { + return EMPTY_STRING_SET; + } else { + return mBigramMap.get(word1); + } + } + + public boolean removeBigram(String word1, String word2) { + final HashSet<String> set = getBigrams(word1); + if (set.isEmpty()) { + return false; + } + if (set.contains(word2)) { + set.remove(word2); + --mSize; + return true; + } + return false; + } +} diff --git a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java index eb3881726..9cd8c6778 100644 --- a/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/UserHistoryForgettingCurveUtils.java @@ -16,14 +16,91 @@ package com.android.inputmethod.latin; +import android.text.format.DateUtils; +import android.util.Log; + public class UserHistoryForgettingCurveUtils { + private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); + private static final boolean DEBUG = false; private static final int FC_FREQ_MAX = 127; /* package */ static final int COUNT_MAX = 3; private static final int FC_LEVEL_MAX = 3; /* package */ static final int ELAPSED_TIME_MAX = 15; private static final int ELAPSED_TIME_INTERVAL_HOURS = 6; + private static final long ELAPSED_TIME_INTERVAL_MILLIS = ELAPSED_TIME_INTERVAL_HOURS + * DateUtils.HOUR_IN_MILLIS; private static final int HALF_LIFE_HOURS = 48; + private UserHistoryForgettingCurveUtils() { + // This utility class is not publicly instantiable. + } + + public static class ForgettingCurveParams { + private byte mFc; + long mLastTouchedTime = 0; + private final boolean mIsValid; + + private void updateLastTouchedTime() { + mLastTouchedTime = System.currentTimeMillis(); + } + + public ForgettingCurveParams(boolean isValid) { + this(System.currentTimeMillis(), isValid); + } + + private ForgettingCurveParams(long now, boolean isValid) { + this((int)pushCount((byte)0, isValid), now, now, isValid); + } + + /** This constructor is called when the user history bigram dictionary is being restored. */ + public ForgettingCurveParams(int fc, long now, long last) { + // All words with level >= 1 had been saved. + // Invalid words with level == 0 had been saved. + // Valid words words with level == 0 had *not* been saved. + this(fc, now, last, fcToLevel((byte)fc) > 0); + } + + private ForgettingCurveParams(int fc, long now, long last, boolean isValid) { + mIsValid = isValid; + mFc = (byte)fc; + mLastTouchedTime = last; + updateElapsedTime(now); + } + + public boolean isValid() { + return mIsValid; + } + + public byte getFc() { + updateElapsedTime(System.currentTimeMillis()); + return mFc; + } + + public int getFrequency() { + updateElapsedTime(System.currentTimeMillis()); + return UserHistoryForgettingCurveUtils.fcToFreq(mFc); + } + + public int notifyTypedAgainAndGetFrequency() { + updateLastTouchedTime(); + // TODO: Check whether this word is valid or not + mFc = pushCount(mFc, false); + return UserHistoryForgettingCurveUtils.fcToFreq(mFc); + } + + private void updateElapsedTime(long now) { + final int elapsedTimeCount = + (int)((now - mLastTouchedTime) / ELAPSED_TIME_INTERVAL_MILLIS); + if (elapsedTimeCount <= 0) { + return; + } + for (int i = 0; i < elapsedTimeCount; ++i) { + mLastTouchedTime += ELAPSED_TIME_INTERVAL_MILLIS; + mFc = pushElapsedTime(mFc); + } + } + } + /* package */ static int fcToElapsedTime(byte fc) { return fc & 0x0F; } @@ -38,8 +115,8 @@ public class UserHistoryForgettingCurveUtils { private static int calcFreq(int elapsedTime, int count, int level) { if (level <= 0) { - // Reserved words, just return 0 - return 0; + // Reserved words, just return -1 + return -1; } if (count == COUNT_MAX) { // Temporary promote because it's frequently typed recently @@ -87,12 +164,31 @@ public class UserHistoryForgettingCurveUtils { // Upgrade level ++level; count = 0; + if (DEBUG) { + Log.d(TAG, "Upgrade level."); + } } else { ++count; } return calcFc(0, count, level); } + // TODO: isValid should be false for a word whose frequency is 0, + // or that is not in the dictionary. + /** + * Check wheather we should save the bigram to the SQL DB or not + */ + public static boolean needsToSave(byte fc, boolean isValid, boolean addLevel0Bigram) { + int level = fcToLevel(fc); + if (level == 0) { + if (isValid || !addLevel0Bigram) { + return false; + } + } + final int elapsedTime = fcToElapsedTime(fc); + return (elapsedTime < ELAPSED_TIME_MAX - 1 || level > 0); + } + private static class MathUtils { public static final int[][] SCORE_TABLE = new int[FC_LEVEL_MAX][ELAPSED_TIME_MAX + 1]; static { diff --git a/java/src/com/android/inputmethod/latin/Utils.java b/java/src/com/android/inputmethod/latin/Utils.java index 036ff74b8..b3e46baf5 100644 --- a/java/src/com/android/inputmethod/latin/Utils.java +++ b/java/src/com/android/inputmethod/latin/Utils.java @@ -44,8 +44,10 @@ import java.io.IOException; import java.io.PrintWriter; import java.nio.channels.FileChannel; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.Map; public class Utils { private Utils() { @@ -484,4 +486,40 @@ public class Utils { } return sDeviceOverrideValueMap.get(key); } + + private static final HashMap<String, Long> EMPTY_LT_HASH_MAP = new HashMap<String, Long>(); + private static final String LOCALE_AND_TIME_STR_SEPARATER = ","; + public static HashMap<String, Long> localeAndTimeStrToHashMap(String str) { + if (TextUtils.isEmpty(str)) { + return EMPTY_LT_HASH_MAP; + } + final String[] ss = str.split(LOCALE_AND_TIME_STR_SEPARATER); + final int N = ss.length; + if (N < 2 || N % 2 != 0) { + return EMPTY_LT_HASH_MAP; + } + final HashMap<String, Long> retval = new HashMap<String, Long>(); + for (int i = 0; i < N / 2; ++i) { + final String localeStr = ss[i * 2]; + final long time = Long.valueOf(ss[i * 2 + 1]); + retval.put(localeStr, time); + } + return retval; + } + + public static String localeAndTimeHashMapToStr(HashMap<String, Long> map) { + if (map == null || map.isEmpty()) { + return ""; + } + final StringBuilder builder = new StringBuilder(); + for (String localeStr : map.keySet()) { + if (builder.length() > 0) { + builder.append(LOCALE_AND_TIME_STR_SEPARATER); + } + final Long time = map.get(localeStr); + builder.append(localeStr).append(LOCALE_AND_TIME_STR_SEPARATER); + builder.append(String.valueOf(time)); + } + return builder.toString(); + } } diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index e27a546c5..ca9caa1d3 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -313,7 +313,7 @@ public class WordComposer { // `type' should be one of the LastComposedWord.COMMIT_TYPE_* constants above. public LastComposedWord commitWord(final int type, final String committedWord, - final int separatorCode) { + final int separatorCode, final CharSequence prevWord) { // Note: currently, we come here whenever we commit a word. If it's a MANUAL_PICK // or a DECIDED_WORD we may cancel the commit later; otherwise, we should deactivate // the last composed word to ensure this does not happen. @@ -324,7 +324,8 @@ public class WordComposer { mXCoordinates = new int[N]; mYCoordinates = new int[N]; final LastComposedWord lastComposedWord = new LastComposedWord(primaryKeyCodes, - xCoordinates, yCoordinates, mTypedWord.toString(), committedWord, separatorCode); + xCoordinates, yCoordinates, mTypedWord.toString(), committedWord, separatorCode, + prevWord); if (type != LastComposedWord.COMMIT_TYPE_DECIDED_WORD && type != LastComposedWord.COMMIT_TYPE_MANUAL_PICK) { lastComposedWord.deactivate(); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java index 563f8a99b..89c59f809 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java @@ -882,9 +882,9 @@ public class BinaryDictInputOutput { final int indexOfShortcutByteSize = index; index += GROUP_SHORTCUT_LIST_SIZE_SIZE; groupAddress += GROUP_SHORTCUT_LIST_SIZE_SIZE; - final Iterator shortcutIterator = group.mShortcutTargets.iterator(); + final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { - final WeightedString target = (WeightedString)shortcutIterator.next(); + final WeightedString target = shortcutIterator.next(); ++groupAddress; int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), target.mFrequency); @@ -902,9 +902,9 @@ public class BinaryDictInputOutput { } // Write bigrams if (null != group.mBigrams) { - final Iterator bigramIterator = group.mBigrams.iterator(); + final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); while (bigramIterator.hasNext()) { - final WeightedString bigram = (WeightedString)bigramIterator.next(); + final WeightedString bigram = bigramIterator.next(); final CharGroup target = FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord); final int addressOfBigram = target.mCachedAddress; diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index c467ef7d4..8b53c9427 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -296,7 +296,6 @@ public class FusionDictionary implements Iterable<Word> { * @param word the word to add. * @param frequency the frequency of the word, in the range [0..255]. * @param shortcutTargets a list of shortcut targets for this word, or null. - * @param bigrams a list of bigrams, or null. */ public void add(final String word, final int frequency, final ArrayList<WeightedString> shortcutTargets) { @@ -435,6 +434,8 @@ public class FusionDictionary implements Iterable<Word> { } } + private static int ARRAYS_ARE_EQUAL = 0; + /** * Custom comparison of two int arrays taken to contain character codes. * @@ -450,7 +451,6 @@ public class FusionDictionary implements Iterable<Word> { * @param dstOffset the offset in the right-hand side string. * @return the index at which the strings differ, or ARRAYS_ARE_EQUAL = 0 if they don't. */ - private static int ARRAYS_ARE_EQUAL = 0; private static int compareArrays(final int[] src, final int[] dst, int dstOffset) { // We do NOT test the first char, because we come from a method that already // tested it. @@ -469,6 +469,7 @@ public class FusionDictionary implements Iterable<Word> { * This comparator imposes orderings that are inconsistent with equals. */ static private class CharGroupComparator implements java.util.Comparator<CharGroup> { + @Override public int compare(CharGroup c1, CharGroup c2) { if (c1.mChars[0] == c2.mChars[0]) return 0; return c1.mChars[0] < c2.mChars[0] ? -1 : 1; @@ -487,6 +488,8 @@ public class FusionDictionary implements Iterable<Word> { return result >= 0 ? result : -result - 1; } + private static int CHARACTER_NOT_FOUND = -1; + /** * Find the index of a char in a node, if it exists. * @@ -494,7 +497,6 @@ public class FusionDictionary implements Iterable<Word> { * @param character the character to search for. * @return the position of the character if it's there, or CHARACTER_NOT_FOUND = -1 else. */ - private static int CHARACTER_NOT_FOUND = -1; private static int findIndexOfChar(final Node node, int character) { final int insertionIndex = findInsertionIndex(node, character); if (node.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND; diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java index 9807d2892..0e3bf8011 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java @@ -102,13 +102,19 @@ public class AndroidSpellCheckerService extends SpellCheckerService // will never have any suggestions, so it makes no sense checking them. mLanguageToScript = new TreeMap<String, Integer>(); mLanguageToScript.put("en", SCRIPT_LATIN); + mLanguageToScript.put("en_US", SCRIPT_LATIN); + mLanguageToScript.put("en_GB", SCRIPT_LATIN); mLanguageToScript.put("fr", SCRIPT_LATIN); mLanguageToScript.put("de", SCRIPT_LATIN); mLanguageToScript.put("nl", SCRIPT_LATIN); mLanguageToScript.put("cs", SCRIPT_LATIN); mLanguageToScript.put("es", SCRIPT_LATIN); mLanguageToScript.put("it", SCRIPT_LATIN); + mLanguageToScript.put("hr", SCRIPT_LATIN); + mLanguageToScript.put("pt_BR", SCRIPT_LATIN); mLanguageToScript.put("ru", SCRIPT_CYRILLIC); + // TODO: Make a persian proximity, and activate the Farsi subtype. + // mLanguageToScript.put("fa", SCRIPT_PERSIAN); } @Override public void onCreate() { @@ -493,10 +499,6 @@ public class AndroidSpellCheckerService extends SpellCheckerService } mUnigramSuggestionsInfoCache.put(query, new SuggestionsParams(suggestions, flags)); } - - public void remove(String key) { - mUnigramSuggestionsInfoCache.remove(key); - } } AndroidSpellCheckerSession(final AndroidSpellCheckerService service) { @@ -669,6 +671,28 @@ public class AndroidSpellCheckerService extends SpellCheckerService return retval; } + @Override + public SuggestionsInfo[] onGetSuggestionsMultiple(TextInfo[] textInfos, + int suggestionsLimit, boolean sequentialWords) { + final int length = textInfos.length; + final SuggestionsInfo[] retval = new SuggestionsInfo[length]; + for (int i = 0; i < length; ++i) { + final String prevWord; + if (sequentialWords && i > 0) { + final String prevWordCandidate = textInfos[i - 1].getText(); + // Note that an empty string would be used to indicate the initial word + // in the future. + prevWord = TextUtils.isEmpty(prevWordCandidate) ? null : prevWordCandidate; + } else { + prevWord = null; + } + retval[i] = onGetSuggestions(textInfos[i], prevWord, suggestionsLimit); + retval[i].setCookieAndSequence( + textInfos[i].getCookie(), textInfos[i].getSequence()); + } + return retval; + } + // Note : this must be reentrant /** * Gets a list of suggestions for a specific string. This returns a list of possible @@ -678,6 +702,11 @@ public class AndroidSpellCheckerService extends SpellCheckerService @Override public SuggestionsInfo onGetSuggestions(final TextInfo textInfo, final int suggestionsLimit) { + return onGetSuggestions(textInfo, null, suggestionsLimit); + } + + private SuggestionsInfo onGetSuggestions( + final TextInfo textInfo, final String prevWord, final int suggestionsLimit) { try { final String inText = textInfo.getText(); final SuggestionsParams cachedSuggestionsParams = @@ -732,7 +761,7 @@ public class AndroidSpellCheckerService extends SpellCheckerService try { dictInfo = mDictionaryPool.takeOrGetNull(); if (null == dictInfo) return getNotInDictEmptySuggestions(); - dictInfo.mDictionary.getWords(composer, null, suggestionsGatherer, + dictInfo.mDictionary.getWords(composer, prevWord, suggestionsGatherer, dictInfo.mProximityInfo); isInDict = dictInfo.mDictionary.isValidWord(text); if (!isInDict && CAPITALIZE_NONE != capitalizeType) { |