diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin')
32 files changed, 1947 insertions, 1670 deletions
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java index 463d09344..d034515ca 100644 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -55,8 +55,7 @@ abstract public class AbstractDictionaryWriter extends Dictionary { // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve. abstract public void addBigramWords(final String word0, final String word1, - final int frequency, final boolean isValid, - final long lastModifiedTime); + final int frequency, final boolean isValid, final long lastModifiedTime); abstract public void removeBigramWords(final String word0, final String word1); diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java index 722a82961..ad94a0493 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryFileDumper.java @@ -432,8 +432,9 @@ public final class BinaryDictionaryFileDumper { // Actually copy the file final byte[] buffer = new byte[FILE_READ_BUFFER_SIZE]; - for (int readBytes = input.read(buffer); readBytes >= 0; readBytes = input.read(buffer)) + for (int readBytes = input.read(buffer); readBytes >= 0; readBytes = input.read(buffer)) { output.write(buffer, 0, readBytes); + } input.close(); } @@ -478,8 +479,7 @@ public final class BinaryDictionaryFileDumper { * @param context the context for resources and providers. * @param clientId the client ID to use. */ - public static void initializeClientRecordHelper(final Context context, - final String clientId) { + public static void initializeClientRecordHelper(final Context context, final String clientId) { try { final ContentProviderClient client = context.getContentResolver(). acquireContentProviderClient(getProviderUriBuilder("").build()); diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index c4f96016c..9a9653094 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -174,6 +174,7 @@ public final class Constants { public static final int CODE_SLASH = '/'; public static final int CODE_COMMERCIAL_AT = '@'; public static final int CODE_PLUS = '+'; + public static final int CODE_PERCENT = '%'; public static final int CODE_CLOSING_PARENTHESIS = ')'; public static final int CODE_CLOSING_SQUARE_BRACKET = ']'; public static final int CODE_CLOSING_CURLY_BRACKET = '}'; diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index eb8650e6f..d059cc8a9 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -23,7 +23,6 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -53,10 +52,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Whether to print debug output to log */ private static boolean DEBUG = false; - // TODO: Remove. - /** Whether to call binary dictionary dynamically updating methods. */ - public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true; - private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; /** @@ -164,11 +159,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static AbstractDictionaryWriter getDictionaryWriter(final Context context, final String dictType, final boolean isDynamicPersonalizationDictionary) { if (isDynamicPersonalizationDictionary) { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - return null; - } else { - return new DynamicPersonalizationDictionaryWriter(context, dictType); - } + return null; } else { return new DictionaryWriter(context, dictType); } @@ -244,7 +235,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { + if (mDictionaryWriter == null) { mBinaryDictionary.close(); final File file = new File(mContext.getFilesDir(), mFilename); BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), @@ -286,7 +277,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Check whether GC is needed and run GC if required. */ protected void runGCIfRequired(final boolean mindsBlockByGC) { - if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; getExecutor(mFilename).execute(new Runnable() { @Override public void run() { @@ -296,7 +286,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { - if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; // Calls to needsToRunGC() need to be serialized. if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { if (setIsRegeneratingIfNotRegenerating()) { @@ -327,14 +316,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addUnigramWord(word, frequency); - } else { - // TODO: Remove. - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, - isNotAWord); - } + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + mBinaryDictionary.addUnigramWord(word, frequency); } }); } @@ -352,14 +335,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addBigramWords(word0, word1, frequency); - } else { - // TODO: Remove. - mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, - 0 /* lastTouchedTime */); - } + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + mBinaryDictionary.addBigramWords(word0, word1, frequency); } }); } @@ -376,13 +353,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.removeBigramWords(word0, word1); - } else { - // TODO: Remove. - mDictionaryWriter.removeBigramWords(word0, word1); - } + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + mBinaryDictionary.removeBigramWords(word0, word1); } }); } @@ -396,46 +368,20 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (isRegenerating()) { return null; } - final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = new AsyncResultHolder<ArrayList<SuggestedWordInfo>>(); getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - if (mBinaryDictionary == null) { - holder.set(null); - return; - } - final ArrayList<SuggestedWordInfo> binarySuggestion = - mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, additionalFeaturesOptions, - sessionId); - holder.set(binarySuggestion); - } else { - final ArrayList<SuggestedWordInfo> inMemDictSuggestion = - composer.isBatchMode() ? null : - mDictionaryWriter.getSuggestionsWithSessionId(composer, - prevWord, proximityInfo, blockOffensiveWords, - additionalFeaturesOptions, sessionId); - // TODO: Remove checking mIsUpdatable and use native suggestion. - if (mBinaryDictionary != null && !mIsUpdatable) { - final ArrayList<SuggestedWordInfo> binarySuggestion = - mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, - additionalFeaturesOptions, sessionId); - if (inMemDictSuggestion == null) { - holder.set(binarySuggestion); - } else if (binarySuggestion == null) { - holder.set(inMemDictSuggestion); - } else { - binarySuggestion.addAll(inMemDictSuggestion); - holder.set(binarySuggestion); - } - } else { - holder.set(inMemDictSuggestion); - } + if (mBinaryDictionary == null) { + holder.set(null); + return; } + final ArrayList<SuggestedWordInfo> binarySuggestion = + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + sessionId); + holder.set(binarySuggestion); } }); return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); @@ -542,20 +488,16 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { loadDictionaryAsync(); mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } else { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { - final File file = new File(mContext.getFilesDir(), mFilename); - BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); + if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { + final File file = new File(mContext.getFilesDir(), mFilename); + BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); + } else { + if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + mBinaryDictionary.flushWithGC(); } else { - if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { - mBinaryDictionary.flushWithGC(); - } else { - mBinaryDictionary.flush(); - } + mBinaryDictionary.flush(); } - } else { - mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } } } @@ -663,20 +605,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Load the dictionary to memory. - */ - protected void asyncLoadDictionaryToMemory() { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - loadDictionaryAsync(); - } - } - }); - } - - /** * Generate binary dictionary using DictionaryWriter. */ protected void asyncFlashAllBinaryDictionary() { @@ -704,7 +632,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - // TODO: Implement native binary methods once the dynamic dictionary implementation is done. + // TODO: Implement BinaryDictionary.isInDictionary(). @UsedForTesting public boolean isInDictionaryForTests(final String word) { final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); @@ -712,12 +640,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public void run() { if (mDictType == Dictionary.TYPE_USER_HISTORY) { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - holder.set(mBinaryDictionary.isValidWord(word)); - } else { - holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) - .isInBigramListForTests(word)); - } + holder.set(mBinaryDictionary.isValidWord(word)); } } }); @@ -733,4 +656,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public boolean isTerminatedForTests() { return getExecutor(mFilename).isTerminated(); } + + @UsedForTesting + protected void runAfterGcForDebug(final Runnable r) { + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + try { + mBinaryDictionary.flushWithGC(); + r.run(); + } finally { + mFilenameDictionaryUpdateController.mIsRegenerating.set(false); + } + } + }); + } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java deleted file mode 100644 index 95c9bcab9..000000000 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ /dev/null @@ -1,894 +0,0 @@ -/* - * Copyright (C) 2009 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin; - -import android.text.TextUtils; -import android.util.Log; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; - -import java.util.ArrayList; -import java.util.LinkedList; - -/** - * Class for an in-memory dictionary that can grow dynamically and can - * be searched for suggestions and valid words. - */ -// TODO: Remove after binary dictionary supports dynamic update. -public class ExpandableDictionary extends Dictionary { - private static final String TAG = ExpandableDictionary.class.getSimpleName(); - /** - * The weight to give to a word if it's length is the same as the number of typed characters. - */ - private static final int FULL_WORD_SCORE_MULTIPLIER = 2; - - private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH]; - private int mMaxDepth; - private int mInputLength; - - private static final class Node { - char mCode; - int mFrequency; - boolean mTerminal; - Node mParent; - NodeArray mChildren; - ArrayList<char[]> mShortcutTargets; - boolean mShortcutOnly; - LinkedList<NextWord> mNGrams; // Supports ngram - } - - private static final class NodeArray { - Node[] mData; - int mLength = 0; - private static final int INCREMENT = 2; - - NodeArray() { - mData = new Node[INCREMENT]; - } - - void add(final Node n) { - if (mLength + 1 > mData.length) { - Node[] tempData = new Node[mLength + INCREMENT]; - if (mLength > 0) { - System.arraycopy(mData, 0, tempData, 0, mLength); - } - mData = tempData; - } - mData[mLength++] = n; - } - } - - public interface NextWord { - public Node getWordNode(); - public int getFrequency(); - public ForgettingCurveParams getFcParams(); - public int notifyTypedAgainAndGetFrequency(); - } - - private static final class NextStaticWord implements NextWord { - public final Node mWord; - private final int mFrequency; - public NextStaticWord(Node word, int frequency) { - mWord = word; - mFrequency = frequency; - } - - @Override - public Node getWordNode() { - return mWord; - } - - @Override - public int getFrequency() { - return mFrequency; - } - - @Override - public ForgettingCurveParams getFcParams() { - return null; - } - - @Override - public int notifyTypedAgainAndGetFrequency() { - return mFrequency; - } - } - - private static final class NextHistoryWord implements NextWord { - public final Node mWord; - public final ForgettingCurveParams mFcp; - - public NextHistoryWord(Node word, ForgettingCurveParams fcp) { - mWord = word; - mFcp = fcp; - } - - @Override - public Node getWordNode() { - return mWord; - } - - @Override - public int getFrequency() { - return mFcp.getFrequency(); - } - - @Override - public ForgettingCurveParams getFcParams() { - return mFcp; - } - - @Override - public int notifyTypedAgainAndGetFrequency() { - return mFcp.notifyTypedAgainAndGetFrequency(); - } - } - - private NodeArray mRoots; - - private int[][] mCodes; - - public ExpandableDictionary(final String dictType) { - super(dictType); - clearDictionary(); - mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][]; - } - - public int getMaxWordLength() { - return Constants.DICTIONARY_MAX_WORD_LENGTH; - } - - /** - * Add a word with an optional shortcut to the dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - */ - public void addWord(final String word, final String shortcutTarget, final int frequency, - final int shortcutFreq) { - if (word.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH) { - return; - } - addWordRec(mRoots, word, 0, shortcutTarget, frequency, shortcutFreq, null); - } - - /** - * Add a word, recursively searching for its correct place in the trie tree. - * @param children The node to recursively search for addition. Initially, the root of the tree. - * @param word The word to add. - * @param depth The current depth in the tree. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param parentNode The parent node, for up linking. Initially null, as the root has no parent. - */ - private void addWordRec(final NodeArray children, final String word, final int depth, - final String shortcutTarget, final int frequency, final int shortcutFreq, - final Node parentNode) { - final int wordLength = word.length(); - if (wordLength <= depth) return; - final char c = word.charAt(depth); - // Does children have the current character? - final int childrenLength = children.mLength; - Node childNode = null; - for (int i = 0; i < childrenLength; i++) { - final Node node = children.mData[i]; - if (node.mCode == c) { - childNode = node; - break; - } - } - final boolean isShortcutOnly = (null != shortcutTarget); - if (childNode == null) { - childNode = new Node(); - childNode.mCode = c; - childNode.mParent = parentNode; - childNode.mShortcutOnly = isShortcutOnly; - children.add(childNode); - } - if (wordLength == depth + 1) { - // Terminate this word - childNode.mTerminal = true; - if (isShortcutOnly) { - if (null == childNode.mShortcutTargets) { - childNode.mShortcutTargets = CollectionUtils.newArrayList(); - } - childNode.mShortcutTargets.add(shortcutTarget.toCharArray()); - } else { - childNode.mShortcutOnly = false; - } - childNode.mFrequency = Math.max(frequency, childNode.mFrequency); - if (childNode.mFrequency > 255) childNode.mFrequency = 255; - return; - } - if (childNode.mChildren == null) { - childNode.mChildren = new NodeArray(); - } - addWordRec(childNode.mChildren, word, depth + 1, shortcutTarget, frequency, shortcutFreq, - childNode); - } - - @Override - public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { - if (composer.size() > 1) { - if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) { - return null; - } - final ArrayList<SuggestedWordInfo> suggestions = - getWordsInner(composer, prevWord, proximityInfo); - return suggestions; - } else { - if (TextUtils.isEmpty(prevWord)) return null; - final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); - runBigramReverseLookUp(prevWord, suggestions); - return suggestions; - } - } - - private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, - final String prevWordForBigrams, final ProximityInfo proximityInfo) { - final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); - mInputLength = codes.size(); - if (mCodes.length < mInputLength) mCodes = new int[mInputLength][]; - final InputPointers ips = codes.getInputPointers(); - final int[] xCoordinates = ips.getXCoordinates(); - final int[] yCoordinates = ips.getYCoordinates(); - // Cache the codes so that we don't have to lookup an array list - for (int i = 0; i < mInputLength; i++) { - // TODO: Calculate proximity info here. - if (mCodes[i] == null || mCodes[i].length < 1) { - mCodes[i] = new int[ProximityInfo.MAX_PROXIMITY_CHARS_SIZE]; - } - final int x = xCoordinates != null && i < xCoordinates.length ? - xCoordinates[i] : Constants.NOT_A_COORDINATE; - final int y = xCoordinates != null && i < yCoordinates.length ? - yCoordinates[i] : Constants.NOT_A_COORDINATE; - proximityInfo.fillArrayWithNearestKeyCodes(x, y, codes.getCodeAt(i), mCodes[i]); - } - mMaxDepth = mInputLength * 3; - getWordsRec(mRoots, codes, mWordBuilder, 0, false, 1, 0, -1, suggestions); - for (int i = 0; i < mInputLength; i++) { - getWordsRec(mRoots, codes, mWordBuilder, 0, false, 1, 0, i, suggestions); - } - return suggestions; - } - - @Override - public synchronized boolean isValidWord(final String word) { - final Node node = searchNode(mRoots, word, 0, word.length()); - // If node is null, we didn't find the word, so it's not valid. - // If node.mShortcutOnly is true, then it exists as a shortcut but not as a word, - // so that means it's not a valid word. - // If node.mShortcutOnly is false, then it exists as a word (it may also exist as - // a shortcut, but this does not matter), so it's a valid word. - return (node == null) ? false : !node.mShortcutOnly; - } - - public boolean removeBigram(final String word0, final String word1) { - // Refer to addOrSetBigram() about word1.toLowerCase() - final Node firstWord = searchWord(mRoots, word0.toLowerCase(), 0, null); - final Node secondWord = searchWord(mRoots, word1, 0, null); - LinkedList<NextWord> bigrams = firstWord.mNGrams; - NextWord bigramNode = null; - if (bigrams == null || bigrams.size() == 0) { - return false; - } else { - for (NextWord nw : bigrams) { - if (nw.getWordNode() == secondWord) { - bigramNode = nw; - break; - } - } - } - if (bigramNode == null) { - return false; - } - return bigrams.remove(bigramNode); - } - - /** - * Returns the word's frequency or -1 if not found - */ - @UsedForTesting - public int getWordFrequency(final String word) { - // Case-sensitive search - final Node node = searchNode(mRoots, word, 0, word.length()); - return (node == null) ? -1 : node.mFrequency; - } - - public NextWord getBigramWord(final String word0, final String word1) { - // Refer to addOrSetBigram() about word0.toLowerCase() - final Node firstWord = searchWord(mRoots, word0.toLowerCase(), 0, null); - final Node secondWord = searchWord(mRoots, word1, 0, null); - LinkedList<NextWord> bigrams = firstWord.mNGrams; - if (bigrams == null || bigrams.size() == 0) { - return null; - } else { - for (NextWord nw : bigrams) { - if (nw.getWordNode() == secondWord) { - return nw; - } - } - } - return null; - } - - private static int computeSkippedWordFinalFreq(final int freq, final int snr, - final int inputLength) { - // The computation itself makes sense for >= 2, but the == 2 case returns 0 - // anyway so we may as well test against 3 instead and return the constant - if (inputLength >= 3) { - return (freq * snr * (inputLength - 2)) / (inputLength - 1); - } else { - return 0; - } - } - - /** - * Helper method to add a word and its shortcuts. - * - * @param node the terminal node - * @param word the word to insert, as an array of code points - * @param depth the depth of the node in the tree - * @param finalFreq the frequency for this word - * @param suggestions the suggestion collection to add the suggestions to - * @return whether there is still space for more words. - */ - private boolean addWordAndShortcutsFromNode(final Node node, final char[] word, final int depth, - final int finalFreq, final ArrayList<SuggestedWordInfo> suggestions) { - if (finalFreq > 0 && !node.mShortcutOnly) { - // Use KIND_CORRECTION always. This dictionary does not really have a notion of - // COMPLETION against CORRECTION; we could artificially add one by looking at - // the respective size of the typed word and the suggestion if it matters sometime - // in the future. - suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, - SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, - SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); - if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; - } - if (null != node.mShortcutTargets) { - final int length = node.mShortcutTargets.size(); - for (int shortcutIndex = 0; shortcutIndex < length; ++shortcutIndex) { - final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); - suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), - finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, - SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); - if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; - } - } - return true; - } - - /** - * Recursively traverse the tree for words that match the input. Input consists of - * a list of arrays. Each item in the list is one input character position. An input - * character is actually an array of multiple possible candidates. This function is not - * optimized for speed, assuming that the user dictionary will only be a few hundred words in - * size. - * @param roots node whose children have to be search for matches - * @param codes the input character codes - * @param word the word being composed as a possible match - * @param depth the depth of traversal - the length of the word being composed thus far - * @param completion whether the traversal is now in completion mode - meaning that we've - * exhausted the input and we're looking for all possible suffixes. - * @param snr current weight of the word being formed - * @param inputIndex position in the input characters. This can be off from the depth in - * case we skip over some punctuations such as apostrophe in the traversal. That is, if you type - * "wouldve", it could be matching "would've", so the depth will be one more than the - * inputIndex - * @param suggestions the list in which to add suggestions - */ - // TODO: Share this routine with the native code for BinaryDictionary - private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word, - final int depth, final boolean completion, final int snr, final int inputIndex, - final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) { - final int count = roots.mLength; - final int codeSize = mInputLength; - // Optimization: Prune out words that are too long compared to how much was typed. - if (depth > mMaxDepth) { - return; - } - final int[] currentChars; - if (codeSize <= inputIndex) { - currentChars = null; - } else { - currentChars = mCodes[inputIndex]; - } - - for (int i = 0; i < count; i++) { - final Node node = roots.mData[i]; - final char c = node.mCode; - final char lowerC = toLowerCase(c); - final boolean terminal = node.mTerminal; - final NodeArray children = node.mChildren; - final int freq = node.mFrequency; - if (completion || currentChars == null) { - word[depth] = c; - if (terminal) { - final int finalFreq; - if (skipPos < 0) { - finalFreq = freq * snr; - } else { - finalFreq = computeSkippedWordFinalFreq(freq, snr, mInputLength); - } - if (!addWordAndShortcutsFromNode(node, word, depth, finalFreq, suggestions)) { - // No space left in the queue, bail out - return; - } - } - if (children != null) { - getWordsRec(children, codes, word, depth + 1, true, snr, inputIndex, - skipPos, suggestions); - } - } else if ((c == Constants.CODE_SINGLE_QUOTE - && currentChars[0] != Constants.CODE_SINGLE_QUOTE) || depth == skipPos) { - // Skip the ' and continue deeper - word[depth] = c; - if (children != null) { - getWordsRec(children, codes, word, depth + 1, completion, snr, inputIndex, - skipPos, suggestions); - } - } else { - // Don't use alternatives if we're looking for missing characters - final int alternativesSize = skipPos >= 0 ? 1 : currentChars.length; - for (int j = 0; j < alternativesSize; j++) { - final int addedAttenuation = (j > 0 ? 1 : 2); - final int currentChar = currentChars[j]; - if (currentChar == Constants.NOT_A_CODE) { - break; - } - if (currentChar == lowerC || currentChar == c) { - word[depth] = c; - - if (codeSize == inputIndex + 1) { - if (terminal) { - final int finalFreq; - if (skipPos < 0) { - finalFreq = freq * snr * addedAttenuation - * FULL_WORD_SCORE_MULTIPLIER; - } else { - finalFreq = computeSkippedWordFinalFreq(freq, - snr * addedAttenuation, mInputLength); - } - if (!addWordAndShortcutsFromNode(node, word, depth, finalFreq, - suggestions)) { - // No space left in the queue, bail out - return; - } - } - if (children != null) { - getWordsRec(children, codes, word, depth + 1, - true, snr * addedAttenuation, inputIndex + 1, - skipPos, suggestions); - } - } else if (children != null) { - getWordsRec(children, codes, word, depth + 1, - false, snr * addedAttenuation, inputIndex + 1, - skipPos, suggestions); - } - } - } - } - } - } - - public int setBigramAndGetFrequency(final String word0, final String word1, - final int frequency) { - return setBigramAndGetFrequency(word0, word1, frequency, null /* unused */); - } - - public int setBigramAndGetFrequency(final String word0, final String word1, - final ForgettingCurveParams fcp) { - return setBigramAndGetFrequency(word0, word1, 0 /* unused */, fcp); - } - - /** - * Adds bigrams to the in-memory trie structure that is being used to retrieve any word - * @param word0 the first word of this bigram - * @param word1 the second word of this bigram - * @param frequency frequency for this bigram - * @param fcp an instance of ForgettingCurveParams to use for decay policy - * @return returns the final bigram frequency - */ - private int setBigramAndGetFrequency(final String word0, final String word1, - final int frequency, final ForgettingCurveParams fcp) { - if (TextUtils.isEmpty(word0)) { - Log.e(TAG, "Invalid bigram previous word: " + word0); - return frequency; - } - // We don't want results to be different according to case of the looked up left hand side - // word. We do want however to return the correct case for the right hand side. - // So we want to squash the case of the left hand side, and preserve that of the right - // hand side word. - final String word0Lower = word0.toLowerCase(); - if (TextUtils.isEmpty(word0Lower) || TextUtils.isEmpty(word1)) { - Log.e(TAG, "Invalid bigram pair: " + word0 + ", " + word0Lower + ", " + word1); - return frequency; - } - final Node firstWord = searchWord(mRoots, word0Lower, 0, null); - final Node secondWord = searchWord(mRoots, word1, 0, null); - LinkedList<NextWord> bigrams = firstWord.mNGrams; - if (bigrams == null || bigrams.size() == 0) { - firstWord.mNGrams = CollectionUtils.newLinkedList(); - bigrams = firstWord.mNGrams; - } else { - for (NextWord nw : bigrams) { - if (nw.getWordNode() == secondWord) { - return nw.notifyTypedAgainAndGetFrequency(); - } - } - } - if (fcp != null) { - // history - firstWord.mNGrams.add(new NextHistoryWord(secondWord, fcp)); - } else { - firstWord.mNGrams.add(new NextStaticWord(secondWord, frequency)); - } - return frequency; - } - - /** - * Searches for the word and add the word if it does not exist. - * @return Returns the terminal node of the word we are searching for. - */ - private Node searchWord(final NodeArray children, final String word, final int depth, - final Node parentNode) { - final int wordLength = word.length(); - final char c = word.charAt(depth); - // Does children have the current character? - final int childrenLength = children.mLength; - Node childNode = null; - for (int i = 0; i < childrenLength; i++) { - final Node node = children.mData[i]; - if (node.mCode == c) { - childNode = node; - break; - } - } - if (childNode == null) { - childNode = new Node(); - childNode.mCode = c; - childNode.mParent = parentNode; - children.add(childNode); - } - if (wordLength == depth + 1) { - // Terminate this word - childNode.mTerminal = true; - return childNode; - } - if (childNode.mChildren == null) { - childNode.mChildren = new NodeArray(); - } - return searchWord(childNode.mChildren, word, depth + 1, childNode); - } - - private void runBigramReverseLookUp(final String previousWord, - final ArrayList<SuggestedWordInfo> suggestions) { - // Search for the lowercase version of the word only, because that's where bigrams - // store their sons. - final Node prevWord = searchNode(mRoots, previousWord.toLowerCase(), 0, - previousWord.length()); - if (prevWord != null && prevWord.mNGrams != null) { - reverseLookUp(prevWord.mNGrams, suggestions); - } - } - - // Local to reverseLookUp, but do not allocate each time. - private final char[] mLookedUpString = new char[Constants.DICTIONARY_MAX_WORD_LENGTH]; - - /** - * reverseLookUp retrieves the full word given a list of terminal nodes and adds those words - * to the suggestions list passed as an argument. - * @param terminalNodes list of terminal nodes we want to add - * @param suggestions the suggestion collection to add the word to - */ - private void reverseLookUp(final LinkedList<NextWord> terminalNodes, - final ArrayList<SuggestedWordInfo> suggestions) { - Node node; - int freq; - for (NextWord nextWord : terminalNodes) { - node = nextWord.getWordNode(); - freq = nextWord.getFrequency(); - int index = Constants.DICTIONARY_MAX_WORD_LENGTH; - do { - --index; - mLookedUpString[index] = node.mCode; - node = node.mParent; - } while (node != null && index > 0); - - // If node is null, we have a word longer than MAX_WORD_LENGTH in the dictionary. - // It's a little unclear how this can happen, but just in case it does it's safer - // to ignore the word in this case. - if (freq >= 0 && node == null) { - suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, - Constants.DICTIONARY_MAX_WORD_LENGTH - index), - freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, - SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); - } - } - } - - /** - * Recursively search for the terminal node of the word. - * - * One iteration takes the full word to search for and the current index of the recursion. - * - * @param children the node of the trie to search under. - * @param word the word to search for. Only read [offset..length] so there may be trailing chars - * @param offset the index in {@code word} this recursion should operate on. - * @param length the length of the input word. - * @return Returns the terminal node of the word if the word exists - */ - private Node searchNode(final NodeArray children, final CharSequence word, final int offset, - final int length) { - final int count = children.mLength; - final char currentChar = word.charAt(offset); - for (int j = 0; j < count; j++) { - final Node node = children.mData[j]; - if (node.mCode == currentChar) { - if (offset == length - 1) { - if (node.mTerminal) { - return node; - } - } else { - if (node.mChildren != null) { - Node returnNode = searchNode(node.mChildren, word, offset + 1, length); - if (returnNode != null) return returnNode; - } - } - } - } - return null; - } - - public void clearDictionary() { - mRoots = new NodeArray(); - } - - private static char toLowerCase(final char c) { - char baseChar = c; - if (c < BASE_CHARS.length) { - baseChar = BASE_CHARS[c]; - } - if (baseChar >= 'A' && baseChar <= 'Z') { - return (char)(baseChar | 32); - } else if (baseChar > 127) { - return Character.toLowerCase(baseChar); - } - return baseChar; - } - - /** - * Table mapping most combined Latin, Greek, and Cyrillic characters - * to their base characters. If c is in range, BASE_CHARS[c] == c - * if c is not a combined character, or the base character if it - * is combined. - * - * cf. native/jni/src/utils/char_utils.cpp - */ - private static final char BASE_CHARS[] = { - /* U+0000 */ 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, - /* U+0008 */ 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, - /* U+0010 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, - /* U+0018 */ 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, - /* U+0020 */ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, - /* U+0028 */ 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, - /* U+0030 */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, - /* U+0038 */ 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, - /* U+0040 */ 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, - /* U+0048 */ 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, - /* U+0050 */ 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, - /* U+0058 */ 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, - /* U+0060 */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, - /* U+0068 */ 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, - /* U+0070 */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, - /* U+0078 */ 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, - /* U+0080 */ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, - /* U+0088 */ 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, - /* U+0090 */ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, - /* U+0098 */ 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, - /* U+00A0 */ 0x0020, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, - /* U+00A8 */ 0x0020, 0x00A9, 0x0061, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0020, - /* U+00B0 */ 0x00B0, 0x00B1, 0x0032, 0x0033, 0x0020, 0x03BC, 0x00B6, 0x00B7, - /* U+00B8 */ 0x0020, 0x0031, 0x006F, 0x00BB, 0x0031, 0x0031, 0x0033, 0x00BF, - /* U+00C0 */ 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x0041, 0x00C6, 0x0043, - /* U+00C8 */ 0x0045, 0x0045, 0x0045, 0x0045, 0x0049, 0x0049, 0x0049, 0x0049, - /* U+00D0 */ 0x00D0, 0x004E, 0x004F, 0x004F, 0x004F, 0x004F, 0x004F, 0x00D7, - /* U+00D8 */ 0x004F, 0x0055, 0x0055, 0x0055, 0x0055, 0x0059, 0x00DE, 0x0073, - // U+00D8: Manually changed from 00D8 to 004F - // TODO: Check if it's really acceptable to consider Ø a diacritical variant of O - // U+00DF: Manually changed from 00DF to 0073 - /* U+00E0 */ 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x0061, 0x00E6, 0x0063, - /* U+00E8 */ 0x0065, 0x0065, 0x0065, 0x0065, 0x0069, 0x0069, 0x0069, 0x0069, - /* U+00F0 */ 0x00F0, 0x006E, 0x006F, 0x006F, 0x006F, 0x006F, 0x006F, 0x00F7, - /* U+00F8 */ 0x006F, 0x0075, 0x0075, 0x0075, 0x0075, 0x0079, 0x00FE, 0x0079, - // U+00F8: Manually changed from 00F8 to 006F - // TODO: Check if it's really acceptable to consider ø a diacritical variant of o - /* U+0100 */ 0x0041, 0x0061, 0x0041, 0x0061, 0x0041, 0x0061, 0x0043, 0x0063, - /* U+0108 */ 0x0043, 0x0063, 0x0043, 0x0063, 0x0043, 0x0063, 0x0044, 0x0064, - /* U+0110 */ 0x0110, 0x0111, 0x0045, 0x0065, 0x0045, 0x0065, 0x0045, 0x0065, - /* U+0118 */ 0x0045, 0x0065, 0x0045, 0x0065, 0x0047, 0x0067, 0x0047, 0x0067, - /* U+0120 */ 0x0047, 0x0067, 0x0047, 0x0067, 0x0048, 0x0068, 0x0126, 0x0127, - /* U+0128 */ 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, 0x0049, 0x0069, - /* U+0130 */ 0x0049, 0x0131, 0x0049, 0x0069, 0x004A, 0x006A, 0x004B, 0x006B, - /* U+0138 */ 0x0138, 0x004C, 0x006C, 0x004C, 0x006C, 0x004C, 0x006C, 0x004C, - /* U+0140 */ 0x006C, 0x004C, 0x006C, 0x004E, 0x006E, 0x004E, 0x006E, 0x004E, - // U+0141: Manually changed from 0141 to 004C - // U+0142: Manually changed from 0142 to 006C - /* U+0148 */ 0x006E, 0x02BC, 0x014A, 0x014B, 0x004F, 0x006F, 0x004F, 0x006F, - /* U+0150 */ 0x004F, 0x006F, 0x0152, 0x0153, 0x0052, 0x0072, 0x0052, 0x0072, - /* U+0158 */ 0x0052, 0x0072, 0x0053, 0x0073, 0x0053, 0x0073, 0x0053, 0x0073, - /* U+0160 */ 0x0053, 0x0073, 0x0054, 0x0074, 0x0054, 0x0074, 0x0166, 0x0167, - /* U+0168 */ 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, - /* U+0170 */ 0x0055, 0x0075, 0x0055, 0x0075, 0x0057, 0x0077, 0x0059, 0x0079, - /* U+0178 */ 0x0059, 0x005A, 0x007A, 0x005A, 0x007A, 0x005A, 0x007A, 0x0073, - /* U+0180 */ 0x0180, 0x0181, 0x0182, 0x0183, 0x0184, 0x0185, 0x0186, 0x0187, - /* U+0188 */ 0x0188, 0x0189, 0x018A, 0x018B, 0x018C, 0x018D, 0x018E, 0x018F, - /* U+0190 */ 0x0190, 0x0191, 0x0192, 0x0193, 0x0194, 0x0195, 0x0196, 0x0197, - /* U+0198 */ 0x0198, 0x0199, 0x019A, 0x019B, 0x019C, 0x019D, 0x019E, 0x019F, - /* U+01A0 */ 0x004F, 0x006F, 0x01A2, 0x01A3, 0x01A4, 0x01A5, 0x01A6, 0x01A7, - /* U+01A8 */ 0x01A8, 0x01A9, 0x01AA, 0x01AB, 0x01AC, 0x01AD, 0x01AE, 0x0055, - /* U+01B0 */ 0x0075, 0x01B1, 0x01B2, 0x01B3, 0x01B4, 0x01B5, 0x01B6, 0x01B7, - /* U+01B8 */ 0x01B8, 0x01B9, 0x01BA, 0x01BB, 0x01BC, 0x01BD, 0x01BE, 0x01BF, - /* U+01C0 */ 0x01C0, 0x01C1, 0x01C2, 0x01C3, 0x0044, 0x0044, 0x0064, 0x004C, - /* U+01C8 */ 0x004C, 0x006C, 0x004E, 0x004E, 0x006E, 0x0041, 0x0061, 0x0049, - /* U+01D0 */ 0x0069, 0x004F, 0x006F, 0x0055, 0x0075, 0x0055, 0x0075, 0x0055, - // U+01D5: Manually changed from 00DC to 0055 - // U+01D6: Manually changed from 00FC to 0075 - // U+01D7: Manually changed from 00DC to 0055 - /* U+01D8 */ 0x0075, 0x0055, 0x0075, 0x0055, 0x0075, 0x01DD, 0x0041, 0x0061, - // U+01D8: Manually changed from 00FC to 0075 - // U+01D9: Manually changed from 00DC to 0055 - // U+01DA: Manually changed from 00FC to 0075 - // U+01DB: Manually changed from 00DC to 0055 - // U+01DC: Manually changed from 00FC to 0075 - // U+01DE: Manually changed from 00C4 to 0041 - // U+01DF: Manually changed from 00E4 to 0061 - /* U+01E0 */ 0x0041, 0x0061, 0x00C6, 0x00E6, 0x01E4, 0x01E5, 0x0047, 0x0067, - // U+01E0: Manually changed from 0226 to 0041 - // U+01E1: Manually changed from 0227 to 0061 - /* U+01E8 */ 0x004B, 0x006B, 0x004F, 0x006F, 0x004F, 0x006F, 0x01B7, 0x0292, - // U+01EC: Manually changed from 01EA to 004F - // U+01ED: Manually changed from 01EB to 006F - /* U+01F0 */ 0x006A, 0x0044, 0x0044, 0x0064, 0x0047, 0x0067, 0x01F6, 0x01F7, - /* U+01F8 */ 0x004E, 0x006E, 0x0041, 0x0061, 0x00C6, 0x00E6, 0x004F, 0x006F, - // U+01FA: Manually changed from 00C5 to 0041 - // U+01FB: Manually changed from 00E5 to 0061 - // U+01FE: Manually changed from 00D8 to 004F - // TODO: Check if it's really acceptable to consider Ø a diacritical variant of O - // U+01FF: Manually changed from 00F8 to 006F - // TODO: Check if it's really acceptable to consider ø a diacritical variant of o - /* U+0200 */ 0x0041, 0x0061, 0x0041, 0x0061, 0x0045, 0x0065, 0x0045, 0x0065, - /* U+0208 */ 0x0049, 0x0069, 0x0049, 0x0069, 0x004F, 0x006F, 0x004F, 0x006F, - /* U+0210 */ 0x0052, 0x0072, 0x0052, 0x0072, 0x0055, 0x0075, 0x0055, 0x0075, - /* U+0218 */ 0x0053, 0x0073, 0x0054, 0x0074, 0x021C, 0x021D, 0x0048, 0x0068, - /* U+0220 */ 0x0220, 0x0221, 0x0222, 0x0223, 0x0224, 0x0225, 0x0041, 0x0061, - /* U+0228 */ 0x0045, 0x0065, 0x004F, 0x006F, 0x004F, 0x006F, 0x004F, 0x006F, - // U+022A: Manually changed from 00D6 to 004F - // U+022B: Manually changed from 00F6 to 006F - // U+022C: Manually changed from 00D5 to 004F - // U+022D: Manually changed from 00F5 to 006F - /* U+0230 */ 0x004F, 0x006F, 0x0059, 0x0079, 0x0234, 0x0235, 0x0236, 0x0237, - // U+0230: Manually changed from 022E to 004F - // U+0231: Manually changed from 022F to 006F - /* U+0238 */ 0x0238, 0x0239, 0x023A, 0x023B, 0x023C, 0x023D, 0x023E, 0x023F, - /* U+0240 */ 0x0240, 0x0241, 0x0242, 0x0243, 0x0244, 0x0245, 0x0246, 0x0247, - /* U+0248 */ 0x0248, 0x0249, 0x024A, 0x024B, 0x024C, 0x024D, 0x024E, 0x024F, - /* U+0250 */ 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0255, 0x0256, 0x0257, - /* U+0258 */ 0x0258, 0x0259, 0x025A, 0x025B, 0x025C, 0x025D, 0x025E, 0x025F, - /* U+0260 */ 0x0260, 0x0261, 0x0262, 0x0263, 0x0264, 0x0265, 0x0266, 0x0267, - /* U+0268 */ 0x0268, 0x0269, 0x026A, 0x026B, 0x026C, 0x026D, 0x026E, 0x026F, - /* U+0270 */ 0x0270, 0x0271, 0x0272, 0x0273, 0x0274, 0x0275, 0x0276, 0x0277, - /* U+0278 */ 0x0278, 0x0279, 0x027A, 0x027B, 0x027C, 0x027D, 0x027E, 0x027F, - /* U+0280 */ 0x0280, 0x0281, 0x0282, 0x0283, 0x0284, 0x0285, 0x0286, 0x0287, - /* U+0288 */ 0x0288, 0x0289, 0x028A, 0x028B, 0x028C, 0x028D, 0x028E, 0x028F, - /* U+0290 */ 0x0290, 0x0291, 0x0292, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, - /* U+0298 */ 0x0298, 0x0299, 0x029A, 0x029B, 0x029C, 0x029D, 0x029E, 0x029F, - /* U+02A0 */ 0x02A0, 0x02A1, 0x02A2, 0x02A3, 0x02A4, 0x02A5, 0x02A6, 0x02A7, - /* U+02A8 */ 0x02A8, 0x02A9, 0x02AA, 0x02AB, 0x02AC, 0x02AD, 0x02AE, 0x02AF, - /* U+02B0 */ 0x0068, 0x0266, 0x006A, 0x0072, 0x0279, 0x027B, 0x0281, 0x0077, - /* U+02B8 */ 0x0079, 0x02B9, 0x02BA, 0x02BB, 0x02BC, 0x02BD, 0x02BE, 0x02BF, - /* U+02C0 */ 0x02C0, 0x02C1, 0x02C2, 0x02C3, 0x02C4, 0x02C5, 0x02C6, 0x02C7, - /* U+02C8 */ 0x02C8, 0x02C9, 0x02CA, 0x02CB, 0x02CC, 0x02CD, 0x02CE, 0x02CF, - /* U+02D0 */ 0x02D0, 0x02D1, 0x02D2, 0x02D3, 0x02D4, 0x02D5, 0x02D6, 0x02D7, - /* U+02D8 */ 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x02DE, 0x02DF, - /* U+02E0 */ 0x0263, 0x006C, 0x0073, 0x0078, 0x0295, 0x02E5, 0x02E6, 0x02E7, - /* U+02E8 */ 0x02E8, 0x02E9, 0x02EA, 0x02EB, 0x02EC, 0x02ED, 0x02EE, 0x02EF, - /* U+02F0 */ 0x02F0, 0x02F1, 0x02F2, 0x02F3, 0x02F4, 0x02F5, 0x02F6, 0x02F7, - /* U+02F8 */ 0x02F8, 0x02F9, 0x02FA, 0x02FB, 0x02FC, 0x02FD, 0x02FE, 0x02FF, - /* U+0300 */ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, - /* U+0308 */ 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, - /* U+0310 */ 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, - /* U+0318 */ 0x0318, 0x0319, 0x031A, 0x031B, 0x031C, 0x031D, 0x031E, 0x031F, - /* U+0320 */ 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, - /* U+0328 */ 0x0328, 0x0329, 0x032A, 0x032B, 0x032C, 0x032D, 0x032E, 0x032F, - /* U+0330 */ 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, - /* U+0338 */ 0x0338, 0x0339, 0x033A, 0x033B, 0x033C, 0x033D, 0x033E, 0x033F, - /* U+0340 */ 0x0300, 0x0301, 0x0342, 0x0313, 0x0308, 0x0345, 0x0346, 0x0347, - /* U+0348 */ 0x0348, 0x0349, 0x034A, 0x034B, 0x034C, 0x034D, 0x034E, 0x034F, - /* U+0350 */ 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, - /* U+0358 */ 0x0358, 0x0359, 0x035A, 0x035B, 0x035C, 0x035D, 0x035E, 0x035F, - /* U+0360 */ 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, - /* U+0368 */ 0x0368, 0x0369, 0x036A, 0x036B, 0x036C, 0x036D, 0x036E, 0x036F, - /* U+0370 */ 0x0370, 0x0371, 0x0372, 0x0373, 0x02B9, 0x0375, 0x0376, 0x0377, - /* U+0378 */ 0x0378, 0x0379, 0x0020, 0x037B, 0x037C, 0x037D, 0x003B, 0x037F, - /* U+0380 */ 0x0380, 0x0381, 0x0382, 0x0383, 0x0020, 0x00A8, 0x0391, 0x00B7, - /* U+0388 */ 0x0395, 0x0397, 0x0399, 0x038B, 0x039F, 0x038D, 0x03A5, 0x03A9, - /* U+0390 */ 0x03CA, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, - /* U+0398 */ 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, - /* U+03A0 */ 0x03A0, 0x03A1, 0x03A2, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, - /* U+03A8 */ 0x03A8, 0x03A9, 0x0399, 0x03A5, 0x03B1, 0x03B5, 0x03B7, 0x03B9, - /* U+03B0 */ 0x03CB, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, - /* U+03B8 */ 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, - /* U+03C0 */ 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, - /* U+03C8 */ 0x03C8, 0x03C9, 0x03B9, 0x03C5, 0x03BF, 0x03C5, 0x03C9, 0x03CF, - /* U+03D0 */ 0x03B2, 0x03B8, 0x03A5, 0x03D2, 0x03D2, 0x03C6, 0x03C0, 0x03D7, - /* U+03D8 */ 0x03D8, 0x03D9, 0x03DA, 0x03DB, 0x03DC, 0x03DD, 0x03DE, 0x03DF, - /* U+03E0 */ 0x03E0, 0x03E1, 0x03E2, 0x03E3, 0x03E4, 0x03E5, 0x03E6, 0x03E7, - /* U+03E8 */ 0x03E8, 0x03E9, 0x03EA, 0x03EB, 0x03EC, 0x03ED, 0x03EE, 0x03EF, - /* U+03F0 */ 0x03BA, 0x03C1, 0x03C2, 0x03F3, 0x0398, 0x03B5, 0x03F6, 0x03F7, - /* U+03F8 */ 0x03F8, 0x03A3, 0x03FA, 0x03FB, 0x03FC, 0x03FD, 0x03FE, 0x03FF, - /* U+0400 */ 0x0415, 0x0415, 0x0402, 0x0413, 0x0404, 0x0405, 0x0406, 0x0406, - /* U+0408 */ 0x0408, 0x0409, 0x040A, 0x040B, 0x041A, 0x0418, 0x0423, 0x040F, - /* U+0410 */ 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, - /* U+0418 */ 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, - // U+0419: Manually changed from 0418 to 0419 - /* U+0420 */ 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, - /* U+0428 */ 0x0428, 0x0429, 0x042C, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, - // U+042A: Manually changed from 042A to 042C - /* U+0430 */ 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, - /* U+0438 */ 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, - // U+0439: Manually changed from 0438 to 0439 - /* U+0440 */ 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, - /* U+0448 */ 0x0448, 0x0449, 0x044C, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, - // U+044A: Manually changed from 044A to 044C - /* U+0450 */ 0x0435, 0x0435, 0x0452, 0x0433, 0x0454, 0x0455, 0x0456, 0x0456, - /* U+0458 */ 0x0458, 0x0459, 0x045A, 0x045B, 0x043A, 0x0438, 0x0443, 0x045F, - /* U+0460 */ 0x0460, 0x0461, 0x0462, 0x0463, 0x0464, 0x0465, 0x0466, 0x0467, - /* U+0468 */ 0x0468, 0x0469, 0x046A, 0x046B, 0x046C, 0x046D, 0x046E, 0x046F, - /* U+0470 */ 0x0470, 0x0471, 0x0472, 0x0473, 0x0474, 0x0475, 0x0474, 0x0475, - /* U+0478 */ 0x0478, 0x0479, 0x047A, 0x047B, 0x047C, 0x047D, 0x047E, 0x047F, - /* U+0480 */ 0x0480, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, - /* U+0488 */ 0x0488, 0x0489, 0x048A, 0x048B, 0x048C, 0x048D, 0x048E, 0x048F, - /* U+0490 */ 0x0490, 0x0491, 0x0492, 0x0493, 0x0494, 0x0495, 0x0496, 0x0497, - /* U+0498 */ 0x0498, 0x0499, 0x049A, 0x049B, 0x049C, 0x049D, 0x049E, 0x049F, - /* U+04A0 */ 0x04A0, 0x04A1, 0x04A2, 0x04A3, 0x04A4, 0x04A5, 0x04A6, 0x04A7, - /* U+04A8 */ 0x04A8, 0x04A9, 0x04AA, 0x04AB, 0x04AC, 0x04AD, 0x04AE, 0x04AF, - /* U+04B0 */ 0x04B0, 0x04B1, 0x04B2, 0x04B3, 0x04B4, 0x04B5, 0x04B6, 0x04B7, - /* U+04B8 */ 0x04B8, 0x04B9, 0x04BA, 0x04BB, 0x04BC, 0x04BD, 0x04BE, 0x04BF, - /* U+04C0 */ 0x04C0, 0x0416, 0x0436, 0x04C3, 0x04C4, 0x04C5, 0x04C6, 0x04C7, - /* U+04C8 */ 0x04C8, 0x04C9, 0x04CA, 0x04CB, 0x04CC, 0x04CD, 0x04CE, 0x04CF, - /* U+04D0 */ 0x0410, 0x0430, 0x0410, 0x0430, 0x04D4, 0x04D5, 0x0415, 0x0435, - /* U+04D8 */ 0x04D8, 0x04D9, 0x04D8, 0x04D9, 0x0416, 0x0436, 0x0417, 0x0437, - /* U+04E0 */ 0x04E0, 0x04E1, 0x0418, 0x0438, 0x0418, 0x0438, 0x041E, 0x043E, - /* U+04E8 */ 0x04E8, 0x04E9, 0x04E8, 0x04E9, 0x042D, 0x044D, 0x0423, 0x0443, - /* U+04F0 */ 0x0423, 0x0443, 0x0423, 0x0443, 0x0427, 0x0447, 0x04F6, 0x04F7, - /* U+04F8 */ 0x042B, 0x044B, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF, - }; -} diff --git a/java/src/com/android/inputmethod/latin/InputAttributes.java b/java/src/com/android/inputmethod/latin/InputAttributes.java index 8caf6f17f..fcf043031 100644 --- a/java/src/com/android/inputmethod/latin/InputAttributes.java +++ b/java/src/com/android/inputmethod/latin/InputAttributes.java @@ -52,8 +52,7 @@ public final class InputAttributes { } else if (inputClass == 0) { // TODO: is this check still necessary? Log.w(TAG, String.format("Unexpected input class: inputType=0x%08x" - + " imeOptions=0x%08x", - inputType, editorInfo.imeOptions)); + + " imeOptions=0x%08x", inputType, editorInfo.imeOptions)); } mIsSettingsSuggestionStripOn = false; mInputTypeNoAutoCorrect = false; @@ -204,8 +203,7 @@ public final class InputAttributes { public static boolean inPrivateImeOptions(String packageName, String key, EditorInfo editorInfo) { if (editorInfo == null) return false; - final String findingKey = (packageName != null) ? packageName + "." + key - : key; + final String findingKey = (packageName != null) ? packageName + "." + key : key; return StringUtils.containsInCommaSplittableText(findingKey, editorInfo.privateImeOptions); } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 608bb3cea..955b83556 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -196,9 +196,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private int mLastSelectionStart = NOT_A_CURSOR_POSITION; private int mLastSelectionEnd = NOT_A_CURSOR_POSITION; - // Whether we are expecting an onUpdateSelection event to fire. If it does when we don't - // "expect" it, it means the user actually moved the cursor. - private boolean mExpectingUpdateSelection; private int mDeleteCount; private long mLastKeyTime; private final TreeSet<Long> mCurrentlyPressedHardwareKeys = CollectionUtils.newTreeSet(); @@ -725,8 +722,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen LatinImeLogger.commit(); LatinImeLogger.onDestroy(); if (mInputUpdater != null) { - mInputUpdater.onDestroy(); - mInputUpdater = null; + mInputUpdater.quitLooper(); } super.onDestroy(); } @@ -761,8 +757,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen .findViewById(android.R.id.extractArea); mKeyPreviewBackingView = view.findViewById(R.id.key_preview_backing); mSuggestionStripView = (SuggestionStripView)view.findViewById(R.id.suggestion_strip_view); - if (mSuggestionStripView != null) + if (mSuggestionStripView != null) { mSuggestionStripView.setListener(this, view); + } if (LatinImeLogger.sVISUALDEBUG) { mKeyPreviewBackingView.setBackgroundColor(0x10FF0000); } @@ -811,6 +808,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen super.onStartInputView(editorInfo, restarting); mRichImm.clearSubtypeCaches(); final KeyboardSwitcher switcher = mKeyboardSwitcher; + switcher.updateKeyboardTheme(); final MainKeyboardView mainKeyboardView = switcher.getMainKeyboardView(); // If we are starting input in a different text field from before, we'll have to reload // settings, so currentSettingsValues can't be final. @@ -907,9 +905,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // refresh later. final boolean canReachInputConnection; if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(editorInfo.initialSelStart, - false /* shouldFinishComposition */)) { + editorInfo.initialSelEnd, false /* shouldFinishComposition */)) { // We try resetting the caches up to 5 times before giving up. mHandler.postResetCaches(isDifferentTextField, 5 /* remainingTries */); + // mLastSelection{Start,End} are reset later in this method, don't need to do it here canReachInputConnection = false; } else { if (isDifferentTextField) { @@ -989,10 +988,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (textLength > mLastSelectionStart || (textLength < Constants.EDITOR_CONTENTS_CACHE_SIZE && mLastSelectionStart < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { + // It should not be possible to have only one of those variables be + // NOT_A_CURSOR_POSITION, so if they are equal, either the selection is zero-sized + // (simple cursor, no selection) or there is no cursor/we don't know its pos + final boolean wasEqual = mLastSelectionStart == mLastSelectionEnd; mLastSelectionStart = textLength; // We can't figure out the value of mLastSelectionEnd :( - // But at least if it's smaller than mLastSelectionStart something is wrong - if (mLastSelectionStart > mLastSelectionEnd) { + // But at least if it's smaller than mLastSelectionStart something is wrong, + // and if they used to be equal we also don't want to make it look like there is a + // selection. + if (wasEqual || mLastSelectionStart > mLastSelectionEnd) { mLastSelectionEnd = mLastSelectionStart; } } @@ -1081,16 +1086,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen + ", ce=" + composingSpanEnd); } if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - final boolean expectingUpdateSelectionFromLogger = - ResearchLogger.getAndClearLatinIMEExpectingUpdateSelection(); ResearchLogger.latinIME_onUpdateSelection(mLastSelectionStart, mLastSelectionEnd, oldSelStart, oldSelEnd, newSelStart, newSelEnd, composingSpanStart, - composingSpanEnd, mExpectingUpdateSelection, - expectingUpdateSelectionFromLogger, mConnection); - if (expectingUpdateSelectionFromLogger) { - // TODO: Investigate. Quitting now sounds wrong - we won't do the resetting work - return; - } + composingSpanEnd, mConnection); } final boolean selectionChanged = mLastSelectionStart != newSelStart @@ -1109,14 +1107,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // TODO: revisit this when LatinIME supports hardware keyboards. // NOTE: the test harness subclasses LatinIME and overrides isInputViewShown(). // TODO: find a better way to simulate actual execution. - if (isInputViewShown() && !mExpectingUpdateSelection - && !mConnection.isBelatedExpectedUpdate(oldSelStart, newSelStart)) { - // TAKE CARE: there is a race condition when we enter this test even when the user - // did not explicitly move the cursor. This happens when typing fast, where two keys - // turn this flag on in succession and both onUpdateSelection() calls arrive after - // the second one - the first call successfully avoids this test, but the second one - // enters. For the moment we rely on noComposingSpan to further reduce the impact. - + if (isInputViewShown() && !mConnection.isBelatedExpectedUpdate(oldSelStart, newSelStart, + oldSelEnd, newSelEnd)) { // TODO: the following is probably better done in resetEntireInputState(). // it should only happen when the cursor moved, and the very purpose of the // test below is to narrow down whether this happened or not. Likewise with @@ -1142,13 +1134,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Another option would be to send suggestions each time we set the composing // text, but that is probably too expensive to do, so we decided to leave things // as is. - resetEntireInputState(newSelStart); + resetEntireInputState(newSelStart, newSelEnd); } else { - // resetEntireInputState calls resetCachesUponCursorMove, but with the second - // argument as true. But in all cases where we don't reset the entire input state, - // we still want to tell the rich input connection about the new cursor position so - // that it can update its caches. - mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, + // resetEntireInputState calls resetCachesUponCursorMove, but forcing the + // composition to end. But in all cases where we don't reset the entire input + // state, we still want to tell the rich input connection about the new cursor + // position so that it can update its caches. + mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, newSelEnd, false /* shouldFinishComposition */); } @@ -1161,7 +1153,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mRecapitalizeStatus.deactivate(); mKeyboardSwitcher.updateShiftState(); } - mExpectingUpdateSelection = false; // Make a note of the cursor position mLastSelectionStart = newSelStart; @@ -1347,8 +1338,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public boolean onEvaluateFullscreenMode() { // Reread resource value here, because this method is called by framework anytime as needed. - final boolean isFullscreenModeAllowed = - Settings.readUseFullscreenMode(getResources()); + final boolean isFullscreenModeAllowed = Settings.readUseFullscreenMode(getResources()); if (super.onEvaluateFullscreenMode() && isFullscreenModeAllowed) { // TODO: Remove this hack. Actually we should not really assume NO_EXTRACT_UI // implies NO_FULLSCREEN. However, the framework mistakenly does. i.e. NO_EXTRACT_UI @@ -1373,7 +1363,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This will reset the whole input state to the starting state. It will clear // the composing word, reset the last composed word, tell the inputconnection about it. - private void resetEntireInputState(final int newCursorPosition) { + private void resetEntireInputState(final int newSelStart, final int newSelEnd) { final boolean shouldFinishComposition = mWordComposer.isComposingWord(); resetComposingState(true /* alsoResetLastComposedWord */); final SettingsValues settingsValues = mSettings.getCurrent(); @@ -1382,14 +1372,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { setSuggestedWords(settingsValues.mSuggestPuncList, false); } - mConnection.resetCachesUponCursorMoveAndReturnSuccess(newCursorPosition, + mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, newSelEnd, shouldFinishComposition); } private void resetComposingState(final boolean alsoResetLastComposedWord) { mWordComposer.reset(); - if (alsoResetLastComposedWord) + if (alsoResetLastComposedWord) { mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; + } } private void commitTyped(final String separatorString) { @@ -1436,15 +1427,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (0 != (auto & TextUtils.CAP_MODE_CHARACTERS)) { return WordComposer.CAPS_MODE_AUTO_SHIFT_LOCKED; } - if (0 != auto) return WordComposer.CAPS_MODE_AUTO_SHIFTED; + if (0 != auto) { + return WordComposer.CAPS_MODE_AUTO_SHIFTED; + } return WordComposer.CAPS_MODE_OFF; } private void swapSwapperAndSpace() { final CharSequence lastTwo = mConnection.getTextBeforeCursor(2, 0); // It is guaranteed lastTwo.charAt(1) is a swapper - else this method is not called. - if (lastTwo != null && lastTwo.length() == 2 - && lastTwo.charAt(0) == Constants.CODE_SPACE) { + if (lastTwo != null && lastTwo.length() == 2 && lastTwo.charAt(0) == Constants.CODE_SPACE) { mConnection.deleteSurroundingText(2, 0); final String text = lastTwo.charAt(1) + " "; mConnection.commitText(text, 1); @@ -1457,7 +1449,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean maybeDoubleSpacePeriod() { final SettingsValues currentSettingsValues = mSettings.getCurrent(); - if (!currentSettingsValues.mCorrectionEnabled) return false; if (!currentSettingsValues.mUseDoubleSpacePeriod) return false; if (!mHandler.isAcceptingDoubleSpacePeriod()) return false; // We only do this when we see two spaces and an accepted code point before the cursor. @@ -1502,6 +1493,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen || codePoint == Constants.CODE_CLOSING_CURLY_BRACKET || codePoint == Constants.CODE_CLOSING_ANGLE_BRACKET || codePoint == Constants.CODE_PLUS + || codePoint == Constants.CODE_PERCENT || Character.getType(codePoint) == Character.OTHER_SYMBOL; } @@ -1723,7 +1715,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the character at the current cursor position. - resetEntireInputState(mLastSelectionStart); + resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); } else { commitTyped(LastComposedWord.NOT_A_SEPARATOR); } @@ -1739,7 +1731,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } handleCharacter(primaryCode, keyX, keyY, spaceState); } - mExpectingUpdateSelection = true; return didAutoCorrect; } @@ -1792,7 +1783,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the batch input at the current cursor position. - resetEntireInputState(mLastSelectionStart); + resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); } else if (wordComposerSize <= 1) { // We auto-correct the previous (typed, not gestured) string iff it's one character // long. The reason for this is, even in the middle of gesture typing, you'll still @@ -1805,7 +1796,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { commitTyped(LastComposedWord.NOT_A_SEPARATOR); } - mExpectingUpdateSelection = true; } final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); if (Character.isLetterOrDigit(codePointBeforeCursor) @@ -1816,13 +1806,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); } - private static final class InputUpdater implements Handler.Callback { + static final class InputUpdater implements Handler.Callback { private final Handler mHandler; private final LatinIME mLatinIme; private final Object mLock = new Object(); private boolean mInBatchInput; // synchronized using {@link #mLock}. - private InputUpdater(final LatinIME latinIme) { + InputUpdater(final LatinIME latinIme) { final HandlerThread handlerThread = new HandlerThread( InputUpdater.class.getSimpleName()); handlerThread.start(); @@ -1939,7 +1929,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen .sendToTarget(); } - private void onDestroy() { + void quitLooper() { mHandler.removeMessages(MSG_GET_SUGGESTED_WORDS); mHandler.removeMessages(MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP); mHandler.getLooper().quit(); @@ -1996,8 +1986,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This method must run in UI Thread. public void onEndBatchInputAsyncInternal(final SuggestedWords suggestedWords) { - final String batchInputText = suggestedWords.isEmpty() - ? null : suggestedWords.getWord(0); + final String batchInputText = suggestedWords.isEmpty() ? null : suggestedWords.getWord(0); if (TextUtils.isEmpty(batchInputText)) { return; } @@ -2019,7 +2008,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mWordComposer.setBatchInputWord(batchInputText); mConnection.setComposingText(batchInputText, 1); } - mExpectingUpdateSelection = true; mConnection.endBatchEdit(); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_onEndBatchInput(batchInputText, 0, suggestedWords); @@ -2073,9 +2061,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } private void handleBackspace(final int spaceState) { - // We revert these in this method if the deletion doesn't happen. mDeleteCount++; - mExpectingUpdateSelection = true; // In many cases, we may have to put the keyboard in auto-shift state again. However // we want to wait a few milliseconds before doing it to avoid the keyboard flashing @@ -2085,7 +2071,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can remove the character at the current cursor position. - resetEntireInputState(mLastSelectionStart); + resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); // When we exit this if-clause, mWordComposer.isComposingWord() will return false. } if (mWordComposer.isComposingWord()) { @@ -2168,10 +2154,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); if (codePointBeforeCursor == Constants.NOT_A_CODE) { - // Nothing to delete before the cursor. We have to revert the deletion states - // that were updated at the beginning of this method. - mDeleteCount--; - mExpectingUpdateSelection = false; + // Nothing to delete before the cursor. return; } final int lengthToDelete = @@ -2219,8 +2202,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen /* * Strip a trailing space if necessary and returns whether it's a swap weak space situation. */ - private boolean maybeStripSpace(final int code, - final int spaceState, final boolean isFromSuggestionStrip) { + private boolean maybeStripSpace(final int code, final int spaceState, + final boolean isFromSuggestionStrip) { if (Constants.CODE_ENTER == code && SPACE_STATE_SWAP_PUNCTUATION == spaceState) { mConnection.removeTrailingSpace(); return false; @@ -2235,8 +2218,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return false; } - private void handleCharacter(final int primaryCode, final int x, - final int y, final int spaceState) { + private void handleCharacter(final int primaryCode, final int x, final int y, + final int spaceState) { // TODO: refactor this method to stop flipping isComposingWord around all the time, and // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter // which has the same name as other handle* methods but is not the same. @@ -2256,7 +2239,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the character at the current cursor position. - resetEntireInputState(mLastSelectionStart); + resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); isComposingWord = false; } // We want to find out whether to start composing a new word with this character. If so, @@ -2303,8 +2286,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } mConnection.setComposingText(getTextWithUnderline(mWordComposer.getTypedWord()), 1); } else { - final boolean swapWeakSpace = maybeStripSpace(primaryCode, - spaceState, Constants.SUGGESTION_STRIP_COORDINATE == x); + final boolean swapWeakSpace = maybeStripSpace(primaryCode, spaceState, + Constants.SUGGESTION_STRIP_COORDINATE == x); sendKeyCodePoint(primaryCode); @@ -2342,9 +2325,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!mRecapitalizeStatus.isSetAt(mLastSelectionStart, mLastSelectionEnd)) { mLastSelectionStart = mRecapitalizeStatus.getNewCursorStart(); mLastSelectionEnd = mRecapitalizeStatus.getNewCursorEnd(); - mConnection.setSelection(mLastSelectionStart, mLastSelectionEnd); } } + mConnection.finishComposingText(); mRecapitalizeStatus.rotate(); final int numCharsDeleted = mLastSelectionEnd - mLastSelectionStart; mConnection.setSelection(mLastSelectionEnd, mLastSelectionEnd); @@ -2368,7 +2351,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the separator at the current cursor position. - resetEntireInputState(mLastSelectionStart); + resetEntireInputState(mLastSelectionStart, mLastSelectionEnd); } if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing if (currentSettings.mCorrectionEnabled) { @@ -2541,6 +2524,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } + private String getPreviousWordForSuggestion(final SettingsValues currentSettings) { + if (currentSettings.mCurrentLanguageHasSpaces) { + // If we are typing in a language with spaces we can just look up the previous + // word from textview. + return mConnection.getNthPreviousWord(currentSettings.mWordSeparators, + mWordComposer.isComposingWord() ? 2 : 1); + } else { + return LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null + : mLastComposedWord.mCommittedWord; + } + } + private void getSuggestedWords(final int sessionId, final int sequenceNumber, final OnGetSuggestedWordsCallback callback) { final Keyboard keyboard = mKeyboardSwitcher.getKeyboard(); @@ -2554,16 +2549,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // should just skip whitespace if any, so 1. final SettingsValues currentSettings = mSettings.getCurrent(); final int[] additionalFeaturesOptions = currentSettings.mAdditionalFeaturesSettingValues; - final String prevWord; - if (currentSettings.mCurrentLanguageHasSpaces) { - // If we are typing in a language with spaces we can just look up the previous - // word from textview. - prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, - mWordComposer.isComposingWord() ? 2 : 1); - } else { - prevWord = LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null - : mLastComposedWord.mCommittedWord; - } + final String prevWord = getPreviousWordForSuggestion(currentSettings); suggest.getSuggestedWords(mWordComposer, prevWord, keyboard.getProximityInfo(), currentSettings.mBlockPotentiallyOffensive, currentSettings.mCorrectionEnabled, additionalFeaturesOptions, sessionId, sequenceNumber, callback); @@ -2681,7 +2667,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.latinIme_commitCurrentAutoCorrection(typedWord, autoCorrection, separator, mWordComposer.isBatchMode(), suggestedWords); } - mExpectingUpdateSelection = true; commitChosenWord(autoCorrection, LastComposedWord.COMMIT_TYPE_DECIDED_WORD, separator); if (!typedWord.equals(autoCorrection)) { @@ -2752,7 +2737,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // typed word. final String replacedWord = mWordComposer.getTypedWord(); LatinImeLogger.logOnManualSuggestion(replacedWord, suggestion, index, suggestedWords); - mExpectingUpdateSelection = true; commitChosenWord(suggestion, LastComposedWord.COMMIT_TYPE_MANUAL_PICK, LastComposedWord.NOT_A_SEPARATOR); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -2999,7 +2983,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen * @param remainingTries How many times we may try again before giving up. */ private void retryResetCaches(final boolean tryResumeSuggestions, final int remainingTries) { - if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, false)) { + if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, + mLastSelectionEnd, false)) { if (0 < remainingTries) { mHandler.postResetCaches(tryResumeSuggestions, remainingTries - 1); return; @@ -3026,8 +3011,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen throw new RuntimeException("revertCommit, but we are composing a word"); } final CharSequence wordBeforeCursor = - mConnection.getTextBeforeCursor(deleteLength, 0) - .subSequence(0, cancelLength); + mConnection.getTextBeforeCursor(deleteLength, 0).subSequence(0, cancelLength); if (!TextUtils.equals(committedWord, wordBeforeCursor)) { throw new RuntimeException("revertCommit check failed: we thought we were " + "reverting \"" + committedWord @@ -3228,8 +3212,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Intent intent = IntentUtils.getInputLanguageSelectionIntent( mRichImm.getInputMethodIdOfThisIme(), Intent.FLAG_ACTIVITY_NEW_TASK - | Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED - | Intent.FLAG_ACTIVITY_CLEAR_TOP); + | Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED + | Intent.FLAG_ACTIVITY_CLEAR_TOP); startActivity(intent); break; case 1: @@ -3238,9 +3222,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } }; - final AlertDialog.Builder builder = new AlertDialog.Builder(this) - .setItems(items, listener) - .setTitle(title); + final AlertDialog.Builder builder = + new AlertDialog.Builder(this).setItems(items, listener).setTitle(title); showOptionDialog(builder.create()); } @@ -3301,11 +3284,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final Printer p = new PrintWriterPrinter(fout); p.println("LatinIME state :"); + p.println(" VersionCode = " + ApplicationUtils.getVersionCode(this)); + p.println(" VersionName = " + ApplicationUtils.getVersionName(this)); final Keyboard keyboard = mKeyboardSwitcher.getKeyboard(); final int keyboardMode = keyboard != null ? keyboard.mId.mMode : -1; p.println(" Keyboard mode = " + keyboardMode); final SettingsValues settingsValues = mSettings.getCurrent(); - p.println(" mIsSuggestionsSuggestionsRequested = " + p.println(" mIsSuggestionsRequested = " + settingsValues.isSuggestionsRequested(mDisplayOrientation)); p.println(" mCorrectionEnabled=" + settingsValues.mCorrectionEnabled); p.println(" isComposingWord=" + mWordComposer.isComposingWord()); diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index c212f9c81..4a7f530f9 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -57,14 +57,19 @@ public final class RichInputConnection { private static final int INVALID_CURSOR_POSITION = -1; /** - * This variable contains an expected value for the cursor position. This is where the - * cursor may end up after all the keyboard-triggered updates have passed. We keep this to - * compare it to the actual cursor position to guess whether the move was caused by a - * keyboard command or not. - * It's not really the cursor position: the cursor may not be there yet, and it's also expected - * there be cases where it never actually comes to be there. + * This variable contains an expected value for the selection start position. This is where the + * cursor or selection start may end up after all the keyboard-triggered updates have passed. We + * keep this to compare it to the actual selection start to guess whether the move was caused by + * a keyboard command or not. + * It's not really the selection start position: the selection start may not be there yet, and + * in some cases, it may never arrive there. */ - private int mExpectedCursorPosition = INVALID_CURSOR_POSITION; // in chars, not code points + private int mExpectedSelStart = INVALID_CURSOR_POSITION; // in chars, not code points + /** + * The expected selection end. Only differs from mExpectedSelStart if a non-empty selection is + * expected. The same caveats as mExpectedSelStart apply. + */ + private int mExpectedSelEnd = INVALID_CURSOR_POSITION; // in chars, not code points /** * This contains the committed text immediately preceding the cursor and the composing * text if any. It is refreshed when the cursor moves by calling upon the TextView. @@ -103,16 +108,16 @@ public final class RichInputConnection { final String reference = (beforeCursor.length() <= actualLength) ? beforeCursor.toString() : beforeCursor.subSequence(beforeCursor.length() - actualLength, beforeCursor.length()).toString(); - if (et.selectionStart != mExpectedCursorPosition + if (et.selectionStart != mExpectedSelStart || !(reference.equals(internal.toString()))) { - final String context = "Expected cursor position = " + mExpectedCursorPosition - + "\nActual cursor position = " + et.selectionStart + final String context = "Expected selection start = " + mExpectedSelStart + + "\nActual selection start = " + et.selectionStart + "\nExpected text = " + internal.length() + " " + internal + "\nActual text = " + reference.length() + " " + reference; ((LatinIME)mParent).debugDumpStateAndCrashWithException(context); } else { Log.e(TAG, DebugLogUtils.getStackTrace(2)); - Log.e(TAG, "Exp <> Actual : " + mExpectedCursorPosition + " <> " + et.selectionStart); + Log.e(TAG, "Exp <> Actual : " + mExpectedSelStart + " <> " + et.selectionStart); } } @@ -150,16 +155,50 @@ public final class RichInputConnection { * data, so we empty the cache and note that we don't know the new cursor position, and we * return false so that the caller knows about this and can retry later. * - * @param newCursorPosition The new position of the cursor, as received from the system. - * @param shouldFinishComposition Whether we should finish the composition in progress. + * @param newSelStart the new position of the selection start, as received from the system. + * @param newSelEnd the new position of the selection end, as received from the system. + * @param shouldFinishComposition whether we should finish the composition in progress. * @return true if we were able to connect to the editor successfully, false otherwise. When * this method returns false, the caches could not be correctly refreshed so they were only * reset: the caller should try again later to return to normal operation. */ - public boolean resetCachesUponCursorMoveAndReturnSuccess(final int newCursorPosition, - final boolean shouldFinishComposition) { - mExpectedCursorPosition = newCursorPosition; + public boolean resetCachesUponCursorMoveAndReturnSuccess(final int newSelStart, + final int newSelEnd, final boolean shouldFinishComposition) { + mExpectedSelStart = newSelStart; + mExpectedSelEnd = newSelEnd; mComposingText.setLength(0); + final boolean didReloadTextSuccessfully = reloadTextCache(); + if (!didReloadTextSuccessfully) { + Log.d(TAG, "Will try to retrieve text later."); + return false; + } + final int lengthOfTextBeforeCursor = mCommittedTextBeforeComposingText.length(); + if (lengthOfTextBeforeCursor > newSelStart + || (lengthOfTextBeforeCursor < Constants.EDITOR_CONTENTS_CACHE_SIZE + && newSelStart < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { + // newSelStart and newSelEnd may be lying -- when rotating the device (probably a + // framework bug). If we have less chars than we asked for, then we know how many chars + // we have, and if we got more than newSelStart says, then we know it was lying. In both + // cases the length is more reliable. Note that we only have to check newSelStart (not + // newSelEnd) since if newSelEnd is wrong, the newSelStart will be wrong as well. + mExpectedSelStart = lengthOfTextBeforeCursor; + mExpectedSelEnd = lengthOfTextBeforeCursor; + } + if (null != mIC && shouldFinishComposition) { + mIC.finishComposingText(); + if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { + ResearchLogger.richInputConnection_finishComposingText(); + } + } + return true; + } + + /** + * Reload the cached text from the InputConnection. + * + * @return true if successful + */ + private boolean reloadTextCache() { mCommittedTextBeforeComposingText.setLength(0); mIC = mParent.getCurrentInputConnection(); // Call upon the inputconnection directly since our own method is using the cache, and @@ -169,27 +208,12 @@ public final class RichInputConnection { if (null == textBeforeCursor) { // For some reason the app thinks we are not connected to it. This looks like a // framework bug... Fall back to ground state and return false. - mExpectedCursorPosition = INVALID_CURSOR_POSITION; - Log.e(TAG, "Unable to connect to the editor to retrieve text... will retry later"); + mExpectedSelStart = INVALID_CURSOR_POSITION; + mExpectedSelEnd = INVALID_CURSOR_POSITION; + Log.e(TAG, "Unable to connect to the editor to retrieve text."); return false; } mCommittedTextBeforeComposingText.append(textBeforeCursor); - final int lengthOfTextBeforeCursor = textBeforeCursor.length(); - if (lengthOfTextBeforeCursor > newCursorPosition - || (lengthOfTextBeforeCursor < Constants.EDITOR_CONTENTS_CACHE_SIZE - && newCursorPosition < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { - // newCursorPosition may be lying -- when rotating the device (probably a framework - // bug). If we have less chars than we asked for, then we know how many chars we have, - // and if we got more than newCursorPosition says, then we know it was lying. In both - // cases the length is more reliable - mExpectedCursorPosition = lengthOfTextBeforeCursor; - } - if (null != mIC && shouldFinishComposition) { - mIC.finishComposingText(); - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.richInputConnection_finishComposingText(); - } - } return true; } @@ -218,7 +242,7 @@ public final class RichInputConnection { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); mCommittedTextBeforeComposingText.append(text); - mExpectedCursorPosition += text.length() - mComposingText.length(); + mExpectedSelStart += text.length() - mComposingText.length(); mComposingText.setLength(0); if (null != mIC) { mIC.commitText(text, i); @@ -231,7 +255,7 @@ public final class RichInputConnection { } public boolean canDeleteCharacters() { - return mExpectedCursorPosition > 0; + return mExpectedSelStart > 0; } /** @@ -268,11 +292,10 @@ public final class RichInputConnection { // heavy pressing of delete, for example DEFAULT_TEXT_CACHE_SIZE - 5 times or so. // getCapsMode should be updated to be able to return a "not enough info" result so that // we can get more context only when needed. - if (TextUtils.isEmpty(mCommittedTextBeforeComposingText) && 0 != mExpectedCursorPosition) { - final CharSequence textBeforeCursor = getTextBeforeCursor( - Constants.EDITOR_CONTENTS_CACHE_SIZE, 0); - if (!TextUtils.isEmpty(textBeforeCursor)) { - mCommittedTextBeforeComposingText.append(textBeforeCursor); + if (TextUtils.isEmpty(mCommittedTextBeforeComposingText) && 0 != mExpectedSelStart) { + if (!reloadTextCache()) { + Log.w(TAG, "Unable to connect to the editor. " + + "Setting caps mode without knowing text."); } } // This never calls InputConnection#getCapsMode - in fact, it's a static method that @@ -292,7 +315,11 @@ public final class RichInputConnection { mCommittedTextBeforeComposingText.length() + mComposingText.length(); // If we have enough characters to satisfy the request, or if we have all characters in // the text field, then we can return the cached version right away. - if (cachedLength >= n || cachedLength >= mExpectedCursorPosition) { + // However, if we don't have an expected cursor position, then we should always + // go fetch the cache again (as it happens, INVALID_CURSOR_POSITION < 0, so we need to + // test for this explicitly) + if (INVALID_CURSOR_POSITION != mExpectedSelStart + && (cachedLength >= n || cachedLength >= mExpectedSelStart)) { final StringBuilder s = new StringBuilder(mCommittedTextBeforeComposingText); // We call #toString() here to create a temporary object. // In some situations, this method is called on a worker thread, and it's possible @@ -332,10 +359,14 @@ public final class RichInputConnection { + remainingChars, 0); mCommittedTextBeforeComposingText.setLength(len); } - if (mExpectedCursorPosition > beforeLength) { - mExpectedCursorPosition -= beforeLength; + if (mExpectedSelStart > beforeLength) { + mExpectedSelStart -= beforeLength; + mExpectedSelEnd -= beforeLength; } else { - mExpectedCursorPosition = 0; + // There are fewer characters before the cursor in the buffer than we are being asked to + // delete. Only delete what is there. + mExpectedSelStart = 0; + mExpectedSelEnd -= mExpectedSelStart; } if (null != mIC) { mIC.deleteSurroundingText(beforeLength, afterLength); @@ -369,7 +400,8 @@ public final class RichInputConnection { switch (keyEvent.getKeyCode()) { case KeyEvent.KEYCODE_ENTER: mCommittedTextBeforeComposingText.append("\n"); - mExpectedCursorPosition += 1; + mExpectedSelStart += 1; + mExpectedSelEnd = mExpectedSelStart; break; case KeyEvent.KEYCODE_DEL: if (0 == mComposingText.length()) { @@ -381,18 +413,24 @@ public final class RichInputConnection { } else { mComposingText.delete(mComposingText.length() - 1, mComposingText.length()); } - if (mExpectedCursorPosition > 0) mExpectedCursorPosition -= 1; + if (mExpectedSelStart > 0 && mExpectedSelStart == mExpectedSelEnd) { + // TODO: Handle surrogate pairs. + mExpectedSelStart -= 1; + } + mExpectedSelEnd = mExpectedSelStart; break; case KeyEvent.KEYCODE_UNKNOWN: if (null != keyEvent.getCharacters()) { mCommittedTextBeforeComposingText.append(keyEvent.getCharacters()); - mExpectedCursorPosition += keyEvent.getCharacters().length(); + mExpectedSelStart += keyEvent.getCharacters().length(); + mExpectedSelEnd = mExpectedSelStart; } break; default: final String text = new String(new int[] { keyEvent.getUnicodeChar() }, 0, 1); mCommittedTextBeforeComposingText.append(text); - mExpectedCursorPosition += text.length(); + mExpectedSelStart += text.length(); + mExpectedSelEnd = mExpectedSelStart; break; } } @@ -426,10 +464,12 @@ public final class RichInputConnection { public void setComposingText(final CharSequence text, final int newCursorPosition) { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); - mExpectedCursorPosition += text.length() - mComposingText.length(); + mExpectedSelStart += text.length() - mComposingText.length(); + mExpectedSelEnd = mExpectedSelStart; mComposingText.setLength(0); mComposingText.append(text); - // TODO: support values of i != 1. At this time, this is never called with i != 1. + // TODO: support values of newCursorPosition != 1. At this time, this is never called with + // newCursorPosition != 1. if (null != mIC) { mIC.setComposingText(text, newCursorPosition); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -439,19 +479,30 @@ public final class RichInputConnection { if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); } - public void setSelection(final int start, final int end) { + /** + * Set the selection of the text editor. + * + * Calls through to {@link InputConnection#setSelection(int, int)}. + * + * @param start the character index where the selection should start. + * @param end the character index where the selection should end. + * @return Returns true on success, false if the input connection is no longer valid either when + * setting the selection or when retrieving the text cache at that point. + */ + public boolean setSelection(final int start, final int end) { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); + mExpectedSelStart = start; if (null != mIC) { - mIC.setSelection(start, end); + final boolean isIcValid = mIC.setSelection(start, end); + if (!isIcValid) { + return false; + } if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.richInputConnection_setSelection(start, end); } } - mExpectedCursorPosition = start; - mCommittedTextBeforeComposingText.setLength(0); - mCommittedTextBeforeComposingText.append( - getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0)); + return reloadTextCache(); } public void commitCorrection(final CorrectionInfo correctionInfo) { @@ -472,7 +523,7 @@ public final class RichInputConnection { // text should never be null, but just in case, it's better to insert nothing than to crash if (null == text) text = ""; mCommittedTextBeforeComposingText.append(text); - mExpectedCursorPosition += text.length() - mComposingText.length(); + mExpectedSelStart += text.length() - mComposingText.length(); mComposingText.setLength(0); if (null != mIC) { mIC.commitCompletion(completionInfo); @@ -754,20 +805,30 @@ public final class RichInputConnection { * this update and not the ones in-between. This is almost impossible to achieve even trying * very very hard. * - * @param oldSelStart The value of the old cursor position in the update. - * @param newSelStart The value of the new cursor position in the update. + * @param oldSelStart The value of the old selection in the update. + * @param newSelStart The value of the new selection in the update. + * @param oldSelEnd The value of the old selection end in the update. + * @param newSelEnd The value of the new selection end in the update. * @return whether this is a belated expected update or not. */ - public boolean isBelatedExpectedUpdate(final int oldSelStart, final int newSelStart) { - // If this is an update that arrives at our expected position, it's a belated update. - if (newSelStart == mExpectedCursorPosition) return true; - // If this is an update that moves the cursor from our expected position, it must be - // an explicit move. - if (oldSelStart == mExpectedCursorPosition) return false; - // The following returns true if newSelStart is between oldSelStart and - // mCurrentCursorPosition. We assume that if the updated position is between the old - // position and the expected position, then it must be a belated update. - return (newSelStart - oldSelStart) * (mExpectedCursorPosition - newSelStart) >= 0; + public boolean isBelatedExpectedUpdate(final int oldSelStart, final int newSelStart, + final int oldSelEnd, final int newSelEnd) { + // This update is "belated" if we are expecting it. That is, mExpectedSelStart and + // mExpectedSelEnd match the new values that the TextView is updating TO. + if (mExpectedSelStart == newSelStart && mExpectedSelEnd == newSelEnd) return true; + // This update is not belated if mExpectedSelStart and mExpeectedSelend match the old + // values, and one of newSelStart or newSelEnd is updated to a different value. In this + // case, there is likely something other than the IME that has moved the selection endpoint + // to the new value. + if (mExpectedSelStart == oldSelStart && mExpectedSelEnd == oldSelEnd + && (oldSelStart != newSelStart || oldSelEnd != newSelEnd)) return false; + // If nether of the above two cases holds, then the system may be having trouble keeping up + // with updates. If 1) the selection is a cursor, 2) newSelStart is between oldSelStart + // and mExpectedSelStart, and 3) newSelEnd is between oldSelEnd and mExpectedSelEnd, then + // assume a belated update. + return (newSelStart == newSelEnd) + && (newSelStart - oldSelStart) * (mExpectedSelStart - newSelStart) >= 0 + && (newSelEnd - oldSelEnd) * (mExpectedSelEnd - newSelEnd) >= 0; } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index 9f7f502ea..fda97dafc 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -60,7 +60,8 @@ public abstract class AbstractDictDecoder implements DictDecoder { 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE), + 0 != (optionsFlags & FormatSpec.CONTAINS_TIMESTAMP_FLAG))); return header; } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 216492b4d..8a8ceaa8c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -169,6 +169,14 @@ public final class BinaryDictDecoderUtils { return size; } + static int getCharArraySize(final int[] chars, final int start, final int end) { + int size = 0; + for (int i = start; i < end; ++i) { + size += getCharSize(chars[i]); + } + return size; + } + /** * Writes a char array to a byte buffer. * @@ -200,8 +208,7 @@ public final class BinaryDictDecoderUtils { * @param word the string to write. * @return the size written, in bytes. */ - static int writeString(final byte[] buffer, final int origin, - final String word) { + static int writeString(final byte[] buffer, final int origin, final String word) { final int length = word.length(); int index = origin; for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { @@ -223,22 +230,62 @@ public final class BinaryDictDecoderUtils { * * This will also write the terminator byte. * - * @param buffer the OutputStream to write to. + * @param stream the OutputStream to write to. * @param word the string to write. + * @return the size written, in bytes. */ - static void writeString(final OutputStream buffer, final String word) throws IOException { + static int writeString(final OutputStream stream, final String word) throws IOException { final int length = word.length(); + int written = 0; for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer.write((byte) codePoint); + final int charSize = getCharSize(codePoint); + if (1 == charSize) { + stream.write((byte) codePoint); } else { - buffer.write((byte) (0xFF & (codePoint >> 16))); - buffer.write((byte) (0xFF & (codePoint >> 8))); - buffer.write((byte) (0xFF & codePoint)); + stream.write((byte) (0xFF & (codePoint >> 16))); + stream.write((byte) (0xFF & (codePoint >> 8))); + stream.write((byte) (0xFF & codePoint)); } + written += charSize; + } + stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + written += FormatSpec.PTNODE_TERMINATOR_SIZE; + return written; + } + + /** + * Writes an array of code points with our character format to an OutputStream. + * + * This will also write the terminator byte. + * + * @param stream the OutputStream to write to. + * @param codePoints the array of code points + * @return the size written, in bytes. + */ + // TODO: Merge this method with writeCharArray and rename the various write* methods to + // make the difference clear. + static int writeCodePoints(final OutputStream stream, final int[] codePoints, + final int startIndex, final int endIndex) + throws IOException { + int written = 0; + for (int i = startIndex; i < endIndex; ++i) { + final int codePoint = codePoints[i]; + final int charSize = getCharSize(codePoint); + if (1 == charSize) { + stream.write((byte) codePoint); + } else { + stream.write((byte) (0xFF & (codePoint >> 16))); + stream.write((byte) (0xFF & (codePoint >> 8))); + stream.write((byte) (0xFF & codePoint)); + } + written += charSize; + } + if (endIndex - startIndex > 1) { + stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + written += FormatSpec.PTNODE_TERMINATOR_SIZE; } - buffer.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + return written; } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index f761829de..c0dad3db2 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; @@ -245,6 +246,26 @@ public class BinaryDictEncoderUtils { } } + static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value, + final int size) { + switch(size) { + case 4: + dictBuffer.put((byte) ((value >> 24) & 0xFF)); + /* fall through */ + case 3: + dictBuffer.put((byte) ((value >> 16) & 0xFF)); + /* fall through */ + case 2: + dictBuffer.put((byte) ((value >> 8) & 0xFF)); + /* fall through */ + case 1: + dictBuffer.put((byte) (value & 0xFF)); + break; + default: + /* nop */ + } + } + // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time @@ -690,6 +711,13 @@ public class BinaryDictEncoderUtils { + word + " is " + unigramFrequency); bigramFrequency = unigramFrequency; } + bigramFlags += getBigramFrequencyDiff(unigramFrequency, bigramFrequency) + & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; + return bigramFlags; + } + + public static int getBigramFrequencyDiff(final int unigramFrequency, + final int bigramFrequency) { // We compute the difference between 255 (which means probability = 1) and the // unigram score. We split this into a number of discrete steps. // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 @@ -723,9 +751,7 @@ public class BinaryDictEncoderUtils { // include this bigram in the dictionary. For now, register as 0, and live with the // small over-estimation that we get in this case. TODO: actually remove this bigram // if discretizedFrequency < 0. - final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; - bigramFlags += finalBigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; - return bigramFlags; + return discretizedFrequency > 0 ? discretizedFrequency : 0; } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 0f7d2f6c9..8d14e4d60 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -245,8 +245,7 @@ public final class BinaryDictIOUtils { /** * @return the size written, in bytes. Always 3 bytes. */ - static int writeSInt24ToBuffer(final DictBuffer dictBuffer, - final int value) { + static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) { final int absValue = Math.abs(value); dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); dictBuffer.put((byte)((absValue >> 8) & 0xFF)); @@ -301,35 +300,6 @@ public final class BinaryDictIOUtils { } /** - * Write a string to a stream. - * - * @param destination the stream to write. - * @param word the string to be written. - * @return the size written, in bytes. - * @throws IOException - */ - private static int writeString(final OutputStream destination, final String word) - throws IOException { - int size = 0; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (CharEncoding.getCharSize(codePoint) == 1) { - destination.write((byte)codePoint); - size++; - } else { - destination.write((byte)(0xFF & (codePoint >> 16))); - destination.write((byte)(0xFF & (codePoint >> 8))); - destination.write((byte)(0xFF & codePoint)); - size += 3; - } - } - destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); - size += FormatSpec.PTNODE_TERMINATOR_SIZE; - return size; - } - - /** * Write a PtNode to an output stream from a PtNodeInfo. * A PtNode is an in-memory representation of a node in the patricia trie. * A PtNode info is a container for low-level information about how the @@ -387,7 +357,7 @@ public final class BinaryDictIOUtils { destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; - size += writeString(destination, target.mWord); + size += CharEncoding.writeString(destination, target.mWord); } } @@ -445,6 +415,25 @@ public final class BinaryDictIOUtils { } /** + * Writes a PtNodeCount to the stream. + * + * @param destination the stream to write. + * @param ptNodeCount the count. + * @return the size written in bytes. + */ + static int writePtNodeCount(final OutputStream destination, final int ptNodeCount) + throws IOException { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + // the count must fit on one byte or two bytes. + // Please see comments in FormatSpec. + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); + } + BinaryDictEncoderUtils.writeUIntToStream(destination, ptNodeCount, countSize); + return countSize; + } + + /** * Write a node array to the stream. * * @param destination the stream to write. @@ -454,18 +443,7 @@ public final class BinaryDictIOUtils { */ static int writeNodes(final OutputStream destination, final PtNodeInfo[] infos) throws IOException { - int size = getPtNodeCountSize(infos.length); - switch (getPtNodeCountSize(infos.length)) { - case 1: - destination.write((byte)infos.length); - break; - case 2: - destination.write((byte)(infos.length >> 8)); - destination.write((byte)(infos.length & 0xFF)); - break; - default: - throw new RuntimeException("Invalid node count size."); - } + int size = writePtNodeCount(destination, infos.length); for (final PtNodeInfo info : infos) size += writePtNode(destination, info); writeSInt24ToStream(destination, FormatSpec.NO_FORWARD_LINK_ADDRESS); return size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 3dbeee099..91543986d 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -35,6 +35,7 @@ import java.util.TreeMap; /** * An interface of binary dictionary decoders. */ +// TODO: Straighten out responsibility for the buffer's file pointer. public interface DictDecoder { /** @@ -43,7 +44,7 @@ public interface DictDecoder { public FileHeader readHeader() throws IOException, UnsupportedFormatException; /** - * Reads PtNode from nodeAddress. + * Reads PtNode from ptNodePos. * @param ptNodePos the position of PtNode. * @param formatOptions the format options. * @return PtNodeInfo. diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 336277196..971b4ff9f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -22,6 +22,7 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.IOException; import java.io.OutputStream; @@ -36,7 +37,7 @@ import java.util.Arrays; @UsedForTesting public final class DynamicBinaryDictIOUtils { private static final boolean DBG = false; - private static final int MAX_JUMPS = 10000; + static final int MAX_JUMPS = 10000; private DynamicBinaryDictIOUtils() { // This utility class is not publicly instantiable. @@ -217,6 +218,25 @@ public final class DynamicBinaryDictIOUtils { } /** + * Converts a list of WeightedString to a list of PendingAttribute. + */ + public static ArrayList<PendingAttribute> resolveBigramPositions(final DictUpdater dictUpdater, + final ArrayList<WeightedString> bigramStrings) + throws IOException, UnsupportedFormatException { + if (bigramStrings == null) return CollectionUtils.newArrayList(); + final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); + for (final WeightedString bigram : bigramStrings) { + final int pos = dictUpdater.getTerminalPosition(bigram.mWord); + if (pos == FormatSpec.NOT_VALID_WORD) { + // TODO: figure out what is the correct thing to do here. + } else { + bigrams.add(new PendingAttribute(bigram.mFrequency, pos)); + } + } + return bigrams; + } + + /** * Insert a word into a binary dictionary. * * @param dictUpdater the dict updater. @@ -238,18 +258,9 @@ public final class DynamicBinaryDictIOUtils { final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { - final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); + final ArrayList<PendingAttribute> bigrams = resolveBigramPositions(dictUpdater, + bigramStrings); final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); - if (bigramStrings != null) { - for (final WeightedString bigram : bigramStrings) { - int position = dictUpdater.getTerminalPosition(bigram.mWord); - if (position == FormatSpec.NOT_VALID_WORD) { - // TODO: figure out what is the correct thing to do here. - } else { - bigrams.add(new PendingAttribute(bigram.mFrequency, position)); - } - } - } final boolean isTerminal = true; final boolean hasBigrams = !bigrams.isEmpty(); diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 5a5d7af6b..5aee135ad 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -37,13 +37,15 @@ public final class FormatSpec { * sion * * o | - * p | not used 4 bits - * t | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG - * i | FRENCH_LIGATURE_PROCESSING_FLAG - * o | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE - * n | GERMAN_UMLAUT_PROCESSING_FLAG - * f | - * lags + * p | not used 3 bits + * t | each unigram and bigram entry has a time stamp? + * i | 1 bit, 1 = yes, 0 = no : CONTAINS_TIMESTAMP_FLAG + * o | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG + * n | FRENCH_LIGATURE_PROCESSING_FLAG + * f | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE + * l | GERMAN_UMLAUT_PROCESSING_FLAG + * a | + * gs * * h | * e | size of the file header, 4bytes @@ -211,6 +213,7 @@ public final class FormatSpec { static final int SUPPORTS_DYNAMIC_UPDATE = 0x2; static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; static final int CONTAINS_BIGRAMS_FLAG = 0x8; + static final int CONTAINS_TIMESTAMP_FLAG = 0x10; // TODO: Make this value adaptative to content data, store it in the header, and // use it in the reading code. @@ -261,8 +264,9 @@ public final class FormatSpec { static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; // These values are used only by version 4 or later. - static final String TRIE_FILE_EXTENSION = ".trie"; + public static final String TRIE_FILE_EXTENSION = ".trie"; static final String FREQ_FILE_EXTENSION = ".freq"; + static final String UNIGRAM_TIMESTAMP_FILE_EXTENSION = ".timestamp"; // tat = Terminal Address Table static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; static final String BIGRAM_FILE_EXTENSION = ".bigram"; @@ -271,19 +275,25 @@ public final class FormatSpec { static final String CONTENT_TABLE_FILE_SUFFIX = "_index"; static final int FREQUENCY_AND_FLAGS_SIZE = 2; static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3; + static final int UNIGRAM_TIMESTAMP_SIZE = 4; // With the English main dictionary as of October 2013, the size of bigram address table is - // is 584KB with the block size being 4. - // This is 91% of that of full address table. - static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4; - static final int BIGRAM_CONTENT_COUNT = 1; + // is 345KB with the block size being 16. + // This is 54% of that of full address table. + static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 16; + static final int BIGRAM_CONTENT_COUNT = 2; static final int BIGRAM_FREQ_CONTENT_INDEX = 0; + static final int BIGRAM_TIMESTAMP_CONTENT_INDEX = 1; static final String BIGRAM_FREQ_CONTENT_ID = "_freq"; + static final String BIGRAM_TIMESTAMP_CONTENT_ID = "_timestamp"; + static final int BIGRAM_TIMESTAMP_SIZE = 4; + static final int BIGRAM_COUNTER_SIZE = 1; + static final int BIGRAM_LEVEL_SIZE = 1; static final int SHORTCUT_CONTENT_COUNT = 1; static final int SHORTCUT_CONTENT_INDEX = 0; // With the English main dictionary as of October 2013, the size of shortcut address table is - // 29KB with the block size being 64. + // 26KB with the block size being 64. // This is only 4.4% of that of full address table. static final int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64; static final String SHORTCUT_CONTENT_ID = "_shortcut"; @@ -321,6 +331,7 @@ public final class FormatSpec { public final int mVersion; public final boolean mSupportsDynamicUpdate; public final boolean mHasTerminalId; + public final boolean mHasTimestamp; @UsedForTesting public FormatOptions(final int version) { this(version, false); @@ -328,6 +339,11 @@ public final class FormatSpec { @UsedForTesting public FormatOptions(final int version, final boolean supportsDynamicUpdate) { + this(version, supportsDynamicUpdate, false /* hasTimestamp */); + } + + public FormatOptions(final int version, final boolean supportsDynamicUpdate, + final boolean hasTimestamp) { mVersion = version; if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) { throw new RuntimeException("Dynamic updates are only supported with versions " @@ -335,6 +351,7 @@ public final class FormatSpec { } mSupportsDynamicUpdate = supportsDynamicUpdate; mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); + mHasTimestamp = hasTimestamp; } } diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java new file mode 100644 index 000000000..06088b651 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * An auxiliary class for reading SparseTable and data written by SparseTableContentWriter. + */ +public class SparseTableContentReader { + + /** + * An interface of a function which is passed to SparseTableContentReader.read. + */ + public interface SparseTableContentReaderInterface { + /** + * Reads data. + * + * @param buffer the DictBuffer. The position of the buffer is set to the head of data. + */ + public void read(final DictBuffer buffer); + } + + protected final int mContentCount; + protected final int mBlockSize; + protected final File mBaseDir; + protected final File mLookupTableFile; + protected final File[] mAddressTableFiles; + protected final File[] mContentFiles; + protected DictBuffer mLookupTableBuffer; + protected final DictBuffer[] mAddressTableBuffers; + private final DictBuffer[] mContentBuffers; + protected final DictionaryBufferFactory mFactory; + + /** + * Sole constructor of SparseTableContentReader. + * + * @param name the name of SparseTable. + * @param blockSize the block size of the content table. + * @param baseDir the directory which contains the files of the content table. + * @param contentFilenames the file names of content files. + * @param contentIds the ids of contents. These ids are used for a suffix of a name of + * address files and content files. + * @param factory the DictionaryBufferFactory which is used for opening the files. + */ + public SparseTableContentReader(final String name, final int blockSize, final File baseDir, + final String[] contentFilenames, final String[] contentIds, + final DictionaryBufferFactory factory) { + if (contentFilenames.length != contentIds.length) { + throw new RuntimeException("The length of contentFilenames and the length of" + + " contentIds are different " + contentFilenames.length + ", " + + contentIds.length); + } + mBlockSize = blockSize; + mBaseDir = baseDir; + mFactory = factory; + mContentCount = contentFilenames.length; + mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + mAddressTableFiles = new File[mContentCount]; + mContentFiles = new File[mContentCount]; + for (int i = 0; i < mContentCount; ++i) { + mAddressTableFiles[i] = new File(mBaseDir, + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + } + mAddressTableBuffers = new DictBuffer[mContentCount]; + mContentBuffers = new DictBuffer[mContentCount]; + } + + public void openBuffers() throws FileNotFoundException, IOException { + mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); + for (int i = 0; i < mContentCount; ++i) { + mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); + mContentBuffers[i] = mFactory.getDictionaryBuffer(mContentFiles[i]); + } + } + + protected void read(final int contentIndex, final int index, + final SparseTableContentReaderInterface reader) { + if (index < 0 || (index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES + >= mLookupTableBuffer.limit()) { + return; + } + + mLookupTableBuffer.position((index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); + final int posInAddressTable = mLookupTableBuffer.readInt(); + if (posInAddressTable == SparseTable.NOT_EXIST) { + return; + } + + mAddressTableBuffers[contentIndex].position( + (posInAddressTable + index % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); + final int address = mAddressTableBuffers[contentIndex].readInt(); + if (address == SparseTable.NOT_EXIST) { + return; + } + + mContentBuffers[contentIndex].position(address); + reader.read(mContentBuffers[contentIndex]); + } +}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java new file mode 100644 index 000000000..4518f21b9 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentUpdater.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * An auxiliary class for updating data associated with SparseTable. + */ +public class SparseTableContentUpdater extends SparseTableContentReader { + protected OutputStream mLookupTableOutStream; + protected OutputStream[] mAddressTableOutStreams; + protected OutputStream[] mContentOutStreams; + + public SparseTableContentUpdater(final String name, final int blockSize, + final File baseDir, final String[] contentFilenames, final String[] contentIds, + final DictionaryBufferFactory factory) { + super(name, blockSize, baseDir, contentFilenames, contentIds, factory); + mAddressTableOutStreams = new OutputStream[mContentCount]; + mContentOutStreams = new OutputStream[mContentCount]; + } + + protected void openStreamsAndBuffers() throws IOException { + openBuffers(); + mLookupTableOutStream = new FileOutputStream(mLookupTableFile, true /* append */); + for (int i = 0; i < mContentCount; ++i) { + mAddressTableOutStreams[i] = new FileOutputStream(mAddressTableFiles[i], + true /* append */); + mContentOutStreams[i] = new FileOutputStream(mContentFiles[i], true /* append */); + } + } + + /** + * Set the contentIndex-th elements of contentId-th table. + * + * @param contentId the id of the content table. + * @param contentIndex the index where to set the valie. + * @param value the value to set. + */ + protected void setContentValue(final int contentId, final int contentIndex, final int value) + throws IOException { + if ((contentIndex / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES + >= mLookupTableBuffer.limit()) { + // Need to extend the lookup table + final int currentSize = mLookupTableBuffer.limit() + / SparseTable.SIZE_OF_INT_IN_BYTES; + final int target = contentIndex / mBlockSize + 1; + for (int i = currentSize; i < target; ++i) { + BinaryDictEncoderUtils.writeUIntToStream(mLookupTableOutStream, + SparseTable.NOT_EXIST, SparseTable.SIZE_OF_INT_IN_BYTES); + } + // We need to reopen the byte buffer of the lookup table because a MappedByteBuffer in + // Java isn't expanded automatically when the underlying file is expanded. + reopenLookupTable(); + } + + mLookupTableBuffer.position((contentIndex / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); + int posInAddressTable = mLookupTableBuffer.readInt(); + if (posInAddressTable == SparseTable.NOT_EXIST) { + // Need to extend the address table + mLookupTableBuffer.position(mLookupTableBuffer.position() + - SparseTable.SIZE_OF_INT_IN_BYTES); + posInAddressTable = mAddressTableBuffers[0].limit() / mBlockSize; + BinaryDictEncoderUtils.writeUIntToDictBuffer(mLookupTableBuffer, + posInAddressTable, SparseTable.SIZE_OF_INT_IN_BYTES); + for (int i = 0; i < mContentCount; ++i) { + for (int j = 0; j < mBlockSize; ++j) { + BinaryDictEncoderUtils.writeUIntToStream(mAddressTableOutStreams[i], + SparseTable.NOT_EXIST, SparseTable.SIZE_OF_INT_IN_BYTES); + } + } + // We need to reopen the byte buffers of the address tables because a MappedByteBuffer + // in Java isn't expanded automatically when the underlying file is expanded. + reopenAddressTables(); + } + posInAddressTable += (contentIndex % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES; + + mAddressTableBuffers[contentId].position(posInAddressTable); + BinaryDictEncoderUtils.writeUIntToDictBuffer(mAddressTableBuffers[contentId], + value, SparseTable.SIZE_OF_INT_IN_BYTES); + } + + private void reopenLookupTable() throws IOException { + mLookupTableOutStream.flush(); + mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); + } + + private void reopenAddressTables() throws IOException { + for (int i = 0; i < mContentCount; ++i) { + mAddressTableOutStreams[i].flush(); + mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); + } + } + + protected void close() throws IOException { + mLookupTableOutStream.close(); + for (final OutputStream stream : mAddressTableOutStreams) { + stream.close(); + } + for (final OutputStream stream : mContentOutStreams) { + stream.close(); + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java new file mode 100644 index 000000000..49f0fd624 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * An auxiliary class for writing data associated with SparseTable to files. + */ +public class SparseTableContentWriter { + public interface SparseTableContentWriterInterface { + public void write(final OutputStream outStream) throws IOException; + } + + private final int mContentCount; + private final SparseTable mSparseTable; + private final File mLookupTableFile; + protected final File mBaseDir; + private final File[] mAddressTableFiles; + private final File[] mContentFiles; + protected final OutputStream[] mContentOutStreams; + + /** + * Sole constructor of SparseTableContentWriter. + * + * @param name the name of SparseTable. + * @param initialCapacity the initial capacity of SparseTable. + * @param blockSize the block size of the content table. + * @param baseDir the directory which contains the files of the content table. + * @param contentFilenames the file names of content files. + * @param contentIds the ids of contents. These ids are used for a suffix of a name of address + * files and content files. + */ + public SparseTableContentWriter(final String name, final int initialCapacity, + final int blockSize, final File baseDir, final String[] contentFilenames, + final String[] contentIds) { + if (contentFilenames.length != contentIds.length) { + throw new RuntimeException("The length of contentFilenames and the length of" + + " contentIds are different " + contentFilenames.length + ", " + + contentIds.length); + } + mContentCount = contentFilenames.length; + mSparseTable = new SparseTable(initialCapacity, blockSize, mContentCount); + mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + mAddressTableFiles = new File[mContentCount]; + mContentFiles = new File[mContentCount]; + mBaseDir = baseDir; + for (int i = 0; i < mContentCount; ++i) { + mAddressTableFiles[i] = new File(mBaseDir, + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + } + mContentOutStreams = new OutputStream[mContentCount]; + } + + public void openStreams() throws FileNotFoundException { + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); + } + } + + protected void write(final int contentIndex, final int index, + final SparseTableContentWriterInterface writer) throws IOException { + mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); + writer.write(mContentOutStreams[contentIndex]); + mContentOutStreams[contentIndex].flush(); + } + + public void closeStreams() throws IOException { + mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i].close(); + } + } +}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 53729075f..f0fed3fda 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -40,21 +40,45 @@ import java.util.Arrays; public class Ver4DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver4DictDecoder.class.getSimpleName(); - private static final int FILETYPE_TRIE = 1; - private static final int FILETYPE_FREQUENCY = 2; - private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; - private static final int FILETYPE_BIGRAM_FREQ = 4; - private static final int FILETYPE_SHORTCUT = 5; - - private final File mDictDirectory; - private final DictionaryBufferFactory mBufferFactory; + protected static final int FILETYPE_TRIE = 1; + protected static final int FILETYPE_FREQUENCY = 2; + protected static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; + protected static final int FILETYPE_BIGRAM_FREQ = 4; + protected static final int FILETYPE_SHORTCUT = 5; + + protected final File mDictDirectory; + protected final DictionaryBufferFactory mBufferFactory; protected DictBuffer mDictBuffer; - private DictBuffer mFrequencyBuffer; - private DictBuffer mTerminalAddressTableBuffer; - private DictBuffer mBigramBuffer; - private DictBuffer mShortcutBuffer; - private SparseTable mBigramAddressTable; - private SparseTable mShortcutAddressTable; + protected DictBuffer mFrequencyBuffer; + protected DictBuffer mTerminalAddressTableBuffer; + private BigramContentReader mBigramReader; + private ShortcutContentReader mShortcutReader; + + /** + * Raw PtNode info straight out of a trie file in version 4 dictionary. + */ + protected static final class Ver4PtNodeInfo { + public final int mFlags; + public final int[] mCharacters; + public final int mTerminalId; + public final int mChildrenPos; + public final int mParentPos; + public final int mNodeSize; + public int mStartIndexOfCharacters; + public int mEndIndexOfCharacters; // exclusive + + public Ver4PtNodeInfo(final int flags, final int[] characters, final int terminalId, + final int childrenPos, final int parentPos, final int nodeSize) { + mFlags = flags; + mCharacters = characters; + mTerminalId = terminalId; + mChildrenPos = childrenPos; + mParentPos = parentPos; + mNodeSize = nodeSize; + mStartIndexOfCharacters = 0; + mEndIndexOfCharacters = characters.length; + } + } @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { @@ -79,7 +103,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder { mDictBuffer = mFrequencyBuffer = null; } - private File getFile(final int fileType) { + protected File getFile(final int fileType) { if (fileType == FILETYPE_TRIE) { return new File(mDictDirectory, mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION); @@ -108,10 +132,12 @@ public class Ver4DictDecoder extends AbstractDictDecoder { mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); - mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM_FREQ)); - loadBigramAddressSparseTable(); - mShortcutBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_SHORTCUT)); - loadShortcutAddressSparseTable(); + mBigramReader = new BigramContentReader(mDictDirectory.getName(), + mDictDirectory, mBufferFactory, false); + mBigramReader.openBuffers(); + mShortcutReader = new ShortcutContentReader(mDictDirectory.getName(), mDictDirectory, + mBufferFactory); + mShortcutReader.openBuffers(); } @Override @@ -119,6 +145,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder { return mDictBuffer != null; } + @UsedForTesting /* package */ DictBuffer getDictBuffer() { return mDictBuffer; } @@ -136,25 +163,113 @@ public class Ver4DictDecoder extends AbstractDictDecoder { return header; } - private void loadBigramAddressSparseTable() throws IOException { - final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - final File freqsFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX - + FormatSpec.BIGRAM_FREQ_CONTENT_ID); - mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { freqsFile }, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); + /** + * An auxiliary class for reading bigrams. + */ + protected static class BigramContentReader extends SparseTableContentReader { + private final boolean mHasTimestamp; + + public BigramContentReader(final String name, final File baseDir, + final DictionaryBufferFactory factory, final boolean hasTimestamp) { + super(name + FormatSpec.BIGRAM_FILE_EXTENSION, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + getContentFilenames(name, hasTimestamp), getContentIds(hasTimestamp), factory); + mHasTimestamp = hasTimestamp; + } + + // TODO: Consolidate this method and BigramContentWriter.getContentFilenames. + protected static String[] getContentFilenames(final String name, + final boolean hasTimestamp) { + final String[] contentFilenames; + if (hasTimestamp) { + contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION, + name + FormatSpec.BIGRAM_FILE_EXTENSION }; + } else { + contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }; + } + return contentFilenames; + } + + // TODO: Consolidate this method and BigramContentWriter.getContentIds. + protected static String[] getContentIds(final boolean hasTimestamp) { + final String[] contentIds; + if (hasTimestamp) { + contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID, + FormatSpec.BIGRAM_TIMESTAMP_CONTENT_ID }; + } else { + contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }; + } + return contentIds; + } + + public ArrayList<PendingAttribute> readTargetsAndFrequencies(final int terminalId, + final DictBuffer terminalAddressTableBuffer) { + final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); + read(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, + new SparseTableContentReaderInterface() { + @Override + public void read(final DictBuffer buffer) { + while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, + // remaining bigram entries are ignored. + final int bigramFlags = buffer.readUnsignedByte(); + final int targetTerminalId = buffer.readUnsignedInt24(); + terminalAddressTableBuffer.position(targetTerminalId + * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + final int targetAddress = + terminalAddressTableBuffer.readUnsignedInt24(); + bigrams.add(new PendingAttribute(bigramFlags + & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + targetAddress)); + if (0 == (bigramFlags + & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { + break; + } + } + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode (" + + bigrams.size() + " but max is " + + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); + } + } + }); + if (bigrams.isEmpty()) return null; + return bigrams; + } } - // TODO: Let's have something like SparseTableContentsReader in this class. - private void loadShortcutAddressSparseTable() throws IOException { - final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - final File contentFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX - + FormatSpec.SHORTCUT_CONTENT_ID); - mShortcutAddressTable = SparseTable.readFromFiles(lookupIndexFile, - new File[] { contentFile }, FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE); + /** + * An auxiliary class for reading shortcuts. + */ + protected static class ShortcutContentReader extends SparseTableContentReader { + public ShortcutContentReader(final String name, final File baseDir, + final DictionaryBufferFactory factory) { + super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, + FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, + new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, factory); + } + + public ArrayList<WeightedString> readShortcuts(final int terminalId) { + final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList(); + read(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, + new SparseTableContentReaderInterface() { + @Override + public void read(final DictBuffer buffer) { + while (true) { + final int flags = buffer.readUnsignedByte(); + final String word = CharEncoding.readString(buffer); + shortcuts.add(new WeightedString(word, + flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { + break; + } + } + } + }); + if (shortcuts.isEmpty()) return null; + return shortcuts; + } } protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { @@ -168,102 +283,82 @@ public class Ver4DictDecoder extends AbstractDictDecoder { } } - private ArrayList<WeightedString> readShortcuts(final int terminalId) { - if (mShortcutAddressTable.get(0, terminalId) == SparseTable.NOT_EXIST) return null; - - final ArrayList<WeightedString> ret = CollectionUtils.newArrayList(); - final int posOfShortcuts = mShortcutAddressTable.get(FormatSpec.SHORTCUT_CONTENT_INDEX, - terminalId); - mShortcutBuffer.position(posOfShortcuts); - while (true) { - final int flags = mShortcutBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(mShortcutBuffer); - ret.add(new WeightedString(word, - flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return ret; - } + private final int[] mCharacterBufferForReadingVer4PtNodeInfo + = new int[FormatSpec.MAX_WORD_LENGTH]; + /** + * Reads PtNode from ptNodePos in the trie file and returns Ver4PtNodeInfo. + * + * @param ptNodePos the position of PtNode. + * @param options the format options. + * @return Ver4PtNodeInfo. + */ // TODO: Make this buffer thread safe. // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH. - private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; - @Override - public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { - int addressPointer = ptNodePos; + protected Ver4PtNodeInfo readVer4PtNodeInfo(final int ptNodePos, final FormatOptions options) { + int readingPos = ptNodePos; final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; + readingPos += FormatSpec.PTNODE_FLAGS_SIZE; - final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + final int parentPos = PtNodeReader.readParentAddress(mDictBuffer, options); if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += FormatSpec.PARENT_ADDRESS_SIZE; + readingPos += FormatSpec.PARENT_ADDRESS_SIZE; } final int characters[]; if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { int index = 0; int character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character); + readingPos += CharEncoding.getCharSize(character); while (FormatSpec.INVALID_CHARACTER != character && index < FormatSpec.MAX_WORD_LENGTH) { - mCharacterBuffer[index++] = character; + mCharacterBufferForReadingVer4PtNodeInfo[index++] = character; character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character); + readingPos += CharEncoding.getCharSize(character); } - characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + characters = Arrays.copyOfRange(mCharacterBufferForReadingVer4PtNodeInfo, 0, index); } else { final int character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character); + readingPos += CharEncoding.getCharSize(character); characters = new int[] { character }; } final int terminalId; if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { terminalId = PtNodeReader.readTerminalId(mDictBuffer); - addressPointer += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + readingPos += FormatSpec.PTNODE_TERMINAL_ID_SIZE; } else { terminalId = PtNode.NOT_A_TERMINAL; } + int childrenPos = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenPos += readingPos; + } + readingPos += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + + return new Ver4PtNodeInfo(flags, characters, terminalId, childrenPos, parentPos, + readingPos - ptNodePos); + } + + @Override + public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { + final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(ptNodePos, options); + final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - frequency = PtNodeReader.readFrequency(mFrequencyBuffer, terminalId); + if (0 != (FormatSpec.FLAG_IS_TERMINAL & nodeInfo.mFlags)) { + frequency = PtNodeReader.readFrequency(mFrequencyBuffer, nodeInfo.mTerminalId); } else { frequency = PtNode.NOT_A_TERMINAL; } - int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - final ArrayList<WeightedString> shortcutTargets = readShortcuts(terminalId); - - final ArrayList<PendingAttribute> bigrams; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId); - mBigramBuffer.position(posOfBigrams); - while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, - // remaining bigram entries are ignored. - final int bigramFlags = mBigramBuffer.readUnsignedByte(); - final int targetTerminalId = mBigramBuffer.readUnsignedInt24(); - mTerminalAddressTableBuffer.position( - targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - final int targetAddress = mTerminalAddressTableBuffer.readUnsignedInt24(); - bigrams.add(new PendingAttribute( - bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, - targetAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() - + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); - } - } else { - bigrams = null; - } - return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); + + final ArrayList<WeightedString> shortcutTargets = mShortcutReader.readShortcuts( + nodeInfo.mTerminalId); + final ArrayList<PendingAttribute> bigrams = mBigramReader.readTargetsAndFrequencies( + nodeInfo.mTerminalId, mTerminalAddressTableBuffer); + + return new PtNodeInfo(ptNodePos, ptNodePos + nodeInfo.mNodeSize, nodeInfo.mFlags, + nodeInfo.mCharacters, frequency, nodeInfo.mParentPos, nodeInfo.mChildrenPos, + shortcutTargets, bigrams); } private void deleteDictFiles() { @@ -314,10 +409,14 @@ public class Ver4DictDecoder extends AbstractDictDecoder { @Override public boolean readAndFollowForwardLink() { - final int nextAddress = mDictBuffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { - mDictBuffer.position(nextAddress); - return true; + final int forwardLinkPos = mDictBuffer.position(); + int nextRelativePos = BinaryDictDecoderUtils.readSInt24(mDictBuffer); + if (nextRelativePos != FormatSpec.NO_FORWARD_LINK_ADDRESS) { + final int nextPos = forwardLinkPos + nextRelativePos; + if (nextPos >= 0 && nextPos < mDictBuffer.limit()) { + mDictBuffer.position(nextPos); + return true; + } } return false; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index f9dcacf77..4b3acdc8e 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -25,6 +25,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.FileNotFoundException; @@ -32,6 +33,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; /** @@ -45,6 +48,7 @@ public class Ver4DictEncoder implements DictEncoder { private int mHeaderSize; private OutputStream mTrieOutStream; private OutputStream mFreqOutStream; + private OutputStream mUnigramTimestampOutStream; private OutputStream mTerminalAddressTableOutStream; private File mDictDir; private String mBaseFilename; @@ -56,73 +60,41 @@ public class Ver4DictEncoder implements DictEncoder { mDictPlacedDir = dictPlacedDir; } - private interface SparseTableContentWriterInterface { - public void write(final OutputStream outStream) throws IOException; - } + private static class BigramContentWriter extends SparseTableContentWriter { + private final boolean mWriteTimestamp; - private static class SparseTableContentWriter { - private final int mContentCount; - private final SparseTable mSparseTable; - private final File mLookupTableFile; - protected final File mBaseDir; - private final File[] mAddressTableFiles; - private final File[] mContentFiles; - protected final OutputStream[] mContentOutStreams; - - public SparseTableContentWriter(final String name, final int contentCount, - final int initialCapacity, final int blockSize, final File baseDir, - final String[] contentFilenames, final String[] contentIds) { - if (contentFilenames.length != contentIds.length) { - throw new RuntimeException("The length of contentFilenames and the length of" - + " contentIds are different " + contentFilenames.length + ", " - + contentIds.length); - } - mContentCount = contentCount; - mSparseTable = new SparseTable(initialCapacity, blockSize, contentCount); - mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - mAddressTableFiles = new File[mContentCount]; - mContentFiles = new File[mContentCount]; - mBaseDir = baseDir; - for (int i = 0; i < mContentCount; ++i) { - mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); - } - mContentOutStreams = new OutputStream[mContentCount]; + public BigramContentWriter(final String name, final int initialCapacity, + final File baseDir, final boolean writeTimestamp) { + super(name + FormatSpec.BIGRAM_FILE_EXTENSION, initialCapacity, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + getContentFilenames(name, writeTimestamp), getContentIds(writeTimestamp)); + mWriteTimestamp = writeTimestamp; } - public void openStreams() throws FileNotFoundException { - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); + private static String[] getContentFilenames(final String name, + final boolean writeTimestamp) { + final String[] contentFilenames; + if (writeTimestamp) { + contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION, + name + FormatSpec.BIGRAM_FILE_EXTENSION }; + } else { + contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }; } + return contentFilenames; } - protected void write(final int contentIndex, final int index, - final SparseTableContentWriterInterface writer) throws IOException { - mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); - writer.write(mContentOutStreams[contentIndex]); - mContentOutStreams[contentIndex].flush(); - } - - public void closeStreams() throws IOException { - mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i].close(); + private static String[] getContentIds(final boolean writeTimestamp) { + final String[] contentIds; + if (writeTimestamp) { + contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID, + FormatSpec.BIGRAM_TIMESTAMP_CONTENT_ID }; + } else { + contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }; } - } - } - - private static class BigramContentWriter extends SparseTableContentWriter { - - public BigramContentWriter(final String name, final int initialCapacity, - final File baseDir) { - super(name + FormatSpec.BIGRAM_FILE_EXTENSION, FormatSpec.BIGRAM_CONTENT_COUNT, - initialCapacity, FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, - new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }, - new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }); + return contentIds; } - public void writeBigramsForOneWord(final int terminalId, + public void writeBigramsForOneWord(final int terminalId, final int bigramCount, final Iterator<WeightedString> bigramIterator, final FusionDictionary dict) throws IOException { write(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, @@ -130,8 +102,16 @@ public class Ver4DictEncoder implements DictEncoder { @Override public void write(final OutputStream outStream) throws IOException { writeBigramsForOneWordInternal(outStream, bigramIterator, dict); - } - }); + }}); + if (mWriteTimestamp) { + write(FormatSpec.BIGRAM_TIMESTAMP_CONTENT_INDEX, terminalId, + new SparseTableContentWriterInterface() { + @Override + public void write(final OutputStream outStream) throws IOException { + initBigramTimestampsCountersAndLevelsForOneWordInternal(outStream, + bigramCount); + }}); + } } private void writeBigramsForOneWordInternal(final OutputStream outStream, @@ -151,13 +131,26 @@ public class Ver4DictEncoder implements DictEncoder { FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); } } + + private void initBigramTimestampsCountersAndLevelsForOneWordInternal( + final OutputStream outStream, final int bigramCount) throws IOException { + for (int i = 0; i < bigramCount; ++i) { + // TODO: Figure out what initial values should be. + BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */, + FormatSpec.BIGRAM_TIMESTAMP_SIZE); + BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */, + FormatSpec.BIGRAM_COUNTER_SIZE); + BinaryDictEncoderUtils.writeUIntToStream(outStream, 0 /* value */, + FormatSpec.BIGRAM_LEVEL_SIZE); + } + } } private static class ShortcutContentWriter extends SparseTableContentWriter { public ShortcutContentWriter(final String name, final int initialCapacity, final File baseDir) { - super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, FormatSpec.SHORTCUT_CONTENT_COUNT, - initialCapacity, FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, initialCapacity, + FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, new String[] { FormatSpec.SHORTCUT_CONTENT_ID }); } @@ -193,18 +186,20 @@ public class Ver4DictEncoder implements DictEncoder { mDictDir = new File(mDictPlacedDir, mBaseFilename); final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION); final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION); + final File timestampFile = new File(mDictDir, + mBaseFilename + FormatSpec.UNIGRAM_TIMESTAMP_FILE_EXTENSION); final File terminalAddressTableFile = new File(mDictDir, mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); if (!mDictDir.isDirectory()) { if (mDictDir.exists()) mDictDir.delete(); mDictDir.mkdirs(); } - if (!trieFile.exists()) trieFile.createNewFile(); - if (!freqFile.exists()) freqFile.createNewFile(); - if (!terminalAddressTableFile.exists()) terminalAddressTableFile.createNewFile(); mTrieOutStream = new FileOutputStream(trieFile); mFreqOutStream = new FileOutputStream(freqFile); mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile); + if (formatOptions.mHasTimestamp) { + mUnigramTimestampOutStream = new FileOutputStream(timestampFile); + } } private void close() throws IOException { @@ -218,6 +213,9 @@ public class Ver4DictEncoder implements DictEncoder { if (mTerminalAddressTableOutStream != null) { mTerminalAddressTableOutStream.close(); } + if (mUnigramTimestampOutStream != null) { + mUnigramTimestampOutStream.close(); + } } finally { mTrieOutStream = null; mFreqOutStream = null; @@ -246,10 +244,29 @@ public class Ver4DictEncoder implements DictEncoder { MakedictLog.i("Flattening the tree..."); ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); int terminalCount = 0; + final ArrayList<PtNode> nodes = CollectionUtils.newArrayList(); for (final PtNodeArray array : flatNodes) { for (final PtNode node : array.mData) { - if (node.isTerminal()) node.mTerminalId = terminalCount++; + if (node.isTerminal()) { + nodes.add(node); + node.mTerminalId = terminalCount++; + } + } + } + Collections.sort(nodes, new Comparator<PtNode>() { + @Override + public int compare(final PtNode lhs, final PtNode rhs) { + if (lhs.mFrequency != rhs.mFrequency) { + return lhs.mFrequency < rhs.mFrequency ? -1 : 1; + } + if (lhs.mTerminalId < rhs.mTerminalId) return -1; + if (lhs.mTerminalId > rhs.mTerminalId) return 1; + return 0; } + }); + int count = 0; + for (final PtNode node : nodes) { + node.mTerminalId = count++; } MakedictLog.i("Computing addresses..."); @@ -257,7 +274,11 @@ public class Ver4DictEncoder implements DictEncoder { if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); writeTerminalData(flatNodes, terminalCount); - mBigramWriter = new BigramContentWriter(mBaseFilename, terminalCount, mDictDir); + if (formatOptions.mHasTimestamp) { + initUnigramTimestamps(terminalCount); + } + mBigramWriter = new BigramContentWriter(mBaseFilename, terminalCount, mDictDir, + formatOptions.mHasTimestamp); writeBigrams(flatNodes, dict); mShortcutWriter = new ShortcutContentWriter(mBaseFilename, terminalCount, mDictDir); writeShortcuts(flatNodes); @@ -348,7 +369,7 @@ public class Ver4DictEncoder implements DictEncoder { for (final PtNodeArray nodeArray : flatNodes) { for (final PtNode ptNode : nodeArray.mData) { if (ptNode.mBigrams != null) { - mBigramWriter.writeBigramsForOneWord(ptNode.mTerminalId, + mBigramWriter.writeBigramsForOneWord(ptNode.mTerminalId, ptNode.mBigrams.size(), ptNode.mBigrams.iterator(), dict); } } @@ -408,4 +429,11 @@ public class Ver4DictEncoder implements DictEncoder { mFreqOutStream.write(freqBuf); mTerminalAddressTableOutStream.write(terminalAddressTableBuf); } + + private void initUnigramTimestamps(final int terminalCount) throws IOException { + // Initial value of time stamps for each word is 0. + final byte[] unigramTimestampBuf = + new byte[terminalCount * FormatSpec.UNIGRAM_TIMESTAMP_SIZE]; + mUnigramTimestampOutStream.write(unigramTimestampBuf); + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java index 3d8f186ba..65860ee72 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -17,23 +17,124 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.CollectionUtils; + +import android.util.Log; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; /** * An implementation of DictUpdater for version 4 binary dictionary. */ @UsedForTesting public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { + private static final String TAG = Ver4DictUpdater.class.getSimpleName(); + + private OutputStream mDictStream; + private final File mFrequencyFile; @UsedForTesting public Ver4DictUpdater(final File dictDirectory, final int factoryType) { // DictUpdater must have an updatable DictBuffer. super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); + mFrequencyFile = getFile(FILETYPE_FREQUENCY); + } + + private static class BigramContentUpdater extends SparseTableContentUpdater { + private final boolean mHasTimestamp; + + public BigramContentUpdater(final String name, final File baseDir, + final boolean hasTimestamp) { + super(name + FormatSpec.BIGRAM_FILE_EXTENSION, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + BigramContentReader.getContentFilenames(name, hasTimestamp), + BigramContentReader.getContentIds(hasTimestamp), + new DictionaryBufferFromWritableByteBufferFactory()); + mHasTimestamp = hasTimestamp; + } + + public void insertBigramEntries(final int terminalId, final int frequency, + final ArrayList<PendingAttribute> entries) throws IOException { + if (terminalId < 0) { + throw new RuntimeException("Invalid terminal id : " + terminalId); + } + openStreamsAndBuffers(); + + if (entries == null || entries.isEmpty()) { + setContentValue(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, + SparseTable.NOT_EXIST); + return; + } + final int positionOfEntries = + (int) mContentFiles[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX].length(); + setContentValue(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, positionOfEntries); + + final Iterator<PendingAttribute> bigramIterator = entries.iterator(); + while (bigramIterator.hasNext()) { + final PendingAttribute entry = bigramIterator.next(); + final int flags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + 0 /* offset */, entry.mFrequency, frequency, "" /* word */); + BinaryDictEncoderUtils.writeUIntToStream( + mContentOutStreams[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX], flags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + BinaryDictEncoderUtils.writeUIntToStream( + mContentOutStreams[FormatSpec.BIGRAM_FREQ_CONTENT_INDEX], entry.mAddress, + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); + } + close(); + } + } + + private static class ShortcutContentUpdater extends SparseTableContentUpdater { + public ShortcutContentUpdater(final String name, final File baseDir) { + super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, + FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, + new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, + new DictionaryBufferFromWritableByteBufferFactory()); + } + + public void insertShortcuts(final int terminalId, + final ArrayList<WeightedString> shortcuts) throws IOException { + if (terminalId < 0) { + throw new RuntimeException("Invalid terminal id : " + terminalId); + } + openStreamsAndBuffers(); + if (shortcuts == null || shortcuts.isEmpty()) { + setContentValue(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, + SparseTable.NOT_EXIST); + return; + } + + final int positionOfShortcuts = + (int) mContentFiles[FormatSpec.SHORTCUT_CONTENT_INDEX].length(); + setContentValue(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, positionOfShortcuts); + + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), target.mFrequency); + BinaryDictEncoderUtils.writeUIntToStream( + mContentOutStreams[FormatSpec.SHORTCUT_CONTENT_INDEX], shortcutFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + CharEncoding.writeString(mContentOutStreams[FormatSpec.SHORTCUT_CONTENT_INDEX], + target.mWord); + } + close(); + } } @Override @@ -49,11 +150,622 @@ public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { } } - @Override + private int getNewTerminalId() { + // The size of frequency file is FormatSpec.FREQUENCY_AND_FLAGS_SIZE * number of terminals + // because each terminal always has a frequency. + // So we can get a fresh terminal id by this logic. + // CAVEAT: we are reading the file size from the disk each time: beware of race conditions, + // even on one thread. + return (int) (mFrequencyFile.length() / FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + } + + private void updateParentPosIfNotMoved(final int nodePos, final int newParentPos, + final FormatOptions formatOptions) { + final int originalPos = getPosition(); + setPosition(nodePos); + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + if (!BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { + final int parentOffset = newParentPos - nodePos; + BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, parentOffset); + } + setPosition(originalPos); + } + + private void updateParentPositions(final int nodeArrayPos, final int newParentPos, + final FormatOptions formatOptions) { + final int originalPos = mDictBuffer.position(); + mDictBuffer.position(nodeArrayPos); + int jumpCount = 0; + do { + final int count = readPtNodeCount(); + for (int i = 0; i < count; ++i) { + updateParentPosIfNotMoved(getPosition(), newParentPos, formatOptions); + skipPtNode(formatOptions); + } + if (!readAndFollowForwardLink()) break; + } while (jumpCount++ < DynamicBinaryDictIOUtils.MAX_JUMPS); + setPosition(originalPos); + } + + private void updateChildrenPos(final int nodePos, final int newChildrenPos, + final FormatOptions options) { + final int originalPos = getPosition(); + setPosition(nodePos); + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + PtNodeReader.readParentAddress(mDictBuffer, options); + BinaryDictIOUtils.skipString(mDictBuffer, + (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer); + final int basePos = getPosition(); + BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, newChildrenPos - basePos); + setPosition(originalPos); + } + + private void updateTerminalPosition(final int terminalId, final int position) { + if (terminalId == PtNode.NOT_A_TERMINAL + || terminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE + >= mTerminalAddressTableBuffer.limit()) return; + mTerminalAddressTableBuffer.position(terminalId + * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + BinaryDictEncoderUtils.writeUIntToDictBuffer(mTerminalAddressTableBuffer, position, + FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + } + + private void updateForwardLink(final int nodeArrayPos, final int newForwardLink, + final FormatOptions formatOptions) { + final int originalPos = getPosition(); + setPosition(nodeArrayPos); + int jumpCount = 0; + while (jumpCount++ < DynamicBinaryDictIOUtils.MAX_JUMPS) { + final int ptNodeCount = readPtNodeCount(); + for (int i = 0; i < ptNodeCount; ++i) { + skipPtNode(formatOptions); + } + final int forwardLinkPos = getPosition(); + if (!readAndFollowForwardLink()) { + setPosition(forwardLinkPos); + BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, newForwardLink - forwardLinkPos); + break; + } + } + setPosition(originalPos); + } + + private void markPtNodeAsMoved(final int nodePos, final int newNodePos, + final FormatOptions options) { + final int originalPos = getPosition(); + updateParentPosIfNotMoved(nodePos, newNodePos, options); + setPosition(nodePos); + final int currentFlags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + setPosition(nodePos); + mDictBuffer.put((byte) (FormatSpec.FLAG_IS_MOVED + | (currentFlags & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); + final int offset = newNodePos - nodePos; + BinaryDictIOUtils.writeSInt24ToBuffer(mDictBuffer, offset); + setPosition(originalPos); + } + + /** + * Writes a PtNode to an output stream from a Ver4PtNodeInfo. + * + * @param nodePos the position of the head of the PtNode. + * @param info the PtNode info to be written. + * @return the size written, in bytes. + */ + private int writePtNode(final int nodePos, final Ver4PtNodeInfo info) throws IOException { + int written = 0; + + // Write flags. + mDictStream.write((byte) (info.mFlags & 0xFF)); + written += FormatSpec.PTNODE_FLAGS_SIZE; + + // Write the parent position. + final int parentOffset = info.mParentPos == FormatSpec.NO_PARENT_ADDRESS ? + FormatSpec.NO_PARENT_ADDRESS : info.mParentPos - nodePos; + BinaryDictIOUtils.writeSInt24ToStream(mDictStream, parentOffset); + written += FormatSpec.PARENT_ADDRESS_SIZE; + + // Write a string. + if (((info.mFlags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0) + != (info.mEndIndexOfCharacters - info.mStartIndexOfCharacters > 1)) { + throw new RuntimeException("Inconsistent flags : hasMultipleChars = " + + ((info.mFlags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0) + ", length = " + + (info.mEndIndexOfCharacters - info.mStartIndexOfCharacters)); + } + written += CharEncoding.writeCodePoints(mDictStream, info.mCharacters, + info.mStartIndexOfCharacters, info.mEndIndexOfCharacters); + + // Write the terminal id. + if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { + BinaryDictEncoderUtils.writeUIntToStream(mDictStream, info.mTerminalId, + FormatSpec.PTNODE_TERMINAL_ID_SIZE); + written += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } + + // Write the children position. + final int childrenOffset = info.mChildrenPos == FormatSpec.NO_CHILDREN_ADDRESS + ? 0 : info.mChildrenPos - (nodePos + written); + BinaryDictIOUtils.writeSInt24ToStream(mDictStream, childrenOffset); + written += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + + return written; + } + + /** + * Helper method to split and move PtNode. + * + * @param ptNodeArrayPos the position of PtNodeArray which contains the split and moved PtNode. + * @param splittedPtNodeToMovePos the position of the split and moved PtNode. + * @param newParent the parent PtNode after splitting. + * @param newChildren the children PtNodes after splitting. + * @param newParentStartPos where to write the new parent. + * @param formatOptions the format options. + */ + private void writeSplittedPtNodes(final int ptNodeArrayPos, final int splittedPtNodeToMovePos, + final Ver4PtNodeInfo newParent, final Ver4PtNodeInfo[] newChildren, + final int newParentStartPos, + final FormatOptions formatOptions) throws IOException { + updateTerminalPosition(newParent.mTerminalId, + newParentStartPos + 1 /* size of PtNodeCount */); + int written = writePtNodeArray(newParentStartPos, new Ver4PtNodeInfo[] { newParent }, + FormatSpec.NO_FORWARD_LINK_ADDRESS); + final int childrenStartPos = newParentStartPos + written; + writePtNodeArray(childrenStartPos, newChildren, FormatSpec.NO_FORWARD_LINK_ADDRESS); + int childrenNodePos = childrenStartPos + 1 /* size of PtNodeCount */; + for (final Ver4PtNodeInfo info : newChildren) { + updateTerminalPosition(info.mTerminalId, childrenNodePos); + childrenNodePos += computePtNodeSize(info.mCharacters, info.mStartIndexOfCharacters, + info.mEndIndexOfCharacters, + (info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0); + } + + // Mark as moved. + markPtNodeAsMoved(splittedPtNodeToMovePos, newParentStartPos + 1 /* size of PtNodeCount */, + formatOptions); + updateForwardLink(ptNodeArrayPos, newParentStartPos, formatOptions); + } + + /** + * Writes a node array to the stream. + * + * @param nodeArrayPos the position of the head of the node array. + * @param infos an array of Ver4PtNodeInfo to be written. + * @return the written length in bytes. + */ + private int writePtNodeArray(final int nodeArrayPos, final Ver4PtNodeInfo[] infos, + final int forwardLink) throws IOException { + int written = BinaryDictIOUtils.writePtNodeCount(mDictStream, infos.length); + for (int i = 0; i < infos.length; ++i) { + written += writePtNode(nodeArrayPos + written, infos[i]); + } + BinaryDictIOUtils.writeSInt24ToStream(mDictStream, forwardLink); + written += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + return written; + } + + private int computePtNodeSize(final int[] codePoints, final int startIndex, final int endIndex, + final boolean isTerminal) { + return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + CharEncoding.getCharArraySize(codePoints, startIndex, endIndex) + + (endIndex - startIndex > 1 ? FormatSpec.PTNODE_TERMINATOR_SIZE : 0) + + (isTerminal ? FormatSpec.PTNODE_TERMINAL_ID_SIZE : 0) + + FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + } + + private void writeNewSinglePtNodeWithAttributes(final int[] codePoints, + final boolean hasShortcuts, final int terminalId, final boolean hasBigrams, + final boolean isNotAWord, final boolean isBlackListEntry, final int parentPos, + final FormatOptions formatOptions) throws IOException { + final int newNodeArrayPos = mDictBuffer.limit(); + final int newNodeFlags = BinaryDictEncoderUtils.makePtNodeFlags(codePoints.length > 1, + terminalId != PtNode.NOT_A_TERMINAL, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, + hasBigrams, isNotAWord, isBlackListEntry, formatOptions); + final Ver4PtNodeInfo info = new Ver4PtNodeInfo(newNodeFlags, codePoints, terminalId, + FormatSpec.NO_CHILDREN_ADDRESS, parentPos, 0 /* nodeSize */); + writePtNodeArray(newNodeArrayPos, new Ver4PtNodeInfo[] { info }, + FormatSpec.NO_FORWARD_LINK_ADDRESS); + } + + private int setMultipleCharsInFlags(final int currentFlags, final boolean hasMultipleChars) { + final int flags; + if (hasMultipleChars) { + flags = currentFlags | FormatSpec.FLAG_HAS_MULTIPLE_CHARS; + } else { + flags = currentFlags & (~FormatSpec.FLAG_HAS_MULTIPLE_CHARS); + } + return flags; + } + + private int setIsNotAWordInFlags(final int currentFlags, final boolean isNotAWord) { + final int flags; + if (isNotAWord) { + flags = currentFlags | FormatSpec.FLAG_IS_NOT_A_WORD; + } else { + flags = currentFlags & (~FormatSpec.FLAG_IS_NOT_A_WORD); + } + return flags; + } + + private int setIsBlackListEntryInFlags(final int currentFlags, final boolean isBlackListEntry) { + final int flags; + if (isBlackListEntry) { + flags = currentFlags | FormatSpec.FLAG_IS_BLACKLISTED; + } else { + flags = currentFlags & (~FormatSpec.FLAG_IS_BLACKLISTED); + } + return flags; + } + + /** + * Splits a PtNode. + * + * abcd - ef + * + * -> inserting "abc" + * + * abc - d - ef + * + * @param nodeArrayToSplitPos the position of PtNodeArray which contains the PtNode to split. + * @param nodeToSplitPos the position of the PtNode to split. + * @param nodeToSplitInfo the information of the PtNode to split. + * @param indexToSplit the index where to split in the code points array. + * @param parentOfNodeToSplitPos the absolute position of a parent of the node to split. + * @param newTerminalId the terminal id of the inserted node (corresponds to "d"). + * @param hasShortcuts whether the inserted word should have shortcuts. + * @param hasBigrams whether the inserted word should have bigrams. + * @param isNotAWord whether the inserted word should be not a word. + * @param isBlackListEntry whether the inserted word should be a black list entry. + * @param formatOptions the format options. + */ + private void splitOnly(final int nodeArrayToSplitPos, final int nodeToSplitPos, + final Ver4PtNodeInfo nodeToSplitInfo, final int indexToSplit, + final int parentOfNodeToSplitPos, final int newTerminalId, final boolean hasShortcuts, + final boolean hasBigrams, final boolean isNotAWord, final boolean isBlackListEntry, + final FormatOptions formatOptions) throws IOException { + final int parentNodeArrayStartPos = mDictBuffer.limit(); + final int parentNodeStartPos = parentNodeArrayStartPos + 1 /* size of PtNodeCount */; + final int parentFlags = BinaryDictEncoderUtils.makePtNodeFlags(indexToSplit > 1, + true /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, formatOptions); + final Ver4PtNodeInfo parentInfo = new Ver4PtNodeInfo(parentFlags, + nodeToSplitInfo.mCharacters, newTerminalId, parentNodeStartPos + + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, true) + + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, + parentOfNodeToSplitPos, 0 /* nodeSize */); + parentInfo.mStartIndexOfCharacters = 0; + parentInfo.mEndIndexOfCharacters = indexToSplit; + + // Write the child. + final int childrenFlags = setMultipleCharsInFlags(nodeToSplitInfo.mFlags, + nodeToSplitInfo.mCharacters.length - indexToSplit > 1); + final Ver4PtNodeInfo childrenInfo = new Ver4PtNodeInfo(childrenFlags, + nodeToSplitInfo.mCharacters, nodeToSplitInfo.mTerminalId, + nodeToSplitInfo.mChildrenPos, parentNodeStartPos, 0 /* nodeSize */); + childrenInfo.mStartIndexOfCharacters = indexToSplit; + childrenInfo.mEndIndexOfCharacters = nodeToSplitInfo.mCharacters.length; + if (nodeToSplitInfo.mChildrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { + updateParentPositions(nodeToSplitInfo.mChildrenPos, + parentInfo.mChildrenPos + 1 /* size of PtNodeCount */, formatOptions); + } + + writeSplittedPtNodes(nodeArrayToSplitPos, nodeToSplitPos, parentInfo, + new Ver4PtNodeInfo[] { childrenInfo }, parentNodeArrayStartPos, formatOptions); + } + + /** + * Split and branch a PtNode. + * + * ab - cd + * + * -> inserting "ac" + * + * a - b - cd + * | + * - c + * + * @param nodeArrayToSplitPos the position of PtNodeArray which contains the PtNode to split. + * @param nodeToSplitPos the position of the PtNode to split. + * @param nodeToSplitInfo the information of the PtNode to split. + * @param indexToSplit the index where to split in the code points array. + * @param parentOfNodeToSplitPos the absolute position of parent of the node to split. + * @param newWordSuffixCodePoints the suffix of the newly inserted word (corresponds to "c"). + * @param startIndexOfNewWordSuffixCodePoints the start index in newWordSuffixCodePoints where + * the suffix starts. + * @param newTerminalId the terminal id of the inserted node (correspond to "c"). + * @param hasShortcuts whether the inserted word should have shortcuts. + * @param hasBigrams whether the inserted word should have bigrams. + * @param isNotAWord whether the inserted word should be not a word. + * @param isBlackListEntry whether the inserted word should be a black list entry. + * @param formatOptions the format options. + */ + private void splitAndBranch(final int nodeArrayToSplitPos, final int nodeToSplitPos, + final Ver4PtNodeInfo nodeToSplitInfo, final int indexToSplit, + final int parentOfNodeToSplitPos, final int[] newWordSuffixCodePoints, + final int startIndexOfNewWordSuffixCodePoints, + final int newTerminalId, + final boolean hasShortcuts, final boolean hasBigrams, final boolean isNotAWord, + final boolean isBlackListEntry, final FormatOptions formatOptions) throws IOException { + final int parentNodeArrayStartPos = mDictBuffer.limit(); + final int parentNodeStartPos = parentNodeArrayStartPos + 1 /* size of PtNodeCount */; + final int parentFlags = BinaryDictEncoderUtils.makePtNodeFlags( + indexToSplit > 1, + false /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, + false /* hasShortcut */, false /* hasBigrams */, + false /* isNotAWord */, false /* isBlackListEntry */, formatOptions); + final Ver4PtNodeInfo parentInfo = new Ver4PtNodeInfo(parentFlags, + nodeToSplitInfo.mCharacters, PtNode.NOT_A_TERMINAL, + parentNodeStartPos + + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, false) + + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, + parentOfNodeToSplitPos, 0 /* nodeSize */); + parentInfo.mStartIndexOfCharacters = 0; + parentInfo.mEndIndexOfCharacters = indexToSplit; + + final int childrenNodeArrayStartPos = parentNodeStartPos + + computePtNodeSize(nodeToSplitInfo.mCharacters, 0, indexToSplit, false) + + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + final int firstChildrenFlags = BinaryDictEncoderUtils.makePtNodeFlags( + newWordSuffixCodePoints.length - startIndexOfNewWordSuffixCodePoints > 1, + true /* isTerminal */, FormatSpec.FLAG_IS_NOT_MOVED, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, formatOptions); + final Ver4PtNodeInfo firstChildrenInfo = new Ver4PtNodeInfo(firstChildrenFlags, + newWordSuffixCodePoints, newTerminalId, + FormatSpec.NO_CHILDREN_ADDRESS, parentNodeStartPos, + 0 /* nodeSize */); + firstChildrenInfo.mStartIndexOfCharacters = startIndexOfNewWordSuffixCodePoints; + firstChildrenInfo.mEndIndexOfCharacters = newWordSuffixCodePoints.length; + + final int secondChildrenStartPos = childrenNodeArrayStartPos + 1 /* size of ptNodeCount */ + + computePtNodeSize(newWordSuffixCodePoints, startIndexOfNewWordSuffixCodePoints, + newWordSuffixCodePoints.length, true /* isTerminal */); + final int secondChildrenFlags = setMultipleCharsInFlags(nodeToSplitInfo.mFlags, + nodeToSplitInfo.mCharacters.length - indexToSplit > 1); + final Ver4PtNodeInfo secondChildrenInfo = new Ver4PtNodeInfo(secondChildrenFlags, + nodeToSplitInfo.mCharacters, nodeToSplitInfo.mTerminalId, + nodeToSplitInfo.mChildrenPos, parentNodeStartPos, 0 /* nodeSize */); + secondChildrenInfo.mStartIndexOfCharacters = indexToSplit; + secondChildrenInfo.mEndIndexOfCharacters = nodeToSplitInfo.mCharacters.length; + if (nodeToSplitInfo.mChildrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { + updateParentPositions(nodeToSplitInfo.mChildrenPos, secondChildrenStartPos, + formatOptions); + } + + writeSplittedPtNodes(nodeArrayToSplitPos, nodeToSplitPos, parentInfo, + new Ver4PtNodeInfo[] { firstChildrenInfo, secondChildrenInfo }, + parentNodeArrayStartPos, formatOptions); + } + + /** + * Inserts a word into the trie file and returns the position of inserted terminal node. + * If the insertion is failed, returns FormatSpec.NOT_VALID_WORD. + */ + @UsedForTesting + private int insertWordToTrie(final String word, final int newTerminalId, + final boolean isNotAWord, final boolean isBlackListEntry, final boolean hasBigrams, + final boolean hasShortcuts) throws IOException, UnsupportedFormatException { + setPosition(0); + final FileHeader header = readHeader(); + + final int[] codePoints = FusionDictionary.getCodePoints(word); + final int wordLen = codePoints.length; + + int wordPos = 0; + for (int depth = 0; depth < FormatSpec.MAX_WORD_LENGTH; /* nop */) { + final int nodeArrayPos = getPosition(); + final int ptNodeCount = readPtNodeCount(); + boolean goToChildren = false; + int parentPos = FormatSpec.NO_PARENT_ADDRESS; + for (int i = 0; i < ptNodeCount; ++i) { + final int nodePos = getPosition(); + final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(nodePos, header.mFormatOptions); + if (BinaryDictIOUtils.isMovedPtNode(nodeInfo.mFlags, header.mFormatOptions)) { + continue; + } + if (nodeInfo.mParentPos != FormatSpec.NO_PARENT_ADDRESS) { + parentPos = nodePos + nodeInfo.mParentPos; + } + + final boolean firstCharacterMatched = + codePoints[wordPos] == nodeInfo.mCharacters[0]; + boolean allCharactersMatched = true; + int firstDifferentCharacterIndex = -1; + for (int p = 0; p < nodeInfo.mCharacters.length; ++p) { + if (wordPos + p >= codePoints.length) break; + if (codePoints[wordPos + p] != nodeInfo.mCharacters[p]) { + if (firstDifferentCharacterIndex == -1) { + firstDifferentCharacterIndex = p; + } + allCharactersMatched = false; + } + } + + if (!firstCharacterMatched) { + // Go to the next sibling node. + continue; + } + + if (!allCharactersMatched) { + final int parentNodeArrayStartPos = mDictBuffer.limit(); + splitAndBranch(nodeArrayPos, nodePos, nodeInfo, firstDifferentCharacterIndex, + parentPos, codePoints, wordPos + firstDifferentCharacterIndex, + newTerminalId, hasShortcuts, hasBigrams, isNotAWord, + isBlackListEntry, header.mFormatOptions); + + return parentNodeArrayStartPos + computePtNodeSize(codePoints, wordPos, + wordPos + firstDifferentCharacterIndex, false) + + FormatSpec.FORWARD_LINK_ADDRESS_SIZE + 1 /* size of PtNodeCount */; + } + + if (wordLen - wordPos < nodeInfo.mCharacters.length) { + final int parentNodeArrayStartPos = mDictBuffer.limit(); + splitOnly(nodeArrayPos, nodePos, nodeInfo, wordLen - wordPos, parentPos, + newTerminalId, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, + header.mFormatOptions); + + // Return the position of the inserted word. + return parentNodeArrayStartPos + 1 /* size of PtNodeCount */; + } + + wordPos += nodeInfo.mCharacters.length; + if (wordPos == wordLen) { + // This dictionary already contains the word. + Log.e(TAG, "Something went wrong. If the word is already contained, " + + " there is no need to insert new PtNode."); + return FormatSpec.NOT_VALID_WORD; + } + if (nodeInfo.mChildrenPos == FormatSpec.NO_CHILDREN_ADDRESS) { + // There are no children. + // We need to add a new node as a child of this node. + final int newNodeArrayPos = mDictBuffer.limit(); + final int[] newNodeCodePoints = Arrays.copyOfRange(codePoints, wordPos, + codePoints.length); + writeNewSinglePtNodeWithAttributes(newNodeCodePoints, hasShortcuts, + newTerminalId, hasBigrams, isNotAWord, isBlackListEntry, nodePos, + header.mFormatOptions); + updateChildrenPos(nodePos, newNodeArrayPos, header.mFormatOptions); + return newNodeArrayPos + 1 /* size of PtNodeCount */; + } else { + // Found the matched node. + // Go to the children of this node. + setPosition(nodeInfo.mChildrenPos); + goToChildren = true; + depth++; + break; + } + } + + if (goToChildren) continue; + if (!readAndFollowForwardLink()) { + // Add a new node that contains [wordPos, word.length()-1]. + // and update the forward link. + final int newNodeArrayPos = mDictBuffer.limit(); + final int[] newCodePoints = Arrays.copyOfRange(codePoints, wordPos, + codePoints.length); + writeNewSinglePtNodeWithAttributes(newCodePoints, hasShortcuts, newTerminalId, + hasBigrams, isNotAWord, isBlackListEntry, parentPos, header.mFormatOptions); + updateForwardLink(nodeArrayPos, newNodeArrayPos, header.mFormatOptions); + return newNodeArrayPos + 1 /* size of PtNodeCount */; + } + } + return FormatSpec.NOT_VALID_WORD; + } + + private void updateFrequency(final int terminalId, final int frequency) { + mFrequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + BinaryDictEncoderUtils.writeUIntToDictBuffer(mFrequencyBuffer, frequency, + FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + } + + private void insertFrequency(final int frequency) throws IOException { + final OutputStream frequencyStream = new FileOutputStream(mFrequencyFile, + true /* append */); + BinaryDictEncoderUtils.writeUIntToStream(frequencyStream, frequency, + FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + frequencyStream.close(); + } + + private void insertTerminalPosition(final int posOfTerminal) throws IOException { + final OutputStream terminalPosStream = new FileOutputStream( + getFile(FILETYPE_TERMINAL_ADDRESS_TABLE), true /* append */); + BinaryDictEncoderUtils.writeUIntToStream(terminalPosStream, posOfTerminal, + FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + terminalPosStream.close(); + } + + private void insertBigrams(final int terminalId, final int frequency, + final ArrayList<PendingAttribute> bigramAddresses) + throws IOException, UnsupportedFormatException { + openDictBuffer(); + final BigramContentUpdater updater = new BigramContentUpdater(mDictDirectory.getName(), + mDictDirectory, false); + + // Convert addresses to terminal ids. + final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); + mDictBuffer.position(0); + final FileHeader header = readHeader(); + for (PendingAttribute attr : bigramAddresses) { + mDictBuffer.position(attr.mAddress); + final Ver4PtNodeInfo info = readVer4PtNodeInfo(attr.mAddress, header.mFormatOptions); + if (info.mTerminalId == PtNode.NOT_A_TERMINAL) { + throw new RuntimeException("We can't have a bigram target that's not a terminal."); + } + bigrams.add(new PendingAttribute(frequency, info.mTerminalId)); + } + updater.insertBigramEntries(terminalId, frequency, bigrams); + close(); + } + + private void insertShortcuts(final int terminalId, final ArrayList<WeightedString> shortcuts) + throws IOException { + final ShortcutContentUpdater updater = new ShortcutContentUpdater(mDictDirectory.getName(), + mDictDirectory); + updater.insertShortcuts(terminalId, shortcuts); + } + + private void openBuffersAndStream() throws IOException { + openDictBuffer(); + mDictStream = new FileOutputStream(getFile(FILETYPE_TRIE), true /* append */); + } + + private void close() throws IOException { + if (mDictStream != null) { + mDictStream.close(); + mDictStream = null; + } + mDictBuffer = null; + mFrequencyBuffer = null; + mTerminalAddressTableBuffer = null; + } + + private void updateAttributes(final int posOfWord, final int frequency, + final ArrayList<WeightedString> bigramStrings, + final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, + final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { + mDictBuffer.position(0); + final FileHeader header = readHeader(); + mDictBuffer.position(posOfWord); + final Ver4PtNodeInfo info = readVer4PtNodeInfo(posOfWord, header.mFormatOptions); + final int terminalId = info.mTerminalId; + + // Update the flags. + final int newFlags = setIsNotAWordInFlags( + setIsBlackListEntryInFlags(info.mFlags, isBlackListEntry), isNotAWord); + mDictBuffer.position(posOfWord); + mDictBuffer.put((byte) newFlags); + + updateFrequency(terminalId, frequency); + insertBigrams(terminalId, frequency, + DynamicBinaryDictIOUtils.resolveBigramPositions(this, bigramStrings)); + insertShortcuts(terminalId, shortcuts); + } + + @Override @UsedForTesting public void insertWord(final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { - // TODO: Implement this method. + final int newTerminalId = getNewTerminalId(); + + openBuffersAndStream(); + final int posOfWord = getTerminalPosition(word); + if (posOfWord != FormatSpec.NOT_VALID_WORD) { + // The word is already contained in the dictionary. + updateAttributes(posOfWord, frequency, bigramStrings, shortcuts, isNotAWord, + isBlackListEntry); + close(); + return; + } + + // Insert new PtNode into trie. + final int posOfTerminal = insertWordToTrie(word, newTerminalId, isNotAWord, + isBlackListEntry, bigramStrings != null && !bigramStrings.isEmpty(), + shortcuts != null && !shortcuts.isEmpty()); + insertFrequency(frequency); + insertTerminalPosition(posOfTerminal); + close(); + + insertBigrams(newTerminalId, frequency, + DynamicBinaryDictIOUtils.resolveBigramPositions(this, bigramStrings)); + insertShortcuts(newTerminalId, shortcuts); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 1de15a333..e7a25d216 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -46,6 +46,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName(); public static final boolean DBG_SAVE_RESTORE = false; private static final boolean DBG_STRESS_TEST = false; + private static final boolean DBG_DUMP_ON_CLOSE = false; private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; /** Any pair being typed or picked */ @@ -75,15 +76,14 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB mFileName = fileName; mPrefs = sp; if (mLocale != null && mLocale.length() > 1) { - asyncLoadDictionaryToMemory(); reloadDictionaryIfRequired(); } } @Override public void close() { - if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - closeBinaryDictionary(); + if (DBG_DUMP_ON_CLOSE) { + dumpAllWordsForDebug(); } // Flush pending writes. // TODO: Remove after this class become to use a dynamic binary dictionary. @@ -126,9 +126,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { return; } - final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? - (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) : - FREQUENCY_FOR_TYPED; + final int frequency = isValid ? + FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS; addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */, false /* isNotAWord */); // Do not insert a word as a bigram of itself @@ -222,6 +221,55 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB } @UsedForTesting + public void dumpAllWordsForDebug() { + runAfterGcForDebug(new Runnable() { + @Override + public void run() { + dumpAllWordsForDebugLocked(); + } + }); + } + + private void dumpAllWordsForDebugLocked() { + Log.d(TAG, "dumpAllWordsForDebug started."); + final OnAddWordListener listener = new OnAddWordListener() { + @Override + public void setUnigram(final String word, final String shortcutTarget, + final int frequency, final int shortcutFreq) { + Log.d(TAG, "load unigram: " + word + "," + frequency); + } + + @Override + public void setBigram(final String word0, final String word1, final int frequency) { + if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH + && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { + Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); + } else { + Log.d(TAG, "Skip inserting a too long bigram: " + word0 + "," + word1 + "," + + frequency); + } + } + }; + + // Load the dictionary from binary file + final File dictFile = new File(mContext.getFilesDir(), mFileName); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, + DictDecoder.USE_BYTEARRAY); + if (dictDecoder == null) { + // This is an expected condition: we don't have a user history dictionary for this + // language yet. It will be created sometime later. + return; + } + + try { + dictDecoder.openDictBuffer(); + UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); + } catch (IOException e) { + Log.d(TAG, "IOException on opening a bytebuffer", e); + } + } + + @UsedForTesting public void clearAndFlushDictionary() { // Clear the node structure on memory clear(); diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java deleted file mode 100644 index 6f152bb91..000000000 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.personalization; - -import android.content.Context; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.compat.ActivityManagerCompatUtils; -import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.AbstractDictionaryWriter; -import com.android.inputmethod.latin.ExpandableDictionary; -import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.ExpandableDictionary.NextWord; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Map; - -// Currently this class is used to implement dynamic prodiction dictionary. -// TODO: Move to native code. -public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { - private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); - /** Maximum number of pairs. Pruning will start when databases goes above this number. */ - public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000; - public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000; - - /** Any pair being typed or picked */ - private static final int FREQUENCY_FOR_TYPED = 2; - - private static final int BINARY_DICT_VERSION = 3; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); - - private final UserHistoryDictionaryBigramList mBigramList = - new UserHistoryDictionaryBigramList(); - private final ExpandableDictionary mExpandableDictionary; - private final int mMaxHistoryBigrams; - - public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { - super(context, dictType); - mExpandableDictionary = new ExpandableDictionary(dictType); - final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context); - mMaxHistoryBigrams = isLowRamDevice ? - LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS; - } - - @Override - public void clear() { - mBigramList.evictAll(); - mExpandableDictionary.clearDictionary(); - } - - /** - * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes - * are done to update the binary dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - @Override - public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, - final int shortcutFreq, final boolean isNotAWord) { - if (mBigramList.size() > mMaxHistoryBigrams * 2) { - // Too many entries: just stop adding new vocabulary and wait next refresh. - return; - } - mExpandableDictionary.addWord(word, shortcutTarget, frequency, shortcutFreq); - mBigramList.addBigram(null, word, (byte)frequency); - } - - @Override - public void addBigramWords(final String word0, final String word1, final int frequency, - final boolean isValid, final long lastModifiedTime) { - if (mBigramList.size() > mMaxHistoryBigrams * 2) { - // Too many entries: just stop adding new vocabulary and wait next refresh. - return; - } - if (lastModifiedTime > 0) { - mExpandableDictionary.setBigramAndGetFrequency(word0, word1, - new ForgettingCurveParams(frequency, System.currentTimeMillis(), - lastModifiedTime)); - mBigramList.addBigram(word0, word1, (byte)frequency); - } else { - mExpandableDictionary.setBigramAndGetFrequency(word0, word1, - new ForgettingCurveParams(isValid)); - mBigramList.addBigram(word0, word1, (byte)frequency); - } - } - - @Override - public void removeBigramWords(final String word0, final String word1) { - if (mBigramList.removeBigram(word0, word1)) { - mExpandableDictionary.removeBigram(word0, word1); - } - } - - @Override - protected void writeDictionary(final DictEncoder dictEncoder, - final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException { - UserHistoryDictIOUtils.writeDictionary(dictEncoder, - new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams), - mBigramList, FORMAT_OPTIONS); - } - - private static class FrequencyProvider implements BigramDictionaryInterface { - private final UserHistoryDictionaryBigramList mBigramList; - private final ExpandableDictionary mExpandableDictionary; - private final int mMaxHistoryBigrams; - - public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, - final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) { - mBigramList = bigramList; - mExpandableDictionary = expandableDictionary; - mMaxHistoryBigrams = maxHistoryBigrams; - } - - @Override - public int getFrequency(final String word0, final String word1) { - final int freq; - if (word0 == null) { // unigram - freq = FREQUENCY_FOR_TYPED; - } else { // bigram - final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1); - if (nw != null) { - final ForgettingCurveParams forgettingCurveParams = nw.getFcParams(); - final byte prevFc = mBigramList.getBigrams(word0).get(word1); - final byte fc = forgettingCurveParams.getFc(); - final boolean isValid = forgettingCurveParams.isValid(); - if (prevFc > 0 && prevFc == fc) { - freq = fc & 0xFF; - } else if (UserHistoryForgettingCurveUtils. - needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) { - freq = fc & 0xFF; - } else { - // Delete this entry - freq = -1; - } - } else { - // Delete this entry - freq = -1; - } - } - return freq; - } - } - - @Override - public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, - boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { - return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords, additionalFeaturesOptions); - } - - @Override - public boolean isValidWord(final String word) { - return mExpandableDictionary.isValidWord(word); - } - - @UsedForTesting - public boolean isInBigramListForTests(final String word) { - // TODO: Use native method to determine whether the word is in dictionary or not - return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word); - } -} diff --git a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java index 1b592b565..da1fb73fe 100644 --- a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java +++ b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java @@ -113,9 +113,7 @@ public final class DebugSettings extends PreferenceFragment mServiceNeedsRestart = true; } } else if (key.equals(PREF_FORCE_NON_DISTINCT_MULTITOUCH) - || key.equals(KeyboardSwitcher.PREF_KEYBOARD_LAYOUT)) { - mServiceNeedsRestart = true; - } else if (key.equals(PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG)) { + || key.equals(PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG)) { mServiceNeedsRestart = true; } } diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java index acd47450b..0e1a33a6c 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java @@ -75,7 +75,7 @@ public final class MoreSuggestions extends Keyboard { while (index < size) { final String word = suggestedWords.getWord(index); // TODO: Should take care of text x-scaling. - mWidths[index] = (int)(TypefaceUtils.getLabelWidth(word, paint) + padding); + mWidths[index] = (int)(TypefaceUtils.getStringWidth(word, paint) + padding); final int numColumn = index - rowStartIndex + 1; final int columnWidth = (maxWidth - mDividerWidth * (numColumn - 1)) / numColumn; diff --git a/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java b/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java index d87f6f3c4..ef1d0f42c 100644 --- a/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java @@ -17,21 +17,25 @@ package com.android.inputmethod.latin.utils; import static com.android.inputmethod.latin.Constants.Subtype.KEYBOARD_MODE; +import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.EMOJI_CAPABLE; import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.IS_ADDITIONAL_SUBTYPE; import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.KEYBOARD_LAYOUT_SET; import static com.android.inputmethod.latin.Constants.Subtype.ExtraValue.UNTRANSLATABLE_STRING_IN_SUBTYPE_NAME; import android.os.Build; import android.text.TextUtils; +import android.util.Log; import android.view.inputmethod.InputMethodSubtype; import com.android.inputmethod.compat.InputMethodSubtypeCompatUtils; -import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; import java.util.ArrayList; +import java.util.Arrays; public final class AdditionalSubtypeUtils { + private static final String TAG = AdditionalSubtypeUtils.class.getSimpleName(); + private static final InputMethodSubtype[] EMPTY_SUBTYPE_ARRAY = new InputMethodSubtype[0]; private AdditionalSubtypeUtils() { @@ -43,6 +47,11 @@ public final class AdditionalSubtypeUtils { } private static final String LOCALE_AND_LAYOUT_SEPARATOR = ":"; + private static final int INDEX_OF_LOCALE = 0; + private static final int INDEX_OF_KEYBOARD_LAYOUT = 1; + private static final int INDEX_OF_EXTRA_VALUE = 2; + private static final int LENGTH_WITHOUT_EXTRA_VALUE = (INDEX_OF_KEYBOARD_LAYOUT + 1); + private static final int LENGTH_WITH_EXTRA_VALUE = (INDEX_OF_EXTRA_VALUE + 1); private static final String PREF_SUBTYPE_SEPARATOR = ";"; public static InputMethodSubtype createAdditionalSubtype(final String localeString, @@ -79,17 +88,6 @@ public final class AdditionalSubtypeUtils { : basePrefSubtype + LOCALE_AND_LAYOUT_SEPARATOR + extraValue; } - public static InputMethodSubtype createAdditionalSubtype(final String prefSubtype) { - final String elems[] = prefSubtype.split(LOCALE_AND_LAYOUT_SEPARATOR); - if (elems.length < 2 || elems.length > 3) { - throw new RuntimeException("Unknown additional subtype specified: " + prefSubtype); - } - final String localeString = elems[0]; - final String keyboardLayoutSetName = elems[1]; - final String extraValue = (elems.length == 3) ? elems[2] : null; - return createAdditionalSubtype(localeString, keyboardLayoutSetName, extraValue); - } - public static InputMethodSubtype[] createAdditionalSubtypesArray(final String prefSubtypes) { if (TextUtils.isEmpty(prefSubtypes)) { return EMPTY_SUBTYPE_ARRAY; @@ -98,7 +96,19 @@ public final class AdditionalSubtypeUtils { final ArrayList<InputMethodSubtype> subtypesList = CollectionUtils.newArrayList(prefSubtypeArray.length); for (final String prefSubtype : prefSubtypeArray) { - final InputMethodSubtype subtype = createAdditionalSubtype(prefSubtype); + final String elems[] = prefSubtype.split(LOCALE_AND_LAYOUT_SEPARATOR); + if (elems.length != LENGTH_WITHOUT_EXTRA_VALUE + && elems.length != LENGTH_WITH_EXTRA_VALUE) { + Log.w(TAG, "Unknown additional subtype specified: " + prefSubtype + " in " + + prefSubtypes); + continue; + } + final String localeString = elems[INDEX_OF_LOCALE]; + final String keyboardLayoutSetName = elems[INDEX_OF_KEYBOARD_LAYOUT]; + final String extraValue = (elems.length == LENGTH_WITH_EXTRA_VALUE) + ? elems[INDEX_OF_EXTRA_VALUE] : null; + final InputMethodSubtype subtype = createAdditionalSubtype( + localeString, keyboardLayoutSetName, extraValue); if (subtype.getNameResId() == SubtypeLocaleUtils.UNKNOWN_KEYBOARD_LAYOUT) { // Skip unknown keyboard layout subtype. This may happen when predefined keyboard // layout has been removed. @@ -137,31 +147,36 @@ public final class AdditionalSubtypeUtils { return sb.toString(); } - private static InputMethodSubtype buildInputMethodSubtype(int nameId, String localeString, - String layoutExtraValue, String additionalSubtypeExtraValue) { - // CAVEAT! If you want to change subtypeId after changing the extra values, - // you must change "getInputMethodSubtypeId". But it will remove the additional keyboard - // from the current users. So, you should be really careful to change it. - final int subtypeId = getInputMethodSubtypeId(nameId, localeString, layoutExtraValue, - additionalSubtypeExtraValue); + private static InputMethodSubtype buildInputMethodSubtype(final int nameId, + final String localeString, final String layoutExtraValue, + final String additionalSubtypeExtraValue) { + // To preserve additional subtype settings and user's selection across OS updates, subtype + // id shouldn't be changed. New attributes, such as emojiCapable, are carefully excluded + // from the calculation of subtype id. + final String compatibleExtraValue = StringUtils.joinCommaSplittableText( + layoutExtraValue, additionalSubtypeExtraValue); + final int compatibleSubtypeId = getInputMethodSubtypeId(localeString, compatibleExtraValue); final String extraValue; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) { - extraValue = layoutExtraValue + "," + additionalSubtypeExtraValue - + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE - + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE; + // Color Emoji is supported from KitKat. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + extraValue = StringUtils.appendToCommaSplittableTextIfNotExists( + EMOJI_CAPABLE, compatibleExtraValue); } else { - extraValue = layoutExtraValue + "," + additionalSubtypeExtraValue; + extraValue = compatibleExtraValue; } return InputMethodSubtypeCompatUtils.newInputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, localeString, KEYBOARD_MODE, extraValue, - false, false, subtypeId); + false, false, compatibleSubtypeId); } - private static int getInputMethodSubtypeId(int nameId, String localeString, - String layoutExtraValue, String additionalSubtypeExtraValue) { - // TODO: Use InputMethodSubtypeBuilder once we use SDK version 19. - return (new InputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, - localeString, KEYBOARD_MODE, layoutExtraValue + "," + additionalSubtypeExtraValue, - false, false)).hashCode(); + private static int getInputMethodSubtypeId(final String localeString, final String extraValue) { + // From the compatibility point of view, the calculation of subtype id has been copied from + // {@link InputMethodSubtype} of JellyBean MR2. + return Arrays.hashCode(new Object[] { + localeString, + KEYBOARD_MODE, + extraValue, + false /* isAuxiliary */, + false /* overrideImplicitlyEnabledSubtype */ }); } } diff --git a/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java b/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java index 08a2a8c5a..e521ec807 100644 --- a/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/ApplicationUtils.java @@ -62,4 +62,22 @@ public final class ApplicationUtils { } return ""; } + + /** + * A utility method to get the application's PackageInfo.versionCode + * @return the application's PackageInfo.versionCode + */ + public static int getVersionCode(final Context context) { + try { + if (context == null) { + return 0; + } + final String packageName = context.getPackageName(); + final PackageInfo info = context.getPackageManager().getPackageInfo(packageName, 0); + return info.versionCode; + } catch (final NameNotFoundException e) { + Log.e(TAG, "Could not find version info.", e); + } + return 0; + } } diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index a36548392..4cc89d0a7 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -39,6 +39,8 @@ public final class StringUtils { public static final int CAPITALIZE_FIRST = 1; // First only public static final int CAPITALIZE_ALL = 2; // All caps + private static final String EMPTY_STRING = ""; + private StringUtils() { // This utility class is not publicly instantiable. } @@ -80,6 +82,20 @@ public final class StringUtils { return containsInArray(text, extraValues.split(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT)); } + public static String joinCommaSplittableText(final String head, final String tail) { + if (TextUtils.isEmpty(head) && TextUtils.isEmpty(tail)) { + return EMPTY_STRING; + } + // Here either head or tail is not null. + if (TextUtils.isEmpty(head)) { + return tail; + } + if (TextUtils.isEmpty(tail)) { + return head; + } + return head + SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT + tail; + } + public static String appendToCommaSplittableTextIfNotExists(final String text, final String extraValues) { if (TextUtils.isEmpty(extraValues)) { @@ -94,7 +110,7 @@ public final class StringUtils { public static String removeFromCommaSplittableTextIfExists(final String text, final String extraValues) { if (TextUtils.isEmpty(extraValues)) { - return ""; + return EMPTY_STRING; } final String[] elements = extraValues.split(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT); if (!containsInArray(text, elements)) { @@ -380,7 +396,7 @@ public final class StringUtils { @UsedForTesting public static String byteArrayToHexString(byte[] bytes) { if (bytes == null || bytes.length == 0) { - return ""; + return EMPTY_STRING; } final StringBuilder sb = new StringBuilder(); for (byte b : bytes) { @@ -444,7 +460,7 @@ public final class StringUtils { public static String listToJsonStr(List<Object> list) { if (list == null || list.isEmpty()) { - return ""; + return EMPTY_STRING; } final StringWriter sw = new StringWriter(); final JsonWriter writer = new JsonWriter(sw); @@ -470,6 +486,6 @@ public final class StringUtils { } catch (IOException e) { } } - return ""; + return EMPTY_STRING; } } diff --git a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java index 102a41b4e..fdbe81ab6 100644 --- a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java @@ -197,7 +197,9 @@ public final class SubtypeLocaleUtils { // es_US spanish F Español (EE.UU.) exception // fr azerty F Français // fr_CA qwerty F Français (Canada) + // fr_CH swiss F Français (Suisse) // de qwertz F Deutsch + // de_CH swiss T Deutsch (Schweiz) // zz qwerty F No language (QWERTY) in system locale // fr qwertz T Français (QWERTZ) // de qwerty T Deutsch (QWERTY) @@ -298,7 +300,9 @@ public final class SubtypeLocaleUtils { // es_US spanish F Es Español Español (EE.UU.) exception // fr azerty F Fr Français Français // fr_CA qwerty F Fr Français Français (Canada) + // fr_CH swiss F Fr Français Français (Suisse) // de qwertz F De Deutsch Deutsch + // de_CH swiss T De Deutsch Deutsch (Schweiz) // zz qwerty F QWERTY QWERTY // fr qwertz T Fr Français Français // de qwerty T De Deutsch Deutsch diff --git a/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java b/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java index 47ea1ea75..087a7f255 100644 --- a/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/TypefaceUtils.java @@ -22,6 +22,9 @@ import android.graphics.Typeface; import android.util.SparseArray; public final class TypefaceUtils { + private static final char[] KEY_LABEL_REFERENCE_CHAR = { 'M' }; + private static final char[] KEY_NUMERIC_HINT_LABEL_REFERENCE_CHAR = { '8' }; + private TypefaceUtils() { // This utility class is not publicly instantiable. } @@ -31,7 +34,7 @@ public final class TypefaceUtils { // Working variable for the following method. private static final Rect sTextHeightBounds = new Rect(); - public static float getCharHeight(final char[] referenceChar, final Paint paint) { + private static float getCharHeight(final char[] referenceChar, final Paint paint) { final int key = getCharGeometryCacheKey(referenceChar[0], paint); synchronized (sTextHeightCache) { final Float cachedValue = sTextHeightCache.get(key); @@ -51,7 +54,7 @@ public final class TypefaceUtils { // Working variable for the following method. private static final Rect sTextWidthBounds = new Rect(); - public static float getCharWidth(final char[] referenceChar, final Paint paint) { + private static float getCharWidth(final char[] referenceChar, final Paint paint) { final int key = getCharGeometryCacheKey(referenceChar[0], paint); synchronized (sTextWidthCache) { final Float cachedValue = sTextWidthCache.get(key); @@ -66,11 +69,6 @@ public final class TypefaceUtils { } } - public static float getStringWidth(final String string, final Paint paint) { - paint.getTextBounds(string, 0, string.length(), sTextWidthBounds); - return sTextWidthBounds.width(); - } - private static int getCharGeometryCacheKey(final char referenceChar, final Paint paint) { final int labelSize = (int)paint.getTextSize(); final Typeface face = paint.getTypeface(); @@ -86,9 +84,25 @@ public final class TypefaceUtils { } } - public static float getLabelWidth(final String label, final Paint paint) { - final Rect textBounds = new Rect(); - paint.getTextBounds(label, 0, label.length(), textBounds); - return textBounds.width(); + public static float getReferenceCharHeight(final Paint paint) { + return getCharHeight(KEY_LABEL_REFERENCE_CHAR, paint); + } + + public static float getReferenceCharWidth(final Paint paint) { + return getCharWidth(KEY_LABEL_REFERENCE_CHAR, paint); + } + + public static float getReferenceDigitWidth(final Paint paint) { + return getCharWidth(KEY_NUMERIC_HINT_LABEL_REFERENCE_CHAR, paint); + } + + // Working variable for the following method. + private static final Rect sStringWidthBounds = new Rect(); + + public static float getStringWidth(final String string, final Paint paint) { + synchronized (sStringWidthBounds) { + paint.getTextBounds(string, 0, string.length(), sStringWidthBounds); + return sStringWidthBounds.width(); + } } } diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index 635afe7cc..7b81e8be1 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -70,6 +70,7 @@ public final class UserHistoryDictIOUtils { /** * Writes dictionary to file. */ + @UsedForTesting public static void writeDictionary(final DictEncoder dictEncoder, final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams, final FormatOptions formatOptions) { diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java index 1992b2f5d..677035ed6 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java @@ -20,6 +20,9 @@ import android.util.Log; import java.util.concurrent.TimeUnit; +import com.android.inputmethod.annotations.UsedForTesting; + +@UsedForTesting public final class UserHistoryForgettingCurveUtils { private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); private static final boolean DEBUG = false; @@ -118,18 +121,22 @@ public final class UserHistoryForgettingCurveUtils { } } + @UsedForTesting /* package */ static int fcToElapsedTime(byte fc) { return fc & 0x0F; } + @UsedForTesting /* package */ static int fcToCount(byte fc) { return (fc >> 4) & 0x03; } + @UsedForTesting /* package */ static int fcToLevel(byte fc) { return (fc >> 6) & 0x03; } + @UsedForTesting private static int calcFreq(int elapsedTime, int count, int level) { if (level <= 0) { // Reserved words, just return -1 @@ -158,6 +165,7 @@ public final class UserHistoryForgettingCurveUtils { return calcFreq(elapsedTime, count, level); } + @UsedForTesting public static byte pushElapsedTime(byte fc) { int elapsedTime = fcToElapsedTime(fc); int count = fcToCount(fc); @@ -173,6 +181,7 @@ public final class UserHistoryForgettingCurveUtils { return calcFc(elapsedTime, count, level); } + @UsedForTesting public static byte pushCount(byte fc, boolean isValid) { final int elapsedTime = fcToElapsedTime(fc); int count = fcToCount(fc); |