diff options
Diffstat (limited to 'java/src')
19 files changed, 467 insertions, 299 deletions
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index d181bf697..eb19ef932 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -179,7 +179,8 @@ public final class BinaryDictionary extends Dictionary { // TODO: check that all users of the `kind' parameter are ready to accept // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), - score, kind, mDictType)); + score, kind, this /* sourceDict */, + mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */)); } } return suggestions; diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 0f5fc71cb..8f6b848bb 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,21 +21,16 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.LocaleUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.BufferUnderflowException; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; @@ -233,9 +228,9 @@ final public class BinaryDictionaryGetter { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { try { // Read the version of the file - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(f); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); final FileHeader header = dictDecoder.readHeader(); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 7c3e4a740..d9ded7ceb 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -28,9 +28,26 @@ import java.util.ArrayList; public abstract class Dictionary { public static final int NOT_A_PROBABILITY = -1; + // The following types do not actually come from real dictionary instances, so we create + // corresponding instances. public static final String TYPE_USER_TYPED = "user_typed"; + public static final Dictionary DICTIONARY_USER_TYPED = new PhonyDictionary(TYPE_USER_TYPED); + public static final String TYPE_APPLICATION_DEFINED = "application_defined"; + public static final Dictionary DICTIONARY_APPLICATION_DEFINED = + new PhonyDictionary(TYPE_APPLICATION_DEFINED); + public static final String TYPE_HARDCODED = "hardcoded"; // punctuation signs and such + public static final Dictionary DICTIONARY_HARDCODED = + new PhonyDictionary(TYPE_HARDCODED); + + // Spawned by resuming suggestions. Comes from a span that was in the TextView. + public static final String TYPE_RESUMED = "resumed"; + public static final Dictionary DICTIONARY_RESUMED = + new PhonyDictionary(TYPE_RESUMED); + + // The following types of dictionary have actual functional instances. We don't need final + // phony dictionary instances for them. public static final String TYPE_MAIN = "main"; public static final String TYPE_CONTACTS = "contacts"; // User dictionary, the system-managed one. @@ -42,9 +59,7 @@ public abstract class Dictionary { // Personalization prediction dictionary internal to LatinIME's Java code. public static final String TYPE_PERSONALIZATION_PREDICTION_IN_JAVA = "personalization_prediction_in_java"; - // Spawned by resuming suggestions. Comes from a span that was in the TextView. - public static final String TYPE_RESUMED = "resumed"; - protected final String mDictType; + public final String mDictType; public Dictionary(final String dictType) { mDictType = dictType; @@ -114,8 +129,40 @@ public abstract class Dictionary { /** * Subclasses may override to indicate that this Dictionary is not yet properly initialized. */ - public boolean isInitialized() { return true; } + + /** + * Whether we think this suggestion should trigger an auto-commit. + */ + public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { + // If we don't have support for auto-commit, or if we don't know, we return false to + // avoid auto-committing stuff. Implementations of the Dictionary class that know to + // determine whether we should auto-commit will override this. + return false; + } + + /** + * Not a true dictionary. A placeholder used to indicate suggestions that don't come from any + * real dictionary. + */ + private static class PhonyDictionary extends Dictionary { + // This class is not publicly instantiable. + private PhonyDictionary(final String type) { + super(type); + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + final boolean blockOffensiveWords) { + return null; + } + + @Override + public boolean isValidWord(String word) { + return false; + } + } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 1ececd5c1..b3a572809 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -20,7 +20,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -87,7 +87,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override protected void writeBinaryDictionary(final FileOutputStream out) throws IOException, UnsupportedFormatException { - BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + BinaryDictEncoderUtils.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); } @Override diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index bd2d70365..516b8426c 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -403,7 +403,8 @@ public class ExpandableDictionary extends Dictionary { // the respective size of the typed word and the suggestion if it matters sometime // in the future. suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, - SuggestedWordInfo.KIND_CORRECTION, mDictType)); + SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; } if (null != node.mShortcutTargets) { @@ -411,7 +412,8 @@ public class ExpandableDictionary extends Dictionary { for (int shortcutIndex = 0; shortcutIndex < length; ++shortcutIndex) { final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), - finalFreq, SuggestedWordInfo.KIND_SHORTCUT, mDictType)); + finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; } } @@ -657,7 +659,8 @@ public class ExpandableDictionary extends Dictionary { if (freq >= 0 && node == null) { suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, Constants.DICTIONARY_MAX_WORD_LENGTH - index), - freq, SuggestedWordInfo.KIND_CORRECTION, mDictType)); + freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 709156951..d67195b0d 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1798,6 +1798,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onUpdateBatchInput(final InputPointers batchPointers) { + final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate(); + if (null != candidate) { + if (candidate.mSourceDict.shouldAutoCommit(candidate)) { + // TODO: implement auto-commit + } + } BatchInputUpdater.getInstance().onUpdateBatchInput(batchPointers); } @@ -2046,10 +2052,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen (!mConnection.isCursorTouchingWord(currentSettings) || !currentSettings.mCurrentLanguageHasSpaces)) { // Reset entirely the composing state anyway, then start composing a new word unless - // the character is a single quote. The idea here is, single quote is not a - // separator and it should be treated as a normal character, except in the first - // position where it should not start composing a word. - isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode); + // the character is a single quote or a dash. The idea here is, single quote and dash + // are not separators and they should be treated as normal characters, except in the + // first position where they should not start composing a word. + isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode + && Constants.CODE_DASH != primaryCode); // Here we don't need to reset the last composed word. It will be reset // when we commit this one, if we ever do; if on the other hand we backspace // it entirely and resume suggestions on the previous word, we'd like to still @@ -2486,7 +2493,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_pickSuggestionManually(replacedWord, index, suggestion, mWordComposer.isBatchMode(), suggestionInfo.mScore, suggestionInfo.mKind, - suggestionInfo.mSourceDict); + suggestionInfo.mSourceDict.mDictType); } mConnection.endBatchEdit(); // Don't allow cancellation of manual pick @@ -2615,7 +2622,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!TextUtils.equals(s, typedWord)) { suggestions.add(new SuggestedWordInfo(s, SuggestionStripView.MAX_SUGGESTIONS - i, - SuggestedWordInfo.KIND_RESUMED, Dictionary.TYPE_RESUMED)); + SuggestedWordInfo.KIND_RESUMED, Dictionary.DICTIONARY_RESUMED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 55b70c6f8..8766e0fc1 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -303,13 +303,15 @@ public final class Suggest { for (int i = 0; i < suggestionsCount; ++i) { final SuggestedWordInfo wordInfo = suggestionsContainer.get(i); - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), + wordInfo.mSourceDict.mDictType); } if (!TextUtils.isEmpty(typedWord)) { suggestionsContainer.add(0, new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, - Dictionary.TYPE_USER_TYPED)); + Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } SuggestedWordInfo.removeDups(suggestionsContainer); @@ -352,7 +354,7 @@ public final class Suggest { } for (SuggestedWordInfo wordInfo : suggestionsSet) { - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType); } final ArrayList<SuggestedWordInfo> suggestionsContainer = @@ -453,7 +455,7 @@ public final class Suggest { sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE); } return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind, - wordInfo.mSourceDict); + wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord); } public void close() { diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index 22beaefee..b27fd81e9 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -113,7 +113,8 @@ public final class SuggestedWords { if (null == text) continue; final SuggestedWordInfo suggestedWordInfo = new SuggestedWordInfo(text.toString(), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_APP_DEFINED, - Dictionary.TYPE_APPLICATION_DEFINED); + Dictionary.DICTIONARY_APPLICATION_DEFINED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */); result.add(suggestedWordInfo); } return result; @@ -126,7 +127,8 @@ public final class SuggestedWords { final ArrayList<SuggestedWordInfo> suggestionsList = CollectionUtils.newArrayList(); final HashSet<String> alreadySeen = CollectionUtils.newHashSet(); suggestionsList.add(new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, - SuggestedWordInfo.KIND_TYPED, Dictionary.TYPE_USER_TYPED)); + SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); alreadySeen.add(typedWord.toString()); final int previousSize = previousSuggestions.size(); for (int index = 1; index < previousSize; index++) { @@ -141,7 +143,14 @@ public final class SuggestedWords { return suggestionsList; } + public SuggestedWordInfo getAutoCommitCandidate() { + if (mSuggestedWordInfoList.size() <= 0) return null; + final SuggestedWordInfo candidate = mSuggestedWordInfoList.get(0); + return candidate.isEligibleForAutoCommit() ? candidate : null; + } + public static final class SuggestedWordInfo { + public static final int NOT_AN_INDEX = -1; public static final int MAX_SCORE = Integer.MAX_VALUE; public static final int KIND_MASK_KIND = 0xFF; // Mask to get only the kind public static final int KIND_TYPED = 0; // What user typed @@ -166,18 +175,26 @@ public final class SuggestedWords { public final int mScore; public final int mKind; // one of the KIND_* constants above public final int mCodePointCount; - public final String mSourceDict; + public final Dictionary mSourceDict; + // For auto-commit. This keeps track of the index inside the touch coordinates array + // passed to native code to get suggestions for a gesture that corresponds to the first + // letter of the second word. + public final int mIndexOfTouchPointOfSecondWord; private String mDebugString = ""; public SuggestedWordInfo(final String word, final int score, final int kind, - final String sourceDict) { + final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord) { mWord = word; mScore = score; mKind = kind; mSourceDict = sourceDict; mCodePointCount = StringUtils.codePointCount(mWord); + mIndexOfTouchPointOfSecondWord = indexOfTouchPointOfSecondWord; } + public boolean isEligibleForAutoCommit() { + return (KIND_CORRECTION == mKind && NOT_AN_INDEX != mIndexOfTouchPointOfSecondWord); + } public void setDebugString(final String str) { if (null == str) throw new NullPointerException("Debug info is null"); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index d1974c8d4..5d3695eff 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -40,7 +40,7 @@ import java.util.TreeMap; * * All the methods in this class are static. * - * TODO: Remove calls from classes except BinaryDictDecoder + * TODO: Remove calls from classes except Ver3DictDecoder * TODO: Move this file to makedict/internal. */ public final class BinaryDictDecoderUtils { @@ -278,6 +278,12 @@ public final class BinaryDictDecoderUtils { // Input methods: Read a binary dictionary to memory. // readDictionaryBinary is the public entry point for them. + static int readSInt24(final DictBuffer dictBuffer) { + final int retval = dictBuffer.readUnsignedInt24(); + final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (retval & FormatSpec.SINT24_MAX); + } + static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { if (options.mSupportsDynamicUpdate) { @@ -314,103 +320,6 @@ public final class BinaryDictDecoderUtils { } } - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final DictBuffer dictBuffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = dictBuffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(dictBuffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = dictBuffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(dictBuffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = dictBuffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - dictBuffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += dictBuffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (dictBuffer.readUnsignedByte() << 16) - + dictBuffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - /** * Reads and returns the char group count out of a buffer and forwards the pointer. */ @@ -427,24 +336,25 @@ public final class BinaryDictDecoderUtils { /** * Finds, as a string, the word at the address passed as an argument. * - * @param dictBuffer the buffer to read from. + * @param dictDecoder the dict decoder. * @param headerSize the size of the header. * @param address the address to seek. * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ /* package for tests */ static WeightedString getWordAtAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + final Ver3DictDecoder dictDecoder, final int headerSize, final int address, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final WeightedString result; final int originalPointer = dictBuffer.position(); dictBuffer.position(address); if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(dictBuffer, headerSize, address, + result = getWordAtAddressWithParentAddress(dictDecoder, headerSize, address, formatOptions); } else { - result = getWordAtAddressWithoutParentAddress(dictBuffer, headerSize, address, + result = getWordAtAddressWithoutParentAddress(dictDecoder, headerSize, address, formatOptions); } @@ -454,8 +364,9 @@ public final class BinaryDictDecoderUtils { @SuppressWarnings("unused") private static WeightedString getWordAtAddressWithParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + final Ver3DictDecoder dictDecoder, final int headerSize, final int address, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int currentAddress = address; int frequency = Integer.MIN_VALUE; final StringBuilder builder = new StringBuilder(); @@ -465,7 +376,7 @@ public final class BinaryDictDecoderUtils { int loopCounter = 0; do { dictBuffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(dictBuffer, currentAddress, options); + currentInfo = dictDecoder.readPtNode(currentAddress, options); if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } @@ -483,8 +394,9 @@ public final class BinaryDictDecoderUtils { } private static WeightedString getWordAtAddressWithoutParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + final Ver3DictDecoder dictDecoder, final int headerSize, final int address, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(headerSize); final int count = readCharGroupCount(dictBuffer); int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); @@ -493,7 +405,7 @@ public final class BinaryDictDecoderUtils { CharGroupInfo last = null; for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); + CharGroupInfo info = dictDecoder.readPtNode(groupOffset, options); groupOffset = info.mEndAddress; if (info.mOriginalAddress == address) { builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); @@ -532,17 +444,18 @@ public final class BinaryDictDecoderUtils { * This will recursively read other node arrays into the structure, populating the reverse * maps on the fly and using them to keep track of already read nodes. * - * @param dictBuffer the buffer, correctly positioned at the start of a node array. + * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. * @param headerSize the size, in bytes, of the file header. * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. * @param reverseGroupMap a mapping from addresses to already read character groups. * @param options file format options. * @return the read node array with all his children already read. */ - private static PtNodeArray readNodeArray(final DictBuffer dictBuffer, + private static PtNodeArray readNodeArray(final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, final Map<Integer, CharGroup> reverseGroupMap, final FormatOptions options) throws IOException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final ArrayList<CharGroup> nodeArrayContents = new ArrayList<CharGroup>(); final int nodeArrayOrigin = dictBuffer.position() - headerSize; @@ -551,15 +464,15 @@ public final class BinaryDictDecoderUtils { final int count = readCharGroupCount(dictBuffer); int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); + CharGroupInfo info = dictDecoder.readPtNode(groupOffset, options); if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; ArrayList<WeightedString> bigrams = null; if (null != info.mBigrams) { bigrams = new ArrayList<WeightedString>(); for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - dictBuffer, headerSize, bigram.mAddress, options); + final WeightedString word = getWordAtAddress(dictDecoder, headerSize, + bigram.mAddress, options); final int reconstructedFrequency = BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); @@ -571,7 +484,7 @@ public final class BinaryDictDecoderUtils { if (null == children) { final int currentPosition = dictBuffer.position(); dictBuffer.position(info.mChildrenAddress + headerSize); - children = readNodeArray(dictBuffer, headerSize, reverseNodeArrayMap, + children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, reverseGroupMap, options); dictBuffer.position(currentPosition); } @@ -649,13 +562,13 @@ public final class BinaryDictDecoderUtils { * @return the created (or merged) dictionary. */ @UsedForTesting - public static FusionDictionary readDictionaryBinary(final BinaryDictDecoder dictDecoder, + public static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder, final FusionDictionary dict) throws FileNotFoundException, IOException, UnsupportedFormatException { // if the buffer has not been opened, open the buffer with bytebuffer. if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); if (dictDecoder.getDictBuffer() == null) { MakedictLog.e("Cannot open the buffer"); } @@ -665,7 +578,7 @@ public final class BinaryDictDecoderUtils { Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final PtNodeArray root = readNodeArray(dictDecoder.getDictBuffer(), fileHeader.mHeaderSize, + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index ff11cde39..3fe3ae6ce 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -34,11 +34,11 @@ import java.util.Iterator; * * All the methods in this class are static. */ -public class BinaryDictEncoder { +public class BinaryDictEncoderUtils { private static final boolean DBG = MakedictLog.DBG; - private BinaryDictEncoder() { + private BinaryDictEncoderUtils() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 1abc779d0..a54fc8c21 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -62,10 +62,11 @@ public final class BinaryDictIOUtils { * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner( - final DictBuffer dictBuffer, final int headerSize, + final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack<Position> stack = new Stack<Position>(); @@ -94,8 +95,7 @@ public final class BinaryDictIOUtils { stack.pop(); continue; } - CharGroupInfo info = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - p.mAddress - headerSize, formatOptions); + CharGroupInfo info = dictDecoder.readPtNode(p.mAddress - headerSize, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; } @@ -148,13 +148,13 @@ public final class BinaryDictIOUtils { * @throws IOException if the file can't be read. * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final BinaryDictDecoder dictDecoder, + public static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header final FileHeader header = dictDecoder.readHeader(); - readUnigramsAndBigramsBinaryInner(dictDecoder.getDictBuffer(), header.mHeaderSize, words, + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, frequencies, bigrams, header.mFormatOptions); } @@ -169,7 +169,7 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final BinaryDictDecoder dictDecoder, + public static int getTerminalPosition(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; @@ -186,8 +186,8 @@ public final class BinaryDictIOUtils { boolean foundNextCharGroup = false; for (int i = 0; i < charGroupCount; ++i) { final int charGroupPos = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup( - dictBuffer, dictBuffer.position(), header.mFormatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(charGroupPos, + header.mFormatOptions); final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, header.mFormatOptions); final boolean isDeletedGroup = isDeletedGroup(currentInfo.mFlags, @@ -272,7 +272,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictEncoder.getByteSize(value)) { + switch (BinaryDictEncoderUtils.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -286,7 +286,7 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictEncoder.getByteSize(value); + return BinaryDictEncoderUtils.getByteSize(value); } static void skipCharGroup(final DictBuffer dictBuffer, @@ -413,14 +413,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictEncoder.makeShortcutFlags( + destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -429,7 +429,7 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictEncoder#computeActualNodeArraySize + // in BinaryDictEncoderUtils#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; @@ -439,7 +439,7 @@ public final class BinaryDictIOUtils { final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictEncoder.getByteSize(bigramOffset)) { + switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { case 1: bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; break; @@ -463,18 +463,18 @@ public final class BinaryDictIOUtils { */ static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) { int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters) + + BinaryDictEncoderUtils.getGroupCharactersSize(info.mCharacters) + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { size += FormatSpec.GROUP_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictEncoder.getByteSize(attr.mAddress); + size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); } } return size; @@ -508,7 +508,7 @@ public final class BinaryDictIOUtils { } /** - * Find a word using the BinaryDictDecoder. + * Find a word using the Ver3DictDecoder. * * @param dictDecoder the dict reader * @param word the word searched @@ -517,7 +517,7 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException */ @UsedForTesting - public static CharGroupInfo findWordByBinaryDictReader(final BinaryDictDecoder dictDecoder, + public static CharGroupInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = getTerminalPosition(dictDecoder, word); final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); @@ -525,8 +525,7 @@ public final class BinaryDictIOUtils { dictBuffer.position(0); final FileHeader header = dictDecoder.readHeader(); dictBuffer.position(position); - return BinaryDictDecoderUtils.readCharGroup(dictBuffer, position, - header.mFormatOptions); + return dictDecoder.readPtNode(position, header.mFormatOptions); } return null; } @@ -545,8 +544,8 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); - dictDecoder.openDictBuffer(new BinaryDictDecoder.DictionaryBufferFactory() { + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + dictDecoder.openDictBuffer(new DictDecoder.DictionaryBufferFactory() { @Override public DictBuffer getDictionaryBuffer(File file) throws FileNotFoundException, IOException { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 2007b6284..144f91618 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -17,12 +17,10 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; -import com.android.inputmethod.latin.utils.JniUtils; import java.io.File; import java.io.FileInputStream; @@ -31,18 +29,17 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.HashMap; -// TODO: Rename this class to "Ver3DictDecoder" or something, and make an interface "DictDecoder". -@UsedForTesting -public class BinaryDictDecoder { - - static { - JniUtils.loadNativeLibrary(); - } - - // TODO: implement something sensical instead of just a phony method - private static native int doNothing(); +/** + * An interface of binary dictionary decoder. + */ +public interface DictDecoder { + public FileHeader readHeader() throws IOException, UnsupportedFormatException; + /** + * Reads a PtNode and returns CharGroupInfo. + */ + public CharGroupInfo readPtNode(final int originalGroupAddress, + final FormatOptions formatOptions); public interface DictionaryBufferFactory { public DictBuffer getDictionaryBuffer(final File file) @@ -133,81 +130,4 @@ public class BinaryDictDecoder { return null; } } - - private final static class HeaderReader { - protected static int readVersion(final DictBuffer dictBuffer) - throws IOException, UnsupportedFormatException { - return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); - } - - protected static int readOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedShort(); - } - - protected static int readHeaderSize(final DictBuffer dictBuffer) { - return dictBuffer.readInt(); - } - - protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, - final int headerSize) { - final HashMap<String, String> attributes = new HashMap<String, String>(); - while (dictBuffer.position() < headerSize) { - // We can avoid an infinite loop here since dictBuffer.position() is always - // increased by calling CharEncoding.readString. - final String key = CharEncoding.readString(dictBuffer); - final String value = CharEncoding.readString(dictBuffer); - attributes.put(key, value); - } - dictBuffer.position(headerSize); - return attributes; - } - } - - private final File mDictionaryBinaryFile; - private DictBuffer mDictBuffer; - - public BinaryDictDecoder(final File file) { - mDictionaryBinaryFile = file; - mDictBuffer = null; - } - - public void openDictBuffer(final DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); - } - - public DictBuffer getDictBuffer() { - return mDictBuffer; - } - - @UsedForTesting - public DictBuffer openAndGetDictBuffer( - final DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - openDictBuffer(factory); - return getDictBuffer(); - } - - // TODO : Define public functions of decoders - public FileHeader readHeader() throws IOException, UnsupportedFormatException { - final int version = HeaderReader.readVersion(mDictBuffer); - final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); - - final int headerSize = HeaderReader.readHeaderSize(mDictBuffer); - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final HashMap<String, String> attributes = HeaderReader.readAttributes(mDictBuffer, - headerSize); - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 6c1e75cbb..f976c8152 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -55,7 +55,7 @@ public final class DynamicBinaryDictIOUtils { * @throws UnsupportedFormatException */ @UsedForTesting - public static void deleteWord(final BinaryDictDecoder dictDecoder, final String word) + public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(0); @@ -253,7 +253,7 @@ public final class DynamicBinaryDictIOUtils { // TODO: Support batch insertion. // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. @UsedForTesting - public static void insertWord(final BinaryDictDecoder dictDecoder, + public static void insertWord(final Ver3DictDecoder dictDecoder, final OutputStream destination, final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, @@ -293,8 +293,8 @@ public final class DynamicBinaryDictIOUtils { for (int i = 0; i < charGroupCount; ++i) { address = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - dictBuffer.position(), fileHeader.mFormatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(address, + fileHeader.mFormatOptions); final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, fileHeader.mFormatOptions); if (isMovedGroup) continue; @@ -314,7 +314,7 @@ public final class DynamicBinaryDictIOUtils { * abc - d - ef */ final int newNodeAddress = dictBuffer.limit(); - final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(p > 1, isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, false /* isBlackListEntry */, fileHeader.mFormatOptions); int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, @@ -353,7 +353,7 @@ public final class DynamicBinaryDictIOUtils { final int childrenAddress = currentInfo.mChildrenAddress; // move prefix - final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int prefixFlags = BinaryDictEncoderUtils.makeCharGroupFlags(p > 1, false /* isTerminal */, 0 /* childrenAddressSize*/, false /* hasShortcut */, false /* hasBigrams */, false /* isNotAWord */, false /* isBlackListEntry */, @@ -369,7 +369,7 @@ public final class DynamicBinaryDictIOUtils { updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } - final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags( + final int suffixFlags = BinaryDictEncoderUtils.makeCharGroupFlags( suffixCharacters.length > 1, (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, 0 /* childrenAddressSize */, @@ -387,7 +387,7 @@ public final class DynamicBinaryDictIOUtils { final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, codePoints.length); - final int flags = BinaryDictEncoder.makeCharGroupFlags( + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags( newCharacters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); @@ -410,7 +410,7 @@ public final class DynamicBinaryDictIOUtils { // only update group. final int newNodeAddress = dictBuffer.limit(); final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, @@ -440,7 +440,7 @@ public final class DynamicBinaryDictIOUtils { fileHeader.mFormatOptions); final int newGroupAddress = newNodeAddress + 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); @@ -485,7 +485,7 @@ public final class DynamicBinaryDictIOUtils { BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(characters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java new file mode 100644 index 000000000..8373ae0bd --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -0,0 +1,274 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.JniUtils; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +/** + * An implementation of DictDecoder for version 3 binary dictionary. + */ +@UsedForTesting +public class Ver3DictDecoder implements DictDecoder { + + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + + private final static class HeaderReader { + protected static int readVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); + } + + protected static int readOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedShort(); + } + + protected static int readHeaderSize(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + + protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, + final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (dictBuffer.position() < headerSize) { + // We can avoid an infinite loop here since dictBuffer.position() is always + // increased by calling CharEncoding.readString. + final String key = CharEncoding.readString(dictBuffer); + final String value = CharEncoding.readString(dictBuffer); + attributes.put(key, value); + } + dictBuffer.position(headerSize); + return attributes; + } + } + + private final static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readFrequency(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigrams(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + final int offset = (dictBuffer.readUnsignedByte() << 16) + + dictBuffer.readUnsignedShort(); + bigramAddress += sign * offset; + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + return readLength; + } + } + + private final File mDictionaryBinaryFile; + private DictBuffer mDictBuffer; + + public Ver3DictDecoder(final File file) { + mDictionaryBinaryFile = file; + mDictBuffer = null; + } + + public void openDictBuffer(final DictDecoder.DictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); + } + + public DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @UsedForTesting + public DictBuffer openAndGetDictBuffer(final DictDecoder.DictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + openDictBuffer(factory); + return getDictBuffer(); + } + + @Override + public FileHeader readHeader() throws IOException, UnsupportedFormatException { + final int version = HeaderReader.readVersion(mDictBuffer); + final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(mDictBuffer); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = HeaderReader.readAttributes(mDictBuffer, + headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public CharGroupInfo readPtNode(final int originalGroupAddress, + final FormatOptions options) { + int addressPointer = originalGroupAddress; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + ++addressPointer; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += 3; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (-1 != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + ++addressPointer; + frequency = PtNodeReader.readFrequency(mDictBuffer); + } else { + frequency = CharGroup.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + ArrayList<WeightedString> shortcutTargets = null; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } + ArrayList<PendingAttribute> bigrams = null; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigrams(mDictBuffer, bigrams, addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + MakedictLog.d("too many bigrams in a group."); + } + } + return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 8160501a9..b565b2f9f 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -28,8 +28,8 @@ import com.android.inputmethod.latin.ExpandableDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; @@ -241,10 +241,10 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona }; // Load the dictionary from binary file - final BinaryDictDecoder reader = new BinaryDictDecoder( + final Ver3DictDecoder reader = new Ver3DictDecoder( new File(getContext().getFilesDir(), fileName)); try { - reader.openDictBuffer(new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + reader.openDictBuffer(new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); } catch (FileNotFoundException e) { // This is an expected condition: we don't have a user history dictionary for this diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 674933019..32730d23f 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -306,7 +306,8 @@ public final class SettingsValues { // TODO: Stop using KeySpceParser.getLabel(). puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, - Dictionary.TYPE_HARDCODED)); + Dictionary.DICTIONARY_HARDCODED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } return new SuggestedWords(puncList, diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index bcf64a8e8..ed0812583 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -125,24 +125,12 @@ final class SuggestionStripLayoutHelper { R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripViewStyle); mSuggestionStripOption = a.getInt( R.styleable.SuggestionStripView_suggestionStripOption, 0); - final float alphaValidTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaValidTypedWord, 1.0f); - final float alphaTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaTypedWord, 1.0f); - final float alphaAutoCorrect = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaAutoCorrect, 1.0f); - final float alphaSuggested = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); mAlphaObsoleted = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); - mColorValidTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorValidTypedWord, 0), alphaValidTypedWord); - mColorTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorTypedWord, 0), alphaTypedWord); - mColorAutoCorrect = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorAutoCorrect, 0), alphaAutoCorrect); - mColorSuggested = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorSuggested, 0), alphaSuggested); + R.styleable.SuggestionStripView_alphaObsoleted, 1.0f); + mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0); + mColorTypedWord = a.getColor(R.styleable.SuggestionStripView_colorTypedWord, 0); + mColorAutoCorrect = a.getColor(R.styleable.SuggestionStripView_colorAutoCorrect, 0); + mColorSuggested = a.getColor(R.styleable.SuggestionStripView_colorSuggested, 0); mSuggestionsCountInStrip = a.getInt( R.styleable.SuggestionStripView_suggestionsCountInStrip, DEFAULT_SUGGESTIONS_COUNT_IN_STRIP); diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index dd7f534dc..771db3a47 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -19,14 +19,14 @@ package com.android.inputmethod.latin.utils; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; @@ -62,7 +62,7 @@ public final class UserHistoryDictIOUtils { final FormatOptions formatOptions) { final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); try { - BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryBinary(destination, fusionDict, formatOptions); Log.d(TAG, "end writing"); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); @@ -118,7 +118,7 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final BinaryDictDecoder dictDecoder, + public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, final OnAddWordListener dict) { final Map<Integer, String> unigrams = CollectionUtils.newTreeMap(); final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); diff --git a/java/src/com/android/inputmethod/research/JsonUtils.java b/java/src/com/android/inputmethod/research/JsonUtils.java index 977f843e9..2beebdfae 100644 --- a/java/src/com/android/inputmethod/research/JsonUtils.java +++ b/java/src/com/android/inputmethod/research/JsonUtils.java @@ -103,7 +103,7 @@ import java.util.Map; jsonWriter.name("word").value(wordInfo.toString()); jsonWriter.name("score").value(wordInfo.mScore); jsonWriter.name("kind").value(wordInfo.mKind); - jsonWriter.name("sourceDict").value(wordInfo.mSourceDict); + jsonWriter.name("sourceDict").value(wordInfo.mSourceDict.mDictType); jsonWriter.endObject(); } jsonWriter.endArray(); |