diff options
43 files changed, 627 insertions, 432 deletions
diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml index 5c59f5f68..a96cfec1e 100644 --- a/java/res/values/attrs.xml +++ b/java/res/values/attrs.xml @@ -169,10 +169,6 @@ <attr name="colorTypedWord" format="color" /> <attr name="colorAutoCorrect" format="color" /> <attr name="colorSuggested" format="color" /> - <attr name="alphaValidTypedWord" format="fraction" /> - <attr name="alphaTypedWord" format="fraction" /> - <attr name="alphaAutoCorrect" format="fraction" /> - <attr name="alphaSuggested" format="fraction" /> <attr name="alphaObsoleted" format="fraction" /> <attr name="suggestionsCountInStrip" format="integer" /> <attr name="centerSuggestionPercentile" format="fraction" /> diff --git a/java/res/values/colors.xml b/java/res/values/colors.xml index daa167c8a..598a5d050 100644 --- a/java/res/values/colors.xml +++ b/java/res/values/colors.xml @@ -41,7 +41,9 @@ <color name="spacebar_text_shadow_color_stone">#D0FFFFFF</color> <!-- Color resources for IceCreamSandwich theme. --> <!-- android:color/holo_blue_light value is #FF33B5E5 --> - <color name="highlight_color_ics">@android:color/holo_blue_light</color> + <color name="highlight_color_ics">#FF33B5E5</color> + <color name="typed_word_color_ics">#D833B5E5</color> + <color name="suggested_word_color_ics">#B233B5E5</color> <color name="highlight_translucent_color_ics">#9933B5E5</color> <color name="key_text_color_ics">@android:color/white</color> <color name="key_text_shadow_color_ics">@android:color/transparent</color> @@ -52,7 +54,6 @@ <color name="key_shifted_letter_hint_activated_color_ics">@android:color/white</color> <color name="spacebar_text_color_ics">#FFC0C0C0</color> <color name="spacebar_text_shadow_color_ics">#80000000</color> - <color name="typed_word_color_ics">@color/highlight_color_ics</color> <!-- Color resources for setup wizard and tutorial --> <color name="setup_background">#FFEBEBEB</color> <color name="setup_text_dark">#FF707070</color> diff --git a/java/res/values/styles.xml b/java/res/values/styles.xml index 37c6a9553..d175a12c0 100644 --- a/java/res/values/styles.xml +++ b/java/res/values/styles.xml @@ -382,13 +382,10 @@ > <item name="android:background">@drawable/keyboard_suggest_strip_holo</item> <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item> - <item name="colorValidTypedWord">@color/highlight_color_ics</item> - <item name="colorTypedWord">@color/highlight_color_ics</item> + <item name="colorValidTypedWord">@color/typed_word_color_ics</item> + <item name="colorTypedWord">@color/typed_word_color_ics</item> <item name="colorAutoCorrect">@color/highlight_color_ics</item> - <item name="colorSuggested">@color/highlight_color_ics</item> - <item name="alphaValidTypedWord">85%</item> - <item name="alphaTypedWord">85%</item> - <item name="alphaSuggested">70%</item> + <item name="colorSuggested">@color/suggested_word_color_ics</item> <item name="alphaObsoleted">70%</item> <item name="suggestionsCountInStrip">@integer/suggestions_count_in_strip</item> <item name="centerSuggestionPercentile">@fraction/center_suggestion_percentile</item> diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index d181bf697..eb19ef932 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -179,7 +179,8 @@ public final class BinaryDictionary extends Dictionary { // TODO: check that all users of the `kind' parameter are ready to accept // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), - score, kind, mDictType)); + score, kind, this /* sourceDict */, + mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */)); } } return suggestions; diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 0f5fc71cb..8f6b848bb 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,21 +21,16 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.LocaleUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.BufferUnderflowException; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; @@ -233,9 +228,9 @@ final public class BinaryDictionaryGetter { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { try { // Read the version of the file - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(f); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); final FileHeader header = dictDecoder.readHeader(); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 7c3e4a740..d9ded7ceb 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -28,9 +28,26 @@ import java.util.ArrayList; public abstract class Dictionary { public static final int NOT_A_PROBABILITY = -1; + // The following types do not actually come from real dictionary instances, so we create + // corresponding instances. public static final String TYPE_USER_TYPED = "user_typed"; + public static final Dictionary DICTIONARY_USER_TYPED = new PhonyDictionary(TYPE_USER_TYPED); + public static final String TYPE_APPLICATION_DEFINED = "application_defined"; + public static final Dictionary DICTIONARY_APPLICATION_DEFINED = + new PhonyDictionary(TYPE_APPLICATION_DEFINED); + public static final String TYPE_HARDCODED = "hardcoded"; // punctuation signs and such + public static final Dictionary DICTIONARY_HARDCODED = + new PhonyDictionary(TYPE_HARDCODED); + + // Spawned by resuming suggestions. Comes from a span that was in the TextView. + public static final String TYPE_RESUMED = "resumed"; + public static final Dictionary DICTIONARY_RESUMED = + new PhonyDictionary(TYPE_RESUMED); + + // The following types of dictionary have actual functional instances. We don't need final + // phony dictionary instances for them. public static final String TYPE_MAIN = "main"; public static final String TYPE_CONTACTS = "contacts"; // User dictionary, the system-managed one. @@ -42,9 +59,7 @@ public abstract class Dictionary { // Personalization prediction dictionary internal to LatinIME's Java code. public static final String TYPE_PERSONALIZATION_PREDICTION_IN_JAVA = "personalization_prediction_in_java"; - // Spawned by resuming suggestions. Comes from a span that was in the TextView. - public static final String TYPE_RESUMED = "resumed"; - protected final String mDictType; + public final String mDictType; public Dictionary(final String dictType) { mDictType = dictType; @@ -114,8 +129,40 @@ public abstract class Dictionary { /** * Subclasses may override to indicate that this Dictionary is not yet properly initialized. */ - public boolean isInitialized() { return true; } + + /** + * Whether we think this suggestion should trigger an auto-commit. + */ + public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { + // If we don't have support for auto-commit, or if we don't know, we return false to + // avoid auto-committing stuff. Implementations of the Dictionary class that know to + // determine whether we should auto-commit will override this. + return false; + } + + /** + * Not a true dictionary. A placeholder used to indicate suggestions that don't come from any + * real dictionary. + */ + private static class PhonyDictionary extends Dictionary { + // This class is not publicly instantiable. + private PhonyDictionary(final String type) { + super(type); + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + final boolean blockOffensiveWords) { + return null; + } + + @Override + public boolean isValidWord(String word) { + return false; + } + } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 1ececd5c1..b3a572809 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -20,7 +20,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -87,7 +87,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override protected void writeBinaryDictionary(final FileOutputStream out) throws IOException, UnsupportedFormatException { - BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + BinaryDictEncoderUtils.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); } @Override diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index bd2d70365..516b8426c 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -403,7 +403,8 @@ public class ExpandableDictionary extends Dictionary { // the respective size of the typed word and the suggestion if it matters sometime // in the future. suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, - SuggestedWordInfo.KIND_CORRECTION, mDictType)); + SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; } if (null != node.mShortcutTargets) { @@ -411,7 +412,8 @@ public class ExpandableDictionary extends Dictionary { for (int shortcutIndex = 0; shortcutIndex < length; ++shortcutIndex) { final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), - finalFreq, SuggestedWordInfo.KIND_SHORTCUT, mDictType)); + finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; } } @@ -657,7 +659,8 @@ public class ExpandableDictionary extends Dictionary { if (freq >= 0 && node == null) { suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, Constants.DICTIONARY_MAX_WORD_LENGTH - index), - freq, SuggestedWordInfo.KIND_CORRECTION, mDictType)); + freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 709156951..d67195b0d 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1798,6 +1798,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onUpdateBatchInput(final InputPointers batchPointers) { + final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate(); + if (null != candidate) { + if (candidate.mSourceDict.shouldAutoCommit(candidate)) { + // TODO: implement auto-commit + } + } BatchInputUpdater.getInstance().onUpdateBatchInput(batchPointers); } @@ -2046,10 +2052,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen (!mConnection.isCursorTouchingWord(currentSettings) || !currentSettings.mCurrentLanguageHasSpaces)) { // Reset entirely the composing state anyway, then start composing a new word unless - // the character is a single quote. The idea here is, single quote is not a - // separator and it should be treated as a normal character, except in the first - // position where it should not start composing a word. - isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode); + // the character is a single quote or a dash. The idea here is, single quote and dash + // are not separators and they should be treated as normal characters, except in the + // first position where they should not start composing a word. + isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode + && Constants.CODE_DASH != primaryCode); // Here we don't need to reset the last composed word. It will be reset // when we commit this one, if we ever do; if on the other hand we backspace // it entirely and resume suggestions on the previous word, we'd like to still @@ -2486,7 +2493,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_pickSuggestionManually(replacedWord, index, suggestion, mWordComposer.isBatchMode(), suggestionInfo.mScore, suggestionInfo.mKind, - suggestionInfo.mSourceDict); + suggestionInfo.mSourceDict.mDictType); } mConnection.endBatchEdit(); // Don't allow cancellation of manual pick @@ -2615,7 +2622,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!TextUtils.equals(s, typedWord)) { suggestions.add(new SuggestedWordInfo(s, SuggestionStripView.MAX_SUGGESTIONS - i, - SuggestedWordInfo.KIND_RESUMED, Dictionary.TYPE_RESUMED)); + SuggestedWordInfo.KIND_RESUMED, Dictionary.DICTIONARY_RESUMED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 55b70c6f8..8766e0fc1 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -303,13 +303,15 @@ public final class Suggest { for (int i = 0; i < suggestionsCount; ++i) { final SuggestedWordInfo wordInfo = suggestionsContainer.get(i); - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), + wordInfo.mSourceDict.mDictType); } if (!TextUtils.isEmpty(typedWord)) { suggestionsContainer.add(0, new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, - Dictionary.TYPE_USER_TYPED)); + Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } SuggestedWordInfo.removeDups(suggestionsContainer); @@ -352,7 +354,7 @@ public final class Suggest { } for (SuggestedWordInfo wordInfo : suggestionsSet) { - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType); } final ArrayList<SuggestedWordInfo> suggestionsContainer = @@ -453,7 +455,7 @@ public final class Suggest { sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE); } return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind, - wordInfo.mSourceDict); + wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord); } public void close() { diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index 22beaefee..b27fd81e9 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -113,7 +113,8 @@ public final class SuggestedWords { if (null == text) continue; final SuggestedWordInfo suggestedWordInfo = new SuggestedWordInfo(text.toString(), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_APP_DEFINED, - Dictionary.TYPE_APPLICATION_DEFINED); + Dictionary.DICTIONARY_APPLICATION_DEFINED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */); result.add(suggestedWordInfo); } return result; @@ -126,7 +127,8 @@ public final class SuggestedWords { final ArrayList<SuggestedWordInfo> suggestionsList = CollectionUtils.newArrayList(); final HashSet<String> alreadySeen = CollectionUtils.newHashSet(); suggestionsList.add(new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, - SuggestedWordInfo.KIND_TYPED, Dictionary.TYPE_USER_TYPED)); + SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); alreadySeen.add(typedWord.toString()); final int previousSize = previousSuggestions.size(); for (int index = 1; index < previousSize; index++) { @@ -141,7 +143,14 @@ public final class SuggestedWords { return suggestionsList; } + public SuggestedWordInfo getAutoCommitCandidate() { + if (mSuggestedWordInfoList.size() <= 0) return null; + final SuggestedWordInfo candidate = mSuggestedWordInfoList.get(0); + return candidate.isEligibleForAutoCommit() ? candidate : null; + } + public static final class SuggestedWordInfo { + public static final int NOT_AN_INDEX = -1; public static final int MAX_SCORE = Integer.MAX_VALUE; public static final int KIND_MASK_KIND = 0xFF; // Mask to get only the kind public static final int KIND_TYPED = 0; // What user typed @@ -166,18 +175,26 @@ public final class SuggestedWords { public final int mScore; public final int mKind; // one of the KIND_* constants above public final int mCodePointCount; - public final String mSourceDict; + public final Dictionary mSourceDict; + // For auto-commit. This keeps track of the index inside the touch coordinates array + // passed to native code to get suggestions for a gesture that corresponds to the first + // letter of the second word. + public final int mIndexOfTouchPointOfSecondWord; private String mDebugString = ""; public SuggestedWordInfo(final String word, final int score, final int kind, - final String sourceDict) { + final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord) { mWord = word; mScore = score; mKind = kind; mSourceDict = sourceDict; mCodePointCount = StringUtils.codePointCount(mWord); + mIndexOfTouchPointOfSecondWord = indexOfTouchPointOfSecondWord; } + public boolean isEligibleForAutoCommit() { + return (KIND_CORRECTION == mKind && NOT_AN_INDEX != mIndexOfTouchPointOfSecondWord); + } public void setDebugString(final String str) { if (null == str) throw new NullPointerException("Debug info is null"); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index d1974c8d4..5d3695eff 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -40,7 +40,7 @@ import java.util.TreeMap; * * All the methods in this class are static. * - * TODO: Remove calls from classes except BinaryDictDecoder + * TODO: Remove calls from classes except Ver3DictDecoder * TODO: Move this file to makedict/internal. */ public final class BinaryDictDecoderUtils { @@ -278,6 +278,12 @@ public final class BinaryDictDecoderUtils { // Input methods: Read a binary dictionary to memory. // readDictionaryBinary is the public entry point for them. + static int readSInt24(final DictBuffer dictBuffer) { + final int retval = dictBuffer.readUnsignedInt24(); + final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (retval & FormatSpec.SINT24_MAX); + } + static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { if (options.mSupportsDynamicUpdate) { @@ -314,103 +320,6 @@ public final class BinaryDictDecoderUtils { } } - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final DictBuffer dictBuffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = dictBuffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(dictBuffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = dictBuffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(dictBuffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = dictBuffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - dictBuffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += dictBuffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (dictBuffer.readUnsignedByte() << 16) - + dictBuffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - /** * Reads and returns the char group count out of a buffer and forwards the pointer. */ @@ -427,24 +336,25 @@ public final class BinaryDictDecoderUtils { /** * Finds, as a string, the word at the address passed as an argument. * - * @param dictBuffer the buffer to read from. + * @param dictDecoder the dict decoder. * @param headerSize the size of the header. * @param address the address to seek. * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ /* package for tests */ static WeightedString getWordAtAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + final Ver3DictDecoder dictDecoder, final int headerSize, final int address, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final WeightedString result; final int originalPointer = dictBuffer.position(); dictBuffer.position(address); if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(dictBuffer, headerSize, address, + result = getWordAtAddressWithParentAddress(dictDecoder, headerSize, address, formatOptions); } else { - result = getWordAtAddressWithoutParentAddress(dictBuffer, headerSize, address, + result = getWordAtAddressWithoutParentAddress(dictDecoder, headerSize, address, formatOptions); } @@ -454,8 +364,9 @@ public final class BinaryDictDecoderUtils { @SuppressWarnings("unused") private static WeightedString getWordAtAddressWithParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + final Ver3DictDecoder dictDecoder, final int headerSize, final int address, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int currentAddress = address; int frequency = Integer.MIN_VALUE; final StringBuilder builder = new StringBuilder(); @@ -465,7 +376,7 @@ public final class BinaryDictDecoderUtils { int loopCounter = 0; do { dictBuffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(dictBuffer, currentAddress, options); + currentInfo = dictDecoder.readPtNode(currentAddress, options); if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } @@ -483,8 +394,9 @@ public final class BinaryDictDecoderUtils { } private static WeightedString getWordAtAddressWithoutParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + final Ver3DictDecoder dictDecoder, final int headerSize, final int address, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(headerSize); final int count = readCharGroupCount(dictBuffer); int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); @@ -493,7 +405,7 @@ public final class BinaryDictDecoderUtils { CharGroupInfo last = null; for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); + CharGroupInfo info = dictDecoder.readPtNode(groupOffset, options); groupOffset = info.mEndAddress; if (info.mOriginalAddress == address) { builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); @@ -532,17 +444,18 @@ public final class BinaryDictDecoderUtils { * This will recursively read other node arrays into the structure, populating the reverse * maps on the fly and using them to keep track of already read nodes. * - * @param dictBuffer the buffer, correctly positioned at the start of a node array. + * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. * @param headerSize the size, in bytes, of the file header. * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. * @param reverseGroupMap a mapping from addresses to already read character groups. * @param options file format options. * @return the read node array with all his children already read. */ - private static PtNodeArray readNodeArray(final DictBuffer dictBuffer, + private static PtNodeArray readNodeArray(final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, final Map<Integer, CharGroup> reverseGroupMap, final FormatOptions options) throws IOException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final ArrayList<CharGroup> nodeArrayContents = new ArrayList<CharGroup>(); final int nodeArrayOrigin = dictBuffer.position() - headerSize; @@ -551,15 +464,15 @@ public final class BinaryDictDecoderUtils { final int count = readCharGroupCount(dictBuffer); int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); + CharGroupInfo info = dictDecoder.readPtNode(groupOffset, options); if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; ArrayList<WeightedString> bigrams = null; if (null != info.mBigrams) { bigrams = new ArrayList<WeightedString>(); for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - dictBuffer, headerSize, bigram.mAddress, options); + final WeightedString word = getWordAtAddress(dictDecoder, headerSize, + bigram.mAddress, options); final int reconstructedFrequency = BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); @@ -571,7 +484,7 @@ public final class BinaryDictDecoderUtils { if (null == children) { final int currentPosition = dictBuffer.position(); dictBuffer.position(info.mChildrenAddress + headerSize); - children = readNodeArray(dictBuffer, headerSize, reverseNodeArrayMap, + children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, reverseGroupMap, options); dictBuffer.position(currentPosition); } @@ -649,13 +562,13 @@ public final class BinaryDictDecoderUtils { * @return the created (or merged) dictionary. */ @UsedForTesting - public static FusionDictionary readDictionaryBinary(final BinaryDictDecoder dictDecoder, + public static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder, final FusionDictionary dict) throws FileNotFoundException, IOException, UnsupportedFormatException { // if the buffer has not been opened, open the buffer with bytebuffer. if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); if (dictDecoder.getDictBuffer() == null) { MakedictLog.e("Cannot open the buffer"); } @@ -665,7 +578,7 @@ public final class BinaryDictDecoderUtils { Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final PtNodeArray root = readNodeArray(dictDecoder.getDictBuffer(), fileHeader.mHeaderSize, + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index ff11cde39..3fe3ae6ce 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -34,11 +34,11 @@ import java.util.Iterator; * * All the methods in this class are static. */ -public class BinaryDictEncoder { +public class BinaryDictEncoderUtils { private static final boolean DBG = MakedictLog.DBG; - private BinaryDictEncoder() { + private BinaryDictEncoderUtils() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 1abc779d0..a54fc8c21 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -62,10 +62,11 @@ public final class BinaryDictIOUtils { * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner( - final DictBuffer dictBuffer, final int headerSize, + final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack<Position> stack = new Stack<Position>(); @@ -94,8 +95,7 @@ public final class BinaryDictIOUtils { stack.pop(); continue; } - CharGroupInfo info = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - p.mAddress - headerSize, formatOptions); + CharGroupInfo info = dictDecoder.readPtNode(p.mAddress - headerSize, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; } @@ -148,13 +148,13 @@ public final class BinaryDictIOUtils { * @throws IOException if the file can't be read. * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final BinaryDictDecoder dictDecoder, + public static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header final FileHeader header = dictDecoder.readHeader(); - readUnigramsAndBigramsBinaryInner(dictDecoder.getDictBuffer(), header.mHeaderSize, words, + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, frequencies, bigrams, header.mFormatOptions); } @@ -169,7 +169,7 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final BinaryDictDecoder dictDecoder, + public static int getTerminalPosition(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; @@ -186,8 +186,8 @@ public final class BinaryDictIOUtils { boolean foundNextCharGroup = false; for (int i = 0; i < charGroupCount; ++i) { final int charGroupPos = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup( - dictBuffer, dictBuffer.position(), header.mFormatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(charGroupPos, + header.mFormatOptions); final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, header.mFormatOptions); final boolean isDeletedGroup = isDeletedGroup(currentInfo.mFlags, @@ -272,7 +272,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictEncoder.getByteSize(value)) { + switch (BinaryDictEncoderUtils.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -286,7 +286,7 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictEncoder.getByteSize(value); + return BinaryDictEncoderUtils.getByteSize(value); } static void skipCharGroup(final DictBuffer dictBuffer, @@ -413,14 +413,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictEncoder.makeShortcutFlags( + destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -429,7 +429,7 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictEncoder#computeActualNodeArraySize + // in BinaryDictEncoderUtils#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; @@ -439,7 +439,7 @@ public final class BinaryDictIOUtils { final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictEncoder.getByteSize(bigramOffset)) { + switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { case 1: bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; break; @@ -463,18 +463,18 @@ public final class BinaryDictIOUtils { */ static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) { int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters) + + BinaryDictEncoderUtils.getGroupCharactersSize(info.mCharacters) + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { size += FormatSpec.GROUP_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictEncoder.getByteSize(attr.mAddress); + size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); } } return size; @@ -508,7 +508,7 @@ public final class BinaryDictIOUtils { } /** - * Find a word using the BinaryDictDecoder. + * Find a word using the Ver3DictDecoder. * * @param dictDecoder the dict reader * @param word the word searched @@ -517,7 +517,7 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException */ @UsedForTesting - public static CharGroupInfo findWordByBinaryDictReader(final BinaryDictDecoder dictDecoder, + public static CharGroupInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = getTerminalPosition(dictDecoder, word); final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); @@ -525,8 +525,7 @@ public final class BinaryDictIOUtils { dictBuffer.position(0); final FileHeader header = dictDecoder.readHeader(); dictBuffer.position(position); - return BinaryDictDecoderUtils.readCharGroup(dictBuffer, position, - header.mFormatOptions); + return dictDecoder.readPtNode(position, header.mFormatOptions); } return null; } @@ -545,8 +544,8 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); - dictDecoder.openDictBuffer(new BinaryDictDecoder.DictionaryBufferFactory() { + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + dictDecoder.openDictBuffer(new DictDecoder.DictionaryBufferFactory() { @Override public DictBuffer getDictionaryBuffer(File file) throws FileNotFoundException, IOException { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 2007b6284..144f91618 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -17,12 +17,10 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; -import com.android.inputmethod.latin.utils.JniUtils; import java.io.File; import java.io.FileInputStream; @@ -31,18 +29,17 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.HashMap; -// TODO: Rename this class to "Ver3DictDecoder" or something, and make an interface "DictDecoder". -@UsedForTesting -public class BinaryDictDecoder { - - static { - JniUtils.loadNativeLibrary(); - } - - // TODO: implement something sensical instead of just a phony method - private static native int doNothing(); +/** + * An interface of binary dictionary decoder. + */ +public interface DictDecoder { + public FileHeader readHeader() throws IOException, UnsupportedFormatException; + /** + * Reads a PtNode and returns CharGroupInfo. + */ + public CharGroupInfo readPtNode(final int originalGroupAddress, + final FormatOptions formatOptions); public interface DictionaryBufferFactory { public DictBuffer getDictionaryBuffer(final File file) @@ -133,81 +130,4 @@ public class BinaryDictDecoder { return null; } } - - private final static class HeaderReader { - protected static int readVersion(final DictBuffer dictBuffer) - throws IOException, UnsupportedFormatException { - return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); - } - - protected static int readOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedShort(); - } - - protected static int readHeaderSize(final DictBuffer dictBuffer) { - return dictBuffer.readInt(); - } - - protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, - final int headerSize) { - final HashMap<String, String> attributes = new HashMap<String, String>(); - while (dictBuffer.position() < headerSize) { - // We can avoid an infinite loop here since dictBuffer.position() is always - // increased by calling CharEncoding.readString. - final String key = CharEncoding.readString(dictBuffer); - final String value = CharEncoding.readString(dictBuffer); - attributes.put(key, value); - } - dictBuffer.position(headerSize); - return attributes; - } - } - - private final File mDictionaryBinaryFile; - private DictBuffer mDictBuffer; - - public BinaryDictDecoder(final File file) { - mDictionaryBinaryFile = file; - mDictBuffer = null; - } - - public void openDictBuffer(final DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); - } - - public DictBuffer getDictBuffer() { - return mDictBuffer; - } - - @UsedForTesting - public DictBuffer openAndGetDictBuffer( - final DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - openDictBuffer(factory); - return getDictBuffer(); - } - - // TODO : Define public functions of decoders - public FileHeader readHeader() throws IOException, UnsupportedFormatException { - final int version = HeaderReader.readVersion(mDictBuffer); - final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); - - final int headerSize = HeaderReader.readHeaderSize(mDictBuffer); - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final HashMap<String, String> attributes = HeaderReader.readAttributes(mDictBuffer, - headerSize); - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 6c1e75cbb..f976c8152 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -55,7 +55,7 @@ public final class DynamicBinaryDictIOUtils { * @throws UnsupportedFormatException */ @UsedForTesting - public static void deleteWord(final BinaryDictDecoder dictDecoder, final String word) + public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(0); @@ -253,7 +253,7 @@ public final class DynamicBinaryDictIOUtils { // TODO: Support batch insertion. // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. @UsedForTesting - public static void insertWord(final BinaryDictDecoder dictDecoder, + public static void insertWord(final Ver3DictDecoder dictDecoder, final OutputStream destination, final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, @@ -293,8 +293,8 @@ public final class DynamicBinaryDictIOUtils { for (int i = 0; i < charGroupCount; ++i) { address = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - dictBuffer.position(), fileHeader.mFormatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(address, + fileHeader.mFormatOptions); final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, fileHeader.mFormatOptions); if (isMovedGroup) continue; @@ -314,7 +314,7 @@ public final class DynamicBinaryDictIOUtils { * abc - d - ef */ final int newNodeAddress = dictBuffer.limit(); - final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(p > 1, isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, false /* isBlackListEntry */, fileHeader.mFormatOptions); int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, @@ -353,7 +353,7 @@ public final class DynamicBinaryDictIOUtils { final int childrenAddress = currentInfo.mChildrenAddress; // move prefix - final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int prefixFlags = BinaryDictEncoderUtils.makeCharGroupFlags(p > 1, false /* isTerminal */, 0 /* childrenAddressSize*/, false /* hasShortcut */, false /* hasBigrams */, false /* isNotAWord */, false /* isBlackListEntry */, @@ -369,7 +369,7 @@ public final class DynamicBinaryDictIOUtils { updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } - final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags( + final int suffixFlags = BinaryDictEncoderUtils.makeCharGroupFlags( suffixCharacters.length > 1, (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, 0 /* childrenAddressSize */, @@ -387,7 +387,7 @@ public final class DynamicBinaryDictIOUtils { final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, codePoints.length); - final int flags = BinaryDictEncoder.makeCharGroupFlags( + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags( newCharacters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); @@ -410,7 +410,7 @@ public final class DynamicBinaryDictIOUtils { // only update group. final int newNodeAddress = dictBuffer.limit(); final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, @@ -440,7 +440,7 @@ public final class DynamicBinaryDictIOUtils { fileHeader.mFormatOptions); final int newGroupAddress = newNodeAddress + 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); @@ -485,7 +485,7 @@ public final class DynamicBinaryDictIOUtils { BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(characters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java new file mode 100644 index 000000000..8373ae0bd --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -0,0 +1,274 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.JniUtils; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +/** + * An implementation of DictDecoder for version 3 binary dictionary. + */ +@UsedForTesting +public class Ver3DictDecoder implements DictDecoder { + + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + + private final static class HeaderReader { + protected static int readVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); + } + + protected static int readOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedShort(); + } + + protected static int readHeaderSize(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + + protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, + final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (dictBuffer.position() < headerSize) { + // We can avoid an infinite loop here since dictBuffer.position() is always + // increased by calling CharEncoding.readString. + final String key = CharEncoding.readString(dictBuffer); + final String value = CharEncoding.readString(dictBuffer); + attributes.put(key, value); + } + dictBuffer.position(headerSize); + return attributes; + } + } + + private final static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readFrequency(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigrams(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + final int offset = (dictBuffer.readUnsignedByte() << 16) + + dictBuffer.readUnsignedShort(); + bigramAddress += sign * offset; + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + return readLength; + } + } + + private final File mDictionaryBinaryFile; + private DictBuffer mDictBuffer; + + public Ver3DictDecoder(final File file) { + mDictionaryBinaryFile = file; + mDictBuffer = null; + } + + public void openDictBuffer(final DictDecoder.DictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); + } + + public DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @UsedForTesting + public DictBuffer openAndGetDictBuffer(final DictDecoder.DictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + openDictBuffer(factory); + return getDictBuffer(); + } + + @Override + public FileHeader readHeader() throws IOException, UnsupportedFormatException { + final int version = HeaderReader.readVersion(mDictBuffer); + final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(mDictBuffer); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = HeaderReader.readAttributes(mDictBuffer, + headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public CharGroupInfo readPtNode(final int originalGroupAddress, + final FormatOptions options) { + int addressPointer = originalGroupAddress; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + ++addressPointer; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += 3; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (-1 != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + ++addressPointer; + frequency = PtNodeReader.readFrequency(mDictBuffer); + } else { + frequency = CharGroup.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + ArrayList<WeightedString> shortcutTargets = null; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } + ArrayList<PendingAttribute> bigrams = null; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigrams(mDictBuffer, bigrams, addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + MakedictLog.d("too many bigrams in a group."); + } + } + return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 8160501a9..b565b2f9f 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -28,8 +28,8 @@ import com.android.inputmethod.latin.ExpandableDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; @@ -241,10 +241,10 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona }; // Load the dictionary from binary file - final BinaryDictDecoder reader = new BinaryDictDecoder( + final Ver3DictDecoder reader = new Ver3DictDecoder( new File(getContext().getFilesDir(), fileName)); try { - reader.openDictBuffer(new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + reader.openDictBuffer(new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); } catch (FileNotFoundException e) { // This is an expected condition: we don't have a user history dictionary for this diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 674933019..32730d23f 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -306,7 +306,8 @@ public final class SettingsValues { // TODO: Stop using KeySpceParser.getLabel(). puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, - Dictionary.TYPE_HARDCODED)); + Dictionary.DICTIONARY_HARDCODED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } return new SuggestedWords(puncList, diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index bcf64a8e8..ed0812583 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -125,24 +125,12 @@ final class SuggestionStripLayoutHelper { R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripViewStyle); mSuggestionStripOption = a.getInt( R.styleable.SuggestionStripView_suggestionStripOption, 0); - final float alphaValidTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaValidTypedWord, 1.0f); - final float alphaTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaTypedWord, 1.0f); - final float alphaAutoCorrect = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaAutoCorrect, 1.0f); - final float alphaSuggested = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); mAlphaObsoleted = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); - mColorValidTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorValidTypedWord, 0), alphaValidTypedWord); - mColorTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorTypedWord, 0), alphaTypedWord); - mColorAutoCorrect = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorAutoCorrect, 0), alphaAutoCorrect); - mColorSuggested = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorSuggested, 0), alphaSuggested); + R.styleable.SuggestionStripView_alphaObsoleted, 1.0f); + mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0); + mColorTypedWord = a.getColor(R.styleable.SuggestionStripView_colorTypedWord, 0); + mColorAutoCorrect = a.getColor(R.styleable.SuggestionStripView_colorAutoCorrect, 0); + mColorSuggested = a.getColor(R.styleable.SuggestionStripView_colorSuggested, 0); mSuggestionsCountInStrip = a.getInt( R.styleable.SuggestionStripView_suggestionsCountInStrip, DEFAULT_SUGGESTIONS_COUNT_IN_STRIP); diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index dd7f534dc..771db3a47 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -19,14 +19,14 @@ package com.android.inputmethod.latin.utils; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; @@ -62,7 +62,7 @@ public final class UserHistoryDictIOUtils { final FormatOptions formatOptions) { final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); try { - BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryBinary(destination, fusionDict, formatOptions); Log.d(TAG, "end writing"); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); @@ -118,7 +118,7 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final BinaryDictDecoder dictDecoder, + public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, final OnAddWordListener dict) { final Map<Integer, String> unigrams = CollectionUtils.newTreeMap(); final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); diff --git a/java/src/com/android/inputmethod/research/JsonUtils.java b/java/src/com/android/inputmethod/research/JsonUtils.java index 977f843e9..2beebdfae 100644 --- a/java/src/com/android/inputmethod/research/JsonUtils.java +++ b/java/src/com/android/inputmethod/research/JsonUtils.java @@ -103,7 +103,7 @@ import java.util.Map; jsonWriter.name("word").value(wordInfo.toString()); jsonWriter.name("score").value(wordInfo.mScore); jsonWriter.name("kind").value(wordInfo.mKind); - jsonWriter.name("sourceDict").value(wordInfo.mSourceDict); + jsonWriter.name("sourceDict").value(wordInfo.mSourceDict.mDictType); jsonWriter.endObject(); } jsonWriter.endArray(); diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 9e7407abd..a51fe3c03 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -43,7 +43,7 @@ LATIN_IME_JNI_SRC_FILES := \ com_android_inputmethod_keyboard_ProximityInfo.cpp \ com_android_inputmethod_latin_BinaryDictionary.cpp \ com_android_inputmethod_latin_DicTraverseSession.cpp \ - com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \ + com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \ jni_common.cpp LATIN_IME_CORE_SRC_FILES := \ diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp b/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp index 457b226b6..15088b65a 100644 --- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp +++ b/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp @@ -14,16 +14,16 @@ * limitations under the License. */ -#define LOG_TAG "LatinIME: jni: BinaryDictDecoder" +#define LOG_TAG "LatinIME: jni: Ver3DictDecoder" -#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h" +#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h" #include "defines.h" #include "jni.h" #include "jni_common.h" namespace latinime { -static int latinime_BinaryDictDecoder_doNothing(JNIEnv *env, jclass clazz) { +static int latinime_Ver3DictDecoder_doNothing(JNIEnv *env, jclass clazz) { // This is a phony method for test - it does nothing. It just returns some value // unlikely to be in memory by chance for testing purposes. // TODO: remove this method. @@ -35,13 +35,13 @@ static const JNINativeMethod sMethods[] = { // TODO: remove this entry when we have one useful method in here const_cast<char *>("doNothing"), const_cast<char *>("()I"), - reinterpret_cast<void *>(latinime_BinaryDictDecoder_doNothing) + reinterpret_cast<void *>(latinime_Ver3DictDecoder_doNothing) }, }; -int register_BinaryDictDecoder(JNIEnv *env) { +int register_Ver3DictDecoder(JNIEnv *env) { const char *const kClassPathName = - "com/android/inputmethod/latin/makedict/BinaryDictDecoder"; + "com/android/inputmethod/latin/makedict/Ver3DictDecoder"; return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods)); } } // namespace latinime diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h b/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.h index 7f3cb67e6..07e80f1d8 100644 --- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h +++ b/native/jni/com_android_inputmethod_latin_makedict_Ver3DictDecoder.h @@ -14,12 +14,12 @@ * limitations under the License. */ -#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H -#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H +#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H +#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H #include "jni.h" namespace latinime { -int register_BinaryDictDecoder(JNIEnv *env); +int register_Ver3DictDecoder(JNIEnv *env); } // namespace latinime -#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H +#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_VER3DICTDECODER_H diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp index d44be6705..3a8f4362d 100644 --- a/native/jni/jni_common.cpp +++ b/native/jni/jni_common.cpp @@ -23,7 +23,7 @@ #include "com_android_inputmethod_latin_BinaryDictionary.h" #include "com_android_inputmethod_latin_DicTraverseSession.h" #endif -#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h" +#include "com_android_inputmethod_latin_makedict_Ver3DictDecoder.h" #include "defines.h" /* @@ -55,8 +55,8 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { return -1; } #endif - if (!latinime::register_BinaryDictDecoder(env)) { - AKLOGE("ERROR: BinaryDictDecoder native registration failed"); + if (!latinime::register_Ver3DictDecoder(env)) { + AKLOGE("ERROR: Ver3DictDecoder native registration failed"); return -1; } /* success -- return valid version number */ diff --git a/native/jni/src/suggest/core/dictionary/probability_utils.h b/native/jni/src/suggest/core/dictionary/probability_utils.h index 219213574..21fe355b8 100644 --- a/native/jni/src/suggest/core/dictionary/probability_utils.h +++ b/native/jni/src/suggest/core/dictionary/probability_utils.h @@ -41,7 +41,7 @@ class ProbabilityUtils { // the unigram probability to be the median value of the 17th step from the top. A value of // 0 for the bigram probability represents the middle of the 16th step from the top, // while a value of 15 represents the middle of the top step. - // See makedict.BinaryDictDecoder for details. + // See makedict.BinaryDictEncoder#makeBigramFlags for details. const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability) / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY); return unigramProbability diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp index cc5252c43..434858d67 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp @@ -22,7 +22,7 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/patricia_trie_policy.h" #include "suggest/policyimpl/dictionary/utils/format_utils.h" -#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h" +#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { @@ -31,7 +31,7 @@ namespace latinime { const int bufOffset, const int size, const bool isUpdatable) { // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of // impl classes of DictionaryStructureWithBufferPolicy. - const MmapedBuffer *const mmapedBuffer = MmapedBuffer::openBuffer(path, pathLength, bufOffset, + const MmappedBuffer *const mmapedBuffer = MmappedBuffer::openBuffer(path, pathLength, bufOffset, size, isUpdatable); if (!mmapedBuffer) { return 0; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index 708967d7b..8ba057b20 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -24,7 +24,7 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h" +#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { @@ -33,7 +33,7 @@ class DicNodeVector; class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: - DynamicPatriciaTriePolicy(const MmapedBuffer *const buffer) + DynamicPatriciaTriePolicy(const MmappedBuffer *const buffer) : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()), mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {} @@ -86,7 +86,7 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); static const int MAX_CHILD_COUNT_TO_AVOID_INFINITE_LOOP; - const MmapedBuffer *const mBuffer; + const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; // TODO: Consolidate mDictRoot. const uint8_t *const mDictRoot; diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index 0d85050f3..d0567fd85 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -24,7 +24,7 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_policy.h" #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_policy.h" -#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h" +#include "suggest/policyimpl/dictionary/utils/mmapped_buffer.h" namespace latinime { @@ -33,7 +33,7 @@ class DicNodeVector; class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { public: - PatriciaTriePolicy(const MmapedBuffer *const buffer) + PatriciaTriePolicy(const MmappedBuffer *const buffer) : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()), mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {} @@ -97,7 +97,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { private: DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy); - const MmapedBuffer *const mBuffer; + const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; const uint8_t *const mDictRoot; const BigramListPolicy mBigramListPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h index 08add03c6..6febd7832 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef LATINIME_MMAPED_BUFFER_H -#define LATINIME_MMAPED_BUFFER_H +#ifndef LATINIME_MMAPPED_BUFFER_H +#define LATINIME_MMAPPED_BUFFER_H #include <cerrno> #include <fcntl.h> @@ -27,39 +27,40 @@ namespace latinime { -class MmapedBuffer { +class MmappedBuffer { public: - static MmapedBuffer* openBuffer(const char *const path, const int pathLength, - const int bufOffset, const int size, const bool isUpdatable) { + static MmappedBuffer* openBuffer(const char *const path, const int pathLength, + const int bufferOffset, const int bufferSize, const bool isUpdatable) { const int openMode = isUpdatable ? O_RDWR : O_RDONLY; - const int fd = open(path, openMode); - if (fd < 0) { + const int mmapFd = open(path, openMode); + if (mmapFd < 0) { AKLOGE("DICT: Can't open the source. path=%s errno=%d", path, errno); return 0; } const int pagesize = getpagesize(); - const int offset = bufOffset % pagesize; - int adjOffset = bufOffset - offset; - int adjSize = size + offset; + const int offset = bufferOffset % pagesize; + int alignedOffset = bufferOffset - offset; + int alignedSize = bufferSize + offset; const int protMode = isUpdatable ? PROT_READ | PROT_WRITE : PROT_READ; - void *const mmapedBuffer = mmap(0, adjSize, protMode, MAP_PRIVATE, fd, adjOffset); - if (mmapedBuffer == MAP_FAILED) { + void *const mmappedBuffer = mmap(0, alignedSize, protMode, MAP_PRIVATE, mmapFd, + alignedOffset); + if (mmappedBuffer == MAP_FAILED) { AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno); - close(fd); + close(mmapFd); return 0; } - uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + offset; + uint8_t *const buffer = static_cast<uint8_t *>(mmappedBuffer) + offset; if (!buffer) { AKLOGE("DICT: buffer is null"); - close(fd); + close(mmapFd); return 0; } - return new MmapedBuffer(buffer, adjSize, fd, offset, isUpdatable); + return new MmappedBuffer(buffer, bufferSize, mmappedBuffer, alignedSize, mmapFd, + isUpdatable); } - ~MmapedBuffer() { - int ret = munmap(static_cast<void *>(mBuffer - mBufferOffset), - mBufferSize + mBufferOffset); + ~MmappedBuffer() { + int ret = munmap(mMmappedBuffer, mAlignedSize); if (ret != 0) { AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); } @@ -82,18 +83,20 @@ class MmapedBuffer { } private: - AK_FORCE_INLINE MmapedBuffer(uint8_t *const buffer, const int bufferSize, const int mmapFd, - const int bufferOffset, const bool isUpdatable) - : mBuffer(buffer), mBufferSize(bufferSize), mMmapFd(mmapFd), - mBufferOffset(bufferOffset), mIsUpdatable(isUpdatable) {} + AK_FORCE_INLINE MmappedBuffer(uint8_t *const buffer, const int bufferSize, + void *const mmappedBuffer, const int alignedSize, const int mmapFd, + const bool isUpdatable) + : mBuffer(buffer), mBufferSize(bufferSize), mMmappedBuffer(mmappedBuffer), + mAlignedSize(alignedSize), mMmapFd(mmapFd), mIsUpdatable(isUpdatable) {} - DISALLOW_IMPLICIT_CONSTRUCTORS(MmapedBuffer); + DISALLOW_IMPLICIT_CONSTRUCTORS(MmappedBuffer); uint8_t *const mBuffer; const int mBufferSize; + void *const mMmappedBuffer; + const int mAlignedSize; const int mMmapFd; - const int mBufferOffset; const bool mIsUpdatable; }; } -#endif /* LATINIME_MMAPED_BUFFER_H */ +#endif /* LATINIME_MMAPPED_BUFFER_H */ diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java index 6cc4befae..fe92be618 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java @@ -305,5 +305,33 @@ public class InputLogicTests extends InputTestsBase { assertEquals("resume suggestion on backspace", 8, BaseInputConnection.getComposingSpanEnd(mEditText.getText())); } + + private void helperTestComposing(final String wordToType, final boolean shouldBeComposing) { + mEditText.setText(""); + type(wordToType); + assertEquals("start composing inside text", shouldBeComposing ? 0 : -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("start composing inside text", shouldBeComposing ? wordToType.length() : -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testStartComposing() { + // Should start composing on a letter + helperTestComposing("a", true); + type(" "); // To reset the composing state + // Should not start composing on quote + helperTestComposing("'", false); + type(" "); + helperTestComposing("'-", false); + type(" "); + // Should not start composing on dash + helperTestComposing("-", false); + type(" "); + helperTestComposing("-'", false); + type(" "); + helperTestComposing("a-", true); + type(" "); + helperTestComposing("a'", true); + } // TODO: Add some tests for non-BMP characters } diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java index 480570e62..0a1c4e963 100644 --- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java +++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java @@ -258,7 +258,8 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { protected void pickSuggestionManually(final int index, final String suggestion) { mLatinIME.pickSuggestionManually(index, new SuggestedWordInfo(suggestion, 1, - SuggestedWordInfo.KIND_CORRECTION, "main")); + SuggestedWordInfo.KIND_CORRECTION, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } // Helper to avoid writing the try{}catch block each time diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java index 8d0fe014c..a5f3685da 100644 --- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java +++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java @@ -34,9 +34,12 @@ public class SuggestedWordsTests extends AndroidTestCase { final int NUMBER_OF_ADDED_SUGGESTIONS = 5; final ArrayList<SuggestedWordInfo> list = CollectionUtils.newArrayList(); list.add(new SuggestedWordInfo(TYPED_WORD, TYPED_WORD_FREQ, - SuggestedWordInfo.KIND_TYPED, "")); + SuggestedWordInfo.KIND_TYPED, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); for (int i = 0; i < NUMBER_OF_ADDED_SUGGESTIONS; ++i) { - list.add(new SuggestedWordInfo("" + i, 1, SuggestedWordInfo.KIND_CORRECTION, "")); + list.add(new SuggestedWordInfo("" + i, 1, SuggestedWordInfo.KIND_CORRECTION, + null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } final SuggestedWords words = new SuggestedWords( diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 028e089f8..d50d1ac96 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -44,7 +44,7 @@ import java.util.Random; import java.util.Set; /** - * Unit tests for BinaryDictDecoderUtils and BinaryDictEncoder. + * Unit tests for BinaryDictDecoderUtils and BinaryDictEncoderUtils. */ @LargeTest public class BinaryDictDecoderEncoderTests extends AndroidTestCase { @@ -120,14 +120,14 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { /** * Makes new DictBuffer according to BUFFER_TYPE. */ - private void getDictBuffer(final BinaryDictDecoder dictDecoder, final int bufferType) + private void getDictBuffer(final Ver3DictDecoder dictDecoder, final int bufferType) throws FileNotFoundException, IOException { if (bufferType == USE_BYTE_BUFFER) { dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); } else if (bufferType == USE_BYTE_ARRAY) { dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); } } @@ -210,7 +210,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // If you need to dump the dict to a textual file, uncomment the line below and the // function above // dumpToCombinedFileForDebug(file, "/tmp/foo"); - BinaryDictEncoder.writeDictionaryBinary(out, dict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryBinary(out, dict, formatOptions); diff = System.currentTimeMillis() - now; out.flush(); @@ -271,7 +271,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final SparseArray<List<Integer>> bigrams, final Map<String, List<String>> shortcutMap, final int bufferType) { long now, diff = -1; - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); FusionDictionary dict = null; try { @@ -409,7 +409,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final Map<Integer, Integer> resultFreqs = CollectionUtils.newTreeMap(); long now = -1, diff = -1; - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); try { getDictBuffer(dictDecoder, bufferType); assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer()); @@ -499,7 +499,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } // Tests for getTerminalPosition - private String getWordFromBinary(final BinaryDictDecoder dictDecoder, final int address) { + private String getWordFromBinary(final Ver3DictDecoder dictDecoder, final int address) { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (dictBuffer.position() != 0) dictBuffer.position(0); @@ -512,11 +512,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return null; } if (fileHeader == null) return null; - return BinaryDictDecoderUtils.getWordAtAddress(dictBuffer, fileHeader.mHeaderSize, + return BinaryDictDecoderUtils.getWordAtAddress(dictDecoder, fileHeader.mHeaderSize, address - fileHeader.mHeaderSize, fileHeader.mFormatOptions).mWord; } - private long runGetTerminalPosition(final BinaryDictDecoder dictDecoder, final String word, + private long runGetTerminalPosition(final Ver3DictDecoder dictDecoder, final String word, int index, boolean contained) { final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; long diff = -1; @@ -552,10 +552,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); try { dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); @@ -613,10 +613,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); try { dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index b6e439561..878417084 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -22,7 +22,7 @@ import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder. +import com.android.inputmethod.latin.makedict.DictDecoder. DictionaryBufferFromWritableByteBufferFactory; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -112,14 +112,15 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { Log.d(TAG, " end address = " + info.mEndAddress); } - private static void printNode(final DictBuffer dictBuffer, + private static void printNode(final Ver3DictDecoder dictDecoder, final FormatSpec.FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); Log.d(TAG, "Node at " + dictBuffer.position()); final int count = BinaryDictDecoderUtils.readCharGroupCount(dictBuffer); Log.d(TAG, " charGroupCount = " + count); for (int i = 0; i < count; ++i) { - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - dictBuffer.position(), formatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(dictBuffer.position(), + formatOptions); printCharGroup(currentInfo); } if (formatOptions.mSupportsDynamicUpdate) { @@ -128,23 +129,23 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } } - private static void printBinaryFile(final BinaryDictDecoder dictDecoder) + private static void printBinaryFile(final Ver3DictDecoder dictDecoder) throws IOException, UnsupportedFormatException { final FileHeader fileHeader = dictDecoder.readHeader(); - final DictBuffer buffer = dictDecoder.getDictBuffer(); - while (buffer.position() < buffer.limit()) { - printNode(buffer, fileHeader.mFormatOptions); + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + while (dictBuffer.position() < dictBuffer.limit()) { + printNode(dictDecoder, fileHeader.mFormatOptions); } } private int getWordPosition(final File file, final String word) { int position = FormatSpec.NOT_VALID_WORD; - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); FileInputStream inStream = null; try { inStream = new FileInputStream(file); dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); position = BinaryDictIOUtils.getTerminalPosition(dictDecoder, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { @@ -161,11 +162,11 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } private CharGroupInfo findWordFromFile(final File file, final String word) { - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); CharGroupInfo info = null; try { dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); info = BinaryDictIOUtils.findWordByBinaryDictReader(dictDecoder, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { @@ -177,7 +178,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList<WeightedString> bigrams, final ArrayList<WeightedString> shortcuts) { - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); BufferedOutputStream outStream = null; long amountOfTime = -1; try { @@ -211,7 +212,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } private void deleteWord(final File file, final String word) { - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); try { dictDecoder.openDictBuffer(new DictionaryBufferFromWritableByteBufferFactory()); DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word); @@ -221,15 +222,14 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } private void checkReverseLookup(final File file, final String word, final int position) { - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); try { final DictBuffer dictBuffer = dictDecoder.openAndGetDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); final FileHeader fileHeader = dictDecoder.readHeader(); assertEquals(word, - BinaryDictDecoderUtils.getWordAtAddress(dictDecoder.getDictBuffer(), - fileHeader.mHeaderSize, position - fileHeader.mHeaderSize, - fileHeader.mFormatOptions).mWord); + BinaryDictDecoderUtils.getWordAtAddress(dictDecoder, fileHeader.mHeaderSize, + position - fileHeader.mHeaderSize, fileHeader.mFormatOptions).mWord); } catch (IOException e) { Log.e(TAG, "Raised an IOException while looking up a word", e); } catch (UnsupportedFormatException e) { @@ -253,7 +253,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { try { final FileOutputStream out = new FileOutputStream(file); - BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); + BinaryDictEncoderUtils.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); out.close(); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); @@ -305,7 +305,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { try { final FileOutputStream out = new FileOutputStream(file); - BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); + BinaryDictEncoderUtils.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); out.close(); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); @@ -343,7 +343,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { try { final FileOutputStream out = new FileOutputStream(file); - BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); + BinaryDictEncoderUtils.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); out.close(); } catch (IOException e) { assertTrue(false); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java index 03742c4c1..20e8b4fda 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver3DictDecoderTests.java @@ -17,12 +17,11 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder.DictionaryBufferFactory; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder. - DictionaryBufferFromByteArrayFactory; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder. +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFromByteArrayFactory; +import com.android.inputmethod.latin.makedict.DictDecoder. DictionaryBufferFromReadOnlyByteBufferFactory; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder. +import com.android.inputmethod.latin.makedict.DictDecoder. DictionaryBufferFromWritableByteBufferFactory; import android.test.AndroidTestCase; @@ -33,10 +32,10 @@ import java.io.FileOutputStream; import java.io.IOException; /** - * Unit tests for BinaryDictDecoder + * Unit tests for Ver3DictDecoder */ -public class BinaryDictDecoderTests extends AndroidTestCase { - private static final String TAG = BinaryDictDecoderTests.class.getSimpleName(); +public class Ver3DictDecoderTests extends AndroidTestCase { + private static final String TAG = Ver3DictDecoderTests.class.getSimpleName(); private final byte[] data = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; @@ -60,8 +59,7 @@ public class BinaryDictDecoderTests extends AndroidTestCase { } @SuppressWarnings("null") - public void runTestOpenBuffer(final String testName, - final DictionaryBufferFactory factory) { + public void runTestOpenBuffer(final String testName, final DictionaryBufferFactory factory) { File testFile = null; try { testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir()); @@ -70,7 +68,7 @@ public class BinaryDictDecoderTests extends AndroidTestCase { } assertNotNull(testFile); - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(testFile); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile); try { dictDecoder.openDictBuffer(factory); } catch (Exception e) { @@ -104,8 +102,7 @@ public class BinaryDictDecoderTests extends AndroidTestCase { } @SuppressWarnings("null") - public void runTestGetBuffer(final String testName, - final DictionaryBufferFactory factory) { + public void runTestGetBuffer(final String testName, final DictionaryBufferFactory factory) { File testFile = null; try { testFile = File.createTempFile(testName, ".tmp", getContext().getCacheDir()); @@ -113,7 +110,7 @@ public class BinaryDictDecoderTests extends AndroidTestCase { Log.e(TAG, "IOException while the creating temporary file", e); } - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(testFile); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(testFile); // the default return value of getBuffer() must be null. assertNull("the default return value of getBuffer() is not null", diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java index 83d9c2122..eca12c0d8 100644 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java @@ -21,10 +21,10 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; @@ -147,10 +147,10 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase } private void readDictFromFile(final File file, final OnAddWordListener listener) { - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); try { dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); } catch (FileNotFoundException e) { Log.e(TAG, "file not found", e); } catch (IOException e) { diff --git a/tools/dicttool/NativeLib.mk b/tools/dicttool/NativeLib.mk index 84e54a931..a3d3c0295 100644 --- a/tools/dicttool/NativeLib.mk +++ b/tools/dicttool/NativeLib.mk @@ -34,7 +34,7 @@ LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(LATINIME_NATIVE_SRC_DIR) # Used in jni_common.cpp to avoid registering useless methods. LATIN_IME_JNI_SRC_FILES := \ - com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \ + com_android_inputmethod_latin_makedict_Ver3DictDecoder.cpp \ jni_common.cpp LATIN_IME_CORE_SRC_FILES := diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index 39ba69b1f..d8fcbeeaf 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -17,9 +17,9 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import org.xml.sax.SAXException; @@ -184,9 +184,9 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(decodedSpec.mFile); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile); dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index f87e9722c..fee445389 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -17,12 +17,12 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.MakedictLog; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import java.io.BufferedWriter; import java.io.File; @@ -266,9 +266,9 @@ public class DictionaryMaker { private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { final File file = new File(binaryFilename); - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null); } @@ -358,7 +358,7 @@ public class DictionaryMaker { throws FileNotFoundException, IOException, UnsupportedFormatException { final File outputFile = new File(outputFilename); final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version); - BinaryDictEncoder.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, + BinaryDictEncoderUtils.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, formatOptions); } diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index b960b035d..da51387be 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -16,14 +16,14 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import junit.framework.TestCase; @@ -58,7 +58,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { Compress.getCompressedStream( new BufferedOutputStream(new FileOutputStream(dst))))); - BinaryDictEncoder.writeDictionaryBinary(out, dict, new FormatOptions(2, false)); + BinaryDictEncoderUtils.writeDictionaryBinary(out, dict, new FormatOptions(2, false)); // Test for an actually compressed dictionary and its contents final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec = @@ -67,9 +67,9 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(decodeSpec.mFile); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile); dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); final FusionDictionary resultDict = BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null /* dict : an optional dictionary to add words to, or null */); assertEquals("Dictionary can't be read back correctly", diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java index fe6738303..55058238c 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java @@ -25,7 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; /** - * Unit tests for BinaryDictEncoder.flattenTree(). + * Unit tests for BinaryDictEncoderUtils.flattenTree(). */ public class BinaryDictEncoderFlattenTreeTests extends TestCase { // Test the flattened array contains the expected number of nodes, and @@ -39,7 +39,8 @@ public class BinaryDictEncoderFlattenTreeTests extends TestCase { dict.add("ftb", 1, null, false /* isNotAWord */); dict.add("bar", 1, null, false /* isNotAWord */); dict.add("fool", 1, null, false /* isNotAWord */); - final ArrayList<PtNodeArray> result = BinaryDictEncoder.flattenTree(dict.mRootNodeArray); + final ArrayList<PtNodeArray> result = + BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); assertEquals(4, result.size()); while (!result.isEmpty()) { final PtNodeArray n = result.remove(0); |