diff options
Diffstat (limited to 'java')
29 files changed, 603 insertions, 423 deletions
diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml index 5c59f5f68..a96cfec1e 100644 --- a/java/res/values/attrs.xml +++ b/java/res/values/attrs.xml @@ -169,10 +169,6 @@ <attr name="colorTypedWord" format="color" /> <attr name="colorAutoCorrect" format="color" /> <attr name="colorSuggested" format="color" /> - <attr name="alphaValidTypedWord" format="fraction" /> - <attr name="alphaTypedWord" format="fraction" /> - <attr name="alphaAutoCorrect" format="fraction" /> - <attr name="alphaSuggested" format="fraction" /> <attr name="alphaObsoleted" format="fraction" /> <attr name="suggestionsCountInStrip" format="integer" /> <attr name="centerSuggestionPercentile" format="fraction" /> diff --git a/java/res/values/colors.xml b/java/res/values/colors.xml index daa167c8a..598a5d050 100644 --- a/java/res/values/colors.xml +++ b/java/res/values/colors.xml @@ -41,7 +41,9 @@ <color name="spacebar_text_shadow_color_stone">#D0FFFFFF</color> <!-- Color resources for IceCreamSandwich theme. --> <!-- android:color/holo_blue_light value is #FF33B5E5 --> - <color name="highlight_color_ics">@android:color/holo_blue_light</color> + <color name="highlight_color_ics">#FF33B5E5</color> + <color name="typed_word_color_ics">#D833B5E5</color> + <color name="suggested_word_color_ics">#B233B5E5</color> <color name="highlight_translucent_color_ics">#9933B5E5</color> <color name="key_text_color_ics">@android:color/white</color> <color name="key_text_shadow_color_ics">@android:color/transparent</color> @@ -52,7 +54,6 @@ <color name="key_shifted_letter_hint_activated_color_ics">@android:color/white</color> <color name="spacebar_text_color_ics">#FFC0C0C0</color> <color name="spacebar_text_shadow_color_ics">#80000000</color> - <color name="typed_word_color_ics">@color/highlight_color_ics</color> <!-- Color resources for setup wizard and tutorial --> <color name="setup_background">#FFEBEBEB</color> <color name="setup_text_dark">#FF707070</color> diff --git a/java/res/values/strings.xml b/java/res/values/strings.xml index aae5b0b70..390f72bfa 100644 --- a/java/res/values/strings.xml +++ b/java/res/values/strings.xml @@ -493,6 +493,8 @@ mobile devices. [CHAR LIMIT=25] --> <string name="prefs_read_external_dictionary">Read external dictionary file</string> <!-- Title of the settings for using only personalization dictionary --> <string name="prefs_use_only_personalization_dictionary" translatable="false">Use only personalization dictionary</string> + <!-- Title of the settings for boosting personalization dictionary --> + <string name="prefs_boost_personalization_dictionary" translatable="false">Boost personalization dictionary</string> <!-- Message to show when there are no files to install as an external dictionary [CHAR LIMIT=100] --> <string name="read_external_dictionary_no_files_message">No dictionary files in the Downloads folder</string> <!-- Title of the dialog that selects a file to install as an external dictionary [CHAR LIMIT=50] --> diff --git a/java/res/values/styles.xml b/java/res/values/styles.xml index 37c6a9553..d175a12c0 100644 --- a/java/res/values/styles.xml +++ b/java/res/values/styles.xml @@ -382,13 +382,10 @@ > <item name="android:background">@drawable/keyboard_suggest_strip_holo</item> <item name="suggestionStripOption">autoCorrectBold|validTypedWordBold</item> - <item name="colorValidTypedWord">@color/highlight_color_ics</item> - <item name="colorTypedWord">@color/highlight_color_ics</item> + <item name="colorValidTypedWord">@color/typed_word_color_ics</item> + <item name="colorTypedWord">@color/typed_word_color_ics</item> <item name="colorAutoCorrect">@color/highlight_color_ics</item> - <item name="colorSuggested">@color/highlight_color_ics</item> - <item name="alphaValidTypedWord">85%</item> - <item name="alphaTypedWord">85%</item> - <item name="alphaSuggested">70%</item> + <item name="colorSuggested">@color/suggested_word_color_ics</item> <item name="alphaObsoleted">70%</item> <item name="suggestionsCountInStrip">@integer/suggestions_count_in_strip</item> <item name="centerSuggestionPercentile">@fraction/center_suggestion_percentile</item> diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index d181bf697..eb19ef932 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -179,7 +179,8 @@ public final class BinaryDictionary extends Dictionary { // TODO: check that all users of the `kind' parameter are ready to accept // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), - score, kind, mDictType)); + score, kind, this /* sourceDict */, + mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */)); } } return suggestions; diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index f9f22ecb4..8f6b848bb 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,17 +21,16 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.LocaleUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.BufferUnderflowException; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; @@ -227,23 +226,14 @@ final public class BinaryDictionaryGetter { // those do not include whitelist entries, the new code with an old version of the dictionary // would lose whitelist functionality. private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { - FileInputStream inStream = null; try { // Read the version of the file - inStream = new FileInputStream(f); - final BinaryDictDecoderUtils.ByteBufferDictBuffer dictBuffer = - new BinaryDictDecoderUtils.ByteBufferDictBuffer(inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, f.length())); - final int magic = dictBuffer.readInt(); - if (magic != FormatSpec.MAGIC_NUMBER) { - return false; - } - final int formatVersion = dictBuffer.readInt(); - final int headerSize = dictBuffer.readInt(); - final HashMap<String, String> options = CollectionUtils.newHashMap(); - BinaryDictDecoderUtils.populateOptions(dictBuffer, headerSize, options); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); + dictDecoder.openDictBuffer( + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + final FileHeader header = dictDecoder.readHeader(); - final String version = options.get(VERSION_KEY); + final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); if (null == version) { // No version in the options : the format is unexpected return false; @@ -259,14 +249,8 @@ final public class BinaryDictionaryGetter { return false; } catch (BufferUnderflowException e) { return false; - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } + } catch (UnsupportedFormatException e) { + return false; } } diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 7c3e4a740..d9ded7ceb 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -28,9 +28,26 @@ import java.util.ArrayList; public abstract class Dictionary { public static final int NOT_A_PROBABILITY = -1; + // The following types do not actually come from real dictionary instances, so we create + // corresponding instances. public static final String TYPE_USER_TYPED = "user_typed"; + public static final Dictionary DICTIONARY_USER_TYPED = new PhonyDictionary(TYPE_USER_TYPED); + public static final String TYPE_APPLICATION_DEFINED = "application_defined"; + public static final Dictionary DICTIONARY_APPLICATION_DEFINED = + new PhonyDictionary(TYPE_APPLICATION_DEFINED); + public static final String TYPE_HARDCODED = "hardcoded"; // punctuation signs and such + public static final Dictionary DICTIONARY_HARDCODED = + new PhonyDictionary(TYPE_HARDCODED); + + // Spawned by resuming suggestions. Comes from a span that was in the TextView. + public static final String TYPE_RESUMED = "resumed"; + public static final Dictionary DICTIONARY_RESUMED = + new PhonyDictionary(TYPE_RESUMED); + + // The following types of dictionary have actual functional instances. We don't need final + // phony dictionary instances for them. public static final String TYPE_MAIN = "main"; public static final String TYPE_CONTACTS = "contacts"; // User dictionary, the system-managed one. @@ -42,9 +59,7 @@ public abstract class Dictionary { // Personalization prediction dictionary internal to LatinIME's Java code. public static final String TYPE_PERSONALIZATION_PREDICTION_IN_JAVA = "personalization_prediction_in_java"; - // Spawned by resuming suggestions. Comes from a span that was in the TextView. - public static final String TYPE_RESUMED = "resumed"; - protected final String mDictType; + public final String mDictType; public Dictionary(final String dictType) { mDictType = dictType; @@ -114,8 +129,40 @@ public abstract class Dictionary { /** * Subclasses may override to indicate that this Dictionary is not yet properly initialized. */ - public boolean isInitialized() { return true; } + + /** + * Whether we think this suggestion should trigger an auto-commit. + */ + public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { + // If we don't have support for auto-commit, or if we don't know, we return false to + // avoid auto-committing stuff. Implementations of the Dictionary class that know to + // determine whether we should auto-commit will override this. + return false; + } + + /** + * Not a true dictionary. A placeholder used to indicate suggestions that don't come from any + * real dictionary. + */ + private static class PhonyDictionary extends Dictionary { + // This class is not publicly instantiable. + private PhonyDictionary(final String type) { + super(type); + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + final boolean blockOffensiveWords) { + return null; + } + + @Override + public boolean isValidWord(String word) { + return false; + } + } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 1ececd5c1..b3a572809 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -20,7 +20,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -87,7 +87,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override protected void writeBinaryDictionary(final FileOutputStream out) throws IOException, UnsupportedFormatException { - BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + BinaryDictEncoderUtils.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); } @Override diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index bd2d70365..516b8426c 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -403,7 +403,8 @@ public class ExpandableDictionary extends Dictionary { // the respective size of the typed word and the suggestion if it matters sometime // in the future. suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, - SuggestedWordInfo.KIND_CORRECTION, mDictType)); + SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; } if (null != node.mShortcutTargets) { @@ -411,7 +412,8 @@ public class ExpandableDictionary extends Dictionary { for (int shortcutIndex = 0; shortcutIndex < length; ++shortcutIndex) { final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), - finalFreq, SuggestedWordInfo.KIND_SHORTCUT, mDictType)); + finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; } } @@ -657,7 +659,8 @@ public class ExpandableDictionary extends Dictionary { if (freq >= 0 && node == null) { suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, Constants.DICTIONARY_MAX_WORD_LENGTH - index), - freq, SuggestedWordInfo.KIND_CORRECTION, mDictType)); + freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index ee7478ca2..d67195b0d 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -95,6 +95,7 @@ import com.android.inputmethod.latin.utils.RecapitalizeStatus; import com.android.inputmethod.latin.utils.StaticInnerHandlerWrapper; import com.android.inputmethod.latin.utils.TargetPackageInfoGetterTask; import com.android.inputmethod.latin.utils.TextRange; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; import com.android.inputmethod.research.ResearchLogger; import java.io.FileDescriptor; @@ -191,7 +192,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean mExpectingUpdateSelection; private int mDeleteCount; private long mLastKeyTime; - private TreeSet<Long> mCurrentlyPressedHardwareKeys = CollectionUtils.newTreeSet(); + private final TreeSet<Long> mCurrentlyPressedHardwareKeys = CollectionUtils.newTreeSet(); + // Personalization debugging params + private boolean mUseOnlyPersonalizationDictionaryForDebug = false; + private boolean mBoostPersonalizationDictionaryForDebug = false; // Member variables for remembering the current device orientation. private int mDisplayOrientation; @@ -635,6 +639,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.getInstance().onDestroy(); } unregisterReceiver(mDictionaryPackInstallReceiver); + PersonalizationDictionarySessionRegister.onDestroy(this); LatinImeLogger.commit(); LatinImeLogger.onDestroy(); super.onDestroy(); @@ -874,9 +879,35 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // be replaced when the user dictionary reports back with the actual word, which ends // up calling #onWordAddedToUserDictionary() in this class. + initPersonalizationDebugSettings(currentSettingsValues); + if (TRACE) Debug.startMethodTracing("/data/trace/latinime"); } + // Initialization of personalization debug settings. This must be called inside + // onStartInputView. + private void initPersonalizationDebugSettings(SettingsValues currentSettingsValues) { + if (mUseOnlyPersonalizationDictionaryForDebug + != currentSettingsValues.mUseOnlyPersonalizationDictionaryForDebug) { + // Only for debug + initSuggest(); + mUseOnlyPersonalizationDictionaryForDebug = + currentSettingsValues.mUseOnlyPersonalizationDictionaryForDebug; + } + + if (mBoostPersonalizationDictionaryForDebug != + currentSettingsValues.mBoostPersonalizationDictionaryForDebug) { + // Only for debug + mBoostPersonalizationDictionaryForDebug = + currentSettingsValues.mBoostPersonalizationDictionaryForDebug; + if (mBoostPersonalizationDictionaryForDebug) { + UserHistoryForgettingCurveUtils.boostMaxFreqForDebug(); + } else { + UserHistoryForgettingCurveUtils.resetMaxFreqForDebug(); + } + } + } + // Callback for the TargetPackageInfoGetterTask @Override public void onTargetPackageInfoKnown(final PackageInfo info) { @@ -1767,6 +1798,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onUpdateBatchInput(final InputPointers batchPointers) { + final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate(); + if (null != candidate) { + if (candidate.mSourceDict.shouldAutoCommit(candidate)) { + // TODO: implement auto-commit + } + } BatchInputUpdater.getInstance().onUpdateBatchInput(batchPointers); } @@ -2015,10 +2052,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen (!mConnection.isCursorTouchingWord(currentSettings) || !currentSettings.mCurrentLanguageHasSpaces)) { // Reset entirely the composing state anyway, then start composing a new word unless - // the character is a single quote. The idea here is, single quote is not a - // separator and it should be treated as a normal character, except in the first - // position where it should not start composing a word. - isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode); + // the character is a single quote or a dash. The idea here is, single quote and dash + // are not separators and they should be treated as normal characters, except in the + // first position where they should not start composing a word. + isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode + && Constants.CODE_DASH != primaryCode); // Here we don't need to reset the last composed word. It will be reset // when we commit this one, if we ever do; if on the other hand we backspace // it entirely and resume suggestions on the previous word, we'd like to still @@ -2455,7 +2493,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_pickSuggestionManually(replacedWord, index, suggestion, mWordComposer.isBatchMode(), suggestionInfo.mScore, suggestionInfo.mKind, - suggestionInfo.mSourceDict); + suggestionInfo.mSourceDict.mDictType); } mConnection.endBatchEdit(); // Don't allow cancellation of manual pick @@ -2584,7 +2622,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!TextUtils.equals(s, typedWord)) { suggestions.add(new SuggestedWordInfo(s, SuggestionStripView.MAX_SUGGESTIONS - i, - SuggestedWordInfo.KIND_RESUMED, Dictionary.TYPE_RESUMED)); + SuggestedWordInfo.KIND_RESUMED, Dictionary.DICTIONARY_RESUMED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 55b70c6f8..8766e0fc1 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -303,13 +303,15 @@ public final class Suggest { for (int i = 0; i < suggestionsCount; ++i) { final SuggestedWordInfo wordInfo = suggestionsContainer.get(i); - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), + wordInfo.mSourceDict.mDictType); } if (!TextUtils.isEmpty(typedWord)) { suggestionsContainer.add(0, new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, - Dictionary.TYPE_USER_TYPED)); + Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } SuggestedWordInfo.removeDups(suggestionsContainer); @@ -352,7 +354,7 @@ public final class Suggest { } for (SuggestedWordInfo wordInfo : suggestionsSet) { - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType); } final ArrayList<SuggestedWordInfo> suggestionsContainer = @@ -453,7 +455,7 @@ public final class Suggest { sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE); } return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind, - wordInfo.mSourceDict); + wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord); } public void close() { diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index 22beaefee..b27fd81e9 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -113,7 +113,8 @@ public final class SuggestedWords { if (null == text) continue; final SuggestedWordInfo suggestedWordInfo = new SuggestedWordInfo(text.toString(), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_APP_DEFINED, - Dictionary.TYPE_APPLICATION_DEFINED); + Dictionary.DICTIONARY_APPLICATION_DEFINED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */); result.add(suggestedWordInfo); } return result; @@ -126,7 +127,8 @@ public final class SuggestedWords { final ArrayList<SuggestedWordInfo> suggestionsList = CollectionUtils.newArrayList(); final HashSet<String> alreadySeen = CollectionUtils.newHashSet(); suggestionsList.add(new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, - SuggestedWordInfo.KIND_TYPED, Dictionary.TYPE_USER_TYPED)); + SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); alreadySeen.add(typedWord.toString()); final int previousSize = previousSuggestions.size(); for (int index = 1; index < previousSize; index++) { @@ -141,7 +143,14 @@ public final class SuggestedWords { return suggestionsList; } + public SuggestedWordInfo getAutoCommitCandidate() { + if (mSuggestedWordInfoList.size() <= 0) return null; + final SuggestedWordInfo candidate = mSuggestedWordInfoList.get(0); + return candidate.isEligibleForAutoCommit() ? candidate : null; + } + public static final class SuggestedWordInfo { + public static final int NOT_AN_INDEX = -1; public static final int MAX_SCORE = Integer.MAX_VALUE; public static final int KIND_MASK_KIND = 0xFF; // Mask to get only the kind public static final int KIND_TYPED = 0; // What user typed @@ -166,18 +175,26 @@ public final class SuggestedWords { public final int mScore; public final int mKind; // one of the KIND_* constants above public final int mCodePointCount; - public final String mSourceDict; + public final Dictionary mSourceDict; + // For auto-commit. This keeps track of the index inside the touch coordinates array + // passed to native code to get suggestions for a gesture that corresponds to the first + // letter of the second word. + public final int mIndexOfTouchPointOfSecondWord; private String mDebugString = ""; public SuggestedWordInfo(final String word, final int score, final int kind, - final String sourceDict) { + final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord) { mWord = word; mScore = score; mKind = kind; mSourceDict = sourceDict; mCodePointCount = StringUtils.codePointCount(mWord); + mIndexOfTouchPointOfSecondWord = indexOfTouchPointOfSecondWord; } + public boolean isEligibleForAutoCommit() { + return (KIND_CORRECTION == mKind && NOT_AN_INDEX != mIndexOfTouchPointOfSecondWord); + } public void setDebugString(final String str) { if (null == str) throw new NullPointerException("Debug info is null"); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index efa491099..b82b41c7d 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -22,7 +22,6 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.makedict.decoder.HeaderReader; import java.io.ByteArrayOutputStream; import java.io.File; @@ -32,8 +31,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -42,7 +39,7 @@ import java.util.TreeMap; * * All the methods in this class are static. * - * TODO: Remove calls from classes except BinaryDictDecoder + * TODO: Remove calls from classes except Ver3DictDecoder * TODO: Move this file to makedict/internal. */ public final class BinaryDictDecoderUtils { @@ -280,6 +277,12 @@ public final class BinaryDictDecoderUtils { // Input methods: Read a binary dictionary to memory. // readDictionaryBinary is the public entry point for them. + static int readSInt24(final DictBuffer dictBuffer) { + final int retval = dictBuffer.readUnsignedInt24(); + final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (retval & FormatSpec.SINT24_MAX); + } + static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { if (options.mSupportsDynamicUpdate) { @@ -316,103 +319,6 @@ public final class BinaryDictDecoderUtils { } } - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final DictBuffer dictBuffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = dictBuffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(dictBuffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = dictBuffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(dictBuffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = dictBuffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - dictBuffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += dictBuffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (dictBuffer.readUnsignedByte() << 16) - + dictBuffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - /** * Reads and returns the char group count out of a buffer and forwards the pointer. */ @@ -427,38 +333,38 @@ public final class BinaryDictDecoderUtils { } /** - * Finds, as a string, the word at the address passed as an argument. + * Finds, as a string, the word at the position passed as an argument. * - * @param dictBuffer the buffer to read from. + * @param dictDecoder the dict decoder. * @param headerSize the size of the header. - * @param address the address to seek. + * @param pos the position to seek. * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ - /* package for tests */ static WeightedString getWordAtAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + /* package for tests */ static WeightedString getWordAtPosition( + final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final WeightedString result; - final int originalPointer = dictBuffer.position(); - dictBuffer.position(address); + final int originalPos = dictBuffer.position(); + dictBuffer.position(pos); if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(dictBuffer, headerSize, address, - formatOptions); + result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions); } else { - result = getWordAtAddressWithoutParentAddress(dictBuffer, headerSize, address, + result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos, formatOptions); } - dictBuffer.position(originalPointer); + dictBuffer.position(originalPos); return result; } @SuppressWarnings("unused") - private static WeightedString getWordAtAddressWithParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, - final FormatOptions options) { - int currentAddress = address; + private static WeightedString getWordAtPositionWithParentAddress( + final Ver3DictDecoder dictDecoder, final int pos, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + int currentPos = pos; int frequency = Integer.MIN_VALUE; final StringBuilder builder = new StringBuilder(); // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH @@ -466,10 +372,10 @@ public final class BinaryDictDecoderUtils { CharGroupInfo currentInfo; int loopCounter = 0; do { - dictBuffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(dictBuffer, currentAddress, options); + dictBuffer.position(currentPos); + currentInfo = dictDecoder.readPtNode(currentPos, options); if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } if (DBG && loopCounter++ > MAX_JUMPS) { MakedictLog.d("Too many jumps - probably a bug"); @@ -479,36 +385,37 @@ public final class BinaryDictDecoderUtils { builder.insert(0, new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } return new WeightedString(builder.toString(), frequency); } - private static WeightedString getWordAtAddressWithoutParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + private static WeightedString getWordAtPositionWithoutParentAddress( + final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(headerSize); final int count = readCharGroupCount(dictBuffer); - int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); + int groupPos = headerSize + BinaryDictIOUtils.getGroupCountSize(count); final StringBuilder builder = new StringBuilder(); WeightedString result = null; CharGroupInfo last = null; for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); - groupOffset = info.mEndAddress; - if (info.mOriginalAddress == address) { + CharGroupInfo info = dictDecoder.readPtNode(groupPos, options); + groupPos = info.mEndAddress; + if (info.mOriginalAddress == pos) { builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); result = new WeightedString(builder.toString(), info.mFrequency); break; // and return } if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { - if (info.mChildrenAddress > address) { + if (info.mChildrenAddress > pos) { if (null == last) continue; builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - dictBuffer.position(last.mChildrenAddress + headerSize); + dictBuffer.position(last.mChildrenAddress); i = readCharGroupCount(dictBuffer); - groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); last = null; continue; } @@ -516,9 +423,9 @@ public final class BinaryDictDecoderUtils { } if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - dictBuffer.position(last.mChildrenAddress + headerSize); + dictBuffer.position(last.mChildrenAddress); i = readCharGroupCount(dictBuffer); - groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); last = null; continue; } @@ -534,34 +441,35 @@ public final class BinaryDictDecoderUtils { * This will recursively read other node arrays into the structure, populating the reverse * maps on the fly and using them to keep track of already read nodes. * - * @param dictBuffer the buffer, correctly positioned at the start of a node array. + * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. * @param headerSize the size, in bytes, of the file header. * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. * @param reverseGroupMap a mapping from addresses to already read character groups. * @param options file format options. * @return the read node array with all his children already read. */ - private static PtNodeArray readNodeArray(final DictBuffer dictBuffer, + private static PtNodeArray readNodeArray(final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, final Map<Integer, CharGroup> reverseGroupMap, final FormatOptions options) throws IOException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final ArrayList<CharGroup> nodeArrayContents = new ArrayList<CharGroup>(); - final int nodeArrayOrigin = dictBuffer.position() - headerSize; + final int nodeArrayOriginPos = dictBuffer.position(); do { // Scan the linked-list node. - final int nodeArrayHeadPosition = dictBuffer.position() - headerSize; + final int nodeArrayHeadPos = dictBuffer.position(); final int count = readCharGroupCount(dictBuffer); - int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); + int groupOffsetPos = nodeArrayHeadPos + BinaryDictIOUtils.getGroupCountSize(count); for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); + CharGroupInfo info = dictDecoder.readPtNode(groupOffsetPos, options); if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; ArrayList<WeightedString> bigrams = null; if (null != info.mBigrams) { bigrams = new ArrayList<WeightedString>(); for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - dictBuffer, headerSize, bigram.mAddress, options); + final WeightedString word = getWordAtPosition(dictDecoder, headerSize, + bigram.mAddress, options); final int reconstructedFrequency = BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); @@ -572,8 +480,8 @@ public final class BinaryDictDecoderUtils { PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); if (null == children) { final int currentPosition = dictBuffer.position(); - dictBuffer.position(info.mChildrenAddress + headerSize); - children = readNodeArray(dictBuffer, headerSize, reverseNodeArrayMap, + dictBuffer.position(info.mChildrenAddress); + children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, reverseGroupMap, options); dictBuffer.position(currentPosition); } @@ -589,7 +497,7 @@ public final class BinaryDictDecoderUtils { 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); } - groupOffset = info.mEndAddress; + groupOffsetPos = info.mEndAddress; } // reach the end of the array. @@ -605,8 +513,8 @@ public final class BinaryDictDecoderUtils { dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); - nodeArray.mCachedAddressBeforeUpdate = nodeArrayOrigin; - nodeArray.mCachedAddressAfterUpdate = nodeArrayOrigin; + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; + nodeArray.mCachedAddressAfterUpdate = nodeArrayOriginPos; reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); return nodeArray; } @@ -640,49 +548,6 @@ public final class BinaryDictDecoderUtils { } /** - * Reads a header from a buffer. - * @param headerReader the header reader - * @throws IOException - * @throws UnsupportedFormatException - */ - public static FileHeader readHeader(final HeaderReader headerReader) - throws IOException, UnsupportedFormatException { - final int version = headerReader.readVersion(); - final int optionsFlags = headerReader.readOptionFlags(); - - final int headerSize = headerReader.readHeaderSize(); - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final HashMap<String, String> attributes = headerReader.readAttributes(headerSize); - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } - - /** - * Reads options from a buffer and populate a map with their contents. - * - * The buffer is read at the current position, so the caller must take care the pointer - * is in the right place before calling this. - */ - public static void populateOptions(final DictBuffer dictBuffer, - final int headerSize, final HashMap<String, String> options) { - while (dictBuffer.position() < headerSize) { - final String key = CharEncoding.readString(dictBuffer); - final String value = CharEncoding.readString(dictBuffer); - options.put(key, value); - } - } - - /** * Reads a buffer and returns the memory representation of the dictionary. * * This high-level method takes a buffer and reads its contents, populating a @@ -694,23 +559,23 @@ public final class BinaryDictDecoderUtils { * @return the created (or merged) dictionary. */ @UsedForTesting - public static FusionDictionary readDictionaryBinary(final BinaryDictDecoder dictDecoder, + public static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder, final FusionDictionary dict) throws FileNotFoundException, IOException, UnsupportedFormatException { // if the buffer has not been opened, open the buffer with bytebuffer. if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer( - new BinaryDictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); + new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); if (dictDecoder.getDictBuffer() == null) { MakedictLog.e("Cannot open the buffer"); } // Read header - final FileHeader fileHeader = readHeader(dictDecoder); + final FileHeader fileHeader = dictDecoder.readHeader(); Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final PtNodeArray root = readNodeArray(dictDecoder.getDictBuffer(), fileHeader.mHeaderSize, + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index ff11cde39..3fe3ae6ce 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -34,11 +34,11 @@ import java.util.Iterator; * * All the methods in this class are static. */ -public class BinaryDictEncoder { +public class BinaryDictEncoderUtils { private static final boolean DBG = MakedictLog.DBG; - private BinaryDictEncoder() { + private BinaryDictEncoderUtils() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 4cf72e915..8e1e14e9e 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -62,10 +62,11 @@ public final class BinaryDictIOUtils { * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner( - final DictBuffer dictBuffer, final int headerSize, + final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack<Position> stack = new Stack<Position>(); @@ -94,8 +95,7 @@ public final class BinaryDictIOUtils { stack.pop(); continue; } - CharGroupInfo info = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - p.mAddress - headerSize, formatOptions); + CharGroupInfo info = dictDecoder.readPtNode(p.mAddress, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; } @@ -131,7 +131,7 @@ public final class BinaryDictIOUtils { } if (!isMovedGroup && hasChildrenAddress(info.mChildrenAddress)) { - Position childrenPos = new Position(info.mChildrenAddress + headerSize, index); + final Position childrenPos = new Position(info.mChildrenAddress, index); stack.push(childrenPos); } } @@ -148,13 +148,13 @@ public final class BinaryDictIOUtils { * @throws IOException if the file can't be read. * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final BinaryDictDecoder dictDecoder, + public static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header - final FileHeader header = BinaryDictDecoderUtils.readHeader(dictDecoder); - readUnigramsAndBigramsBinaryInner(dictDecoder.getDictBuffer(), header.mHeaderSize, words, + final FileHeader header = dictDecoder.readHeader(); + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, frequencies, bigrams, header.mFormatOptions); } @@ -169,13 +169,13 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final BinaryDictDecoder dictDecoder, + public static int getTerminalPosition(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; if (dictBuffer.position() != 0) dictBuffer.position(0); - final FileHeader header = BinaryDictDecoderUtils.readHeader(dictDecoder); + final FileHeader header = dictDecoder.readHeader(); int wordPos = 0; final int wordLen = word.codePointCount(0, word.length()); for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { @@ -186,8 +186,8 @@ public final class BinaryDictIOUtils { boolean foundNextCharGroup = false; for (int i = 0; i < charGroupCount; ++i) { final int charGroupPos = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup( - dictBuffer, dictBuffer.position(), header.mFormatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(charGroupPos, + header.mFormatOptions); final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, header.mFormatOptions); final boolean isDeletedGroup = isDeletedGroup(currentInfo.mFlags, @@ -272,7 +272,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictEncoder.getByteSize(value)) { + switch (BinaryDictEncoderUtils.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -286,7 +286,7 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictEncoder.getByteSize(value); + return BinaryDictEncoderUtils.getByteSize(value); } static void skipCharGroup(final DictBuffer dictBuffer, @@ -413,14 +413,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictEncoder.makeShortcutFlags( + destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -429,7 +429,7 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictEncoder#computeActualNodeArraySize + // in BinaryDictEncoderUtils#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; @@ -439,7 +439,7 @@ public final class BinaryDictIOUtils { final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictEncoder.getByteSize(bigramOffset)) { + switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { case 1: bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; break; @@ -463,18 +463,18 @@ public final class BinaryDictIOUtils { */ static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) { int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters) + + BinaryDictEncoderUtils.getGroupCharactersSize(info.mCharacters) + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { size += FormatSpec.GROUP_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictEncoder.getByteSize(attr.mAddress); + size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); } } return size; @@ -508,7 +508,7 @@ public final class BinaryDictIOUtils { } /** - * Find a word using the BinaryDictDecoder. + * Find a word using the Ver3DictDecoder. * * @param dictDecoder the dict reader * @param word the word searched @@ -517,16 +517,15 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException */ @UsedForTesting - public static CharGroupInfo findWordByBinaryDictReader(final BinaryDictDecoder dictDecoder, + public static CharGroupInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = getTerminalPosition(dictDecoder, word); final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (position != FormatSpec.NOT_VALID_WORD) { dictBuffer.position(0); - final FileHeader header = BinaryDictDecoderUtils.readHeader(dictDecoder); + final FileHeader header = dictDecoder.readHeader(); dictBuffer.position(position); - return BinaryDictDecoderUtils.readCharGroup(dictBuffer, position, - header.mFormatOptions); + return dictDecoder.readPtNode(position, header.mFormatOptions); } return null; } @@ -545,8 +544,8 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final BinaryDictDecoder dictDecoder = new BinaryDictDecoder(file); - dictDecoder.openDictBuffer(new BinaryDictDecoder.DictionaryBufferFactory() { + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + dictDecoder.openDictBuffer(new DictDecoder.DictionaryBufferFactory() { @Override public DictBuffer getDictionaryBuffer(File file) throws FileNotFoundException, IOException { @@ -559,7 +558,7 @@ public final class BinaryDictIOUtils { } } }); - return BinaryDictDecoderUtils.readHeader(dictDecoder); + return dictDecoder.readHeader(); } public static FileHeader getDictionaryFileHeaderOrNull(final File file, final long offset, diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index b86dfe552..c02097fa7 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -17,11 +17,10 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; -import com.android.inputmethod.latin.makedict.decoder.HeaderReader; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; -import com.android.inputmethod.latin.utils.JniUtils; import java.io.File; import java.io.FileInputStream; @@ -30,17 +29,20 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.HashMap; -@UsedForTesting -public class BinaryDictDecoder implements HeaderReader { - - static { - JniUtils.loadNativeLibrary(); - } +/** + * An interface of binary dictionary decoder. + */ +public interface DictDecoder { + public FileHeader readHeader() throws IOException, UnsupportedFormatException; - // TODO: implement something sensical instead of just a phony method - private static native int doNothing(); + /** + * Reads PtNode from nodeAddress. + * @param ptNodePos the position of PtNode. + * @param formatOptions the format options. + * @return CharGroupInfo. + */ + public CharGroupInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); public interface DictionaryBufferFactory { public DictBuffer getDictionaryBuffer(final File file) @@ -131,59 +133,4 @@ public class BinaryDictDecoder implements HeaderReader { return null; } } - - private final File mDictionaryBinaryFile; - private DictBuffer mDictBuffer; - - public BinaryDictDecoder(final File file) { - mDictionaryBinaryFile = file; - mDictBuffer = null; - } - - public void openDictBuffer(final DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); - } - - public DictBuffer getDictBuffer() { - return mDictBuffer; - } - - @UsedForTesting - public DictBuffer openAndGetDictBuffer( - final DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - openDictBuffer(factory); - return getDictBuffer(); - } - - // The implementation of HeaderReader - @Override - public int readVersion() throws IOException, UnsupportedFormatException { - return BinaryDictDecoderUtils.checkFormatVersion(mDictBuffer); - } - - @Override - public int readOptionFlags() { - return mDictBuffer.readUnsignedShort(); - } - - @Override - public int readHeaderSize() { - return mDictBuffer.readInt(); - } - - @Override - public HashMap<String, String> readAttributes(final int headerSize) { - final HashMap<String, String> attributes = new HashMap<String, String>(); - while (mDictBuffer.position() < headerSize) { - // We can avoid infinite loop here since mFusionDictonary.position() is always increased - // by calling CharEncoding.readString. - final String key = CharEncoding.readString(mDictBuffer); - final String value = CharEncoding.readString(mDictBuffer); - attributes.put(key, value); - } - mDictBuffer.position(headerSize); - return attributes; - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 5361d2eba..f976c8152 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -55,11 +55,11 @@ public final class DynamicBinaryDictIOUtils { * @throws UnsupportedFormatException */ @UsedForTesting - public static void deleteWord(final BinaryDictDecoder dictDecoder, final String word) + public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(0); - final FileHeader header = BinaryDictDecoderUtils.readHeader(dictDecoder); + final FileHeader header = dictDecoder.readHeader(); final int wordPosition = BinaryDictIOUtils.getTerminalPosition(dictDecoder, word); if (wordPosition == FormatSpec.NOT_VALID_WORD) return; @@ -253,7 +253,7 @@ public final class DynamicBinaryDictIOUtils { // TODO: Support batch insertion. // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. @UsedForTesting - public static void insertWord(final BinaryDictDecoder dictDecoder, + public static void insertWord(final Ver3DictDecoder dictDecoder, final OutputStream destination, final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, @@ -278,7 +278,7 @@ public final class DynamicBinaryDictIOUtils { // find the insert position of the word. if (dictBuffer.position() != 0) dictBuffer.position(0); - final FileHeader fileHeader = BinaryDictDecoderUtils.readHeader(dictDecoder); + final FileHeader fileHeader = dictDecoder.readHeader(); int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position(); final int[] codePoints = FusionDictionary.getCodePoints(word); @@ -293,8 +293,8 @@ public final class DynamicBinaryDictIOUtils { for (int i = 0; i < charGroupCount; ++i) { address = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - dictBuffer.position(), fileHeader.mFormatOptions); + final CharGroupInfo currentInfo = dictDecoder.readPtNode(address, + fileHeader.mFormatOptions); final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, fileHeader.mFormatOptions); if (isMovedGroup) continue; @@ -314,7 +314,7 @@ public final class DynamicBinaryDictIOUtils { * abc - d - ef */ final int newNodeAddress = dictBuffer.limit(); - final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(p > 1, isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, false /* isBlackListEntry */, fileHeader.mFormatOptions); int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, @@ -353,7 +353,7 @@ public final class DynamicBinaryDictIOUtils { final int childrenAddress = currentInfo.mChildrenAddress; // move prefix - final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int prefixFlags = BinaryDictEncoderUtils.makeCharGroupFlags(p > 1, false /* isTerminal */, 0 /* childrenAddressSize*/, false /* hasShortcut */, false /* hasBigrams */, false /* isNotAWord */, false /* isBlackListEntry */, @@ -369,7 +369,7 @@ public final class DynamicBinaryDictIOUtils { updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } - final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags( + final int suffixFlags = BinaryDictEncoderUtils.makeCharGroupFlags( suffixCharacters.length > 1, (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, 0 /* childrenAddressSize */, @@ -387,7 +387,7 @@ public final class DynamicBinaryDictIOUtils { final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, codePoints.length); - final int flags = BinaryDictEncoder.makeCharGroupFlags( + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags( newCharacters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); @@ -410,7 +410,7 @@ public final class DynamicBinaryDictIOUtils { // only update group. final int newNodeAddress = dictBuffer.limit(); final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, @@ -440,7 +440,7 @@ public final class DynamicBinaryDictIOUtils { fileHeader.mFormatOptions); final int newGroupAddress = newNodeAddress + 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); @@ -485,7 +485,7 @@ public final class DynamicBinaryDictIOUtils { BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1, + final int flags = BinaryDictEncoderUtils.makeCharGroupFlags(characters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java new file mode 100644 index 000000000..8d1b02713 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.JniUtils; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +/** + * An implementation of DictDecoder for version 3 binary dictionary. + */ +@UsedForTesting +public class Ver3DictDecoder implements DictDecoder { + + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + + private final static class HeaderReader { + protected static int readVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); + } + + protected static int readOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedShort(); + } + + protected static int readHeaderSize(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + + protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, + final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (dictBuffer.position() < headerSize) { + // We can avoid an infinite loop here since dictBuffer.position() is always + // increased by calling CharEncoding.readString. + final String key = CharEncoding.readString(dictBuffer); + final String value = CharEncoding.readString(dictBuffer); + attributes.put(key, value); + } + dictBuffer.position(headerSize); + return attributes; + } + } + + private final static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readFrequency(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigrams(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + final int offset = (dictBuffer.readUnsignedByte() << 16) + + dictBuffer.readUnsignedShort(); + bigramAddress += sign * offset; + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + return readLength; + } + } + + private final File mDictionaryBinaryFile; + private DictBuffer mDictBuffer; + + public Ver3DictDecoder(final File file) { + mDictionaryBinaryFile = file; + mDictBuffer = null; + } + + public void openDictBuffer(final DictDecoder.DictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); + } + + public DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @UsedForTesting + public DictBuffer openAndGetDictBuffer(final DictDecoder.DictionaryBufferFactory factory) + throws FileNotFoundException, IOException { + openDictBuffer(factory); + return getDictBuffer(); + } + + @Override + public FileHeader readHeader() throws IOException, UnsupportedFormatException { + final int version = HeaderReader.readVersion(mDictBuffer); + final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(mDictBuffer); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = HeaderReader.readAttributes(mDictBuffer, + headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public CharGroupInfo readPtNode(final int ptNodePos, final FormatOptions options) { + int addressPointer = ptNodePos; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + ++addressPointer; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += 3; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (-1 != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + ++addressPointer; + frequency = PtNodeReader.readFrequency(mDictBuffer); + } else { + frequency = CharGroup.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + ArrayList<WeightedString> shortcutTargets = null; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } + ArrayList<PendingAttribute> bigrams = null; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigrams(mDictBuffer, bigrams, addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + MakedictLog.d("too many bigrams in a group."); + } + } + return new CharGroupInfo(ptNodePos, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReader.java b/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReader.java deleted file mode 100644 index f2badb444..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReader.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict.decoder; - -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; - -import java.io.IOException; -import java.util.HashMap; - -/** - * An interface to read a binary dictionary file header. - */ -public interface HeaderReader { - public int readVersion() throws IOException, UnsupportedFormatException; - public int readOptionFlags(); - public int readHeaderSize(); - public HashMap<String, String> readAttributes(final int headerSize); -} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 916be4481..b565b2f9f 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -28,8 +28,8 @@ import com.android.inputmethod.latin.ExpandableDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; @@ -241,10 +241,10 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona }; // Load the dictionary from binary file - final BinaryDictDecoder reader = new BinaryDictDecoder( + final Ver3DictDecoder reader = new Ver3DictDecoder( new File(getContext().getFilesDir(), fileName)); try { - reader.openDictBuffer(new BinaryDictDecoder.DictionaryBufferFromByteArrayFactory()); + reader.openDictBuffer(new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); } catch (FileNotFoundException e) { // This is an expected condition: we don't have a user history dictionary for this @@ -411,4 +411,18 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { mSessions.remove(session); } + + public void clearAndFlushDictionary() { + // Clear the node structure on memory + clearDictionary(); + mBigramListLock.lock(); + try { + // Clear the bigram list on memory + mBigramList.evictAll(); + } finally { + mBigramListLock.unlock(); + } + // Then flush the cleared state of the dictionary on disk. + flushPendingWrites(); + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java index da59333f5..c1833ff14 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java @@ -31,4 +31,7 @@ public class PersonalizationDictionarySessionRegister { public static void onRemoveData(Context context, String type) { } + + public static void onDestroy(Context context) { + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index 433c69c1c..77f0cdbfa 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -69,6 +69,13 @@ public abstract class PersonalizationDictionaryUpdateSession { dictionary.unRegisterUpdateSession(this); } + public void clearAndFlushPredictionDictionary(Context context) { + final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.clearAndFlushDictionary(); + } public void closeSession(Context context) { unsetPredictionDictionary(); diff --git a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java index b1cd88729..1b592b565 100644 --- a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java +++ b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java @@ -39,6 +39,8 @@ public final class DebugSettings extends PreferenceFragment public static final String PREF_STATISTICS_LOGGING = "enable_logging"; public static final String PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG = "use_only_personalization_dictionary_for_debug"; + public static final String PREF_BOOST_PERSONALIZATION_DICTIONARY_FOR_DEBUG = + "boost_personalization_dictionary_for_debug"; private static final String PREF_READ_EXTERNAL_DICTIONARY = "read_external_dictionary"; private static final boolean SHOW_STATISTICS_LOGGING = false; diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java index 20b675257..fd83865ba 100644 --- a/java/src/com/android/inputmethod/latin/settings/Settings.java +++ b/java/src/com/android/inputmethod/latin/settings/Settings.java @@ -346,6 +346,12 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang DebugSettings.PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false); } + public static boolean readBoostPersonalizationDictionaryForDebug( + final SharedPreferences prefs) { + return prefs.getBoolean( + DebugSettings.PREF_BOOST_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false); + } + public void writeLastUsedPersonalizationToken(byte[] token) { final String tokenStr = StringUtils.byteArrayToHexString(token); mPrefs.edit().putString(PREF_LAST_USED_PERSONALIZATION_TOKEN, tokenStr).apply(); diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index a0b744dd2..32730d23f 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -92,6 +92,8 @@ public final class SettingsValues { public final int mSuggestionVisibility; private final boolean mVoiceKeyEnabled; private final boolean mVoiceKeyOnMain; + public final boolean mBoostPersonalizationDictionaryForDebug; + public final boolean mUseOnlyPersonalizationDictionaryForDebug; // Setting values for additional features public final int[] mAdditionalFeaturesSettingValues = @@ -171,6 +173,10 @@ public final class SettingsValues { AdditionalFeaturesSettingUtils.readAdditionalFeaturesPreferencesIntoArray( prefs, mAdditionalFeaturesSettingValues); mIsInternal = Settings.isInternal(prefs); + mBoostPersonalizationDictionaryForDebug = + Settings.readBoostPersonalizationDictionaryForDebug(prefs); + mUseOnlyPersonalizationDictionaryForDebug = + Settings.readUseOnlyPersonalizationDictionaryForDebug(prefs); } // Only for tests @@ -216,6 +222,8 @@ public final class SettingsValues { mCorrectionEnabled = mAutoCorrectEnabled && !mInputAttributes.mInputTypeNoAutoCorrect; mSuggestionVisibility = 0; mIsInternal = false; + mBoostPersonalizationDictionaryForDebug = false; + mUseOnlyPersonalizationDictionaryForDebug = false; } @UsedForTesting @@ -298,7 +306,8 @@ public final class SettingsValues { // TODO: Stop using KeySpceParser.getLabel(). puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, - Dictionary.TYPE_HARDCODED)); + Dictionary.DICTIONARY_HARDCODED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } return new SuggestedWords(puncList, diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index bcf64a8e8..ed0812583 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -125,24 +125,12 @@ final class SuggestionStripLayoutHelper { R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripViewStyle); mSuggestionStripOption = a.getInt( R.styleable.SuggestionStripView_suggestionStripOption, 0); - final float alphaValidTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaValidTypedWord, 1.0f); - final float alphaTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaTypedWord, 1.0f); - final float alphaAutoCorrect = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaAutoCorrect, 1.0f); - final float alphaSuggested = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); mAlphaObsoleted = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); - mColorValidTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorValidTypedWord, 0), alphaValidTypedWord); - mColorTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorTypedWord, 0), alphaTypedWord); - mColorAutoCorrect = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorAutoCorrect, 0), alphaAutoCorrect); - mColorSuggested = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorSuggested, 0), alphaSuggested); + R.styleable.SuggestionStripView_alphaObsoleted, 1.0f); + mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0); + mColorTypedWord = a.getColor(R.styleable.SuggestionStripView_colorTypedWord, 0); + mColorAutoCorrect = a.getColor(R.styleable.SuggestionStripView_colorAutoCorrect, 0); + mColorSuggested = a.getColor(R.styleable.SuggestionStripView_colorSuggested, 0); mSuggestionsCountInStrip = a.getInt( R.styleable.SuggestionStripView_suggestionsCountInStrip, DEFAULT_SUGGESTIONS_COUNT_IN_STRIP); diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index dd7f534dc..771db3a47 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -19,14 +19,14 @@ package com.android.inputmethod.latin.utils; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoder; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderUtils; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; @@ -62,7 +62,7 @@ public final class UserHistoryDictIOUtils { final FormatOptions formatOptions) { final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); try { - BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryBinary(destination, fusionDict, formatOptions); Log.d(TAG, "end writing"); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); @@ -118,7 +118,7 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final BinaryDictDecoder dictDecoder, + public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, final OnAddWordListener dict) { final Map<Integer, String> unigrams = CollectionUtils.newTreeMap(); final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java index 713a45bda..1992b2f5d 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java @@ -23,7 +23,9 @@ import java.util.concurrent.TimeUnit; public final class UserHistoryForgettingCurveUtils { private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); private static final boolean DEBUG = false; - private static final int FC_FREQ_MAX = 127; + private static final int DEFAULT_FC_FREQ = 127; + private static final int BOOSTED_FC_FREQ = 200; + private static int FC_FREQ_MAX = DEFAULT_FC_FREQ; /* package */ static final int COUNT_MAX = 3; private static final int FC_LEVEL_MAX = 3; /* package */ static final int ELAPSED_TIME_MAX = 15; @@ -33,6 +35,14 @@ public final class UserHistoryForgettingCurveUtils { private static final int HALF_LIFE_HOURS = 48; private static final int MAX_PUSH_ELAPSED = (FC_LEVEL_MAX + 1) * (ELAPSED_TIME_MAX + 1); + public static void boostMaxFreqForDebug() { + FC_FREQ_MAX = BOOSTED_FC_FREQ; + } + + public static void resetMaxFreqForDebug() { + FC_FREQ_MAX = DEFAULT_FC_FREQ; + } + private UserHistoryForgettingCurveUtils() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/research/JsonUtils.java b/java/src/com/android/inputmethod/research/JsonUtils.java index 977f843e9..2beebdfae 100644 --- a/java/src/com/android/inputmethod/research/JsonUtils.java +++ b/java/src/com/android/inputmethod/research/JsonUtils.java @@ -103,7 +103,7 @@ import java.util.Map; jsonWriter.name("word").value(wordInfo.toString()); jsonWriter.name("score").value(wordInfo.mScore); jsonWriter.name("kind").value(wordInfo.mKind); - jsonWriter.name("sourceDict").value(wordInfo.mSourceDict); + jsonWriter.name("sourceDict").value(wordInfo.mSourceDict.mDictType); jsonWriter.endObject(); } jsonWriter.endArray(); |