diff options
Diffstat (limited to 'java/src')
40 files changed, 1176 insertions, 519 deletions
diff --git a/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java b/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java index 684165240..c28d72949 100644 --- a/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java +++ b/java/src/com/android/inputmethod/dictionarypack/DictionarySettingsActivity.java @@ -24,6 +24,8 @@ import android.preference.PreferenceActivity; * Preference screen. */ public final class DictionarySettingsActivity extends PreferenceActivity { + private static final String DEFAULT_FRAGMENT = DictionarySettingsFragment.class.getName(); + @Override protected void onCreate(final Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -32,11 +34,17 @@ public final class DictionarySettingsActivity extends PreferenceActivity { @Override public Intent getIntent() { final Intent modIntent = new Intent(super.getIntent()); - modIntent.putExtra(EXTRA_SHOW_FRAGMENT, DictionarySettingsFragment.class.getName()); + modIntent.putExtra(EXTRA_SHOW_FRAGMENT, DEFAULT_FRAGMENT); modIntent.putExtra(EXTRA_NO_HEADERS, true); // Important note : the original intent should contain a String extra with the key // DictionarySettingsFragment.DICT_SETTINGS_FRAGMENT_CLIENT_ID_ARGUMENT so that the // fragment can know who the client is. return modIntent; } + + // TODO: Uncomment the override annotation once we start using SDK version 19. + // @Override + public boolean isValidFragment(String fragmentName) { + return fragmentName.equals(DEFAULT_FRAGMENT); + } } diff --git a/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java b/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java index fed134eb9..e23131a30 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiCategoryPageIndicatorView.java @@ -50,8 +50,9 @@ public class EmojiCategoryPageIndicatorView extends LinearLayout { @Override protected void onDraw(Canvas canvas) { - if (mCategoryPageSize == 0) { - // If the category is not set yet, just clear and return. + if (mCategoryPageSize <= 1) { + // If the category is not set yet or contains only one category, + // just clear and return. canvas.drawColor(0); return; } diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java index eb48d01f6..db7c845bc 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java @@ -580,10 +580,18 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange } private void setCurrentCategoryId(final int categoryId, final boolean force) { - if (mEmojiCategory.getCurrentCategoryId() == categoryId && !force) { + final int oldCategoryId = mEmojiCategory.getCurrentCategoryId(); + if (oldCategoryId == categoryId && !force) { return; } + if (oldCategoryId == CATEGORY_ID_RECENTS) { + // Needs to save pending updates for recent keys when we get out of the recents + // category because we don't want to move the recent emojis around while the user + // is in the recents category. + mEmojiKeyboardAdapter.flushPendingRecentKeys(); + } + mEmojiCategory.setCurrentCategoryId(categoryId); final int newTabId = mEmojiCategory.getTabIdFromCategoryId(categoryId); final int newCategoryPageId = mEmojiCategory.getPageIdFromCategoryId(categoryId); @@ -612,8 +620,18 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange mRecentsKeyboard = mEmojiCategory.getKeyboard(CATEGORY_ID_RECENTS, 0); } + public void flushPendingRecentKeys() { + mRecentsKeyboard.flushPendingRecentKeys(); + final KeyboardView recentKeyboardView = + mActiveKeyboardView.get(mEmojiCategory.getRecentTabId()); + if (recentKeyboardView != null) { + recentKeyboardView.invalidateAllKeys(); + } + } + public void addRecentKey(final Key key) { if (mEmojiCategory.isInRecentTab()) { + mRecentsKeyboard.addPendingKey(key); return; } mRecentsKeyboard.addKeyFirst(key); diff --git a/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java index 267fad5cd..ceb44e79f 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java @@ -56,7 +56,7 @@ public class EmojiLayoutParams { - (mKeyVerticalGap - mBottomPadding) / 2; mEmojiPagerHeight = defaultKeyboardHeight - mEmojiActionBarHeight - mEmojiCategoryPageIdViewHeight; - mEmojiPagerBottomMargin = mKeyVerticalGap / 2; + mEmojiPagerBottomMargin = 0; mEmojiKeyboardHeight = mEmojiPagerHeight - mEmojiPagerBottomMargin - 1; } @@ -75,9 +75,7 @@ public class EmojiLayoutParams { public void setActionBarProperties(LinearLayout ll) { final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) ll.getLayoutParams(); - lp.height = mEmojiActionBarHeight; - lp.topMargin = 0; - lp.bottomMargin = mBottomPadding; + lp.height = mEmojiActionBarHeight - mBottomPadding; ll.setLayoutParams(lp); } diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java index 3ea68806b..f7ec9509d 100644 --- a/java/src/com/android/inputmethod/keyboard/Key.java +++ b/java/src/com/android/inputmethod/keyboard/Key.java @@ -139,6 +139,8 @@ public class Key implements Comparable<Key> { private final OptionalAttributes mOptionalAttributes; + private static final int DEFAULT_TEXT_COLOR = 0xFFFFFFFF; + private static final class OptionalAttributes { /** Text to output when pressed. This can be multiple characters, like ".com" */ public final String mOutputText; @@ -602,7 +604,22 @@ public class Key implements Comparable<Key> { } public final int selectTextColor(final KeyDrawParams params) { - return isShiftedLetterActivated() ? params.mTextInactivatedColor : params.mTextColor; + if (isShiftedLetterActivated()) { + return params.mTextInactivatedColor; + } + if (params.mTextColorStateList == null) { + return DEFAULT_TEXT_COLOR; + } + final int[] state; + // TODO: Hack!!!!!!!! Consider having a new attribute for the functional text labels. + // Currently, we distinguish "input key" from "functional key" by checking the + // length of the label( > 1) and "functional" attributes (= true). + if (mLabel != null && mLabel.length() > 1) { + state = getCurrentDrawableState(); + } else { + state = KEY_STATE_NORMAL; + } + return params.mTextColorStateList.getColorForState(state, DEFAULT_TEXT_COLOR); } public final int selectHintTextSize(final KeyDrawParams params) { diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index 74edd87cf..ad6e2c0f2 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -155,7 +155,7 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } public void saveKeyboardState() { - if (getKeyboard() != null) { + if (getKeyboard() != null || isShowingEmojiKeyboard()) { mState.onSaveKeyboardState(); } } diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java index ee4ac950c..52f190e77 100644 --- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java +++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java @@ -823,14 +823,16 @@ public final class PointerTracker implements PointerTrackerQueue.Element { final int size = sAggregratedPointers.getPointerSize(); if (size > sLastRecognitionPointSize && stroke.hasRecognitionTimePast(eventTime, sLastRecognitionTime)) { - sLastRecognitionPointSize = size; - sLastRecognitionTime = eventTime; if (DEBUG_LISTENER) { Log.d(TAG, String.format("[%d] onUpdateBatchInput: batchPoints=%d", mPointerId, size)); } mTimerProxy.startUpdateBatchInputTimer(this); mListener.onUpdateBatchInput(sAggregratedPointers); + // The listener may change the size of the pointers (when auto-committing + // for example), so we need to get the size from the pointers again. + sLastRecognitionPointSize = sAggregratedPointers.getPointerSize(); + sLastRecognitionTime = eventTime; } } } diff --git a/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java index c10fdbace..4ccecb2f0 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java +++ b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java @@ -18,6 +18,8 @@ package com.android.inputmethod.keyboard.internal; import com.android.inputmethod.latin.Constants; +import android.text.TextUtils; + /** * The string parser of codesArray specification for <GridRows />. The attribute codesArray is an * array of string. @@ -34,7 +36,7 @@ import com.android.inputmethod.latin.Constants; public final class CodesArrayParser { // Constants for parsing. private static final char COMMA = ','; - private static final char VERTICAL_BAR = '|'; + private static final String VERTICAL_BAR_STRING = "\\|"; private static final String COMMA_STRING = ","; private static final int BASE_HEX = 16; @@ -43,8 +45,11 @@ public final class CodesArrayParser { } private static String getLabelSpec(final String codesArraySpec) { - final int pos = codesArraySpec.indexOf(VERTICAL_BAR); - return (pos < 0) ? codesArraySpec : codesArraySpec.substring(0, pos); + final String[] strs = codesArraySpec.split(VERTICAL_BAR_STRING, -1); + if (strs.length <= 1) { + return codesArraySpec; + } + return strs[0]; } public static String parseLabel(final String codesArraySpec) { @@ -58,8 +63,25 @@ public final class CodesArrayParser { } private static String getCodeSpec(final String codesArraySpec) { - final int pos = codesArraySpec.indexOf(VERTICAL_BAR); - return (pos < 0) ? codesArraySpec : codesArraySpec.substring(pos + 1); + final String[] strs = codesArraySpec.split(VERTICAL_BAR_STRING, -1); + if (strs.length <= 1) { + return codesArraySpec; + } + return TextUtils.isEmpty(strs[1]) ? strs[0] : strs[1]; + } + + // codesArraySpec consists of: + // <label>|<code0>,<code1>,...|<minSupportSdkVersion> + public static int getMinSupportSdkVersion(final String codesArraySpec) { + final String[] strs = codesArraySpec.split(VERTICAL_BAR_STRING, -1); + if (strs.length <= 2) { + return 0; + } + try { + return Integer.parseInt(strs[2]); + } catch (NumberFormatException e) { + return 0; + } } public static int parseCode(final String codesArraySpec) { diff --git a/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java index 0dd71e2ec..587f95a39 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java +++ b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java @@ -39,6 +39,7 @@ public class DynamicGridKeyboard extends Keyboard { private static final String TAG = DynamicGridKeyboard.class.getSimpleName(); private static final int TEMPLATE_KEY_CODE_0 = 0x30; private static final int TEMPLATE_KEY_CODE_1 = 0x31; + private final Object mLock = new Object(); private final SharedPreferences mPrefs; private final int mLeftPadding; @@ -48,6 +49,7 @@ public class DynamicGridKeyboard extends Keyboard { private final int mMaxKeyCount; private final boolean mIsRecents; private final ArrayDeque<GridKey> mGridKeys = CollectionUtils.newArrayDeque(); + private final ArrayDeque<Key> mPendingKeys = CollectionUtils.newArrayDeque(); private Key[] mCachedGridKeys; @@ -74,6 +76,21 @@ public class DynamicGridKeyboard extends Keyboard { throw new RuntimeException("Can't find template key: code=" + code); } + public void addPendingKey(final Key usedKey) { + synchronized (mLock) { + mPendingKeys.addLast(usedKey); + } + } + + public void flushPendingRecentKeys() { + synchronized (mLock) { + while (!mPendingKeys.isEmpty()) { + addKey(mPendingKeys.pollFirst(), true); + } + saveRecentKeys(); + } + } + public void addKeyFirst(final Key usedKey) { addKey(usedKey, true); if (mIsRecents) { @@ -89,7 +106,7 @@ public class DynamicGridKeyboard extends Keyboard { if (usedKey == null) { return; } - synchronized (mGridKeys) { + synchronized (mLock) { mCachedGridKeys = null; final GridKey key = new GridKey(usedKey); while (mGridKeys.remove(key)) { @@ -167,7 +184,7 @@ public class DynamicGridKeyboard extends Keyboard { @Override public Key[] getKeys() { - synchronized (mGridKeys) { + synchronized (mLock) { if (mCachedGridKeys != null) { return mCachedGridKeys; } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyDrawParams.java b/java/src/com/android/inputmethod/keyboard/internal/KeyDrawParams.java index 1716fa049..b528b692e 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyDrawParams.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyDrawParams.java @@ -16,6 +16,7 @@ package com.android.inputmethod.keyboard.internal; +import android.content.res.ColorStateList; import android.graphics.Typeface; import com.android.inputmethod.latin.utils.ResourceUtils; @@ -32,7 +33,7 @@ public final class KeyDrawParams { public int mHintLabelSize; public int mPreviewTextSize; - public int mTextColor; + public ColorStateList mTextColorStateList; public int mTextInactivatedColor; public int mTextShadowColor; public int mHintLetterColor; @@ -57,7 +58,7 @@ public final class KeyDrawParams { mHintLabelSize = copyFrom.mHintLabelSize; mPreviewTextSize = copyFrom.mPreviewTextSize; - mTextColor = copyFrom.mTextColor; + mTextColorStateList = copyFrom.mTextColorStateList; mTextInactivatedColor = copyFrom.mTextInactivatedColor; mTextShadowColor = copyFrom.mTextShadowColor; mHintLetterColor = copyFrom.mHintLetterColor; @@ -89,8 +90,8 @@ public final class KeyDrawParams { attr.mShiftedLetterHintRatio, mShiftedLetterHintSize); mHintLabelSize = selectTextSize(keyHeight, attr.mHintLabelRatio, mHintLabelSize); mPreviewTextSize = selectTextSize(keyHeight, attr.mPreviewTextRatio, mPreviewTextSize); - - mTextColor = selectColor(attr.mTextColor, mTextColor); + mTextColorStateList = + attr.mTextColorStateList != null ? attr.mTextColorStateList : mTextColorStateList; mTextInactivatedColor = selectColor(attr.mTextInactivatedColor, mTextInactivatedColor); mTextShadowColor = selectColor(attr.mTextShadowColor, mTextShadowColor); mHintLetterColor = selectColor(attr.mHintLetterColor, mHintLetterColor); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyVisualAttributes.java b/java/src/com/android/inputmethod/keyboard/internal/KeyVisualAttributes.java index 7a2622cbb..8bdad364c 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyVisualAttributes.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyVisualAttributes.java @@ -16,6 +16,7 @@ package com.android.inputmethod.keyboard.internal; +import android.content.res.ColorStateList; import android.content.res.TypedArray; import android.graphics.Typeface; import android.util.SparseIntArray; @@ -37,7 +38,7 @@ public final class KeyVisualAttributes { public final float mHintLabelRatio; public final float mPreviewTextRatio; - public final int mTextColor; + public final ColorStateList mTextColorStateList; public final int mTextInactivatedColor; public final int mTextShadowColor; public final int mHintLetterColor; @@ -115,7 +116,7 @@ public final class KeyVisualAttributes { mPreviewTextRatio = ResourceUtils.getFraction(keyAttr, R.styleable.Keyboard_Key_keyPreviewTextRatio); - mTextColor = keyAttr.getColor(R.styleable.Keyboard_Key_keyTextColor, 0); + mTextColorStateList = keyAttr.getColorStateList(R.styleable.Keyboard_Key_keyTextColor); mTextInactivatedColor = keyAttr.getColor( R.styleable.Keyboard_Key_keyTextInactivatedColor, 0); mTextShadowColor = keyAttr.getColor(R.styleable.Keyboard_Key_keyTextShadowColor, 0); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java index 22f7a83fc..c1ae65695 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java @@ -20,6 +20,7 @@ import android.content.Context; import android.content.res.Resources; import android.content.res.TypedArray; import android.content.res.XmlResourceParser; +import android.os.Build; import android.util.AttributeSet; import android.util.Log; import android.util.TypedValue; @@ -436,17 +437,24 @@ public class KeyboardBuilder<KP extends KeyboardParams> { final String label; final int code; final String outputText; + final int supportedMinSdkVersion; if (codesArrayId != 0) { final String codeArraySpec = array[i]; label = CodesArrayParser.parseLabel(codeArraySpec); code = CodesArrayParser.parseCode(codeArraySpec); outputText = CodesArrayParser.parseOutputText(codeArraySpec); + supportedMinSdkVersion = + CodesArrayParser.getMinSupportSdkVersion(codeArraySpec); } else { final String textArraySpec = array[i]; // TODO: Utilize KeySpecParser or write more generic TextsArrayParser. label = textArraySpec; code = Constants.CODE_OUTPUT_TEXT; outputText = textArraySpec + (char)Constants.CODE_SPACE; + supportedMinSdkVersion = 0; + } + if (Build.VERSION.SDK_INT < supportedMinSdkVersion) { + continue; } final int x = (int)row.getKeyX(null); final int y = row.getKeyY(); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java index 9f9fdaa6f..506dfa751 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java @@ -178,6 +178,8 @@ public final class KeyboardState { if (!state.mIsAlphabetShiftLocked) { setShifted(state.mShiftMode); } + // TODO: is this the right place to do this? Should we do this in setShift* instead? + mSwitchActions.requestUpdatingShiftState(); } else { mPrevMainKeyboardWasShiftLocked = state.mIsAlphabetShiftLocked; } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index 67553fb75..684cf632b 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -209,8 +209,8 @@ public final class KeyboardTextsSet { /* 104 */ "keylabel_for_tablet_comma", /* 105 */ "keyhintlabel_for_tablet_comma", /* 106 */ "more_keys_for_tablet_comma", - /* 107 */ "keyhintlabel_for_tablet_period", - /* 108 */ "more_keys_for_tablet_period", + /* 107 */ "keyhintlabel_for_period", + /* 108 */ "more_keys_for_period", /* 109 */ "keylabel_for_apostrophe", /* 110 */ "keyhintlabel_for_apostrophe", /* 111 */ "more_keys_for_apostrophe", diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 29c6c0451..541e69788 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -44,9 +44,9 @@ public final class BinaryDictionary extends Dictionary { private static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; // Must be equal to MAX_RESULTS in native/jni/src/defines.h private static final int MAX_RESULTS = 18; - // Required space count for auto commit. - // TODO: Remove this heuristic. - private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3; + // The cutoff returned by native for auto-commit confidence. + // Must be equal to CONFIDENCE_TO_AUTO_COMMIT in native/jni/src/defines.h + private static final int CONFIDENCE_TO_AUTO_COMMIT = 1000000; @UsedForTesting public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; @@ -62,7 +62,8 @@ public final class BinaryDictionary extends Dictionary { private final int[] mSpaceIndices = new int[MAX_RESULTS]; private final int[] mOutputScores = new int[MAX_RESULTS]; private final int[] mOutputTypes = new int[MAX_RESULTS]; - private final int[] mOutputAutoCommitFirstWordConfidence = new int[MAX_RESULTS]; + // Only one result is ever used + private final int[] mOutputAutoCommitFirstWordConfidence = new int[1]; private final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions(); @@ -269,18 +270,11 @@ public final class BinaryDictionary extends Dictionary { return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); } - private void runGCIfRequired() { - if (needsToRunGC(true /* mindsBlockByGC */)) { - flushWithGC(); - } - } - // Add a unigram entry to binary dictionary in native code. public void addUnigramWord(final String word, final int probability) { if (TextUtils.isEmpty(word)) { return; } - runGCIfRequired(); final int[] codePoints = StringUtils.toCodePointArray(word); addUnigramWordNative(mNativeDict, codePoints, probability); } @@ -290,7 +284,6 @@ public final class BinaryDictionary extends Dictionary { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } - runGCIfRequired(); final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability); @@ -301,7 +294,6 @@ public final class BinaryDictionary extends Dictionary { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } - runGCIfRequired(); final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); @@ -351,18 +343,7 @@ public final class BinaryDictionary extends Dictionary { @Override public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { - // TODO: actually use the confidence rather than use this completely broken heuristic - final String word = candidate.mWord; - final int length = word.length(); - int remainingSpaces = SPACE_COUNT_FOR_AUTO_COMMIT; - for (int i = 0; i < length; ++i) { - // This is okay because no low-surrogate and no high-surrogate can ever match the - // space character, so we don't need to take care of iterating on code points. - if (Constants.CODE_SPACE == word.charAt(i)) { - if (0 >= --remainingSpaces) return true; - } - } - return false; + return candidate.mAutoCommitFirstWordConfidence > CONFIDENCE_TO_AUTO_COMMIT; } @Override diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 2d1ca51e2..c79a4ff90 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -249,6 +249,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final File file = new File(mContext.getFilesDir(), mFilename); BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); + mBinaryDictionary = new BinaryDictionary( + file.getAbsolutePath(), 0 /* offset */, file.length(), + true /* useFullEditDistance */, null, mDictType, mIsUpdatable); } else { mDictionaryWriter.clear(); } @@ -274,6 +277,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** + * Check whether GC is needed and run GC if required. + */ + protected void runGCIfRequired(final boolean mindsBlockByGC) { + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + runGCIfRequiredInternalLocked(mindsBlockByGC); + } + }); + } + + private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; + // Calls to needsToRunGC() need to be serialized. + if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { + if (setIsRegeneratingIfNotRegenerating()) { + // Run GC after currently existing time sensitive operations. + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + try { + mBinaryDictionary.flushWithGC(); + } finally { + mFilenameDictionaryUpdateController.mIsRegenerating.set(false); + } + } + }); + } + } + } + + /** * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. */ protected void addWordDynamically(final String word, final String shortcutTarget, @@ -282,11 +318,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename); return; } - getExecutor(mFilename).execute(new Runnable() { @Override public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); mBinaryDictionary.addUnigramWord(word, frequency); } else { // TODO: Remove. @@ -306,11 +342,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { + mFilename); return; } - getExecutor(mFilename).execute(new Runnable() { @Override public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); mBinaryDictionary.addBigramWords(word0, word1, frequency); } else { // TODO: Remove. @@ -330,11 +366,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { + mFilename); return; } - getExecutor(mFilename).execute(new Runnable() { @Override public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); mBinaryDictionary.removeBigramWords(word0, word1); } else { // TODO: Remove. @@ -461,8 +497,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final long length = file.length(); // Build the new binary dictionary - final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length, - true /* useFullEditDistance */, null, mDictType, mIsUpdatable); + final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0 /* offset */, + length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); // Ensure all threads accessing the current dictionary have finished before // swapping in the new one. diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 96e16de0d..d51c63dd3 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -77,6 +77,7 @@ import com.android.inputmethod.keyboard.MainKeyboardView; import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.define.ProductionFlag; +import com.android.inputmethod.latin.personalization.DictionaryDecayBroadcastReciever; import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister; import com.android.inputmethod.latin.personalization.PersonalizationHelper; @@ -567,6 +568,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen newDictFilter.addAction(DictionaryPackConstants.NEW_DICTIONARY_INTENT_ACTION); registerReceiver(mDictionaryPackInstallReceiver, newDictFilter); + DictionaryDecayBroadcastReciever.setUpIntervalAlarmForDictionaryDecaying(this); + mInputUpdater = new InputUpdater(this); } @@ -602,8 +605,24 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } private void initSuggest() { - final Locale subtypeLocale = mSubtypeSwitcher.getCurrentSubtypeLocale(); - final String localeStr = subtypeLocale.toString(); + final Locale switcherSubtypeLocale = mSubtypeSwitcher.getCurrentSubtypeLocale(); + final String switcherLocaleStr = switcherSubtypeLocale.toString(); + final Locale subtypeLocale; + final String localeStr; + if (TextUtils.isEmpty(switcherLocaleStr)) { + // This happens in very rare corner cases - for example, immediately after a switch + // to LatinIME has been requested, about a frame later another switch happens. In this + // case, we are about to go down but we still don't know it, however the system tells + // us there is no current subtype so the locale is the empty string. Take the best + // possible guess instead -- it's bound to have no consequences, and we have no way + // of knowing anyway. + Log.e(TAG, "System is reporting no current subtype."); + subtypeLocale = getResources().getConfiguration().locale; + localeStr = subtypeLocale.toString(); + } else { + subtypeLocale = switcherSubtypeLocale; + localeStr = switcherLocaleStr; + } final Suggest newSuggest = new Suggest(this /* Context */, subtypeLocale, this /* SuggestInitializationListener */); @@ -1435,11 +1454,21 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!settingsValues.mCorrectionEnabled) return false; if (!settingsValues.mUseDoubleSpacePeriod) return false; if (!mHandler.isAcceptingDoubleSpacePeriod()) return false; - final CharSequence lastThree = mConnection.getTextBeforeCursor(3, 0); - if (lastThree != null && lastThree.length() == 3 - && canBeFollowedByDoubleSpacePeriod(lastThree.charAt(0)) - && lastThree.charAt(1) == Constants.CODE_SPACE - && lastThree.charAt(2) == Constants.CODE_SPACE) { + // We only do this when we see two spaces and an accepted code point before the cursor. + // The code point may be a surrogate pair but the two spaces may not, so we need 4 chars. + final CharSequence lastThree = mConnection.getTextBeforeCursor(4, 0); + if (null == lastThree) return false; + final int length = lastThree.length(); + if (length < 3) return false; + if (lastThree.charAt(length - 1) != Constants.CODE_SPACE) return false; + if (lastThree.charAt(length - 2) != Constants.CODE_SPACE) return false; + // We know there are spaces in pos -1 and -2, and we have at least three chars. + // If we have only three chars, isSurrogatePairs can't return true as charAt(1) is a space, + // so this is fine. + final int firstCodePoint = + Character.isSurrogatePair(lastThree.charAt(0), lastThree.charAt(1)) ? + Character.codePointAt(lastThree, 0) : lastThree.charAt(length - 3); + if (canBeFollowedByDoubleSpacePeriod(firstCodePoint)) { mHandler.cancelDoubleSpacePeriodTimer(); mConnection.deleteSurroundingText(2, 0); final String textToInsert = ". "; @@ -1464,7 +1493,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen || codePoint == Constants.CODE_CLOSING_SQUARE_BRACKET || codePoint == Constants.CODE_CLOSING_CURLY_BRACKET || codePoint == Constants.CODE_CLOSING_ANGLE_BRACKET - || codePoint == Constants.CODE_PLUS; + || codePoint == Constants.CODE_PLUS + || Character.getType(codePoint) == Character.OTHER_SYMBOL; } // Callback for the {@link SuggestionStripView}, to call when the "add to dictionary" hint is @@ -2929,6 +2959,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return; } tryFixLyingCursorPosition(); + mKeyboardSwitcher.updateShiftState(); if (tryResumeSuggestions) mHandler.postResumeSuggestions(); } diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java new file mode 100644 index 000000000..9f7f502ea --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -0,0 +1,206 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.TreeMap; + +/** + * A base class of the binary dictionary decoder. + */ +public abstract class AbstractDictDecoder implements DictDecoder { + protected FileHeader readHeader(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + if (dictBuffer == null) { + openDictBuffer(); + } + + final int version = HeaderReader.readVersion(dictBuffer); + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("Unsupported version : " + version); + } + // TODO: Remove this field. + final int optionsFlags = HeaderReader.readOptionFlags(dictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(dictBuffer); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = HeaderReader.readAttributes(dictBuffer, + headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + @Override @UsedForTesting + public int getTerminalPosition(final String word) + throws IOException, UnsupportedFormatException { + if (!isDictBufferOpen()) { + openDictBuffer(); + } + return BinaryDictIOUtils.getTerminalPosition(this, word); + } + + @Override @UsedForTesting + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException { + if (!isDictBufferOpen()) { + openDictBuffer(); + } + BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); + } + + /** + * A utility class for reading a file header. + */ + protected static class HeaderReader { + protected static int readVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); + } + + protected static int readOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedShort(); + } + + protected static int readHeaderSize(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + + protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, + final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (dictBuffer.position() < headerSize) { + // We can avoid an infinite loop here since dictBuffer.position() is always + // increased by calling CharEncoding.readString. + final String key = CharEncoding.readString(dictBuffer); + final String value = CharEncoding.readString(dictBuffer); + attributes.put(key, value); + } + dictBuffer.position(headerSize); + return attributes; + } + } + + /** + * A utility class for reading a PtNode. + */ + protected static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigramAddresses(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + bigramAddress += sign * dictBuffer.readUnsignedInt24(); + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return readLength; + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 665c7a27c..216492b4d 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -23,11 +23,11 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; @@ -219,14 +219,14 @@ public final class BinaryDictDecoderUtils { } /** - * Writes a string with our character format to a ByteArrayOutputStream. + * Writes a string with our character format to an OutputStream. * * This will also write the terminator byte. * - * @param buffer the ByteArrayOutputStream to write to. + * @param buffer the OutputStream to write to. * @param word the string to write. */ - static void writeString(final ByteArrayOutputStream buffer, final String word) { + static void writeString(final OutputStream buffer, final String word) throws IOException { final int length = word.length(); for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); @@ -295,7 +295,6 @@ public final class BinaryDictDecoderUtils { return address; } } - int address; switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return dictBuffer.readUnsignedByte(); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index af61f2979..f761829de 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -278,7 +278,6 @@ public class BinaryDictEncoderUtils { // For future reference, the code to remove duplicate is a simple : list.remove(node); list.add(ptNodeArray); final ArrayList<PtNode> branches = ptNodeArray.mData; - final int nodeSize = branches.size(); for (PtNode ptNode : branches) { if (null != ptNode.mChildren) flattenTreeInner(list, ptNode.mChildren); } @@ -384,8 +383,8 @@ public class BinaryDictEncoderUtils { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, nodeSize + size, ptNode.mChildren)); } - nodeSize += getShortcutListSize(ptNode.mShortcutTargets); if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) { + nodeSize += getShortcutListSize(ptNode.mShortcutTargets); if (null != ptNode.mBigrams) { for (WeightedString bigram : ptNode.mBigrams) { final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, @@ -427,9 +426,6 @@ public class BinaryDictEncoderUtils { nodeCountSize + nodeArrayOffset + nodeffset; nodeffset += ptNode.mCachedSize; } - final int nodeSize = nodeCountSize + nodeffset - + (formatOptions.mSupportsDynamicUpdate - ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); nodeArrayOffset += nodeArray.mCachedSize; } return nodeArrayOffset; @@ -653,8 +649,8 @@ public class BinaryDictEncoderUtils { return flags; } - /* package */ static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress, - final int childrenOffset, final FormatOptions formatOptions) { + /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset, + final FormatOptions formatOptions) { return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0, getByteSize(childrenOffset), node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index a282f595c..0f7d2f6c9 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -288,40 +288,6 @@ public final class BinaryDictIOUtils { return BinaryDictEncoderUtils.getByteSize(value); } - static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) { - final int flags = dictBuffer.readUnsignedByte(); - BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); - skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - BinaryDictDecoderUtils.readChildrenAddress(dictBuffer, flags, formatOptions); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); - if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { - final int shortcutsSize = dictBuffer.readUnsignedShort(); - dictBuffer.position(dictBuffer.position() + shortcutsSize - - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); - } - if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: - dictBuffer.readUnsignedByte(); - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: - dictBuffer.readUnsignedShort(); - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: - dictBuffer.readUnsignedInt24(); - break; - } - if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode."); - } - } - } - static void skipString(final DictBuffer dictBuffer, final boolean hasMultipleChars) { if (hasMultipleChars) { diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 3796a466c..3dbeee099 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -17,11 +17,9 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; @@ -32,50 +30,17 @@ import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; -import java.util.HashMap; import java.util.TreeMap; /** - * The base class of binary dictionary decoders. + * An interface of binary dictionary decoders. */ -public abstract class DictDecoder { - - protected FileHeader readHeader(final DictBuffer dictBuffer) - throws IOException, UnsupportedFormatException { - if (dictBuffer == null) { - openDictBuffer(); - } - - final int version = HeaderReader.readVersion(dictBuffer); - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("Unsupported version : " + version); - } - // TODO: Remove this field. - final int optionsFlags = HeaderReader.readOptionFlags(dictBuffer); - - final int headerSize = HeaderReader.readHeaderSize(dictBuffer); - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final HashMap<String, String> attributes = HeaderReader.readAttributes(dictBuffer, - headerSize); - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } +public interface DictDecoder { /** * Reads and returns the file header. */ - public abstract FileHeader readHeader() throws IOException, UnsupportedFormatException; + public FileHeader readHeader() throws IOException, UnsupportedFormatException; /** * Reads PtNode from nodeAddress. @@ -83,7 +48,7 @@ public abstract class DictDecoder { * @param formatOptions the format options. * @return PtNodeInfo. */ - public abstract PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); /** * Reads a buffer and returns the memory representation of the dictionary. @@ -98,7 +63,7 @@ public abstract class DictDecoder { * @return the created (or merged) dictionary. */ @UsedForTesting - public abstract FusionDictionary readDictionaryBinary(final FusionDictionary dict, + public FusionDictionary readDictionaryBinary(final FusionDictionary dict, final boolean deleteDictIfBroken) throws FileNotFoundException, IOException, UnsupportedFormatException; @@ -113,12 +78,7 @@ public abstract class DictDecoder { */ @UsedForTesting public int getTerminalPosition(final String word) - throws IOException, UnsupportedFormatException { - if (!isDictBufferOpen()) { - openDictBuffer(); - } - return BinaryDictIOUtils.getTerminalPosition(this, word); - } + throws IOException, UnsupportedFormatException; /** * Reads unigrams and bigrams from the binary file. @@ -134,47 +94,42 @@ public abstract class DictDecoder { public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, final TreeMap<Integer, Integer> frequencies, final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) - throws IOException, UnsupportedFormatException { - if (!isDictBufferOpen()) { - openDictBuffer(); - } - BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); - } + throws IOException, UnsupportedFormatException; /** * Sets the position of the buffer to the given value. * * @param newPos the new position */ - public abstract void setPosition(final int newPos); + public void setPosition(final int newPos); /** * Gets the position of the buffer. * * @return the position */ - public abstract int getPosition(); + public int getPosition(); /** * Reads and returns the PtNode count out of a buffer and forwards the pointer. */ - public abstract int readPtNodeCount(); + public int readPtNodeCount(); /** * Reads the forward link and advances the position. * * @return true if this method moves the file pointer, false otherwise. */ - public abstract boolean readAndFollowForwardLink(); - public abstract boolean hasNextPtNodeArray(); + public boolean readAndFollowForwardLink(); + public boolean hasNextPtNodeArray(); /** * Opens the dictionary file and makes DictBuffer. */ @UsedForTesting - public abstract void openDictBuffer() throws FileNotFoundException, IOException; + public void openDictBuffer() throws FileNotFoundException, IOException; @UsedForTesting - public abstract boolean isDictBufferOpen(); + public boolean isDictBufferOpen(); // Constants for DictionaryBufferFactory. public static final int USE_READONLY_BYTEBUFFER = 0x01000000; @@ -272,123 +227,5 @@ public abstract class DictDecoder { } } - /** - * A utility class for reading a file header. - */ - protected static class HeaderReader { - protected static int readVersion(final DictBuffer dictBuffer) - throws IOException, UnsupportedFormatException { - return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); - } - - protected static int readOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedShort(); - } - - protected static int readHeaderSize(final DictBuffer dictBuffer) { - return dictBuffer.readInt(); - } - - protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, - final int headerSize) { - final HashMap<String, String> attributes = new HashMap<String, String>(); - while (dictBuffer.position() < headerSize) { - // We can avoid an infinite loop here since dictBuffer.position() is always - // increased by calling CharEncoding.readString. - final String key = CharEncoding.readString(dictBuffer); - final String value = CharEncoding.readString(dictBuffer); - attributes.put(key, value); - } - dictBuffer.position(headerSize); - return attributes; - } - } - - /** - * A utility class for reading a PtNode. - */ - protected static class PtNodeReader { - protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedByte(); - } - - protected static int readParentAddress(final DictBuffer dictBuffer, - final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - return BinaryDictDecoderUtils.readSInt24(dictBuffer); - } else { - return FormatSpec.NO_PARENT_ADDRESS; - } - } - - protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, - final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); - if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; - return address; - } else { - switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: - return dictBuffer.readUnsignedByte(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: - return dictBuffer.readUnsignedShort(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: - return dictBuffer.readUnsignedInt24(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - } - - // Reads shortcuts and returns the read length. - protected static int readShortcut(final DictBuffer dictBuffer, - final ArrayList<WeightedString> shortcutTargets) { - final int pointerBefore = dictBuffer.position(); - dictBuffer.readUnsignedShort(); // skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return dictBuffer.position() - pointerBefore; - } - - protected static int readBigramAddresses(final DictBuffer dictBuffer, - final ArrayList<PendingAttribute> bigrams, final int baseAddress) { - int readLength = 0; - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++readLength; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = baseAddress + readLength; - switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - readLength += 1; - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - readLength += 2; - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: - bigramAddress += sign * dictBuffer.readUnsignedInt24(); - readLength += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute( - bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return readLength; - } - } + public void skipPtNode(final FormatOptions formatOptions); } diff --git a/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java new file mode 100644 index 000000000..c4f7ec91f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.IOException; +import java.util.ArrayList; + +/** + * An interface of a binary dictionary updater. + */ +@UsedForTesting +public interface DictUpdater extends DictDecoder { + + /** + * Deletes the word from the binary dictionary. + * + * @param word the word to be deleted. + */ + @UsedForTesting + public void deleteWord(final String word) throws IOException, UnsupportedFormatException; + + /** + * Inserts a word into a binary dictionary. + * + * @param word the word to be inserted. + * @param frequency the frequency of the new word. + * @param bigramStrings bigram list, or null if none. + * @param shortcuts shortcut list, or null if none. + * @param isBlackListEntry whether this should be a blacklist entry. + */ + // TODO: Support batch insertion. + @UsedForTesting + public void insertWord(final String word, final int frequency, + final ArrayList<WeightedString> bigramStrings, + final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, + final boolean isBlackListEntry) throws IOException, UnsupportedFormatException; +} diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 411e265b3..336277196 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -42,44 +42,22 @@ public final class DynamicBinaryDictIOUtils { // This utility class is not publicly instantiable. } - private static int markAsDeleted(final int flags) { + /* package */ static int markAsDeleted(final int flags) { return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; } /** - * Delete the word from the binary file. - * - * @param dictDecoder the dict decoder. - * @param word the word we delete - * @throws IOException - * @throws UnsupportedFormatException - */ - @UsedForTesting - public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word) - throws IOException, UnsupportedFormatException { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - dictBuffer.position(0); - final FileHeader header = dictDecoder.readHeader(); - final int wordPosition = dictDecoder.getTerminalPosition(word); - if (wordPosition == FormatSpec.NOT_VALID_WORD) return; - - dictBuffer.position(wordPosition); - final int flags = dictBuffer.readUnsignedByte(); - dictBuffer.position(wordPosition); - dictBuffer.put((byte)markAsDeleted(flags)); - } - - /** * Update a parent address in a PtNode that is referred to by ptNodeOriginAddress. * - * @param dictBuffer the DictBuffer to write. + * @param dictUpdater the DictUpdater to write. * @param ptNodeOriginAddress the address of the PtNode. * @param newParentAddress the absolute address of the parent. * @param formatOptions file format options. */ - private static void updateParentAddress(final DictBuffer dictBuffer, + private static void updateParentAddress(final Ver3DictUpdater dictUpdater, final int ptNodeOriginAddress, final int newParentAddress, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); final int originalPosition = dictBuffer.position(); dictBuffer.position(ptNodeOriginAddress); if (!formatOptions.mSupportsDynamicUpdate) { @@ -104,46 +82,45 @@ public final class DynamicBinaryDictIOUtils { /** * Update parent addresses in a node array stored at ptNodeOriginAddress. * - * @param dictBuffer the DictBuffer to be modified. + * @param dictUpdater the DictUpdater to be modified. * @param ptNodeOriginAddress the address of the node array to update. * @param newParentAddress the address to be written. * @param formatOptions file format options. */ - private static void updateParentAddresses(final DictBuffer dictBuffer, + private static void updateParentAddresses(final Ver3DictUpdater dictUpdater, final int ptNodeOriginAddress, final int newParentAddress, final FormatOptions formatOptions) { - final int originalPosition = dictBuffer.position(); - dictBuffer.position(ptNodeOriginAddress); + final int originalPosition = dictUpdater.getPosition(); + dictUpdater.setPosition(ptNodeOriginAddress); do { - final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + final int count = dictUpdater.readPtNodeCount(); for (int i = 0; i < count; ++i) { - updateParentAddress(dictBuffer, dictBuffer.position(), newParentAddress, + updateParentAddress(dictUpdater, dictUpdater.getPosition(), newParentAddress, formatOptions); - BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); + dictUpdater.skipPtNode(formatOptions); } - final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); - dictBuffer.position(forwardLinkAddress); - } while (formatOptions.mSupportsDynamicUpdate - && dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); - dictBuffer.position(originalPosition); + if (!dictUpdater.readAndFollowForwardLink()) break; + if (dictUpdater.getPosition() == FormatSpec.NO_FORWARD_LINK_ADDRESS) break; + } while (formatOptions.mSupportsDynamicUpdate); + dictUpdater.setPosition(originalPosition); } /** * Update a children address in a PtNode that is addressed by ptNodeOriginAddress. * - * @param dictBuffer the DictBuffer to write. + * @param dictUpdater the DictUpdater to write. * @param ptNodeOriginAddress the address of the PtNode. * @param newChildrenAddress the absolute address of the child. * @param formatOptions file format options. */ - private static void updateChildrenAddress(final DictBuffer dictBuffer, + private static void updateChildrenAddress(final Ver3DictUpdater dictUpdater, final int ptNodeOriginAddress, final int newChildrenAddress, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); final int originalPosition = dictBuffer.position(); dictBuffer.position(ptNodeOriginAddress); final int flags = dictBuffer.readUnsignedByte(); - final int parentAddress = BinaryDictDecoderUtils.readParentAddress(dictBuffer, - formatOptions); + BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); BinaryDictIOUtils.skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS @@ -156,31 +133,33 @@ public final class DynamicBinaryDictIOUtils { * Helper method to move a PtNode to the tail of the file. */ private static int movePtNode(final OutputStream destination, - final DictBuffer dictBuffer, final PtNodeInfo info, + final Ver3DictUpdater dictUpdater, final PtNodeInfo info, final int nodeArrayOriginAddress, final int oldNodeAddress, final FormatOptions formatOptions) throws IOException { - updateParentAddress(dictBuffer, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); + final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); + updateParentAddress(dictUpdater, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); dictBuffer.position(oldNodeAddress); final int currentFlags = dictBuffer.readUnsignedByte(); dictBuffer.position(oldNodeAddress); dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); int size = FormatSpec.PTNODE_FLAGS_SIZE; - updateForwardLink(dictBuffer, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); + updateForwardLink(dictUpdater, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info }); return size; } @SuppressWarnings("unused") - private static void updateForwardLink(final DictBuffer dictBuffer, + private static void updateForwardLink(final Ver3DictUpdater dictUpdater, final int nodeArrayOriginAddress, final int newNodeArrayAddress, final FormatOptions formatOptions) { - dictBuffer.position(nodeArrayOriginAddress); + final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); + dictUpdater.setPosition(nodeArrayOriginAddress); int jumpCount = 0; while (jumpCount++ < MAX_JUMPS) { - final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + final int count = dictUpdater.readPtNodeCount(); for (int i = 0; i < count; ++i) { - BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); + dictUpdater.readPtNode(dictUpdater.getPosition(), formatOptions); } final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { @@ -208,7 +187,7 @@ public final class DynamicBinaryDictIOUtils { * @param shortcutTargets the shortcut targets for this PtNode. * @param bigrams the bigrams for this PtNode. * @param destination the stream representing the tail of the file. - * @param dictBuffer the DictBuffer representing the (constant-size) body of the file. + * @param dictUpdater the DictUpdater. * @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of. * @param oldPtNodeOrigin the old origin where this PtNode used to be stored. * @param formatOptions format options for this dictionary. @@ -219,7 +198,7 @@ public final class DynamicBinaryDictIOUtils { final int length, final int flags, final int frequency, final int parentAddress, final ArrayList<WeightedString> shortcutTargets, final ArrayList<PendingAttribute> bigrams, final OutputStream destination, - final DictBuffer dictBuffer, final int oldPtNodeArrayOrigin, + final Ver3DictUpdater dictUpdater, final int oldPtNodeArrayOrigin, final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException { int size = 0; final int newPtNodeOrigin = fileEndAddress + 1; @@ -232,7 +211,7 @@ public final class DynamicBinaryDictIOUtils { flags, writtenCharacters, frequency, parentAddress, fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, bigrams); - movePtNode(destination, dictBuffer, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, + movePtNode(destination, dictUpdater, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, formatOptions); return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } @@ -240,7 +219,7 @@ public final class DynamicBinaryDictIOUtils { /** * Insert a word into a binary dictionary. * - * @param dictDecoder the dict decoder. + * @param dictUpdater the dict updater. * @param destination a stream to the underlying file, with the pointer at the end of the file. * @param word the word to insert. * @param frequency the frequency of the new word. @@ -253,17 +232,17 @@ public final class DynamicBinaryDictIOUtils { // TODO: Support batch insertion. // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. @UsedForTesting - public static void insertWord(final Ver3DictDecoder dictDecoder, + public static void insertWord(final Ver3DictUpdater dictUpdater, final OutputStream destination, final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + final DictBuffer dictBuffer = dictUpdater.getDictBuffer(); if (bigramStrings != null) { for (final WeightedString bigram : bigramStrings) { - int position = dictDecoder.getTerminalPosition(bigram.mWord); + int position = dictUpdater.getTerminalPosition(bigram.mWord); if (position == FormatSpec.NOT_VALID_WORD) { // TODO: figure out what is the correct thing to do here. } else { @@ -278,7 +257,7 @@ public final class DynamicBinaryDictIOUtils { // find the insert position of the word. if (dictBuffer.position() != 0) dictBuffer.position(0); - final FileHeader fileHeader = dictDecoder.readHeader(); + final FileHeader fileHeader = dictUpdater.readHeader(); int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position(); final int[] codePoints = FusionDictionary.getCodePoints(word); @@ -293,7 +272,7 @@ public final class DynamicBinaryDictIOUtils { for (int i = 0; i < ptNodeCount; ++i) { address = dictBuffer.position(); - final PtNodeInfo currentInfo = dictDecoder.readPtNode(address, + final PtNodeInfo currentInfo = dictUpdater.readPtNode(address, fileHeader.mFormatOptions); final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, fileHeader.mFormatOptions); @@ -319,12 +298,12 @@ public final class DynamicBinaryDictIOUtils { false /* isBlackListEntry */, fileHeader.mFormatOptions); int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags, frequency, nodeParentAddress, shortcuts, bigrams, destination, - dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions); + dictUpdater, nodeOriginAddress, address, fileHeader.mFormatOptions); final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, currentInfo.mCharacters.length); if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, + updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } final PtNodeInfo newInfo2 = new PtNodeInfo( @@ -360,13 +339,13 @@ public final class DynamicBinaryDictIOUtils { fileHeader.mFormatOptions); int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, - destination, dictBuffer, nodeOriginAddress, address, + destination, dictUpdater, nodeOriginAddress, address, fileHeader.mFormatOptions); final int[] suffixCharacters = Arrays.copyOfRange( currentInfo.mCharacters, p, currentInfo.mCharacters.length); if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, + updateParentAddresses(dictUpdater, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags( @@ -417,7 +396,7 @@ public final class DynamicBinaryDictIOUtils { -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, bigrams); - movePtNode(destination, dictBuffer, newInfo, nodeOriginAddress, address, + movePtNode(destination, dictUpdater, newInfo, nodeOriginAddress, address, fileHeader.mFormatOptions); return; } @@ -436,7 +415,7 @@ public final class DynamicBinaryDictIOUtils { * ab - cd - e */ final int newNodeArrayAddress = dictBuffer.limit(); - updateChildrenAddress(dictBuffer, address, newNodeArrayAddress, + updateChildrenAddress(dictUpdater, address, newNodeArrayAddress, fileHeader.mFormatOptions); final int newNodeAddress = newNodeArrayAddress + 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1; diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 9481a8c14..5a5d7af6b 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -266,11 +266,27 @@ public final class FormatSpec { // tat = Terminal Address Table static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; static final String BIGRAM_FILE_EXTENSION = ".bigram"; - static final String BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup"; - static final String BIGRAM_ADDRESS_TABLE_FILE_EXTENSION = ".bigram_index"; + static final String SHORTCUT_FILE_EXTENSION = ".shortcut"; + static final String LOOKUP_TABLE_FILE_SUFFIX = "_lookup"; + static final String CONTENT_TABLE_FILE_SUFFIX = "_index"; static final int FREQUENCY_AND_FLAGS_SIZE = 2; static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3; + + // With the English main dictionary as of October 2013, the size of bigram address table is + // is 584KB with the block size being 4. + // This is 91% of that of full address table. static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4; + static final int BIGRAM_CONTENT_COUNT = 1; + static final int BIGRAM_FREQ_CONTENT_INDEX = 0; + static final String BIGRAM_FREQ_CONTENT_ID = "_freq"; + + static final int SHORTCUT_CONTENT_COUNT = 1; + static final int SHORTCUT_CONTENT_INDEX = 0; + // With the English main dictionary as of October 2013, the size of shortcut address table is + // 29KB with the block size being 64. + // This is only 4.4% of that of full address table. + static final int SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE = 64; + static final String SHORTCUT_CONTENT_ID = "_shortcut"; static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java index 96d057a44..7592a0c13 100644 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.FileInputStream; @@ -37,35 +38,39 @@ public class SparseTable { /** * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize * terminals. - * It contains at index i = j / mBlockSize the index in mContentsTable where the values for - * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array. + * It contains at index i = j / mBlockSize the index in each ArrayList in mContentsTables where + * the values for terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized + * integer array. */ private final ArrayList<Integer> mLookupTable; - private final ArrayList<Integer> mContentTable; + private final ArrayList<ArrayList<Integer>> mContentTables; private final int mBlockSize; + private final int mContentTableCount; public static final int NOT_EXIST = -1; + public static final int SIZE_OF_INT_IN_BYTES = 4; @UsedForTesting - public SparseTable(final int initialCapacity, final int blockSize) { + public SparseTable(final int initialCapacity, final int blockSize, + final int contentTableCount) { mBlockSize = blockSize; final int lookupTableSize = initialCapacity / mBlockSize + (initialCapacity % mBlockSize > 0 ? 1 : 0); mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST)); - mContentTable = new ArrayList<Integer>(); + mContentTableCount = contentTableCount; + mContentTables = CollectionUtils.newArrayList(); + for (int i = 0; i < mContentTableCount; ++i) { + mContentTables.add(new ArrayList<Integer>()); + } } @UsedForTesting - public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) { + public SparseTable(final ArrayList<Integer> lookupTable, + final ArrayList<ArrayList<Integer>> contentTables, final int blockSize) { mBlockSize = blockSize; - mLookupTable = new ArrayList<Integer>(lookupTable.length); - for (int i = 0; i < lookupTable.length; ++i) { - mLookupTable.add(lookupTable[i]); - } - mContentTable = new ArrayList<Integer>(contentTable.length); - for (int i = 0; i < contentTable.length; ++i) { - mContentTable.add(contentTable[i]); - } + mContentTableCount = contentTables.size(); + mLookupTable = lookupTable; + mContentTables = contentTables; } /** @@ -75,8 +80,8 @@ public class SparseTable { * Otherwise, IndexOutOfBoundsException will be raised. */ @UsedForTesting - private static void convertByteArrayToIntegerArray(final byte[] byteArray, - final ArrayList<Integer> integerArray) { + private static ArrayList<Integer> convertByteArrayToIntegerArray(final byte[] byteArray) { + final ArrayList<Integer> integerArray = new ArrayList<Integer>(byteArray.length / 4); for (int i = 0; i < byteArray.length; i += 4) { int value = 0; for (int j = i; j < i + 4; ++j) { @@ -85,39 +90,43 @@ public class SparseTable { } integerArray.add(value); } + return integerArray; } @UsedForTesting - public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) { - mBlockSize = blockSize; - mLookupTable = new ArrayList<Integer>(lookupTable.length / 4); - mContentTable = new ArrayList<Integer>(contentTable.length / 4); - convertByteArrayToIntegerArray(lookupTable, mLookupTable); - convertByteArrayToIntegerArray(contentTable, mContentTable); + public int get(final int contentTableIndex, final int index) { + if (!contains(index)) { + return NOT_EXIST; + } + return mContentTables.get(contentTableIndex).get( + mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); } @UsedForTesting - public int get(final int index) { - if (index < 0 || index / mBlockSize >= mLookupTable.size() - || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { - return NOT_EXIST; + public ArrayList<Integer> getAll(final int index) { + final ArrayList<Integer> ret = CollectionUtils.newArrayList(); + for (int i = 0; i < mContentTableCount; ++i) { + ret.add(get(i, index)); } - return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); + return ret; } @UsedForTesting - public void set(final int index, final int value) { + public void set(final int contentTableIndex, final int index, final int value) { if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) { - mLookupTable.set(index / mBlockSize, mContentTable.size()); - for (int i = 0; i < mBlockSize; ++i) { - mContentTable.add(NOT_EXIST); + mLookupTable.set(index / mBlockSize, mContentTables.get(contentTableIndex).size()); + for (int i = 0; i < mContentTableCount; ++i) { + for (int j = 0; j < mBlockSize; ++j) { + mContentTables.get(i).add(NOT_EXIST); + } } } - mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); + mContentTables.get(contentTableIndex).set( + mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); } - public void remove(final int index) { - set(index, NOT_EXIST); + public void remove(final int indexOfContent, final int index) { + set(indexOfContent, index, NOT_EXIST); } @UsedForTesting @@ -127,7 +136,8 @@ public class SparseTable { @UsedForTesting /* package */ int getContentTableSize() { - return mContentTable.size(); + // This class always has at least one content table. + return mContentTables.get(0).size(); } @UsedForTesting @@ -136,36 +146,51 @@ public class SparseTable { } public boolean contains(final int index) { - return get(index) != NOT_EXIST; + if (index < 0 || index / mBlockSize >= mLookupTable.size() + || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + return false; + } + return true; } @UsedForTesting - public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream) + public void write(final OutputStream lookupOutStream, final OutputStream[] contentOutStreams) throws IOException { + if (contentOutStreams.length != mContentTableCount) { + throw new RuntimeException(contentOutStreams.length + " streams are given, but the" + + " table has " + mContentTableCount + " content tables."); + } for (final int index : mLookupTable) { - BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4); + BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, SIZE_OF_INT_IN_BYTES); } - for (final int index : mContentTable) { - BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4); + for (int i = 0; i < contentOutStreams.length; ++i) { + for (final int data : mContentTables.get(i)) { + BinaryDictEncoderUtils.writeUIntToStream(contentOutStreams[i], data, + SIZE_OF_INT_IN_BYTES); + } } } @UsedForTesting - public void writeToFiles(final File lookupTableFile, final File contentFile) + public void writeToFiles(final File lookupTableFile, final File[] contentFiles) throws IOException { - FileOutputStream lookupTableOutStream = null; - FileOutputStream contentOutStream = null; + FileOutputStream lookupTableOutStream = null; + final FileOutputStream[] contentTableOutStreams = new FileOutputStream[mContentTableCount]; try { lookupTableOutStream = new FileOutputStream(lookupTableFile); - contentOutStream = new FileOutputStream(contentFile); - write(lookupTableOutStream, contentOutStream); + for (int i = 0; i < contentFiles.length; ++i) { + contentTableOutStreams[i] = new FileOutputStream(contentFiles[i]); + } + write(lookupTableOutStream, contentTableOutStreams); } finally { if (lookupTableOutStream != null) { lookupTableOutStream.close(); } - if (contentOutStream != null) { - contentOutStream.close(); + for (int i = 0; i < contentTableOutStreams.length; ++i) { + if (contentTableOutStreams[i] != null) { + contentTableOutStreams[i].close(); + } } } } @@ -185,10 +210,14 @@ public class SparseTable { } @UsedForTesting - public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile, + public static SparseTable readFromFiles(final File lookupTableFile, final File[] contentFiles, final int blockSize) throws IOException { - final byte[] lookupTable = readFileToByteArray(lookupTableFile); - final byte[] content = readFileToByteArray(contentFile); - return new SparseTable(lookupTable, content, blockSize); + final ArrayList<ArrayList<Integer>> contentTables = + new ArrayList<ArrayList<Integer>>(contentFiles.length); + for (int i = 0; i < contentFiles.length; ++i) { + contentTables.add(convertByteArrayToIntegerArray(readFileToByteArray(contentFiles[i]))); + } + return new SparseTable(convertByteArrayToIntegerArray(readFileToByteArray(lookupTableFile)), + contentTables, blockSize); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java index 848277cd4..acab4f8a5 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -37,7 +37,7 @@ import java.util.Arrays; * An implementation of DictDecoder for version 3 binary dictionary. */ @UsedForTesting -public class Ver3DictDecoder extends DictDecoder { +public class Ver3DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver3DictDecoder.class.getSimpleName(); static { @@ -47,15 +47,15 @@ public class Ver3DictDecoder extends DictDecoder { // TODO: implement something sensical instead of just a phony method private static native int doNothing(); - protected static class PtNodeReader extends DictDecoder.PtNodeReader { + protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { private static int readFrequency(final DictBuffer dictBuffer) { return dictBuffer.readUnsignedByte(); } } - private final File mDictionaryBinaryFile; + protected final File mDictionaryBinaryFile; private final DictionaryBufferFactory mBufferFactory; - private DictBuffer mDictBuffer; + protected DictBuffer mDictBuffer; /* package */ Ver3DictDecoder(final File file, final int factoryFlag) { mDictionaryBinaryFile = file; @@ -169,7 +169,8 @@ public class Ver3DictDecoder extends DictDecoder { addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, addressPointer); if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - MakedictLog.d("too many bigrams in a PtNode."); + throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() + + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); } } else { bigrams = null; @@ -231,4 +232,40 @@ public class Ver3DictDecoder extends DictDecoder { public boolean hasNextPtNodeArray() { return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; } + + @Override + public void skipPtNode(final FormatOptions formatOptions) { + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + PtNodeReader.readParentAddress(mDictBuffer, formatOptions); + BinaryDictIOUtils.skipString(mDictBuffer, + (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readFrequency(mDictBuffer); + if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { + final int shortcutsSize = mDictBuffer.readUnsignedShort(); + mDictBuffer.position(mDictBuffer.position() + shortcutsSize + - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = mDictBuffer.readUnsignedByte(); + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + mDictBuffer.readUnsignedByte(); + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + mDictBuffer.readUnsignedShort(); + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + mDictBuffer.readUnsignedInt24(); + break; + } + if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; + } + if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode."); + } + } + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 76f0f4052..d9e19899c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -133,12 +133,10 @@ public class Ver3DictEncoder implements DictEncoder { countSize); } - private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, - final FormatOptions formatOptions) { + private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, - BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, childrenPos, - formatOptions), + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions), FormatSpec.PTNODE_FLAGS_SIZE); } @@ -244,7 +242,7 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writePtNode(final PtNode ptNode, final int parentPosition, final FormatOptions formatOptions, final FusionDictionary dict) { - writePtNodeFlags(ptNode, parentPosition, formatOptions); + writePtNodeFlags(ptNode, formatOptions); writeParentPosition(parentPosition, ptNode, formatOptions); writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); writeFrequency(ptNode.mFrequency); diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java new file mode 100644 index 000000000..07adda625 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; + +/** + * An implementation of DictUpdater for version 3 binary dictionary. + */ +@UsedForTesting +public class Ver3DictUpdater extends Ver3DictDecoder implements DictUpdater { + private OutputStream mOutStream; + + @UsedForTesting + public Ver3DictUpdater(final File dictFile, final int factoryType) { + // DictUpdater must have an updatable DictBuffer. + super(dictFile, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) + ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); + mOutStream = null; + } + + private void openStreamAndBuffer() throws FileNotFoundException, IOException { + super.openDictBuffer(); + mOutStream = new FileOutputStream(mDictionaryBinaryFile, true /* append */); + } + + private void close() throws IOException { + if (mOutStream != null) { + mOutStream.close(); + mOutStream = null; + } + } + + @Override @UsedForTesting + public void deleteWord(final String word) throws IOException, UnsupportedFormatException { + if (mOutStream == null) openStreamAndBuffer(); + mDictBuffer.position(0); + readHeader(); + final int wordPos = getTerminalPosition(word); + if (wordPos != FormatSpec.NOT_VALID_WORD) { + mDictBuffer.position(wordPos); + final int flags = mDictBuffer.readUnsignedByte(); + mDictBuffer.position(wordPos); + mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags)); + } + close(); + } + + @Override @UsedForTesting + public void insertWord(final String word, final int frequency, + final ArrayList<WeightedString> bigramStrings, + final ArrayList<WeightedString> shortcuts, + final boolean isNotAWord, final boolean isBlackListEntry) + throws IOException, UnsupportedFormatException { + if (mOutStream == null) openStreamAndBuffer(); + DynamicBinaryDictIOUtils.insertWord(this, mOutStream, word, frequency, bigramStrings, + shortcuts, isNotAWord, isBlackListEntry); + close(); + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 0aa431966..53729075f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -23,6 +23,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.CollectionUtils; import android.util.Log; @@ -36,21 +37,24 @@ import java.util.Arrays; * An implementation of binary dictionary decoder for version 4 binary dictionary. */ @UsedForTesting -public class Ver4DictDecoder extends DictDecoder { +public class Ver4DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver4DictDecoder.class.getSimpleName(); private static final int FILETYPE_TRIE = 1; private static final int FILETYPE_FREQUENCY = 2; private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; - private static final int FILETYPE_BIGRAM = 4; + private static final int FILETYPE_BIGRAM_FREQ = 4; + private static final int FILETYPE_SHORTCUT = 5; private final File mDictDirectory; private final DictionaryBufferFactory mBufferFactory; - private DictBuffer mDictBuffer; + protected DictBuffer mDictBuffer; private DictBuffer mFrequencyBuffer; private DictBuffer mTerminalAddressTableBuffer; private DictBuffer mBigramBuffer; + private DictBuffer mShortcutBuffer; private SparseTable mBigramAddressTable; + private SparseTable mShortcutAddressTable; @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { @@ -85,9 +89,14 @@ public class Ver4DictDecoder extends DictDecoder { } else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) { return new File(mDictDirectory, mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); - } else if (fileType == FILETYPE_BIGRAM) { + } else if (fileType == FILETYPE_BIGRAM_FREQ) { return new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION); + mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION + + FormatSpec.BIGRAM_FREQ_CONTENT_ID); + } else if (fileType == FILETYPE_SHORTCUT) { + return new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.SHORTCUT_FILE_EXTENSION + + FormatSpec.SHORTCUT_CONTENT_ID); } else { throw new RuntimeException("Unsupported kind of file : " + fileType); } @@ -95,13 +104,14 @@ public class Ver4DictDecoder extends DictDecoder { @Override public void openDictBuffer() throws FileNotFoundException, IOException { - final String filename = mDictDirectory.getName(); mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE)); mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); - mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM)); + mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM_FREQ)); loadBigramAddressSparseTable(); + mShortcutBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_SHORTCUT)); + loadShortcutAddressSparseTable(); } @Override @@ -127,15 +137,27 @@ public class Ver4DictDecoder extends DictDecoder { } private void loadBigramAddressSparseTable() throws IOException { - final File lookupIndexFile = new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION); - final File contentFile = new File(mDictDirectory, - mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION); - mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile, + final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + final File freqsFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + + FormatSpec.BIGRAM_FREQ_CONTENT_ID); + mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { freqsFile }, FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); } - protected static class PtNodeReader extends DictDecoder.PtNodeReader { + // TODO: Let's have something like SparseTableContentsReader in this class. + private void loadShortcutAddressSparseTable() throws IOException { + final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + final File contentFile = new File(mDictDirectory, mDictDirectory.getName() + + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + + FormatSpec.SHORTCUT_CONTENT_ID); + mShortcutAddressTable = SparseTable.readFromFiles(lookupIndexFile, + new File[] { contentFile }, FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE); + } + + protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) { frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1); return frequencyBuffer.readUnsignedByte(); @@ -146,6 +168,23 @@ public class Ver4DictDecoder extends DictDecoder { } } + private ArrayList<WeightedString> readShortcuts(final int terminalId) { + if (mShortcutAddressTable.get(0, terminalId) == SparseTable.NOT_EXIST) return null; + + final ArrayList<WeightedString> ret = CollectionUtils.newArrayList(); + final int posOfShortcuts = mShortcutAddressTable.get(FormatSpec.SHORTCUT_CONTENT_INDEX, + terminalId); + mShortcutBuffer.position(posOfShortcuts); + while (true) { + final int flags = mShortcutBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(mShortcutBuffer); + ret.add(new WeightedString(word, + flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return ret; + } + // TODO: Make this buffer thread safe. // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH. private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; @@ -196,19 +235,12 @@ public class Ver4DictDecoder extends DictDecoder { childrenAddress += addressPointer; } addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - final ArrayList<WeightedString> shortcutTargets; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - // readShortcut will add shortcuts to shortcutTargets. - shortcutTargets = new ArrayList<WeightedString>(); - addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); - } else { - shortcutTargets = null; - } + final ArrayList<WeightedString> shortcutTargets = readShortcuts(terminalId); final ArrayList<PendingAttribute> bigrams; if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { bigrams = new ArrayList<PendingAttribute>(); - final int posOfBigrams = mBigramAddressTable.get(terminalId); + final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId); mBigramBuffer.position(posOfBigrams); while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, @@ -224,7 +256,8 @@ public class Ver4DictDecoder extends DictDecoder { if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; } if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - MakedictLog.d("too many bigrams in a node."); + throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() + + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); } } else { bigrams = null; @@ -293,4 +326,14 @@ public class Ver4DictDecoder extends DictDecoder { public boolean hasNextPtNodeArray() { return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; } + + @Override + public void skipPtNode(final FormatOptions formatOptions) { + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + PtNodeReader.readParentAddress(mDictBuffer, formatOptions); + BinaryDictIOUtils.skipString(mDictBuffer, + (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) PtNodeReader.readTerminalId(mDictBuffer); + PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions); + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 4c25faf88..f9dcacf77 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -26,7 +26,6 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -44,19 +43,149 @@ public class Ver4DictEncoder implements DictEncoder { private byte[] mTrieBuf; private int mTriePos; private int mHeaderSize; - private SparseTable mBigramAddressTable; private OutputStream mTrieOutStream; private OutputStream mFreqOutStream; private OutputStream mTerminalAddressTableOutStream; - private OutputStream mBigramOutStream; private File mDictDir; private String mBaseFilename; + private BigramContentWriter mBigramWriter; + private ShortcutContentWriter mShortcutWriter; @UsedForTesting public Ver4DictEncoder(final File dictPlacedDir) { mDictPlacedDir = dictPlacedDir; } + private interface SparseTableContentWriterInterface { + public void write(final OutputStream outStream) throws IOException; + } + + private static class SparseTableContentWriter { + private final int mContentCount; + private final SparseTable mSparseTable; + private final File mLookupTableFile; + protected final File mBaseDir; + private final File[] mAddressTableFiles; + private final File[] mContentFiles; + protected final OutputStream[] mContentOutStreams; + + public SparseTableContentWriter(final String name, final int contentCount, + final int initialCapacity, final int blockSize, final File baseDir, + final String[] contentFilenames, final String[] contentIds) { + if (contentFilenames.length != contentIds.length) { + throw new RuntimeException("The length of contentFilenames and the length of" + + " contentIds are different " + contentFilenames.length + ", " + + contentIds.length); + } + mContentCount = contentCount; + mSparseTable = new SparseTable(initialCapacity, blockSize, contentCount); + mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + mAddressTableFiles = new File[mContentCount]; + mContentFiles = new File[mContentCount]; + mBaseDir = baseDir; + for (int i = 0; i < mContentCount; ++i) { + mAddressTableFiles[i] = new File(mBaseDir, + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + } + mContentOutStreams = new OutputStream[mContentCount]; + } + + public void openStreams() throws FileNotFoundException { + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); + } + } + + protected void write(final int contentIndex, final int index, + final SparseTableContentWriterInterface writer) throws IOException { + mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); + writer.write(mContentOutStreams[contentIndex]); + mContentOutStreams[contentIndex].flush(); + } + + public void closeStreams() throws IOException { + mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i].close(); + } + } + } + + private static class BigramContentWriter extends SparseTableContentWriter { + + public BigramContentWriter(final String name, final int initialCapacity, + final File baseDir) { + super(name + FormatSpec.BIGRAM_FILE_EXTENSION, FormatSpec.BIGRAM_CONTENT_COUNT, + initialCapacity, FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }, + new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }); + } + + public void writeBigramsForOneWord(final int terminalId, + final Iterator<WeightedString> bigramIterator, final FusionDictionary dict) + throws IOException { + write(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, + new SparseTableContentWriterInterface() { + @Override + public void write(final OutputStream outStream) throws IOException { + writeBigramsForOneWordInternal(outStream, bigramIterator, dict); + } + }); + } + + private void writeBigramsForOneWordInternal(final OutputStream outStream, + final Iterator<WeightedString> bigramIterator, final FusionDictionary dict) + throws IOException { + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int unigramFrequencyForThisWord = target.mFrequency; + final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags( + bigramIterator.hasNext(), 0, bigram.mFrequency, + unigramFrequencyForThisWord, bigram.mWord); + BinaryDictEncoderUtils.writeUIntToStream(outStream, bigramFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + BinaryDictEncoderUtils.writeUIntToStream(outStream, target.mTerminalId, + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); + } + } + } + + private static class ShortcutContentWriter extends SparseTableContentWriter { + public ShortcutContentWriter(final String name, final int initialCapacity, + final File baseDir) { + super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, FormatSpec.SHORTCUT_CONTENT_COUNT, + initialCapacity, FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, + new String[] { FormatSpec.SHORTCUT_CONTENT_ID }); + } + + public void writeShortcutForOneWord(final int terminalId, + final Iterator<WeightedString> shortcutIterator) throws IOException { + write(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, + new SparseTableContentWriterInterface() { + @Override + public void write(final OutputStream outStream) throws IOException { + writeShortcutForOneWordInternal(outStream, shortcutIterator); + } + }); + } + + private void writeShortcutForOneWordInternal(final OutputStream outStream, + final Iterator<WeightedString> shortcutIterator) throws IOException { + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), target.mFrequency); + BinaryDictEncoderUtils.writeUIntToStream(outStream, shortcutFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + CharEncoding.writeString(outStream, target.mWord); + } + } + } + private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions) throws FileNotFoundException, IOException { final FileHeader header = new FileHeader(0, dictOptions, formatOptions); @@ -66,8 +195,6 @@ public class Ver4DictEncoder implements DictEncoder { final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION); final File terminalAddressTableFile = new File(mDictDir, mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); - final File bigramFile = new File(mDictDir, - mBaseFilename + FormatSpec.BIGRAM_FILE_EXTENSION); if (!mDictDir.isDirectory()) { if (mDictDir.exists()) mDictDir.delete(); mDictDir.mkdirs(); @@ -78,7 +205,6 @@ public class Ver4DictEncoder implements DictEncoder { mTrieOutStream = new FileOutputStream(trieFile); mFreqOutStream = new FileOutputStream(freqFile); mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile); - mBigramOutStream = new FileOutputStream(bigramFile); } private void close() throws IOException { @@ -92,14 +218,10 @@ public class Ver4DictEncoder implements DictEncoder { if (mTerminalAddressTableOutStream != null) { mTerminalAddressTableOutStream.close(); } - if (mBigramOutStream != null) { - mBigramOutStream.close(); - } } finally { mTrieOutStream = null; mFreqOutStream = null; mTerminalAddressTableOutStream = null; - mBigramOutStream = null; } } @@ -135,10 +257,10 @@ public class Ver4DictEncoder implements DictEncoder { if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); writeTerminalData(flatNodes, terminalCount); - mBigramAddressTable = new SparseTable(terminalCount, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); + mBigramWriter = new BigramContentWriter(mBaseFilename, terminalCount, mDictDir); writeBigrams(flatNodes, dict); - writeBigramAddressSparseTable(); + mShortcutWriter = new ShortcutContentWriter(mBaseFilename, terminalCount, mDictDir); + writeShortcuts(flatNodes); final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; @@ -181,12 +303,10 @@ public class Ver4DictEncoder implements DictEncoder { countSize); } - private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, - final FormatOptions formatOptions) { + private void writePtNodeFlags(final PtNode ptNode, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, - BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mTriePos, childrenPos, - formatOptions), + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, childrenPos, formatOptions), FormatSpec.PTNODE_FLAGS_SIZE); } @@ -222,65 +342,31 @@ public class Ver4DictEncoder implements DictEncoder { } } - private void writeShortcuts(ArrayList<WeightedString> shortcuts) { - if (null == shortcuts || shortcuts.isEmpty()) return; - - final int indexOfShortcutByteSize = mTriePos; - mTriePos += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), - target.mFrequency); - mTrieBuf[mTriePos++] = (byte)shortcutFlags; - final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos, - target.mWord); - mTriePos += shortcutShift; - } - final int shortcutByteSize = mTriePos - indexOfShortcutByteSize; - if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { - throw new RuntimeException("Shortcut list too large : " + shortcutByteSize); - } - BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, indexOfShortcutByteSize, - shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); - } - private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict) throws IOException { - final ByteArrayOutputStream bigramBuffer = new ByteArrayOutputStream(); - + mBigramWriter.openStreams(); for (final PtNodeArray nodeArray : flatNodes) { for (final PtNode ptNode : nodeArray.mData) { if (ptNode.mBigrams != null) { - final int startPos = bigramBuffer.size(); - mBigramAddressTable.set(ptNode.mTerminalId, startPos); - final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator(); - while (bigramIterator.hasNext()) { - final WeightedString bigram = bigramIterator.next(); - final PtNode target = - FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); - final int unigramFrequencyForThisWord = target.mFrequency; - final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags( - bigramIterator.hasNext(), 0, bigram.mFrequency, - unigramFrequencyForThisWord, bigram.mWord); - BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, bigramFlags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, target.mTerminalId, - FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); - } + mBigramWriter.writeBigramsForOneWord(ptNode.mTerminalId, + ptNode.mBigrams.iterator(), dict); } } } - bigramBuffer.writeTo(mBigramOutStream); + mBigramWriter.closeStreams(); } - private void writeBigramAddressSparseTable() throws IOException { - final File lookupIndexFile = - new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION); - final File contentFile = - new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION); - mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile); + private void writeShortcuts(final ArrayList<PtNodeArray> flatNodes) throws IOException { + mShortcutWriter.openStreams(); + for (final PtNodeArray nodeArray : flatNodes) { + for (final PtNode ptNode : nodeArray.mData) { + if (ptNode.mShortcutTargets != null && !ptNode.mShortcutTargets.isEmpty()) { + mShortcutWriter.writeShortcutForOneWord(ptNode.mTerminalId, + ptNode.mShortcutTargets.iterator()); + } + } + } + mShortcutWriter.closeStreams(); } @Override @@ -292,14 +378,13 @@ public class Ver4DictEncoder implements DictEncoder { @Override public void writePtNode(final PtNode ptNode, final int parentPosition, final FormatOptions formatOptions, final FusionDictionary dict) { - writePtNodeFlags(ptNode, parentPosition, formatOptions); + writePtNodeFlags(ptNode, formatOptions); writeParentPosition(parentPosition, ptNode, formatOptions); writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); if (ptNode.isTerminal()) { writeTerminalId(ptNode.mTerminalId); } writeChildrenPosition(ptNode, formatOptions); - writeShortcuts(ptNode.mShortcutTargets); } private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java new file mode 100644 index 000000000..3d8f186ba --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +/** + * An implementation of DictUpdater for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { + + @UsedForTesting + public Ver4DictUpdater(final File dictDirectory, final int factoryType) { + // DictUpdater must have an updatable DictBuffer. + super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) + ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); + } + + @Override + public void deleteWord(final String word) throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) openDictBuffer(); + readHeader(); + final int wordPos = getTerminalPosition(word); + if (wordPos != FormatSpec.NOT_VALID_WORD) { + mDictBuffer.position(wordPos); + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + mDictBuffer.position(wordPos); + mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags)); + } + } + + @Override + public void insertWord(final String word, final int frequency, + final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, + final boolean isNotAWord, final boolean isBlackListEntry) + throws IOException, UnsupportedFormatException { + // TODO: Implement this method. + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 7cf4f0c88..c8b62b6c8 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -230,10 +230,15 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB mSessions.remove(session); } + @UsedForTesting public void clearAndFlushDictionary() { // Clear the node structure on memory clear(); // Then flush the cleared state of the dictionary on disk. asyncFlashAllBinaryDictionary(); } + + /* package */ void decayIfNeeded() { + runGCIfRequired(false /* mindsBlockByGC */); + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java b/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java new file mode 100644 index 000000000..e9ca662e7 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.app.AlarmManager; +import android.app.PendingIntent; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; + +import java.util.concurrent.TimeUnit; + +/** + * Broadcast receiver for periodically updating decaying dictionaries. + */ +public class DictionaryDecayBroadcastReciever extends BroadcastReceiver { + /** + * The root domain for the personalization. + */ + private static final String PERSONALIZATION_DOMAIN = + "com.android.inputmethod.latin.personalization"; + + /** + * The action of the intent to tell the time to decay dictionaries. + */ + private static final String DICTIONARY_DECAY_INTENT_ACTION = + PERSONALIZATION_DOMAIN + ".DICT_DECAY"; + + /** + * Interval to update for decaying dictionaries. + */ + private static final long DICTIONARY_DECAY_INTERVAL = TimeUnit.MINUTES.toMillis(60); + + public static void setUpIntervalAlarmForDictionaryDecaying(Context context) { + AlarmManager alarmManager = (AlarmManager)context.getSystemService(Context.ALARM_SERVICE); + final Intent updateIntent = new Intent(DICTIONARY_DECAY_INTENT_ACTION); + updateIntent.setClass(context, DictionaryDecayBroadcastReciever.class); + final long alarmTime = System.currentTimeMillis() + DICTIONARY_DECAY_INTERVAL; + final PendingIntent pendingIntent = PendingIntent.getBroadcast(context, 0 /* requestCode */, + updateIntent, PendingIntent.FLAG_CANCEL_CURRENT); + if (null != alarmManager) alarmManager.setInexactRepeating(AlarmManager.RTC, + alarmTime, DICTIONARY_DECAY_INTERVAL, pendingIntent); + } + + @Override + public void onReceive(final Context context, final Intent intent) { + final String action = intent.getAction(); + if (action.equals(DICTIONARY_DECAY_INTENT_ACTION)) { + PersonalizationHelper.tryDecayingAllOpeningUserHistoryDictionary(); + } + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index 8c9484b12..221ddeeba 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentHashMap; public class PersonalizationHelper { private static final String TAG = PersonalizationHelper.class.getSimpleName(); private static final boolean DEBUG = false; - private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>> sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); @@ -62,6 +61,18 @@ public class PersonalizationHelper { } } + public static void tryDecayingAllOpeningUserHistoryDictionary() { + for (final ConcurrentHashMap.Entry<String, SoftReference<UserHistoryDictionary>> entry + : sLangUserHistoryDictCache.entrySet()) { + if (entry.getValue() != null) { + final UserHistoryDictionary dict = entry.getValue().get(); + if (dict != null) { + dict.decayIfNeeded(); + } + } + } + } + public static void registerPersonalizationDictionaryUpdateSession(final Context context, final PersonalizationDictionaryUpdateSession session, String locale) { final PersonalizationPredictionDictionary predictionDictionary = diff --git a/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java b/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java index b499c26b6..ef6ab2a38 100644 --- a/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java +++ b/java/src/com/android/inputmethod/latin/settings/DebugSettingsActivity.java @@ -38,4 +38,10 @@ public final class DebugSettingsActivity extends PreferenceActivity { super.onCreate(savedInstanceState); setTitle(R.string.english_ime_debug_settings); } + + // TODO: Uncomment the override annotation once we start using SDK version 19. + // @Override + public boolean isValidFragment(String fragmentName) { + return fragmentName.equals(DEFAULT_FRAGMENT); + } } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java b/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java index 6c3818651..ad68f8c37 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsActivity.java @@ -32,4 +32,10 @@ public final class SettingsActivity extends PreferenceActivity { intent.putExtra(EXTRA_NO_HEADERS, true); return intent; } + + // TODO: Uncomment the override annotation once we start using SDK version 19. + // @Override + public boolean isValidFragment(String fragmentName) { + return fragmentName.equals(DEFAULT_FRAGMENT); + } } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index ee322e91b..2abcdc7fa 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -45,8 +45,9 @@ import java.util.Locale; */ public final class SettingsValues { private static final String TAG = SettingsValues.class.getSimpleName(); - // "floatNegativeInfinity" is a special marker string for Float.NEGATIVE_INFINITE - // currently used for auto-correction + // "floatMaxValue" and "floatNegativeInfinity" are special marker strings for + // Float.NEGATIVE_INFINITE and Float.MAX_VALUE. Currently used for auto-correction settings. + private static final String FLOAT_MAX_VALUE_MARKER_STRING = "floatMaxValue"; private static final String FLOAT_NEGATIVE_INFINITY_MARKER_STRING = "floatNegativeInfinity"; // From resources: @@ -343,24 +344,28 @@ public final class SettingsValues { final String[] autoCorrectionThresholdValues = res.getStringArray( R.array.auto_correction_threshold_values); // When autoCorrectionThreshold is greater than 1.0, it's like auto correction is off. - float autoCorrectionThreshold = Float.MAX_VALUE; + final float autoCorrectionThreshold; try { final int arrayIndex = Integer.valueOf(currentAutoCorrectionSetting); if (arrayIndex >= 0 && arrayIndex < autoCorrectionThresholdValues.length) { final String val = autoCorrectionThresholdValues[arrayIndex]; - if (FLOAT_NEGATIVE_INFINITY_MARKER_STRING.equals(val)) { + if (FLOAT_MAX_VALUE_MARKER_STRING.equals(val)) { + autoCorrectionThreshold = Float.MAX_VALUE; + } else if (FLOAT_NEGATIVE_INFINITY_MARKER_STRING.equals(val)) { autoCorrectionThreshold = Float.NEGATIVE_INFINITY; } else { autoCorrectionThreshold = Float.parseFloat(val); } + } else { + autoCorrectionThreshold = Float.MAX_VALUE; } - } catch (NumberFormatException e) { + } catch (final NumberFormatException e) { // Whenever the threshold settings are correct, never come here. - autoCorrectionThreshold = Float.MAX_VALUE; Log.w(TAG, "Cannot load auto correction threshold setting." + " currentAutoCorrectionSetting: " + currentAutoCorrectionSetting + ", autoCorrectionThresholdValues: " + Arrays.toString(autoCorrectionThresholdValues), e); + return Float.MAX_VALUE; } return autoCorrectionThreshold; } diff --git a/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java b/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java index 119ca4755..aba563746 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/SpellCheckerSettingsActivity.java @@ -24,6 +24,8 @@ import android.preference.PreferenceActivity; * Spell checker preference screen. */ public final class SpellCheckerSettingsActivity extends PreferenceActivity { + private static final String DEFAULT_FRAGMENT = SpellCheckerSettingsFragment.class.getName(); + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -32,8 +34,14 @@ public final class SpellCheckerSettingsActivity extends PreferenceActivity { @Override public Intent getIntent() { final Intent modIntent = new Intent(super.getIntent()); - modIntent.putExtra(EXTRA_SHOW_FRAGMENT, SpellCheckerSettingsFragment.class.getName()); + modIntent.putExtra(EXTRA_SHOW_FRAGMENT, DEFAULT_FRAGMENT); modIntent.putExtra(EXTRA_NO_HEADERS, true); return modIntent; } + + // TODO: Uncomment the override annotation once we start using SDK version 19. + // @Override + public boolean isValidFragment(String fragmentName) { + return fragmentName.equals(DEFAULT_FRAGMENT); + } } diff --git a/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java b/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java index 44b201642..ff332cdee 100644 --- a/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/AdditionalSubtypeUtils.java @@ -61,10 +61,8 @@ public final class AdditionalSubtypeUtils { StringUtils.appendToCommaSplittableTextIfNotExists( IS_ADDITIONAL_SUBTYPE, layoutDisplayNameExtraValue); final int nameId = SubtypeLocaleUtils.getSubtypeNameId(localeString, keyboardLayoutSetName); - return new InputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, - localeString, KEYBOARD_MODE, layoutExtraValue + "," + additionalSubtypeExtraValue - + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE - + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE, false, false); + return buildInputMethodSubtype( + nameId, localeString, layoutExtraValue, additionalSubtypeExtraValue); } public static String getPrefSubtype(final InputMethodSubtype subtype) { @@ -137,4 +135,27 @@ public final class AdditionalSubtypeUtils { } return sb.toString(); } + + private static InputMethodSubtype buildInputMethodSubtype(int nameId, String localeString, + String layoutExtraValue, String additionalSubtypeExtraValue) { + // CAVEAT! If you want to change subtypeId after changing the extra values, + // you must change "getInputMethodSubtypeId". But it will remove the additional keyboard + // from the current users. So, you should be really careful to change it. + final int subtypeId = getInputMethodSubtypeId(nameId, localeString, layoutExtraValue, + additionalSubtypeExtraValue); + // TODO: Use InputMethodSubtypeBuilder once we use SDK version 19. + return new InputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, + localeString, KEYBOARD_MODE, layoutExtraValue + "," + additionalSubtypeExtraValue + + "," + Constants.Subtype.ExtraValue.ASCII_CAPABLE + + "," + Constants.Subtype.ExtraValue.EMOJI_CAPABLE, false, false, + subtypeId); + } + + private static int getInputMethodSubtypeId(int nameId, String localeString, + String layoutExtraValue, String additionalSubtypeExtraValue) { + // TODO: Use InputMethodSubtypeBuilder once we use SDK version 19. + return (new InputMethodSubtype(nameId, R.drawable.ic_ime_switcher_dark, + localeString, KEYBOARD_MODE, layoutExtraValue + "," + additionalSubtypeExtraValue, + false, false)).hashCode(); + } } |