diff options
Diffstat (limited to 'java/src')
47 files changed, 2189 insertions, 1514 deletions
diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java index 8ce126330..7639432aa 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java @@ -157,7 +157,7 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider // Add the virtual children of the root View. final Keyboard keyboard = mKeyboardView.getKeyboard(); - final Key[] keys = keyboard.mKeys; + final Key[] keys = keyboard.getKeys(); for (Key key : keys) { final int childVirtualViewId = generateVirtualViewIdForKey(key); rootInfo.addChild(mKeyboardView, childVirtualViewId); @@ -300,7 +300,7 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider } mVirtualViewIdToKey.clear(); - final Key[] keys = keyboard.mKeys; + final Key[] keys = keyboard.getKeys(); for (Key key : keys) { final int virtualViewId = generateVirtualViewIdForKey(key); mVirtualViewIdToKey.put(virtualViewId, key); diff --git a/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java b/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java index 73896dfd3..b3bb767af 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java @@ -357,7 +357,6 @@ public final class AccessibleKeyboardViewProxy extends AccessibilityDelegateComp break; case KeyboardId.ELEMENT_ALPHABET_AUTOMATIC_SHIFTED: case KeyboardId.ELEMENT_ALPHABET_MANUAL_SHIFTED: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: text = context.getText(R.string.spoken_description_shiftmode_on); break; default: @@ -389,7 +388,6 @@ public final class AccessibleKeyboardViewProxy extends AccessibilityDelegateComp resId = R.string.spoken_description_mode_alpha; break; case KeyboardId.ELEMENT_SYMBOLS: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_mode_symbol; break; case KeyboardId.ELEMENT_PHONE: diff --git a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java index 58624a2e6..085ca93d5 100644 --- a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java +++ b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java @@ -156,7 +156,6 @@ public final class KeyCodeDescriptionMapper { resId = R.string.spoken_description_to_symbol; break; case KeyboardId.ELEMENT_SYMBOLS: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_to_alpha; break; case KeyboardId.ELEMENT_PHONE: @@ -191,7 +190,6 @@ public final class KeyCodeDescriptionMapper { break; case KeyboardId.ELEMENT_ALPHABET_AUTOMATIC_SHIFTED: case KeyboardId.ELEMENT_ALPHABET_MANUAL_SHIFTED: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_shift_shifted; break; default: diff --git a/java/src/com/android/inputmethod/keyboard/Keyboard.java b/java/src/com/android/inputmethod/keyboard/Keyboard.java index c40ddeabe..23f037fbd 100644 --- a/java/src/com/android/inputmethod/keyboard/Keyboard.java +++ b/java/src/com/android/inputmethod/keyboard/Keyboard.java @@ -51,6 +51,11 @@ public class Keyboard { /** Total width of the keyboard, including the padding and keys */ public final int mOccupiedWidth; + /** Base height of the keyboard, used to calculate rows' height */ + public final int mBaseHeight; + /** Base width of the keyboard, used to calculate keys' width */ + public final int mBaseWidth; + /** The padding above the keyboard */ public final int mTopPadding; /** Default gap between rows */ @@ -69,7 +74,7 @@ public class Keyboard { public final int mMaxMoreKeysKeyboardColumn; /** Array of keys and icons in this keyboard */ - public final Key[] mKeys; + private final Key[] mKeys; public final Key[] mShiftKeys; public final Key[] mAltCodeKeysWhileTyping; public final KeyboardIconsSet mIconsSet; @@ -84,6 +89,8 @@ public class Keyboard { mThemeId = params.mThemeId; mOccupiedHeight = params.mOccupiedHeight; mOccupiedWidth = params.mOccupiedWidth; + mBaseHeight = params.mBaseHeight; + mBaseWidth = params.mBaseWidth; mMostCommonKeyHeight = params.mMostCommonKeyHeight; mMostCommonKeyWidth = params.mMostCommonKeyWidth; mMoreKeysTemplate = params.mMoreKeysTemplate; @@ -104,6 +111,30 @@ public class Keyboard { mProximityCharsCorrectionEnabled = params.mProximityCharsCorrectionEnabled; } + protected Keyboard(final Keyboard keyboard) { + mId = keyboard.mId; + mThemeId = keyboard.mThemeId; + mOccupiedHeight = keyboard.mOccupiedHeight; + mOccupiedWidth = keyboard.mOccupiedWidth; + mBaseHeight = keyboard.mBaseHeight; + mBaseWidth = keyboard.mBaseWidth; + mMostCommonKeyHeight = keyboard.mMostCommonKeyHeight; + mMostCommonKeyWidth = keyboard.mMostCommonKeyWidth; + mMoreKeysTemplate = keyboard.mMoreKeysTemplate; + mMaxMoreKeysKeyboardColumn = keyboard.mMaxMoreKeysKeyboardColumn; + mKeyVisualAttributes = keyboard.mKeyVisualAttributes; + mTopPadding = keyboard.mTopPadding; + mVerticalGap = keyboard.mVerticalGap; + + mKeys = keyboard.mKeys; + mShiftKeys = keyboard.mShiftKeys; + mAltCodeKeysWhileTyping = keyboard.mAltCodeKeysWhileTyping; + mIconsSet = keyboard.mIconsSet; + + mProximityInfo = keyboard.mProximityInfo; + mProximityCharsCorrectionEnabled = keyboard.mProximityCharsCorrectionEnabled; + } + public boolean hasProximityCharsCorrection(final int code) { if (!mProximityCharsCorrectionEnabled) { return false; @@ -120,6 +151,10 @@ public class Keyboard { return mProximityInfo; } + public Key[] getKeys() { + return mKeys; + } + public Key getKey(final int code) { if (code == Constants.CODE_UNSPECIFIED) { return null; @@ -130,7 +165,7 @@ public class Keyboard { return mKeyCache.valueAt(index); } - for (final Key key : mKeys) { + for (final Key key : getKeys()) { if (key.getCode() == code) { mKeyCache.put(code, key); return key; @@ -146,7 +181,7 @@ public class Keyboard { return true; } - for (final Key key : mKeys) { + for (final Key key : getKeys()) { if (key == aKey) { mKeyCache.put(key.getCode(), key); return true; diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardId.java b/java/src/com/android/inputmethod/keyboard/KeyboardId.java index 1dc3c6a4c..53748bb58 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardId.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardId.java @@ -50,10 +50,16 @@ public final class KeyboardId { public static final int ELEMENT_ALPHABET_SHIFT_LOCKED = 3; public static final int ELEMENT_ALPHABET_SHIFT_LOCK_SHIFTED = 4; public static final int ELEMENT_SYMBOLS = 5; - public static final int ELEMENT_SYMBOLS_SHIFTED = 6; public static final int ELEMENT_PHONE = 7; public static final int ELEMENT_PHONE_SYMBOLS = 8; public static final int ELEMENT_NUMBER = 9; + public static final int ELEMENT_EMOJI_RECENTS = 10; + public static final int ELEMENT_EMOJI_CATEGORY1 = 11; + public static final int ELEMENT_EMOJI_CATEGORY2 = 12; + public static final int ELEMENT_EMOJI_CATEGORY3 = 13; + public static final int ELEMENT_EMOJI_CATEGORY4 = 14; + public static final int ELEMENT_EMOJI_CATEGORY5 = 15; + public static final int ELEMENT_EMOJI_CATEGORY6 = 16; public final InputMethodSubtype mSubtype; public final Locale mLocale; @@ -213,10 +219,16 @@ public final class KeyboardId { case ELEMENT_ALPHABET_SHIFT_LOCKED: return "alphabetShiftLocked"; case ELEMENT_ALPHABET_SHIFT_LOCK_SHIFTED: return "alphabetShiftLockShifted"; case ELEMENT_SYMBOLS: return "symbols"; - case ELEMENT_SYMBOLS_SHIFTED: return "symbolsShifted"; case ELEMENT_PHONE: return "phone"; case ELEMENT_PHONE_SYMBOLS: return "phoneSymbols"; case ELEMENT_NUMBER: return "number"; + case ELEMENT_EMOJI_RECENTS: return "emojiRecents"; + case ELEMENT_EMOJI_CATEGORY1: return "emojiCategory1"; + case ELEMENT_EMOJI_CATEGORY2: return "emojiCategory2"; + case ELEMENT_EMOJI_CATEGORY3: return "emojiCategory3"; + case ELEMENT_EMOJI_CATEGORY4: return "emojiCategory4"; + case ELEMENT_EMOJI_CATEGORY5: return "emojiCategory5"; + case ELEMENT_EMOJI_CATEGORY6: return "emojiCategory6"; default: return null; } } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index 1ea0f8b96..098c8b3df 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -58,12 +58,8 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } private static final KeyboardTheme[] KEYBOARD_THEMES = { - new KeyboardTheme(0, R.style.KeyboardTheme), - new KeyboardTheme(1, R.style.KeyboardTheme_HighContrast), - new KeyboardTheme(6, R.style.KeyboardTheme_Stone), - new KeyboardTheme(7, R.style.KeyboardTheme_Stone_Bold), - new KeyboardTheme(8, R.style.KeyboardTheme_Gingerbread), - new KeyboardTheme(5, R.style.KeyboardTheme_IceCreamSandwich), + new KeyboardTheme(0, R.style.KeyboardTheme_ICS), + new KeyboardTheme(1, R.style.KeyboardTheme_GB), }; private SubtypeSwitcher mSubtypeSwitcher; @@ -121,8 +117,9 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } catch (NumberFormatException e) { // Format error, keyboard theme is default to 0. } - Log.w(TAG, "Illegal keyboard theme in preference: " + themeIndex + ", default to 0"); - return KEYBOARD_THEMES[0]; + Log.w(TAG, "Illegal keyboard theme in preference: " + themeIndex + ", default to " + + defaultIndex); + return KEYBOARD_THEMES[Integer.valueOf(defaultIndex)]; } private void setContextThemeWrapper(final Context context, final KeyboardTheme keyboardTheme) { @@ -258,12 +255,6 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { // Implements {@link KeyboardState.SwitchActions}. @Override - public void setSymbolsShiftedKeyboard() { - setKeyboard(mKeyboardLayoutSet.getKeyboard(KeyboardId.ELEMENT_SYMBOLS_SHIFTED)); - } - - // Implements {@link KeyboardState.SwitchActions}. - @Override public void requestUpdatingShiftState() { mState.onUpdateShiftState(mLatinIME.getCurrentAutoCapsState(), mLatinIME.getCurrentRecapitalizeState()); diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java index d049a861e..0ef6802ca 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java @@ -285,7 +285,7 @@ public class KeyboardView extends View { // TODO: Confirm if it's really required to draw all keys when hardware acceleration is on. if (drawAllKeys || isHardwareAccelerated) { // Draw all keys. - for (final Key key : mKeyboard.mKeys) { + for (final Key key : mKeyboard.getKeys()) { onDrawKey(key, canvas, paint); } } else { diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java b/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java index a2001cb8f..6b76e2461 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java @@ -39,7 +39,7 @@ public final class MoreKeysDetector extends KeyDetector { Key nearestKey = null; int nearestDist = (y < 0) ? mSlideAllowanceSquareTop : mSlideAllowanceSquare; - for (final Key key : getKeyboard().mKeys) { + for (final Key key : getKeyboard().getKeys()) { final int dist = key.squaredDistanceToEdge(touchX, touchY); if (dist < nearestDist) { nearestKey = key; diff --git a/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java new file mode 100644 index 000000000..c10fdbace --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import com.android.inputmethod.latin.Constants; + +/** + * The string parser of codesArray specification for <GridRows />. The attribute codesArray is an + * array of string. + * Each element of the array defines a key label by specifying a code point as a hexadecimal string. + * A key label may consist of multiple code points separated by comma. + * Each element of the array optionally can have an output text definition after vertical bar + * marker. An output text may consist of multiple code points separated by comma. + * The format of the codesArray element should be: + * <pre> + * codePointInHex[,codePoint2InHex]*(|outputTextCodePointInHex[,outputTextCodePoint2InHex]*)? + * </pre> + */ +// TODO: Write unit tests for this class. +public final class CodesArrayParser { + // Constants for parsing. + private static final char COMMA = ','; + private static final char VERTICAL_BAR = '|'; + private static final String COMMA_STRING = ","; + private static final int BASE_HEX = 16; + + private CodesArrayParser() { + // This utility class is not publicly instantiable. + } + + private static String getLabelSpec(final String codesArraySpec) { + final int pos = codesArraySpec.indexOf(VERTICAL_BAR); + return (pos < 0) ? codesArraySpec : codesArraySpec.substring(0, pos); + } + + public static String parseLabel(final String codesArraySpec) { + final String labelSpec = getLabelSpec(codesArraySpec); + final StringBuilder sb = new StringBuilder(); + for (final String codeInHex : labelSpec.split(COMMA_STRING)) { + final int codePoint = Integer.parseInt(codeInHex, BASE_HEX); + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } + + private static String getCodeSpec(final String codesArraySpec) { + final int pos = codesArraySpec.indexOf(VERTICAL_BAR); + return (pos < 0) ? codesArraySpec : codesArraySpec.substring(pos + 1); + } + + public static int parseCode(final String codesArraySpec) { + final String codeSpec = getCodeSpec(codesArraySpec); + if (codeSpec.indexOf(COMMA) < 0) { + return Integer.parseInt(codeSpec, BASE_HEX); + } + return Constants.CODE_OUTPUT_TEXT; + } + + public static String parseOutputText(final String codesArraySpec) { + final String codeSpec = getCodeSpec(codesArraySpec); + if (codeSpec.indexOf(COMMA) < 0) { + return null; + } + final StringBuilder sb = new StringBuilder(); + for (final String codeInHex : codeSpec.split(COMMA_STRING)) { + final int codePoint = Integer.parseInt(codeInHex, BASE_HEX); + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } +} diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java index 3f0773e15..8c70389ba 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java @@ -29,6 +29,7 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.KeyboardId; +import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.utils.ResourceUtils; import com.android.inputmethod.latin.utils.RunInLocale; @@ -113,6 +114,7 @@ import java.util.Locale; * </pre> */ +// TODO: Write unit tests for this class. public class KeyboardBuilder<KP extends KeyboardParams> { private static final String BUILDER_TAG = "Keyboard.Builder"; private static final boolean DEBUG = false; @@ -120,6 +122,7 @@ public class KeyboardBuilder<KP extends KeyboardParams> { // Keyboard XML Tags private static final String TAG_KEYBOARD = "Keyboard"; private static final String TAG_ROW = "Row"; + private static final String TAG_GRID_ROWS = "GridRows"; private static final String TAG_KEY = "Key"; private static final String TAG_SPACER = "Spacer"; private static final String TAG_INCLUDE = "include"; @@ -312,6 +315,9 @@ public class KeyboardBuilder<KP extends KeyboardParams> { startRow(row); } parseRowContent(parser, row, skip); + } else if (TAG_GRID_ROWS.equals(tag)) { + if (DEBUG) startTag("<%s>%s", TAG_GRID_ROWS, skip ? " skipped" : ""); + parseGridRows(parser, skip); } else if (TAG_INCLUDE.equals(tag)) { parseIncludeKeyboardContent(parser, skip); } else if (TAG_SWITCH.equals(tag)) { @@ -389,6 +395,73 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } } + private void parseGridRows(final XmlPullParser parser, final boolean skip) + throws XmlPullParserException, IOException { + if (skip) { + XmlParseUtils.checkEndTag(TAG_GRID_ROWS, parser); + if (DEBUG) { + startEndTag("<%s /> skipped", TAG_GRID_ROWS); + } + return; + } + final KeyboardRow gridRows = new KeyboardRow(mResources, mParams, parser, mCurrentY); + final TypedArray gridRowAttr = mResources.obtainAttributes( + Xml.asAttributeSet(parser), R.styleable.Keyboard_GridRows); + final int codesArrayId = gridRowAttr.getResourceId( + R.styleable.Keyboard_GridRows_codesArray, 0); + final int textsArrayId = gridRowAttr.getResourceId( + R.styleable.Keyboard_GridRows_textsArray, 0); + gridRowAttr.recycle(); + if (codesArrayId == 0 && textsArrayId == 0) { + throw new XmlParseUtils.ParseException( + "Missing codesArray or textsArray attributes", parser); + } + if (codesArrayId != 0 && textsArrayId != 0) { + throw new XmlParseUtils.ParseException( + "Both codesArray and textsArray attributes specifed", parser); + } + final String[] array = mResources.getStringArray( + codesArrayId != 0 ? codesArrayId : textsArrayId); + final int counts = array.length; + final float keyWidth = gridRows.getKeyWidth(null, 0.0f); + final int numColumns = (int)(mParams.mOccupiedWidth / keyWidth); + for (int index = 0; index < counts; index += numColumns) { + final KeyboardRow row = new KeyboardRow(mResources, mParams, parser, mCurrentY); + startRow(row); + for (int c = 0; c < numColumns; c++) { + final int i = index + c; + if (i >= counts) { + break; + } + final String label; + final int code; + final String outputText; + if (codesArrayId != 0) { + final String codeArraySpec = array[i]; + label = CodesArrayParser.parseLabel(codeArraySpec); + code = CodesArrayParser.parseCode(codeArraySpec); + outputText = CodesArrayParser.parseOutputText(codeArraySpec); + } else { + final String textArraySpec = array[i]; + // TODO: Utilize KeySpecParser or write more generic TextsArrayParser. + label = textArraySpec; + code = Constants.CODE_OUTPUT_TEXT; + outputText = textArraySpec + (char)Constants.CODE_SPACE; + } + final int x = (int)row.getKeyX(null); + final int y = row.getKeyY(); + final Key key = new Key(mParams, label, null /* hintLabel */, 0 /* iconId */, + code, outputText, x, y, (int)keyWidth, (int)row.getRowHeight(), + row.getDefaultKeyLabelFlags(), row.getDefaultBackgroundType()); + endKey(key); + row.advanceXPos(keyWidth); + } + endRow(row); + } + + XmlParseUtils.checkEndTag(TAG_GRID_ROWS, parser); + } + private void parseKey(final XmlPullParser parser, final KeyboardRow row, final boolean skip) throws XmlPullParserException, IOException { if (skip) { @@ -744,7 +817,10 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } private void endKeyboard() { - // nothing to do here. + // {@link #parseGridRows(XmlPullParser,boolean)} may populate keyboard rows higher than + // previously expected. + final int actualHeight = mCurrentY - mParams.mVerticalGap + mParams.mBottomPadding; + mParams.mOccupiedHeight = Math.max(mParams.mOccupiedHeight, actualHeight); } private void addEdgeSpace(final float width, final KeyboardRow row) { diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java index 164910dd4..0b10a1d1a 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java @@ -46,7 +46,6 @@ public final class KeyboardState { public void setAlphabetShiftLockedKeyboard(); public void setAlphabetShiftLockShiftedKeyboard(); public void setSymbolsKeyboard(); - public void setSymbolsShiftedKeyboard(); /** * Request to call back {@link KeyboardState#onUpdateShiftState(int, int)}. @@ -64,21 +63,17 @@ public final class KeyboardState { private ModifierKeyState mSymbolKeyState = new ModifierKeyState("Symbol"); // TODO: Merge {@link #mSwitchState}, {@link #mIsAlphabetMode}, {@link #mAlphabetShiftState}, - // {@link #mIsSymbolShifted}, {@link #mPrevMainKeyboardWasShiftLocked}, and - // {@link #mPrevSymbolsKeyboardWasShifted} into single state variable. + // {@link #mPrevMainKeyboardWasShiftLocked} into single state variable. private static final int SWITCH_STATE_ALPHA = 0; private static final int SWITCH_STATE_SYMBOL_BEGIN = 1; private static final int SWITCH_STATE_SYMBOL = 2; private static final int SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL = 3; - private static final int SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE = 4; private static final int SWITCH_STATE_MOMENTARY_ALPHA_SHIFT = 5; private int mSwitchState = SWITCH_STATE_ALPHA; private boolean mIsAlphabetMode; private AlphabetShiftState mAlphabetShiftState = new AlphabetShiftState(); - private boolean mIsSymbolShifted; private boolean mPrevMainKeyboardWasShiftLocked; - private boolean mPrevSymbolsKeyboardWasShifted; private int mRecapitalizeMode; // For handling double tap. @@ -100,7 +95,7 @@ public final class KeyboardState { if (mIsAlphabetShiftLocked) return "ALPHABET_SHIFT_LOCKED"; return "ALPHABET_" + shiftModeToString(mShiftMode); } else { - return "SYMBOLS_" + shiftModeToString(mShiftMode); + return "SYMBOLS"; } } } @@ -117,7 +112,6 @@ public final class KeyboardState { // Reset alphabet shift state. mAlphabetShiftState.setShiftLocked(false); mPrevMainKeyboardWasShiftLocked = false; - mPrevSymbolsKeyboardWasShifted = false; mShiftKeyState.onRelease(); mSymbolKeyState.onRelease(); onRestoreKeyboardState(); @@ -137,7 +131,6 @@ public final class KeyboardState { : (mAlphabetShiftState.isShiftedOrShiftLocked() ? MANUAL_SHIFT : UNSHIFT); } else { state.mIsAlphabetShiftLocked = mPrevMainKeyboardWasShiftLocked; - state.mShiftMode = mIsSymbolShifted ? MANUAL_SHIFT : UNSHIFT; } state.mIsValid = true; if (DEBUG_EVENT) { @@ -153,11 +146,7 @@ public final class KeyboardState { if (!state.mIsValid || state.mIsAlphabetMode) { setAlphabetKeyboard(); } else { - if (state.mShiftMode == MANUAL_SHIFT) { - setSymbolsShiftedKeyboard(); - } else { - setSymbolsKeyboard(); - } + setSymbolsKeyboard(); } if (!state.mIsValid) return; @@ -233,14 +222,8 @@ public final class KeyboardState { } if (mIsAlphabetMode) { mPrevMainKeyboardWasShiftLocked = mAlphabetShiftState.isShiftLocked(); - if (mPrevSymbolsKeyboardWasShifted) { - setSymbolsShiftedKeyboard(); - } else { - setSymbolsKeyboard(); - } - mPrevSymbolsKeyboardWasShifted = false; + setSymbolsKeyboard(); } else { - mPrevSymbolsKeyboardWasShifted = mIsSymbolShifted; setAlphabetKeyboard(); if (mPrevMainKeyboardWasShiftLocked) { setShiftLocked(true); @@ -257,7 +240,6 @@ public final class KeyboardState { } if (mIsAlphabetMode) return; - mPrevSymbolsKeyboardWasShifted = mIsSymbolShifted; setAlphabetKeyboard(); if (mPrevMainKeyboardWasShiftLocked) { setShiftLocked(true); @@ -265,14 +247,6 @@ public final class KeyboardState { mPrevMainKeyboardWasShiftLocked = false; } - private void toggleShiftInSymbols() { - if (mIsSymbolShifted) { - setSymbolsKeyboard(); - } else { - setSymbolsShiftedKeyboard(); - } - } - private void setAlphabetKeyboard() { if (DEBUG_ACTION) { Log.d(TAG, "setAlphabetKeyboard"); @@ -280,7 +254,6 @@ public final class KeyboardState { mSwitchActions.setAlphabetKeyboard(); mIsAlphabetMode = true; - mIsSymbolShifted = false; mRecapitalizeMode = RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE; mSwitchState = SWITCH_STATE_ALPHA; mSwitchActions.requestUpdatingShiftState(); @@ -292,19 +265,6 @@ public final class KeyboardState { } mSwitchActions.setSymbolsKeyboard(); mIsAlphabetMode = false; - mIsSymbolShifted = false; - // Reset alphabet shift state. - mAlphabetShiftState.setShiftLocked(false); - mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; - } - - private void setSymbolsShiftedKeyboard() { - if (DEBUG_ACTION) { - Log.d(TAG, "setSymbolsShiftedKeyboard"); - } - mSwitchActions.setSymbolsShiftedKeyboard(); - mIsAlphabetMode = false; - mIsSymbolShifted = true; // Reset alphabet shift state. mAlphabetShiftState.setShiftLocked(false); mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; @@ -357,7 +317,7 @@ public final class KeyboardState { } else if (code == Constants.CODE_CAPSLOCK) { setShiftLocked(!mAlphabetShiftState.isShiftLocked()); } else if (code == Constants.CODE_SWITCH_ALPHA_SYMBOL) { - onReleaseSymbol(withSliding); + onReleaseSymbol(); } } @@ -367,16 +327,11 @@ public final class KeyboardState { mSwitchState = SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL; } - private void onReleaseSymbol(final boolean withSliding) { + private void onReleaseSymbol() { if (mSymbolKeyState.isChording()) { // Switch back to the previous keyboard mode if the user chords the mode change key and // another key, then releases the mode change key. toggleAlphabetAndSymbols(); - } else if (!withSliding) { - // If the mode change key is being released without sliding, we should forget the - // previous symbols keyboard shift state and simply switch back to symbols layout - // (never symbols shifted) next time the mode gets changed to symbols layout. - mPrevSymbolsKeyboardWasShifted = false; } mSymbolKeyState.onRelease(); } @@ -442,47 +397,43 @@ public final class KeyboardState { if (RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE != mRecapitalizeMode) { return; } - if (mIsAlphabetMode) { - mIsInDoubleTapShiftKey = mSwitchActions.isInDoubleTapShiftKeyTimeout(); - if (!mIsInDoubleTapShiftKey) { - // This is first tap. - mSwitchActions.startDoubleTapShiftKeyTimer(); - } - if (mIsInDoubleTapShiftKey) { - if (mAlphabetShiftState.isManualShifted() || mIsInAlphabetUnshiftedFromShifted) { - // Shift key has been double tapped while in manual shifted or automatic - // shifted state. - setShiftLocked(true); - } else { - // Shift key has been double tapped while in normal state. This is the second - // tap to disable shift locked state, so just ignore this. - } + if (!mIsAlphabetMode) { + // There is no shift key on symbols keyboard. + return; + } + mIsInDoubleTapShiftKey = mSwitchActions.isInDoubleTapShiftKeyTimeout(); + if (!mIsInDoubleTapShiftKey) { + // This is first tap. + mSwitchActions.startDoubleTapShiftKeyTimer(); + } + if (mIsInDoubleTapShiftKey) { + if (mAlphabetShiftState.isManualShifted() || mIsInAlphabetUnshiftedFromShifted) { + // Shift key has been double tapped while in manual shifted or automatic + // shifted state. + setShiftLocked(true); } else { - if (mAlphabetShiftState.isShiftLocked()) { - // Shift key is pressed while shift locked state, we will treat this state as - // shift lock shifted state and mark as if shift key pressed while normal state. - setShifted(SHIFT_LOCK_SHIFTED); - mShiftKeyState.onPress(); - } else if (mAlphabetShiftState.isAutomaticShifted()) { - // Shift key is pressed while automatic shifted, we have to move to manual - // shifted. - setShifted(MANUAL_SHIFT); - mShiftKeyState.onPress(); - } else if (mAlphabetShiftState.isShiftedOrShiftLocked()) { - // In manual shifted state, we just record shift key has been pressing while - // shifted state. - mShiftKeyState.onPressOnShifted(); - } else { - // In base layout, chording or manual shifted mode is started. - setShifted(MANUAL_SHIFT); - mShiftKeyState.onPress(); - } + // Shift key has been double tapped while in normal state. This is the second + // tap to disable shift locked state, so just ignore this. } } else { - // In symbol mode, just toggle symbol and symbol more keyboard. - toggleShiftInSymbols(); - mSwitchState = SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE; - mShiftKeyState.onPress(); + if (mAlphabetShiftState.isShiftLocked()) { + // Shift key is pressed while shift locked state, we will treat this state as + // shift lock shifted state and mark as if shift key pressed while normal state. + setShifted(SHIFT_LOCK_SHIFTED); + mShiftKeyState.onPress(); + } else if (mAlphabetShiftState.isAutomaticShifted()) { + // Shift key is pressed while automatic shifted, we have to move to manual shifted. + setShifted(MANUAL_SHIFT); + mShiftKeyState.onPress(); + } else if (mAlphabetShiftState.isShiftedOrShiftLocked()) { + // In manual shifted state, we just record shift key has been pressing while + // shifted state. + mShiftKeyState.onPressOnShifted(); + } else { + // In base layout, chording or manual shifted mode is started. + setShifted(MANUAL_SHIFT); + mShiftKeyState.onPress(); + } } } @@ -537,11 +488,7 @@ public final class KeyboardState { mIsInAlphabetUnshiftedFromShifted = true; } } else { - // In symbol mode, switch back to the previous keyboard mode if the user chords the - // shift key and another key, then releases the shift key. - if (mShiftKeyState.isChording()) { - toggleShiftInSymbols(); - } + // There is no shift key on symbols keyboard. } mShiftKeyState.onRelease(); } @@ -555,9 +502,6 @@ public final class KeyboardState { case SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL: toggleAlphabetAndSymbols(); break; - case SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE: - toggleShiftInSymbols(); - break; case SWITCH_STATE_MOMENTARY_ALPHA_SHIFT: setAlphabetKeyboard(); break; @@ -585,12 +529,6 @@ public final class KeyboardState { } } break; - case SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE: - if (code == Constants.CODE_SHIFT) { - // Detected only the shift key has been pressed on symbol layout, and then released. - mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; - } - break; case SWITCH_STATE_SYMBOL_BEGIN: if (!isSpaceCharacter(code) && (Constants.isLetterCode(code) || code == Constants.CODE_OUTPUT_TEXT)) { @@ -602,7 +540,6 @@ public final class KeyboardState { // characters followed by a space/enter. if (isSpaceCharacter(code)) { toggleAlphabetAndSymbols(); - mPrevSymbolsKeyboardWasShifted = false; } break; } @@ -628,7 +565,6 @@ public final class KeyboardState { case SWITCH_STATE_SYMBOL_BEGIN: return "SYMBOL-BEGIN"; case SWITCH_STATE_SYMBOL: return "SYMBOL"; case SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL: return "MOMENTARY-ALPHA-SYMBOL"; - case SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE: return "MOMENTARY-SYMBOL-MORE"; case SWITCH_STATE_MOMENTARY_ALPHA_SHIFT: return "MOMENTARY-ALPHA_SHIFT"; default: return null; } @@ -636,8 +572,7 @@ public final class KeyboardState { @Override public String toString() { - return "[keyboard=" + (mIsAlphabetMode ? mAlphabetShiftState.toString() - : (mIsSymbolShifted ? "SYMBOLS_SHIFTED" : "SYMBOLS")) + return "[keyboard=" + (mIsAlphabetMode ? mAlphabetShiftState.toString() : "SYMBOLS") + " shift=" + mShiftKeyState + " symbol=" + mSymbolKeyState + " switch=" + switchStateToString(mSwitchState) + "]"; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index bb49f4758..c6d652c0e 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -153,8 +153,8 @@ public final class KeyboardTextsSet { /* 48 */ "single_angle_quotes", /* 49 */ "double_angle_quotes", /* 50 */ "more_keys_for_currency_dollar", - /* 51 */ "keylabel_for_currency_generic", - /* 52 */ "more_keys_for_currency_generic", + /* 51 */ "keylabel_for_currency", + /* 52 */ "more_keys_for_currency", /* 53 */ "more_keys_for_punctuation", /* 54 */ "more_keys_for_star", /* 55 */ "more_keys_for_bullet", @@ -226,32 +226,29 @@ public final class KeyboardTextsSet { /* 121 */ "shortcut_as_more_key", /* 122 */ "action_next_as_more_key", /* 123 */ "action_previous_as_more_key", - /* 124 */ "label_to_more_symbol_key", - /* 125 */ "label_to_more_symbol_for_tablet_key", - /* 126 */ "label_tab_key", - /* 127 */ "label_to_phone_numeric_key", - /* 128 */ "label_to_phone_symbols_key", - /* 129 */ "label_time_am", - /* 130 */ "label_time_pm", - /* 131 */ "label_to_symbol_key_pcqwerty", - /* 132 */ "keylabel_for_popular_domain", - /* 133 */ "more_keys_for_popular_domain", - /* 134 */ "more_keys_for_smiley", - /* 135 */ "single_laqm_raqm", - /* 136 */ "single_laqm_raqm_rtl", - /* 137 */ "single_raqm_laqm", - /* 138 */ "double_laqm_raqm", - /* 139 */ "double_laqm_raqm_rtl", - /* 140 */ "double_raqm_laqm", - /* 141 */ "single_lqm_rqm", - /* 142 */ "single_9qm_lqm", - /* 143 */ "single_9qm_rqm", - /* 144 */ "double_lqm_rqm", - /* 145 */ "double_9qm_lqm", - /* 146 */ "double_9qm_rqm", - /* 147 */ "more_keys_for_single_quote", - /* 148 */ "more_keys_for_double_quote", - /* 149 */ "more_keys_for_tablet_double_quote", + /* 124 */ "label_tab_key", + /* 125 */ "label_to_phone_numeric_key", + /* 126 */ "label_to_phone_symbols_key", + /* 127 */ "label_time_am", + /* 128 */ "label_time_pm", + /* 129 */ "keylabel_for_popular_domain", + /* 130 */ "more_keys_for_popular_domain", + /* 131 */ "more_keys_for_smiley", + /* 132 */ "single_laqm_raqm", + /* 133 */ "single_laqm_raqm_rtl", + /* 134 */ "single_raqm_laqm", + /* 135 */ "double_laqm_raqm", + /* 136 */ "double_laqm_raqm_rtl", + /* 137 */ "double_raqm_laqm", + /* 138 */ "single_lqm_rqm", + /* 139 */ "single_9qm_lqm", + /* 140 */ "single_9qm_rqm", + /* 141 */ "double_lqm_rqm", + /* 142 */ "double_9qm_lqm", + /* 143 */ "double_9qm_rqm", + /* 144 */ "more_keys_for_single_quote", + /* 145 */ "more_keys_for_double_quote", + /* 146 */ "more_keys_for_tablet_double_quote", }; private static final String EMPTY = ""; @@ -289,8 +286,7 @@ public final class KeyboardTextsSet { // U+2666: "♦" BLACK DIAMOND SUIT // U+2663: "♣" BLACK CLUB SUIT /* 55 */ "\u266A,\u2665,\u2660,\u2666,\u2663", - // U+00B1: "±" PLUS-MINUS SIGN - /* 56 */ "\u00B1", + /* 56 */ EMPTY, // The all letters need to be mirrored are found at // http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt /* 57 */ "!fixedColumnOrder!3,<,{,[", @@ -380,28 +376,22 @@ public final class KeyboardTextsSet { /* 121 */ "!icon/shortcut_key|!code/key_shortcut", /* 122 */ "!hasLabels!,!text/label_next_key|!code/key_action_next", /* 123 */ "!hasLabels!,!text/label_previous_key|!code/key_action_previous", - // Label for "switch to more symbol" modifier key. Must be short to fit on key! - /* 124 */ "= \\ <", - // Label for "switch to more symbol" modifier key on tablets. Must be short to fit on key! - /* 125 */ "~ \\ {", // Label for "Tab" key. Must be short to fit on key! - /* 126 */ "Tab", + /* 124 */ "Tab", // Label for "switch to phone numeric" key. Must be short to fit on key! - /* 127 */ "123", + /* 125 */ "123", // Label for "switch to phone symbols" key. Must be short to fit on key! // U+FF0A: "＊" FULLWIDTH ASTERISK // U+FF03: "＃" FULLWIDTH NUMBER SIGN - /* 128 */ "\uFF0A\uFF03", + /* 126 */ "\uFF0A\uFF03", // Key label for "ante meridiem" - /* 129 */ "AM", + /* 127 */ "AM", // Key label for "post meridiem" - /* 130 */ "PM", - // Label for "switch to symbols" key on PC QWERTY layout - /* 131 */ "Sym", - /* 132 */ ".com", + /* 128 */ "PM", + /* 129 */ ".com", // popular web domains for the locale - most popular, displayed on the keyboard - /* 133 */ "!hasLabels!,.net,.org,.gov,.edu", - /* 134 */ "!fixedColumnOrder!5,!hasLabels!,=-O|=-O ,:-P|:-P ,;-)|;-) ,:-(|:-( ,:-)|:-) ,:-!|:-! ,:-$|:-$ ,B-)|B-) ,:O|:O ,:-*|:-* ,:-D|:-D ,:\'(|:\'( ,:-\\\\|:-\\\\ ,O:-)|O:-) ,:-[|:-[ ", + /* 130 */ "!hasLabels!,.net,.org,.gov,.edu", + /* 131 */ "!fixedColumnOrder!5,!hasLabels!,=-O|=-O ,:-P|:-P ,;-)|;-) ,:-(|:-( ,:-)|:-) ,:-!|:-! ,:-$|:-$ ,B-)|B-) ,:O|:O ,:-*|:-* ,:-D|:-D ,:\'(|:\'( ,:-\\\\|:-\\\\ ,O:-)|O:-) ,:-[|:-[ ", // U+2039: "‹" SINGLE LEFT-POINTING ANGLE QUOTATION MARK // U+203A: "›" SINGLE RIGHT-POINTING ANGLE QUOTATION MARK // U+00AB: "«" LEFT-POINTING DOUBLE ANGLE QUOTATION MARK @@ -423,24 +413,24 @@ public final class KeyboardTextsSet { // The following each quotation mark pair consist of // <opening quotation mark>, <closing quotation mark> // and is named after (single|double)_<opening quotation mark>_<closing quotation mark>. - /* 135 */ "\u2039,\u203A", - /* 136 */ "\u2039|\u203A,\u203A|\u2039", - /* 137 */ "\u203A,\u2039", - /* 138 */ "\u00AB,\u00BB", - /* 139 */ "\u00AB|\u00BB,\u00BB|\u00AB", - /* 140 */ "\u00BB,\u00AB", + /* 132 */ "\u2039,\u203A", + /* 133 */ "\u2039|\u203A,\u203A|\u2039", + /* 134 */ "\u203A,\u2039", + /* 135 */ "\u00AB,\u00BB", + /* 136 */ "\u00AB|\u00BB,\u00BB|\u00AB", + /* 137 */ "\u00BB,\u00AB", // The following each quotation mark triplet consists of // <another quotation mark>, <opening quotation mark>, <closing quotation mark> // and is named after (single|double)_<opening quotation mark>_<closing quotation mark>. - /* 141 */ "\u201A,\u2018,\u2019", - /* 142 */ "\u2019,\u201A,\u2018", - /* 143 */ "\u2018,\u201A,\u2019", - /* 144 */ "\u201E,\u201C,\u201D", - /* 145 */ "\u201D,\u201E,\u201C", - /* 146 */ "\u201C,\u201E,\u201D", - /* 147 */ "!fixedColumnOrder!5,!text/single_quotes,!text/single_angle_quotes", - /* 148 */ "!fixedColumnOrder!5,!text/double_quotes,!text/double_angle_quotes", - /* 149 */ "!fixedColumnOrder!6,!text/double_quotes,!text/single_quotes,!text/double_angle_quotes,!text/single_angle_quotes", + /* 138 */ "\u201A,\u2018,\u2019", + /* 139 */ "\u2019,\u201A,\u2018", + /* 140 */ "\u2018,\u201A,\u2019", + /* 141 */ "\u201E,\u201C,\u201D", + /* 142 */ "\u201D,\u201E,\u201C", + /* 143 */ "\u201C,\u201E,\u201D", + /* 144 */ "!fixedColumnOrder!5,!text/single_quotes,!text/single_angle_quotes", + /* 145 */ "!fixedColumnOrder!5,!text/double_quotes,!text/double_angle_quotes", + /* 146 */ "!fixedColumnOrder!6,!text/double_quotes,!text/single_quotes,!text/double_angle_quotes,!text/single_angle_quotes", }; /* Language af: Afrikaans */ @@ -1402,9 +1392,10 @@ public final class KeyboardTextsSet { /* 47 */ null, /* 48 */ "!text/single_laqm_raqm_rtl", /* 49 */ "!text/double_laqm_raqm_rtl", - /* 50~ */ - null, null, null, - /* ~52 */ + /* 50 */ null, + // U+FDFC: "﷼" RIAL SIGN + /* 51 */ "\uFDFC", + /* 52 */ null, // U+061F: "؟" ARABIC QUESTION MARK // U+060C: "،" ARABIC COMMA // U+061B: "؛" ARABIC SEMICOLON @@ -1899,16 +1890,16 @@ public final class KeyboardTextsSet { /* 47 */ "\u201C,\u201D,\u201E", /* 48 */ "!text/single_laqm_raqm_rtl", /* 49 */ "!text/double_laqm_raqm_rtl", - /* 50~ */ - null, null, null, - /* ~52 */ + /* 50 */ null, + // U+20AA: "₪" NEW SHEQEL SIGN + /* 51 */ "\u20AA", + /* 52 */ null, /* 53 */ "!fixedColumnOrder!8,\",\',#,-,:,!,\\,,?,@,&,\\%,+,;,/,(|),)|(", // U+2605: "★" BLACK STAR /* 54 */ "\u2605", /* 55 */ null, - // U+00B1: "±" PLUS-MINUS SIGN // U+FB29: "﬩" HEBREW LETTER ALTERNATIVE PLUS SIGN - /* 56 */ "\u00B1,\uFB29", + /* 56 */ "\uFB29", // The all letters need to be mirrored are found at // http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt /* 57 */ "!fixedColumnOrder!3,<|>,{|},[|]", diff --git a/java/src/com/android/inputmethod/keyboard/internal/ScrollKeyboardView.java b/java/src/com/android/inputmethod/keyboard/internal/ScrollKeyboardView.java new file mode 100644 index 000000000..2628f59a8 --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/ScrollKeyboardView.java @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import android.content.Context; +import android.util.AttributeSet; +import android.view.GestureDetector; +import android.view.MotionEvent; +import android.widget.ScrollView; +import android.widget.Scroller; + +import com.android.inputmethod.keyboard.Key; +import com.android.inputmethod.keyboard.KeyDetector; +import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.keyboard.KeyboardView; +import com.android.inputmethod.latin.R; + +/** + * This is an extended {@link KeyboardView} class that hosts a scroll keyboard. + * Multi-touch unsupported. No {@link PointerTracker}s. No gesture support. + */ +public final class ScrollKeyboardView extends KeyboardView implements + ScrollViewWithNotifier.ScrollListener, GestureDetector.OnGestureListener { + private static final boolean PAGINATION = false; + + public interface OnKeyClickListener { + public void onKeyClick(Key key); + } + + private static final OnKeyClickListener EMPTY_LISTENER = new OnKeyClickListener() { + @Override + public void onKeyClick(final Key key) {} + }; + + private OnKeyClickListener mListener = EMPTY_LISTENER; + private final KeyDetector mKeyDetector = new KeyDetector(0.0f /*keyHysteresisDistance */); + private final GestureDetector mGestureDetector; + + private final Scroller mScroller; + private ScrollViewWithNotifier mScrollView; + + public ScrollKeyboardView(final Context context, final AttributeSet attrs) { + this(context, attrs, R.attr.keyboardViewStyle); + } + + public ScrollKeyboardView(final Context context, final AttributeSet attrs, final int defStyle) { + super(context, attrs, defStyle); + mGestureDetector = new GestureDetector(context, this); + mGestureDetector.setIsLongpressEnabled(false /* isLongpressEnabled */); + mScroller = new Scroller(context); + } + + public void setScrollView(final ScrollViewWithNotifier scrollView) { + mScrollView = scrollView; + scrollView.setScrollListener(this); + } + + private final Runnable mScrollTask = new Runnable() { + @Override + public void run() { + final Scroller scroller = mScroller; + final ScrollView scrollView = mScrollView; + scroller.computeScrollOffset(); + scrollView.scrollTo(0, scroller.getCurrY()); + if (!scroller.isFinished()) { + scrollView.post(this); + } + } + }; + + // {@link ScrollViewWithNotified#ScrollListener} methods. + @Override + public void notifyScrollChanged(final int scrollX, final int scrollY, final int oldX, + final int oldY) { + if (PAGINATION) { + mScroller.forceFinished(true /* finished */); + mScrollView.removeCallbacks(mScrollTask); + final int currentTop = mScrollView.getScrollY(); + final int pageHeight = getKeyboard().mBaseHeight; + final int lastPageNo = currentTop / pageHeight; + final int lastPageTop = lastPageNo * pageHeight; + final int nextPageNo = lastPageNo + 1; + final int nextPageTop = Math.min(nextPageNo * pageHeight, getHeight() - pageHeight); + final int scrollTo = (currentTop - lastPageTop) < (nextPageTop - currentTop) + ? lastPageTop : nextPageTop; + final int deltaY = scrollTo - currentTop; + mScroller.startScroll(0, currentTop, 0, deltaY, 300); + mScrollView.post(mScrollTask); + } + } + + @Override + public void notifyOverScrolled(final int scrollX, final int scrollY, final boolean clampedX, + final boolean clampedY) { + releaseCurrentKey(); + } + + public void setOnKeyClickListener(final OnKeyClickListener listener) { + mListener = listener; + } + + /** + * {@inheritDoc} + */ + @Override + public void setKeyboard(final Keyboard keyboard) { + super.setKeyboard(keyboard); + mKeyDetector.setKeyboard(keyboard, 0 /* correctionX */, 0 /* correctionY */); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean onTouchEvent(final MotionEvent e) { + if (mGestureDetector.onTouchEvent(e)) { + return true; + } + final Key key = getKey(e); + if (key != null && key != mCurrentKey) { + releaseCurrentKey(); + } + return true; + } + + // {@link GestureDetector#OnGestureListener} methods. + private Key mCurrentKey; + + private Key getKey(final MotionEvent e) { + final int index = e.getActionIndex(); + final int x = (int)e.getX(index); + final int y = (int)e.getY(index); + return mKeyDetector.detectHitKey(x, y); + } + + public void releaseCurrentKey() { + final Key currentKey = mCurrentKey; + if (currentKey == null) { + return; + } + currentKey.onReleased(); + invalidateKey(currentKey); + mCurrentKey = null; + } + + @Override + public boolean onDown(final MotionEvent e) { + final Key key = getKey(e); + releaseCurrentKey(); + mCurrentKey = key; + if (key == null) { + return false; + } + // TODO: May call {@link KeyboardActionListener#onPressKey(int,int,boolean)}. + key.onPressed(); + invalidateKey(key); + return false; + } + + @Override + public void onShowPress(final MotionEvent e) { + // User feedback is done at {@link #onDown(MotionEvent)}. + } + + @Override + public boolean onSingleTapUp(final MotionEvent e) { + final Key key = getKey(e); + releaseCurrentKey(); + if (key == null) { + return false; + } + // TODO: May call {@link KeyboardActionListener#onReleaseKey(int,boolean)}. + key.onReleased(); + invalidateKey(key); + mListener.onKeyClick(key); + return true; + } + + @Override + public boolean onScroll(final MotionEvent e1, final MotionEvent e2, final float distanceX, + final float distanceY) { + releaseCurrentKey(); + return false; + } + + @Override + public boolean onFling(final MotionEvent e1, final MotionEvent e2, final float velocityX, + final float velocityY) { + releaseCurrentKey(); + return false; + } + + @Override + public void onLongPress(final MotionEvent e) { + // Long press detection of {@link #mGestureDetector} is disabled and not used. + } +} diff --git a/java/src/com/android/inputmethod/keyboard/internal/ScrollViewWithNotifier.java b/java/src/com/android/inputmethod/keyboard/internal/ScrollViewWithNotifier.java new file mode 100644 index 000000000..d1ccdc7b5 --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/ScrollViewWithNotifier.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import android.content.Context; +import android.util.AttributeSet; +import android.widget.ScrollView; + +/** + * This is an extended {@link ScrollView} that can notify + * {@link ScrollView#onScrollChanged(int,int,int,int} and + * {@link ScrollView#onOverScrolled(int,int,int,int)} to a content view. + */ +public class ScrollViewWithNotifier extends ScrollView { + private ScrollListener mScrollListener = EMPTY_LISTER; + + public interface ScrollListener { + public void notifyScrollChanged(int scrollX, int scrollY, int oldX, int oldY); + public void notifyOverScrolled(int scrollX, int scrollY, boolean clampedX, + boolean clampedY); + } + + private static final ScrollListener EMPTY_LISTER = new ScrollListener() { + @Override + public void notifyScrollChanged(int scrollX, int scrollY, int oldX, int oldY) {} + @Override + public void notifyOverScrolled(int scrollX, int scrollY, boolean clampedX, + boolean clampedY) {} + }; + + public ScrollViewWithNotifier(final Context context, final AttributeSet attrs) { + super(context, attrs); + } + + @Override + protected void onScrollChanged(final int scrollX, final int scrollY, final int oldX, + final int oldY) { + super.onScrollChanged(scrollX, scrollY, oldX, oldY); + mScrollListener.notifyScrollChanged(scrollX, scrollY, oldX, oldY); + } + + @Override + protected void onOverScrolled(final int scrollX, final int scrollY, final boolean clampedX, + final boolean clampedY) { + super.onOverScrolled(scrollX, scrollY, clampedX, clampedY); + mScrollListener.notifyOverScrolled(scrollX, scrollY, clampedX, clampedY); + } + + public void setScrollListener(final ScrollListener listener) { + mScrollListener = listener; + } +} diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java index 269b3a299..55df263fe 100644 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -19,10 +19,11 @@ package com.android.inputmethod.latin; import android.content.Context; import android.util.Log; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; // TODO: Quit extending Dictionary after implementing dynamic binary dictionary. @@ -49,32 +50,21 @@ abstract public class AbstractDictionaryWriter extends Dictionary { abstract public void removeBigramWords(final String word0, final String word1); - abstract protected void writeBinaryDictionary(final FileOutputStream out) + abstract protected void writeDictionary(final DictEncoder dictEncoder) throws IOException, UnsupportedFormatException; public void write(final String fileName) { final String tempFileName = fileName + ".temp"; final File file = new File(mContext.getFilesDir(), fileName); final File tempFile = new File(mContext.getFilesDir(), tempFileName); - FileOutputStream out = null; try { - out = new FileOutputStream(tempFile); - writeBinaryDictionary(out); - out.flush(); - out.close(); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + writeDictionary(dictEncoder); tempFile.renameTo(file); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "Unsupported format", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // ignore - } - } } } } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 8f6b848bb..2b6d983c0 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -229,8 +229,6 @@ final public class BinaryDictionaryGetter { try { // Read the version of the file final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); - dictDecoder.openDictBuffer( - new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); final FileHeader header = dictDecoder.readHeader(); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 1ececd5c1..a97e053d0 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -20,7 +20,7 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -28,7 +28,6 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.utils.CollectionUtils; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -85,9 +84,9 @@ public class DictionaryWriter extends AbstractDictionaryWriter { } @Override - protected void writeBinaryDictionary(final FileOutputStream out) + protected void writeDictionary(final DictEncoder dictEncoder) throws IOException, UnsupportedFormatException { - BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS); } @Override diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 37256770a..939c2a03b 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -23,11 +23,13 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.util.ArrayList; import java.util.HashMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -91,6 +93,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /* A extension for a binary dictionary file. */ public static final String DICT_FILE_EXTENSION = ".dict"; + private final AtomicReference<AsyncWriteBinaryDictionaryTask> mWaitingTask = + new AtomicReference<AsyncWriteBinaryDictionaryTask>(); + /** * Abstract method for loading the unigrams and bigrams of a given dictionary in a background * thread. @@ -118,10 +123,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } private static AbstractDictionaryWriter getDictionaryWriter(final Context context, - final String dictType, final boolean isUpdatable) { - if (isUpdatable) { - // TODO: Employ dynamically updatable DictionaryWriter. - return new DictionaryWriter(context, dictType); + final String dictType, final boolean isDynamicPersonalizationDictionary) { + if (isDynamicPersonalizationDictionary) { + return new DynamicPersonalizationDictionaryWriter(context, dictType); } else { return new DictionaryWriter(context, dictType); } @@ -145,6 +149,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mIsUpdatable = isUpdatable; mBinaryDictionary = null; mSharedDictionaryController = getSharedDictionaryController(filename); + // Currently, only dynamic personalization dictionary is updatable. mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); } @@ -179,6 +184,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } + protected void clear() { + mLocalDictionaryController.writeLock().lock(); + try { + mDictionaryWriter.clear(); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + /** * Adds a word unigram to the dictionary. Used for loading a dictionary. */ @@ -266,7 +280,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final ArrayList<SuggestedWordInfo> inMemDictSuggestion = mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo, blockOffensiveWords); - if (mBinaryDictionary != null) { + // TODO: Remove checking mIsUpdatable and use native suggestion. + if (mBinaryDictionary != null && !mIsUpdatable) { final ArrayList<SuggestedWordInfo> binarySuggestion = mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, blockOffensiveWords); @@ -275,7 +290,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } else if (binarySuggestion == null) { return inMemDictSuggestion; } else { - binarySuggestion.addAll(binarySuggestion); + binarySuggestion.addAll(inMemDictSuggestion); return binarySuggestion; } } else { @@ -401,7 +416,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Reloads the dictionary if required. Reload will occur asynchronously in a separate thread. */ - void asyncReloadDictionaryIfRequired() { + public void asyncReloadDictionaryIfRequired() { if (!isReloadRequired()) return; if (DEBUG) { Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename); @@ -412,7 +427,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Reloads the dictionary if required. */ - protected final void syncReloadDictionaryIfRequired() { + public final void syncReloadDictionaryIfRequired() { if (!isReloadRequired()) return; syncReloadDictionaryInternal(); } @@ -492,6 +507,68 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** + * Load the dictionary to memory. + */ + protected void asyncLoadDictionaryToMemory() { + new AsyncLoadDictionaryToMemoryTask().start(); + } + + /** + * Thread class for asynchronously loading dictionary to memory. + */ + private class AsyncLoadDictionaryToMemoryTask extends Thread { + @Override + public void run() { + mLocalDictionaryController.writeLock().lock(); + try { + mSharedDictionaryController.readLock().lock(); + try { + loadDictionaryAsync(); + } finally { + mSharedDictionaryController.readLock().unlock(); + } + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + } + + /** + * Generate binary dictionary using DictionaryWriter. + */ + protected void asyncWriteBinaryDictionary() { + final AsyncWriteBinaryDictionaryTask newTask = new AsyncWriteBinaryDictionaryTask(); + newTask.start(); + final AsyncWriteBinaryDictionaryTask oldTask = mWaitingTask.getAndSet(newTask); + if (oldTask != null) { + oldTask.interrupt(); + } + } + + /** + * Thread class for asynchronously writing the binary dictionary. + */ + private class AsyncWriteBinaryDictionaryTask extends Thread { + @Override + public void run() { + mSharedDictionaryController.writeLock().lock(); + try { + mLocalDictionaryController.writeLock().lock(); + try { + if (isInterrupted()) { + return; + } + writeBinaryDictionary(); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } finally { + mSharedDictionaryController.writeLock().unlock(); + } + } + } + + /** * Lock for controlling access to a given binary dictionary and for tracking whether the * dictionary is out of date. Can be shared across multiple dictionary instances that access the * same filename. diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index 516b8426c..f5fa5d0d7 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -16,10 +16,10 @@ package com.android.inputmethod.latin; -import android.content.Context; import android.text.TextUtils; import android.util.Log; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -29,9 +29,10 @@ import java.util.ArrayList; import java.util.LinkedList; /** - * Base class for an in-memory dictionary that can grow dynamically and can + * Class for an in-memory dictionary that can grow dynamically and can * be searched for suggestions and valid words. */ +// TODO: Remove after binary dictionary supports dynamic update. public class ExpandableDictionary extends Dictionary { private static final String TAG = ExpandableDictionary.class.getSimpleName(); /** @@ -39,23 +40,11 @@ public class ExpandableDictionary extends Dictionary { */ private static final int FULL_WORD_SCORE_MULTIPLIER = 2; - // Bigram frequency is a fixed point number with 1 meaning 1.2 and 255 meaning 1.8. - protected static final int BIGRAM_MAX_FREQUENCY = 255; - - private Context mContext; private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH]; private int mMaxDepth; private int mInputLength; - private boolean mRequiresReload; - - private boolean mUpdatingDictionary; - - // Use this lock before touching mUpdatingDictionary & mRequiresDownload - private Object mUpdatingLock = new Object(); - private static final class Node { - Node() {} char mCode; int mFrequency; boolean mTerminal; @@ -157,46 +146,12 @@ public class ExpandableDictionary extends Dictionary { private int[][] mCodes; - public ExpandableDictionary(final Context context, final String dictType) { + public ExpandableDictionary(final String dictType) { super(dictType); - mContext = context; clearDictionary(); mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][]; } - public void loadDictionary() { - synchronized (mUpdatingLock) { - startDictionaryLoadingTaskLocked(); - } - } - - public void startDictionaryLoadingTaskLocked() { - if (!mUpdatingDictionary) { - mUpdatingDictionary = true; - mRequiresReload = false; - new LoadDictionaryTask().start(); - } - } - - public void setRequiresReload(final boolean reload) { - synchronized (mUpdatingLock) { - mRequiresReload = reload; - } - } - - public boolean getRequiresReload() { - return mRequiresReload; - } - - /** Override to load your dictionary here, on a background thread. */ - public void loadDictionaryAsync() { - // empty base implementation - } - - public Context getContext() { - return mContext; - } - public int getMaxWordLength() { return Constants.DICTIONARY_MAX_WORD_LENGTH; } @@ -231,7 +186,7 @@ public class ExpandableDictionary extends Dictionary { childNode.mShortcutOnly = isShortcutOnly; children.add(childNode); } - if (wordLength == depth + 1 && shortcutTarget != null) { + if (wordLength == depth + 1) { // Terminate this word childNode.mTerminal = true; if (isShortcutOnly) { @@ -256,7 +211,6 @@ public class ExpandableDictionary extends Dictionary { public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, final boolean blockOffensiveWords) { - if (reloadDictionaryIfRequired()) return null; if (composer.size() > 1) { if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) { return null; @@ -272,17 +226,7 @@ public class ExpandableDictionary extends Dictionary { } } - // This reloads the dictionary if required, and returns whether it's currently updating its - // contents or not. - private boolean reloadDictionaryIfRequired() { - synchronized (mUpdatingLock) { - // If we need to update, start off a background task - if (mRequiresReload) startDictionaryLoadingTaskLocked(); - return mUpdatingDictionary; - } - } - - protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, + private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, final String prevWordForBigrams, final ProximityInfo proximityInfo) { final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); mInputLength = codes.size(); @@ -312,11 +256,6 @@ public class ExpandableDictionary extends Dictionary { @Override public synchronized boolean isValidWord(final String word) { - synchronized (mUpdatingLock) { - // If we need to update, start off a background task - if (mRequiresReload) startDictionaryLoadingTaskLocked(); - if (mUpdatingDictionary) return false; - } final Node node = searchNode(mRoots, word, 0, word.length()); // If node is null, we didn't find the word, so it's not valid. // If node.mShortcutOnly is true, then it exists as a shortcut but not as a word, @@ -326,7 +265,7 @@ public class ExpandableDictionary extends Dictionary { return (node == null) ? false : !node.mShortcutOnly; } - protected boolean removeBigram(final String word1, final String word2) { + public boolean removeBigram(final String word1, final String word2) { // Refer to addOrSetBigram() about word1.toLowerCase() final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); final Node secondWord = searchWord(mRoots, word2, 0, null); @@ -351,13 +290,14 @@ public class ExpandableDictionary extends Dictionary { /** * Returns the word's frequency or -1 if not found */ - protected int getWordFrequency(final String word) { + @UsedForTesting + public int getWordFrequency(final String word) { // Case-sensitive search final Node node = searchNode(mRoots, word, 0, word.length()); return (node == null) ? -1 : node.mFrequency; } - protected NextWord getBigramWord(final String word1, final String word2) { + public NextWord getBigramWord(final String word1, final String word2) { // Refer to addOrSetBigram() about word1.toLowerCase() final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); final Node secondWord = searchWord(mRoots, word2, 0, null); @@ -440,7 +380,7 @@ public class ExpandableDictionary extends Dictionary { * @param suggestions the list in which to add suggestions */ // TODO: Share this routine with the native code for BinaryDictionary - protected void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word, + private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word, final int depth, final boolean completion, final int snr, final int inputIndex, final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) { final int count = roots.mLength; @@ -698,21 +638,10 @@ public class ExpandableDictionary extends Dictionary { return null; } - protected void clearDictionary() { + public void clearDictionary() { mRoots = new NodeArray(); } - private final class LoadDictionaryTask extends Thread { - LoadDictionaryTask() {} - @Override - public void run() { - loadDictionaryAsync(); - synchronized (mUpdatingLock) { - mUpdatingDictionary = false; - } - } - } - private static char toLowerCase(final char c) { char baseChar = c; if (c < BASE_CHARS.length) { diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 816c6daa4..85001c30c 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -73,8 +73,9 @@ import com.android.inputmethod.keyboard.KeyboardSwitcher; import com.android.inputmethod.keyboard.MainKeyboardView; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.define.ProductionFlag; -import com.android.inputmethod.latin.personalization.PersonalizationDictionaryHelper; +import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister; +import com.android.inputmethod.latin.personalization.PersonalizationHelper; import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary; import com.android.inputmethod.latin.settings.Settings; @@ -173,6 +174,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private UserBinaryDictionary mUserDictionary; private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary; private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary; + private PersonalizationDictionary mPersonalizationDictionary; private boolean mIsUserDictionaryAvailable; private LastComposedWord mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; @@ -564,10 +566,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); - mUserHistoryPredictionDictionary = PersonalizationDictionaryHelper + mUserHistoryPredictionDictionary = PersonalizationHelper .getUserHistoryPredictionDictionary(this, localeStr, prefs); newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary); - mPersonalizationPredictionDictionary = PersonalizationDictionaryHelper + mPersonalizationDictionary = PersonalizationHelper + .getPersonalizationDictionary(this, localeStr, prefs); + newSuggest.setPersonalizationDictionary(mPersonalizationDictionary); + mPersonalizationPredictionDictionary = PersonalizationHelper .getPersonalizationPredictionDictionary(this, localeStr, prefs); newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary); @@ -639,6 +644,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.getInstance().onDestroy(); } unregisterReceiver(mDictionaryPackInstallReceiver); + PersonalizationDictionarySessionRegister.onDestroy(this); LatinImeLogger.commit(); LatinImeLogger.onDestroy(); super.onDestroy(); @@ -2051,10 +2057,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen (!mConnection.isCursorTouchingWord(currentSettings) || !currentSettings.mCurrentLanguageHasSpaces)) { // Reset entirely the composing state anyway, then start composing a new word unless - // the character is a single quote. The idea here is, single quote is not a - // separator and it should be treated as a normal character, except in the first - // position where it should not start composing a word. - isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode); + // the character is a single quote or a dash. The idea here is, single quote and dash + // are not separators and they should be treated as normal characters, except in the + // first position where they should not start composing a word. + isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode + && Constants.CODE_DASH != primaryCode); // Here we don't need to reset the last composed word. It will be reset // when we commit this one, if we ever do; if on the other hand we backspace // it entirely and resume suggestions on the previous word, we'd like to still diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 35920f8cb..a031bb3be 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -169,7 +169,6 @@ public final class RichInputConnection { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); mCommittedTextBeforeComposingText.append(mComposingText); - mExpectedCursorPosition += mComposingText.length(); mComposingText.setLength(0); if (null != mIC) { mIC.finishComposingText(); diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 8766e0fc1..c8a151a6c 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -24,6 +24,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary; import com.android.inputmethod.latin.settings.Settings; @@ -200,6 +201,12 @@ public final class Suggest { personalizationPredictionDictionary); } + public void setPersonalizationDictionary( + final PersonalizationDictionary personalizationDictionary) { + addOrReplaceDictionaryInternal(Dictionary.TYPE_PERSONALIZATION, + personalizationDictionary); + } + public void setAutoCorrectionThreshold(float threshold) { mAutoCorrectionThreshold = threshold; } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 995f061f3..ceb8fa81f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -19,7 +19,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -31,7 +31,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; -import java.util.Arrays; import java.util.Map; import java.util.TreeMap; @@ -42,6 +41,7 @@ import java.util.TreeMap; * * TODO: Remove calls from classes except Ver3DictDecoder * TODO: Move this file to makedict/internal. + * TODO: Rename this class to DictDecoderUtils. */ public final class BinaryDictDecoderUtils { @@ -214,7 +214,7 @@ public final class BinaryDictDecoderUtils { buffer[index++] = (byte)(0xFF & codePoint); } } - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; + buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; return index - origin; } @@ -238,7 +238,7 @@ public final class BinaryDictDecoderUtils { buffer.write((byte) (0xFF & codePoint)); } } - buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); + buffer.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); } /** @@ -265,7 +265,7 @@ public final class BinaryDictDecoderUtils { static int readChar(final DictBuffer dictBuffer) { int character = dictBuffer.readUnsignedByte(); if (!fitsOnOneByte(character)) { - if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { + if (FormatSpec.PTNODE_CHARACTERS_TERMINATOR == character) { return FormatSpec.INVALID_CHARACTER; } character <<= 16; @@ -278,6 +278,12 @@ public final class BinaryDictDecoderUtils { // Input methods: Read a binary dictionary to memory. // readDictionaryBinary is the public entry point for them. + static int readSInt24(final DictBuffer dictBuffer) { + final int retval = dictBuffer.readUnsignedInt24(); + final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (retval & FormatSpec.SINT24_MAX); + } + static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, final FormatOptions options) { if (options.mSupportsDynamicUpdate) { @@ -290,14 +296,14 @@ public final class BinaryDictDecoderUtils { } } int address; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return dictBuffer.readUnsignedByte(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: return dictBuffer.readUnsignedShort(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: return dictBuffer.readUnsignedInt24(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: default: return FormatSpec.NO_CHILDREN_ADDRESS; } @@ -314,199 +320,103 @@ public final class BinaryDictDecoderUtils { } } - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final DictBuffer dictBuffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = dictBuffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(dictBuffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(dictBuffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = dictBuffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(dictBuffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = dictBuffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - dictBuffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += dictBuffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (dictBuffer.readUnsignedByte() << 16) - + dictBuffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - /** - * Reads and returns the char group count out of a buffer and forwards the pointer. + * Reads and returns the PtNode count out of a buffer and forwards the pointer. */ - public static int readCharGroupCount(final DictBuffer dictBuffer) { + /* package */ static int readPtNodeCount(final DictBuffer dictBuffer) { final int msb = dictBuffer.readUnsignedByte(); - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= msb) { return msb; } else { - return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) + return ((FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT & msb) << 8) + dictBuffer.readUnsignedByte(); } } /** - * Finds, as a string, the word at the address passed as an argument. + * Finds, as a string, the word at the position passed as an argument. * - * @param dictBuffer the buffer to read from. + * @param dictDecoder the dict decoder. * @param headerSize the size of the header. - * @param address the address to seek. + * @param pos the position to seek. * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ - /* package for tests */ static WeightedString getWordAtAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + /* package for tests */ static WeightedString getWordAtPosition( + final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final WeightedString result; - final int originalPointer = dictBuffer.position(); - dictBuffer.position(address); + final int originalPos = dictBuffer.position(); + dictBuffer.position(pos); if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(dictBuffer, headerSize, address, - formatOptions); + result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions); } else { - result = getWordAtAddressWithoutParentAddress(dictBuffer, headerSize, address, + result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos, formatOptions); } - dictBuffer.position(originalPointer); + dictBuffer.position(originalPos); return result; } @SuppressWarnings("unused") - private static WeightedString getWordAtAddressWithParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, - final FormatOptions options) { - int currentAddress = address; + private static WeightedString getWordAtPositionWithParentAddress( + final Ver3DictDecoder dictDecoder, final int pos, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + int currentPos = pos; int frequency = Integer.MIN_VALUE; final StringBuilder builder = new StringBuilder(); // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { - CharGroupInfo currentInfo; + PtNodeInfo currentInfo; int loopCounter = 0; do { - dictBuffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(dictBuffer, currentAddress, options); - if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + dictBuffer.position(currentPos); + currentInfo = dictDecoder.readPtNode(currentPos, options); + if (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)) { + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } if (DBG && loopCounter++ > MAX_JUMPS) { MakedictLog.d("Too many jumps - probably a bug"); } - } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)); + } while (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)); if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; builder.insert(0, new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; } return new WeightedString(builder.toString(), frequency); } - private static WeightedString getWordAtAddressWithoutParentAddress( - final DictBuffer dictBuffer, final int headerSize, final int address, + private static WeightedString getWordAtPositionWithoutParentAddress( + final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(headerSize); - final int count = readCharGroupCount(dictBuffer); - int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); + final int count = readPtNodeCount(dictBuffer); + int groupPos = headerSize + BinaryDictIOUtils.getPtNodeCountSize(count); final StringBuilder builder = new StringBuilder(); WeightedString result = null; - CharGroupInfo last = null; + PtNodeInfo last = null; for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); - groupOffset = info.mEndAddress; - if (info.mOriginalAddress == address) { + PtNodeInfo info = dictDecoder.readPtNode(groupPos, options); + groupPos = info.mEndAddress; + if (info.mOriginalAddress == pos) { builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); result = new WeightedString(builder.toString(), info.mFrequency); break; // and return } if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { - if (info.mChildrenAddress > address) { + if (info.mChildrenAddress > pos) { if (null == last) continue; builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - dictBuffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(dictBuffer); - groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); + dictBuffer.position(last.mChildrenAddress); + i = readPtNodeCount(dictBuffer); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); last = null; continue; } @@ -514,9 +424,9 @@ public final class BinaryDictDecoderUtils { } if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - dictBuffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(dictBuffer); - groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); + dictBuffer.position(last.mChildrenAddress); + i = readPtNodeCount(dictBuffer); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); last = null; continue; } @@ -532,34 +442,35 @@ public final class BinaryDictDecoderUtils { * This will recursively read other node arrays into the structure, populating the reverse * maps on the fly and using them to keep track of already read nodes. * - * @param dictBuffer the buffer, correctly positioned at the start of a node array. + * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. * @param headerSize the size, in bytes, of the file header. * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. - * @param reverseGroupMap a mapping from addresses to already read character groups. + * @param reversePtNodeMap a mapping from addresses to already read PtNodes. * @param options file format options. * @return the read node array with all his children already read. */ - private static PtNodeArray readNodeArray(final DictBuffer dictBuffer, + private static PtNodeArray readNodeArray(final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, - final Map<Integer, CharGroup> reverseGroupMap, final FormatOptions options) + final Map<Integer, PtNode> reversePtNodeMap, final FormatOptions options) throws IOException { - final ArrayList<CharGroup> nodeArrayContents = new ArrayList<CharGroup>(); - final int nodeArrayOrigin = dictBuffer.position() - headerSize; + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + final ArrayList<PtNode> nodeArrayContents = new ArrayList<PtNode>(); + final int nodeArrayOriginPos = dictBuffer.position(); do { // Scan the linked-list node. - final int nodeArrayHeadPosition = dictBuffer.position() - headerSize; - final int count = readCharGroupCount(dictBuffer); - int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); - for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(dictBuffer, groupOffset, options); - if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; + final int nodeArrayHeadPos = dictBuffer.position(); + final int count = readPtNodeCount(dictBuffer); + int groupOffsetPos = nodeArrayHeadPos + BinaryDictIOUtils.getPtNodeCountSize(count); + for (int i = count; i > 0; --i) { // Scan the array of PtNode. + PtNodeInfo info = dictDecoder.readPtNode(groupOffsetPos, options); + if (BinaryDictIOUtils.isMovedPtNode(info.mFlags, options)) continue; ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; ArrayList<WeightedString> bigrams = null; if (null != info.mBigrams) { bigrams = new ArrayList<WeightedString>(); for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - dictBuffer, headerSize, bigram.mAddress, options); + final WeightedString word = getWordAtPosition(dictDecoder, headerSize, + bigram.mAddress, options); final int reconstructedFrequency = BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); @@ -570,24 +481,24 @@ public final class BinaryDictDecoderUtils { PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); if (null == children) { final int currentPosition = dictBuffer.position(); - dictBuffer.position(info.mChildrenAddress + headerSize); - children = readNodeArray(dictBuffer, headerSize, reverseNodeArrayMap, - reverseGroupMap, options); + dictBuffer.position(info.mChildrenAddress); + children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, + reversePtNodeMap, options); dictBuffer.position(currentPosition); } nodeArrayContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, + new PtNode(info.mCharacters, shortcutTargets, bigrams, info.mFrequency, 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); } else { nodeArrayContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, + new PtNode(info.mCharacters, shortcutTargets, bigrams, info.mFrequency, 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); } - groupOffset = info.mEndAddress; + groupOffsetPos = info.mEndAddress; } // reach the end of the array. @@ -603,8 +514,8 @@ public final class BinaryDictDecoderUtils { dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); - nodeArray.mCachedAddressBeforeUpdate = nodeArrayOrigin; - nodeArray.mCachedAddressAfterUpdate = nodeArrayOrigin; + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; + nodeArray.mCachedAddressAfterUpdate = nodeArrayOriginPos; reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); return nodeArray; } @@ -649,24 +560,15 @@ public final class BinaryDictDecoderUtils { * @return the created (or merged) dictionary. */ @UsedForTesting - public static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder, - final FusionDictionary dict) throws FileNotFoundException, IOException, - UnsupportedFormatException { - - // if the buffer has not been opened, open the buffer with bytebuffer. - if (dictDecoder.getDictBuffer() == null) dictDecoder.openDictBuffer( - new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); - if (dictDecoder.getDictBuffer() == null) { - MakedictLog.e("Cannot open the buffer"); - } - + /* package */ static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder, + final FusionDictionary dict) throws IOException, UnsupportedFormatException { // Read header final FileHeader fileHeader = dictDecoder.readHeader(); Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); - Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final PtNodeArray root = readNodeArray(dictDecoder.getDictBuffer(), fileHeader.mHeaderSize, - reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions); + Map<Integer, PtNode> reversePtNodeMapping = new TreeMap<Integer, PtNode>(); + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, + reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions); FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); if (null != dict) { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index ff11cde39..79f5ad8bd 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -33,12 +33,14 @@ import java.util.Iterator; * Encodes binary files for a FusionDictionary. * * All the methods in this class are static. + * + * TODO: Rename this class to DictEncoderUtils. */ -public class BinaryDictEncoder { +public class BinaryDictEncoderUtils { private static final boolean DBG = MakedictLog.DBG; - private BinaryDictEncoder() { + private BinaryDictEncoderUtils() { // This utility class is not publicly instantiable. } @@ -58,46 +60,46 @@ public class BinaryDictEncoder { * @param characters the character array * @return the size of the char array, including the terminator if any */ - static int getGroupCharactersSize(final int[] characters) { + static int getPtNodeCharactersSize(final int[] characters) { int size = CharEncoding.getCharArraySize(characters); - if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE; + if (characters.length > 1) size += FormatSpec.PTNODE_TERMINATOR_SIZE; return size; } /** - * Compute the binary size of the character array in a group + * Compute the binary size of the character array in a PtNode * * If only one character, this is the size of this character. If many, it's the sum of their * sizes + 1 byte for the terminator. * - * @param group the group + * @param ptNode the PtNode * @return the size of the char array, including the terminator if any */ - private static int getGroupCharactersSize(final CharGroup group) { - return getGroupCharactersSize(group.mChars); + private static int getPtNodeCharactersSize(final PtNode ptNode) { + return getPtNodeCharactersSize(ptNode.mChars); } /** - * Compute the binary size of the group count for a node array. + * Compute the binary size of the PtNode count for a node array. * @param nodeArray the nodeArray - * @return the size of the group count, either 1 or 2 bytes. + * @return the size of the PtNode count, either 1 or 2 bytes. */ - private static int getGroupCountSize(final PtNodeArray nodeArray) { - return BinaryDictIOUtils.getGroupCountSize(nodeArray.mData.size()); + private static int getPtNodeCountSize(final PtNodeArray nodeArray) { + return BinaryDictIOUtils.getPtNodeCountSize(nodeArray.mData.size()); } /** * Compute the size of a shortcut in bytes. */ private static int getShortcutSize(final WeightedString shortcut) { - int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE; + int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; final String word = shortcut.mWord; final int length = word.length(); for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); size += CharEncoding.getCharSize(codePoint); } - size += FormatSpec.GROUP_TERMINATOR_SIZE; + size += FormatSpec.PTNODE_TERMINATOR_SIZE; return size; } @@ -108,8 +110,8 @@ public class BinaryDictEncoder { * like address lists do. */ static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { - if (null == shortcutList) return 0; - int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; + if (null == shortcutList || shortcutList.isEmpty()) return 0; + int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; for (final WeightedString shortcut : shortcutList) { size += getShortcutSize(shortcut); } @@ -117,60 +119,60 @@ public class BinaryDictEncoder { } /** - * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. + * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. * - * @param group the CharGroup to compute the size of. + * @param ptNode the PtNode to compute the size of. * @param options file format options. - * @return the maximum size of the group. + * @return the maximum size of the PtNode. */ - private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) { - int size = getGroupHeaderSize(group, options); + private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { + int size = getNodeHeaderSize(ptNode, options); // If terminal, one byte for the frequency - if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE; - size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address - size += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE - + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) - * group.mBigrams.size(); + if (ptNode.isTerminal()) size += FormatSpec.PTNODE_FREQUENCY_SIZE; + size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address + size += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE + + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE) + * ptNode.mBigrams.size(); } return size; } /** - * Compute the maximum size of each node of a node array, assuming 3-byte addresses for + * Compute the maximum size of each PtNode of a PtNode array, assuming 3-byte addresses for * everything, and caches it in the `mCachedSize' member of the nodes; deduce the size of * the containing node array, and cache it it its 'mCachedSize' member. * - * @param nodeArray the node array to compute the maximum size of. + * @param ptNodeArray the node array to compute the maximum size of. * @param options file format options. */ - private static void calculateNodeArrayMaximumSize(final PtNodeArray nodeArray, + private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray, final FormatOptions options) { - int size = getGroupCountSize(nodeArray); - for (CharGroup g : nodeArray.mData) { - final int groupSize = getCharGroupMaximumSize(g, options); - g.mCachedSize = groupSize; - size += groupSize; + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode node : ptNodeArray.mData) { + final int nodeSize = getPtNodeMaximumSize(node, options); + node.mCachedSize = nodeSize; + size += nodeSize; } if (options.mSupportsDynamicUpdate) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } - nodeArray.mCachedSize = size; + ptNodeArray.mCachedSize = size; } /** - * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. + * Compute the size of the header (flag + [parent address] + characters size) of a PtNode. * - * @param group the group of which to compute the size of the header + * @param ptNode the PtNode of which to compute the size of the header * @param options file format options. */ - private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { + private static int getNodeHeaderSize(final PtNode ptNode, final FormatOptions options) { if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + getGroupCharactersSize(group); + return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + getPtNodeCharactersSize(ptNode); } else { - return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group); + return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode); } } @@ -203,14 +205,14 @@ public class BinaryDictEncoder { // cache performance and dictionary size. /* package for tests */ static ArrayList<PtNodeArray> flattenTree( final PtNodeArray rootNodeArray) { - final int treeSize = FusionDictionary.countCharGroups(rootNodeArray); + final int treeSize = FusionDictionary.countPtNodes(rootNodeArray); MakedictLog.i("Counted nodes : " + treeSize); final ArrayList<PtNodeArray> flatTree = new ArrayList<PtNodeArray>(treeSize); return flattenTreeInner(flatTree, rootNodeArray); } private static ArrayList<PtNodeArray> flattenTreeInner(final ArrayList<PtNodeArray> list, - final PtNodeArray nodeArray) { + final PtNodeArray ptNodeArray) { // Removing the node is necessary if the tails are merged, because we would then // add the same node several times when we only want it once. A number of places in // the code also depends on any node being only once in the list. @@ -228,11 +230,11 @@ public class BinaryDictEncoder { // this simple list.remove operation O(n*n) overall. On Android this overhead is very // high. // For future reference, the code to remove duplicate is a simple : list.remove(node); - list.add(nodeArray); - final ArrayList<CharGroup> branches = nodeArray.mData; + list.add(ptNodeArray); + final ArrayList<PtNode> branches = ptNodeArray.mData; final int nodeSize = branches.size(); - for (CharGroup group : branches) { - if (null != group.mChildren) flattenTreeInner(list, group.mChildren); + for (PtNode ptNode : branches) { + if (null != ptNode.mChildren) flattenTreeInner(list, ptNode.mChildren); } return list; } @@ -248,7 +250,7 @@ public class BinaryDictEncoder { * from the new position in the current node array to the new position in the target node * array. * - * @param currentNodeArray node array containing the CharGroup where the offset will be written + * @param currentNodeArray node array containing the PtNode where the offset will be written * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray * @param targetNodeArray the target node array to get the offset to * @return the offset to the target node array @@ -269,20 +271,20 @@ public class BinaryDictEncoder { } /** - * Get the offset from a position inside a current node array to a target CharGroup, during + * Get the offset from a position inside a current node array to a target PtNode, during * update. * - * @param currentNodeArray node array containing the CharGroup where the offset will be written + * @param currentNodeArray node array containing the PtNode where the offset will be written * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray - * @param targetCharGroup the target CharGroup to get the offset to - * @return the offset to the target CharGroup + * @param targetPtNode the target PtNode to get the offset to + * @return the offset to the target PtNode */ // TODO: is there any way to factorize this method with the one above? - private static int getOffsetToTargetCharGroupDuringUpdate(final PtNodeArray currentNodeArray, - final int offsetFromStartOfCurrentNodeArray, final CharGroup targetCharGroup) { + private static int getOffsetToTargetPtNodeDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNode targetPtNode) { final int oldOffsetBasePoint = currentNodeArray.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNodeArray; - final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate + final boolean isTargetBeforeCurrent = (targetPtNode.mCachedAddressBeforeUpdate < oldOffsetBasePoint); // If the target is before the current node array, then its address has already been // updated. We can use the AfterUpdate member, and compare it to our own member after @@ -292,9 +294,9 @@ public class BinaryDictEncoder { if (isTargetBeforeCurrent) { final int newOffsetBasePoint = currentNodeArray.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNodeArray; - return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint; + return targetPtNode.mCachedAddressAfterUpdate - newOffsetBasePoint; } else { - return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint; + return targetPtNode.mCachedAddressBeforeUpdate - oldOffsetBasePoint; } } @@ -308,49 +310,45 @@ public class BinaryDictEncoder { * contents (as in, any of the addresses stored in the cache fields) have changed with * respect to their previous value. * - * @param nodeArray the node array to compute the size of. + * @param ptNodeArray the node array to compute the size of. * @param dict the dictionary in which the word/attributes are to be found. * @param formatOptions file format options. * @return false if none of the cached addresses inside the node array changed, true otherwise. */ - private static boolean computeActualNodeArraySize(final PtNodeArray nodeArray, + private static boolean computeActualPtNodeArraySize(final PtNodeArray ptNodeArray, final FusionDictionary dict, final FormatOptions formatOptions) { boolean changed = false; - int size = getGroupCountSize(nodeArray); - for (CharGroup group : nodeArray.mData) { - group.mCachedAddressAfterUpdate = nodeArray.mCachedAddressAfterUpdate + size; - if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) { + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode ptNode : ptNodeArray.mData) { + ptNode.mCachedAddressAfterUpdate = ptNodeArray.mCachedAddressAfterUpdate + size; + if (ptNode.mCachedAddressAfterUpdate != ptNode.mCachedAddressBeforeUpdate) { changed = true; } - int groupSize = getGroupHeaderSize(group, formatOptions); - if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE; - if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else if (null != group.mChildren) { - if (formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else { - groupSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(nodeArray, - groupSize + size, group.mChildren)); - } + int nodeSize = getNodeHeaderSize(ptNode, formatOptions); + if (ptNode.isTerminal()) nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (formatOptions.mSupportsDynamicUpdate) { + nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + } else if (null != ptNode.mChildren) { + nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, + nodeSize + size, ptNode.mChildren)); } - groupSize += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - for (WeightedString bigram : group.mBigrams) { - final int offset = getOffsetToTargetCharGroupDuringUpdate(nodeArray, - groupSize + size + FormatSpec.GROUP_FLAGS_SIZE, + nodeSize += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + for (WeightedString bigram : ptNode.mBigrams) { + final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, + nodeSize + size + FormatSpec.PTNODE_FLAGS_SIZE, FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); - groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE; + nodeSize += getByteSize(offset) + FormatSpec.PTNODE_FLAGS_SIZE; } } - group.mCachedSize = groupSize; - size += groupSize; + ptNode.mCachedSize = nodeSize; + size += nodeSize; } if (formatOptions.mSupportsDynamicUpdate) { size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } - if (nodeArray.mCachedSize != size) { - nodeArray.mCachedSize = size; + if (ptNodeArray.mCachedSize != size) { + ptNodeArray.mCachedSize = size; changed = true; } return changed; @@ -363,19 +361,19 @@ public class BinaryDictEncoder { * @param formatOptions file format options. * @return the byte size of the entire stack. */ - private static int initializeNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes, + private static int initializePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) { int nodeArrayOffset = 0; for (final PtNodeArray nodeArray : flatNodes) { nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset; - int groupCountSize = getGroupCountSize(nodeArray); - int groupOffset = 0; - for (final CharGroup g : nodeArray.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate = - groupCountSize + nodeArrayOffset + groupOffset; - groupOffset += g.mCachedSize; + int nodeCountSize = getPtNodeCountSize(nodeArray); + int nodeffset = 0; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate = + nodeCountSize + nodeArrayOffset + nodeffset; + nodeffset += ptNode.mCachedSize; } - final int nodeSize = groupCountSize + groupOffset + final int nodeSize = nodeCountSize + nodeffset + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); nodeArrayOffset += nodeArray.mCachedSize; @@ -388,11 +386,11 @@ public class BinaryDictEncoder { * * @param flatNodes the list of node arrays. */ - private static void updateNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) { + private static void updatePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) { for (final PtNodeArray nodeArray : flatNodes) { nodeArray.mCachedAddressBeforeUpdate = nodeArray.mCachedAddressAfterUpdate; - for (final CharGroup g : nodeArray.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate; } } } @@ -407,38 +405,38 @@ public class BinaryDictEncoder { */ private static void computeParentAddresses(final ArrayList<PtNodeArray> flatNodes) { for (final PtNodeArray nodeArray : flatNodes) { - for (final CharGroup group : nodeArray.mData) { - if (null != group.mChildren) { + for (final PtNode ptNode : nodeArray.mData) { + if (null != ptNode.mChildren) { // Assign my address to children's parent address // Here BeforeUpdate and AfterUpdate addresses have the same value, so it // does not matter which we use. - group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate - - group.mChildren.mCachedAddressAfterUpdate; + ptNode.mChildren.mCachedParentAddress = ptNode.mCachedAddressAfterUpdate + - ptNode.mChildren.mCachedAddressAfterUpdate; } } } } /** - * Compute the addresses and sizes of an ordered list of node arrays. + * Compute the addresses and sizes of an ordered list of PtNode arrays. * - * This method takes a list of node arrays and will update their cached address and size + * This method takes a list of PtNode arrays and will update their cached address and size * values so that they can be written into a file. It determines the smallest size each of the - * nodes arrays can be given the addresses of its children and attributes, and store that into - * each node. - * The order of the node is given by the order of the array. This method makes no effort + * PtNode arrays can be given the addresses of its children and attributes, and store that into + * each PtNode. + * The order of the PtNode is given by the order of the array. This method makes no effort * to find a good order; it only mechanically computes the size this order results in. * * @param dict the dictionary - * @param flatNodes the ordered list of nodes arrays + * @param flatNodes the ordered list of PtNode arrays * @param formatOptions file format options. * @return the same array it was passed. The nodes have been updated for address and size. */ private static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) { // First get the worst possible sizes and offsets - for (final PtNodeArray n : flatNodes) calculateNodeArrayMaximumSize(n, formatOptions); - final int offset = initializeNodeArraysCachedAddresses(flatNodes, formatOptions); + for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions); + final int offset = initializePtNodeArraysCachedAddresses(flatNodes, formatOptions); MakedictLog.i("Compressing the array addresses. Original size : " + offset); MakedictLog.i("(Recursively seen size : " + offset + ")"); @@ -447,19 +445,20 @@ public class BinaryDictEncoder { boolean changesDone = false; do { changesDone = false; - int nodeArrayStartOffset = 0; - for (final PtNodeArray nodeArray : flatNodes) { - nodeArray.mCachedAddressAfterUpdate = nodeArrayStartOffset; - final int oldNodeArraySize = nodeArray.mCachedSize; - final boolean changed = computeActualNodeArraySize(nodeArray, dict, formatOptions); - final int newNodeArraySize = nodeArray.mCachedSize; + int ptNodeArrayStartOffset = 0; + for (final PtNodeArray ptNodeArray : flatNodes) { + ptNodeArray.mCachedAddressAfterUpdate = ptNodeArrayStartOffset; + final int oldNodeArraySize = ptNodeArray.mCachedSize; + final boolean changed = + computeActualPtNodeArraySize(ptNodeArray, dict, formatOptions); + final int newNodeArraySize = ptNodeArray.mCachedSize; if (oldNodeArraySize < newNodeArraySize) { throw new RuntimeException("Increased size ?!"); } - nodeArrayStartOffset += newNodeArraySize; + ptNodeArrayStartOffset += newNodeArraySize; changesDone |= changed; } - updateNodeArraysCachedAddresses(flatNodes); + updatePtNodeArraysCachedAddresses(flatNodes); ++passes; if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); } while (changesDone); @@ -467,10 +466,10 @@ public class BinaryDictEncoder { if (formatOptions.mSupportsDynamicUpdate) { computeParentAddresses(flatNodes); } - final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); MakedictLog.i("Compression complete in " + passes + " passes."); MakedictLog.i("After address compression : " - + (lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize)); + + (lastPtNodeArray.mCachedAddressAfterUpdate + lastPtNodeArray.mCachedSize)); return flatNodes; } @@ -478,25 +477,26 @@ public class BinaryDictEncoder { /** * Sanity-checking method. * - * This method checks a list of node arrays for juxtaposition, that is, it will do + * This method checks a list of PtNode arrays for juxtaposition, that is, it will do * nothing if each node array's cached address is actually the previous node array's address * plus the previous node's size. * If this is not the case, it will throw an exception. * * @param arrays the list of node arrays to check */ - private static void checkFlatNodeArrayList(final ArrayList<PtNodeArray> arrays) { + private static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) { int offset = 0; int index = 0; - for (final PtNodeArray nodeArray : arrays) { + for (final PtNodeArray ptNodeArray : arrays) { // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter // which we use. - if (nodeArray.mCachedAddressAfterUpdate != offset) { + if (ptNodeArray.mCachedAddressAfterUpdate != offset) { throw new RuntimeException("Wrong address for node " + index - + " : expected " + offset + ", got " + nodeArray.mCachedAddressAfterUpdate); + + " : expected " + offset + ", got " + + ptNodeArray.mCachedAddressAfterUpdate); } ++index; - offset += nodeArray.mCachedSize; + offset += ptNodeArray.mCachedSize; } } @@ -552,19 +552,19 @@ public class BinaryDictEncoder { } /** - * Makes the flag value for a char group. + * Makes the flag value for a PtNode. * - * @param hasMultipleChars whether the group has multiple chars. - * @param isTerminal whether the group is terminal. + * @param hasMultipleChars whether the PtNode has multiple chars. + * @param isTerminal whether the PtNode is terminal. * @param childrenAddressSize the size of a children address. - * @param hasShortcuts whether the group has shortcuts. - * @param hasBigrams whether the group has bigrams. - * @param isNotAWord whether the group is not a word. - * @param isBlackListEntry whether the group is a blacklist entry. + * @param hasShortcuts whether the PtNode has shortcuts. + * @param hasBigrams whether the PtNode has bigrams. + * @param isNotAWord whether the PtNode is not a word. + * @param isBlackListEntry whether the PtNode is a blacklist entry. * @param formatOptions file format options. * @return the flags */ - static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal, + static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, final boolean isNotAWord, final boolean isBlackListEntry, final FormatOptions formatOptions) { @@ -576,16 +576,16 @@ public class BinaryDictEncoder { } else if (true) { switch (childrenAddressSize) { case 1: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE; + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE; break; case 2: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES; + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES; break; case 3: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES; break; case 0: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS; + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS; break; default: throw new RuntimeException("Node with a strange address"); @@ -598,11 +598,12 @@ public class BinaryDictEncoder { return flags; } - private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress, + private static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress, final int childrenOffset, final FormatOptions formatOptions) { - return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0, - getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null, - group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions); + return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0, + getByteSize(childrenOffset), + node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), + node.mBigrams != null, node.mIsNotAWord, node.mIsBlacklistEntry, formatOptions); } /** @@ -617,17 +618,17 @@ public class BinaryDictEncoder { */ private static final int makeBigramFlags(final boolean more, final int offset, int bigramFrequency, final int unigramFrequency, final String word) { - int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0); + int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); switch (getByteSize(offset)) { case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; break; case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; break; case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; break; default: throw new RuntimeException("Strange offset size"); @@ -672,7 +673,7 @@ public class BinaryDictEncoder { // small over-estimation that we get in this case. TODO: actually remove this bigram // if discretizedFrequency < 0. final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; - bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; + bigramFlags += finalBigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; return bigramFlags; } @@ -697,8 +698,8 @@ public class BinaryDictEncoder { * @return the flags */ static final int makeShortcutFlags(final boolean more, final int frequency) { - return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY); + return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); } private static final int writeParentAddress(final byte[] buffer, final int index, @@ -721,68 +722,69 @@ public class BinaryDictEncoder { } /** - * Write a node array to memory. The node array is expected to have its final position cached. + * Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached. * * @param dict the dictionary the node array is a part of (for relative offsets). * @param buffer the memory buffer to write to. - * @param nodeArray the node array to write. + * @param ptNodeArray the node array to write. * @param formatOptions file format options. * @return the address of the END of the node. */ @SuppressWarnings("unused") private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, - final PtNodeArray nodeArray, final FormatOptions formatOptions) { - // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup - int index = nodeArray.mCachedAddressAfterUpdate; + final PtNodeArray ptNodeArray, final FormatOptions formatOptions) { + // TODO: Make the code in common with BinaryDictIOUtils#writePtNode + int index = ptNodeArray.mCachedAddressAfterUpdate; - final int groupCount = nodeArray.mData.size(); - final int countSize = getGroupCountSize(nodeArray); - final int parentAddress = nodeArray.mCachedParentAddress; + final int ptNodeCount = ptNodeArray.mData.size(); + final int countSize = getPtNodeCountSize(ptNodeArray); + final int parentAddress = ptNodeArray.mCachedParentAddress; if (1 == countSize) { - buffer[index++] = (byte)groupCount; + buffer[index++] = (byte)ptNodeCount; } else if (2 == countSize) { // We need to signal 2-byte size by setting the top bit of the MSB to 1, so // we | 0x80 to do this. - buffer[index++] = (byte)((groupCount >> 8) | 0x80); - buffer[index++] = (byte)(groupCount & 0xFF); + buffer[index++] = (byte)((ptNodeCount >> 8) | 0x80); + buffer[index++] = (byte)(ptNodeCount & 0xFF); } else { throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); } - int groupAddress = index; - for (int i = 0; i < groupCount; ++i) { - final CharGroup group = nodeArray.mData.get(i); - if (index != group.mCachedAddressAfterUpdate) { + int ptNodeAddress = index; + for (int i = 0; i < ptNodeCount; ++i) { + final PtNode ptNode = ptNodeArray.mData.get(i); + if (index != ptNode.mCachedAddressAfterUpdate) { throw new RuntimeException("Bug: write index is not the same as the cached address " - + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate); + + "of the node : " + index + " <> " + ptNode.mCachedAddressAfterUpdate); } - groupAddress += getGroupHeaderSize(group, formatOptions); + ptNodeAddress += getNodeHeaderSize(ptNode, formatOptions); // Sanity checks. - if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { + if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { throw new RuntimeException("A node has a frequency > " + FormatSpec.MAX_TERMINAL_FREQUENCY - + " : " + group.mFrequency); + + " : " + ptNode.mFrequency); } - if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE; - final int childrenOffset = null == group.mChildren + if (ptNode.mFrequency >= 0) ptNodeAddress += FormatSpec.PTNODE_FREQUENCY_SIZE; + final int childrenOffset = null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS - : group.mChildren.mCachedAddressAfterUpdate - groupAddress; + : ptNode.mChildren.mCachedAddressAfterUpdate - ptNodeAddress; buffer[index++] = - makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions); + makePtNodeFlags(ptNode, ptNodeAddress, childrenOffset, formatOptions); if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { index = writeParentAddress(buffer, index, parentAddress, formatOptions); } else { index = writeParentAddress(buffer, index, parentAddress - + (nodeArray.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate), + + (ptNodeArray.mCachedAddressAfterUpdate + - ptNode.mCachedAddressAfterUpdate), formatOptions); } - index = CharEncoding.writeCharArray(group.mChars, buffer, index); - if (group.hasSeveralChars()) { - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; + index = CharEncoding.writeCharArray(ptNode.mChars, buffer, index); + if (ptNode.hasSeveralChars()) { + buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; } - if (group.mFrequency >= 0) { - buffer[index++] = (byte) group.mFrequency; + if (ptNode.mFrequency >= 0) { + buffer[index++] = (byte) ptNode.mFrequency; } final int shift; @@ -792,23 +794,24 @@ public class BinaryDictEncoder { shift = writeVariableAddress(buffer, index, childrenOffset); } index += shift; - groupAddress += shift; + ptNodeAddress += shift; // Write shortcuts - if (null != group.mShortcutTargets) { + if (null != ptNode.mShortcutTargets && !ptNode.mShortcutTargets.isEmpty()) { final int indexOfShortcutByteSize = index; - index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); + index += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + ptNodeAddress += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = + ptNode.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - ++groupAddress; + ++ptNodeAddress; int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), target.mFrequency); buffer[index++] = (byte)shortcutFlags; final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); index += shortcutShift; - groupAddress += shortcutShift; + ptNodeAddress += shortcutShift; } final int shortcutByteSize = index - indexOfShortcutByteSize; if (shortcutByteSize > 0xFFFF) { @@ -818,22 +821,22 @@ public class BinaryDictEncoder { buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); } // Write bigrams - if (null != group.mBigrams) { - final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); + if (null != ptNode.mBigrams) { + final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator(); while (bigramIterator.hasNext()) { final WeightedString bigram = bigramIterator.next(); - final CharGroup target = + final PtNode target = FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); final int addressOfBigram = target.mCachedAddressAfterUpdate; final int unigramFrequencyForThisWord = target.mFrequency; - ++groupAddress; - final int offset = addressOfBigram - groupAddress; + ++ptNodeAddress; + final int offset = addressOfBigram - ptNodeAddress; int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); buffer[index++] = (byte)bigramFlags; final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); index += bigramShift; - groupAddress += bigramShift; + ptNodeAddress += bigramShift; } } @@ -843,64 +846,64 @@ public class BinaryDictEncoder { = FormatSpec.NO_FORWARD_LINK_ADDRESS; index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } - if (index != nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize) { + if (index != ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize) { throw new RuntimeException( - "Not the same size : written " + (index - nodeArray.mCachedAddressAfterUpdate) - + " bytes from a node that should have " + nodeArray.mCachedSize + " bytes"); + "Not the same size : written " + (index - ptNodeArray.mCachedAddressAfterUpdate) + + " bytes from a node that should have " + ptNodeArray.mCachedSize + " bytes"); } return index; } /** - * Dumps a collection of useful statistics about a list of node arrays. + * Dumps a collection of useful statistics about a list of PtNode arrays. * * This prints purely informative stuff, like the total estimated file size, the - * number of node arrays, of character groups, the repartition of each address size, etc + * number of PtNode arrays, of PtNodes, the repartition of each address size, etc * - * @param nodeArrays the list of node arrays. + * @param ptNodeArrays the list of PtNode arrays. */ - private static void showStatistics(ArrayList<PtNodeArray> nodeArrays) { + private static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) { int firstTerminalAddress = Integer.MAX_VALUE; int lastTerminalAddress = Integer.MIN_VALUE; int size = 0; - int charGroups = 0; - int maxGroups = 0; + int ptNodes = 0; + int maxNodes = 0; int maxRuns = 0; - for (final PtNodeArray nodeArray : nodeArrays) { - if (maxGroups < nodeArray.mData.size()) maxGroups = nodeArray.mData.size(); - for (final CharGroup cg : nodeArray.mData) { - ++charGroups; - if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length; - if (cg.mFrequency >= 0) { - if (nodeArray.mCachedAddressAfterUpdate < firstTerminalAddress) - firstTerminalAddress = nodeArray.mCachedAddressAfterUpdate; - if (nodeArray.mCachedAddressAfterUpdate > lastTerminalAddress) - lastTerminalAddress = nodeArray.mCachedAddressAfterUpdate; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + if (maxNodes < ptNodeArray.mData.size()) maxNodes = ptNodeArray.mData.size(); + for (final PtNode ptNode : ptNodeArray.mData) { + ++ptNodes; + if (ptNode.mChars.length > maxRuns) maxRuns = ptNode.mChars.length; + if (ptNode.mFrequency >= 0) { + if (ptNodeArray.mCachedAddressAfterUpdate < firstTerminalAddress) + firstTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + if (ptNodeArray.mCachedAddressAfterUpdate > lastTerminalAddress) + lastTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; } } - if (nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize > size) { - size = nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize; + if (ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize > size) { + size = ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize; } } - final int[] groupCounts = new int[maxGroups + 1]; + final int[] ptNodeCounts = new int[maxNodes + 1]; final int[] runCounts = new int[maxRuns + 1]; - for (final PtNodeArray nodeArray : nodeArrays) { - ++groupCounts[nodeArray.mData.size()]; - for (final CharGroup cg : nodeArray.mData) { - ++runCounts[cg.mChars.length]; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + ++ptNodeCounts[ptNodeArray.mData.size()]; + for (final PtNode ptNode : ptNodeArray.mData) { + ++runCounts[ptNode.mChars.length]; } } MakedictLog.i("Statistics:\n" + " total file size " + size + "\n" - + " " + nodeArrays.size() + " node arrays\n" - + " " + charGroups + " groups (" + ((float)charGroups / nodeArrays.size()) - + " groups per node)\n" + + " " + ptNodeArrays.size() + " node arrays\n" + + " " + ptNodes + " PtNodes (" + ((float)ptNodes / ptNodeArrays.size()) + + " PtNodes per node)\n" + " first terminal at " + firstTerminalAddress + "\n" + " last terminal at " + lastTerminalAddress + "\n" - + " Group stats : max = " + maxGroups); - for (int i = 0; i < groupCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + groupCounts[i]); + + " PtNode stats : max = " + maxNodes); + for (int i = 0; i < ptNodeCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + ptNodeCounts[i]); } MakedictLog.i(" Character run stats : max = " + maxRuns); for (int i = 0; i < runCounts.length; ++i) { @@ -911,19 +914,17 @@ public class BinaryDictEncoder { /** * Dumps a FusionDictionary to a file. * - * This is the public entry point to write a dictionary to a file. - * * @param destination the stream to write the binary data to. * @param dict the dictionary to write. * @param formatOptions file format options. */ - public static void writeDictionaryBinary(final OutputStream destination, + /* package */ static void writeDictionaryBinary(final OutputStream destination, final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { // Addresses are limited to 3 bytes, but since addresses can be relative to each node // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding - // the order of the node arrays becomes a quite complicated problem, because though the + // the order of the PtNode arrays becomes a quite complicated problem, because though the // dictionary itself does not have a size limit, each node array must still be within 16MB // of all its children and parents. As long as this is ensured, the dictionary file may // grow to any size. @@ -981,8 +982,8 @@ public class BinaryDictEncoder { MakedictLog.i("Computing addresses..."); computeAddresses(dict, flatNodes, formatOptions); - MakedictLog.i("Checking array..."); - if (DBG) checkFlatNodeArrayList(flatNodes); + MakedictLog.i("Checking PtNode array..."); + if (DBG) checkFlatPtNodeArrayList(flatNodes); // Create a buffer that matches the final dictionary size. final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 0fa2cf428..106f02519 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -22,7 +22,7 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; @@ -44,17 +44,17 @@ public final class BinaryDictIOUtils { } private static final class Position { - public static final int NOT_READ_GROUPCOUNT = -1; + public static final int NOT_READ_PTNODE_COUNT = -1; public int mAddress; - public int mNumOfCharGroup; + public int mNumOfPtNode; public int mPosition; public int mLength; public Position(int address, int length) { mAddress = address; mLength = length; - mNumOfCharGroup = NOT_READ_GROUPCOUNT; + mNumOfPtNode = NOT_READ_PTNODE_COUNT; } } @@ -62,10 +62,11 @@ public final class BinaryDictIOUtils { * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner( - final DictBuffer dictBuffer, final int headerSize, + final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack<Position> stack = new Stack<Position>(); @@ -78,46 +79,45 @@ public final class BinaryDictIOUtils { Position p = stack.peek(); if (DBG) { - MakedictLog.d("read: address=" + p.mAddress + ", numOfCharGroup=" + - p.mNumOfCharGroup + ", position=" + p.mPosition + ", length=" + p.mLength); + MakedictLog.d("read: address=" + p.mAddress + ", numOfPtNode=" + + p.mNumOfPtNode + ", position=" + p.mPosition + ", length=" + p.mLength); } if (dictBuffer.position() != p.mAddress) dictBuffer.position(p.mAddress); if (index != p.mLength) index = p.mLength; - if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) { - p.mNumOfCharGroup = BinaryDictDecoderUtils.readCharGroupCount(dictBuffer); - p.mAddress += getGroupCountSize(p.mNumOfCharGroup); + if (p.mNumOfPtNode == Position.NOT_READ_PTNODE_COUNT) { + p.mNumOfPtNode = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + p.mAddress += getPtNodeCountSize(p.mNumOfPtNode); p.mPosition = 0; } - if (p.mNumOfCharGroup == 0) { + if (p.mNumOfPtNode == 0) { stack.pop(); continue; } - CharGroupInfo info = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - p.mAddress - headerSize, formatOptions); + PtNodeInfo info = dictDecoder.readPtNode(p.mAddress, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; } p.mPosition++; - final boolean isMovedGroup = isMovedGroup(info.mFlags, + final boolean isMovedPtNode = isMovedPtNode(info.mFlags, formatOptions); - final boolean isDeletedGroup = isDeletedGroup(info.mFlags, + final boolean isDeletedPtNode = isDeletedPtNode(info.mFlags, formatOptions); - if (!isMovedGroup && !isDeletedGroup - && info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) {// found word + if (!isMovedPtNode && !isDeletedPtNode + && info.mFrequency != FusionDictionary.PtNode.NOT_A_TERMINAL) {// found word words.put(info.mOriginalAddress, new String(pushedChars, 0, index)); frequencies.put(info.mOriginalAddress, info.mFrequency); if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams); } - if (p.mPosition == p.mNumOfCharGroup) { + if (p.mPosition == p.mNumOfPtNode) { if (formatOptions.mSupportsDynamicUpdate) { final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); if (forwardLinkAddress != FormatSpec.NO_FORWARD_LINK_ADDRESS) { // The node array has a forward link. - p.mNumOfCharGroup = Position.NOT_READ_GROUPCOUNT; + p.mNumOfPtNode = Position.NOT_READ_PTNODE_COUNT; p.mAddress = forwardLinkAddress; } else { stack.pop(); @@ -126,12 +126,12 @@ public final class BinaryDictIOUtils { stack.pop(); } } else { - // The node array has more groups. + // The Ptnode array has more PtNodes. p.mAddress = dictBuffer.position(); } - if (!isMovedGroup && hasChildrenAddress(info.mChildrenAddress)) { - Position childrenPos = new Position(info.mChildrenAddress + headerSize, index); + if (!isMovedPtNode && hasChildrenAddress(info.mChildrenAddress)) { + final Position childrenPos = new Position(info.mChildrenAddress, index); stack.push(childrenPos); } } @@ -148,18 +148,18 @@ public final class BinaryDictIOUtils { * @throws IOException if the file can't be read. * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, + /* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header final FileHeader header = dictDecoder.readHeader(); - readUnigramsAndBigramsBinaryInner(dictDecoder.getDictBuffer(), header.mHeaderSize, words, + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, frequencies, bigrams, header.mFormatOptions); } /** - * Gets the address of the last CharGroup of the exact matching word in the dictionary. + * Gets the address of the last PtNode of the exact matching word in the dictionary. * If no match is found, returns NOT_VALID_WORD. * * @param dictDecoder the dict decoder. @@ -169,7 +169,7 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final Ver3DictDecoder dictDecoder, + /* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; @@ -182,17 +182,17 @@ public final class BinaryDictIOUtils { if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; do { - final int charGroupCount = BinaryDictDecoderUtils.readCharGroupCount(dictBuffer); - boolean foundNextCharGroup = false; - for (int i = 0; i < charGroupCount; ++i) { - final int charGroupPos = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup( - dictBuffer, dictBuffer.position(), header.mFormatOptions); - final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, + final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + boolean foundNextPtNode = false; + for (int i = 0; i < ptNodeCount; ++i) { + final int ptNodePos = dictBuffer.position(); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos, + header.mFormatOptions); + final boolean isMovedNode = isMovedPtNode(currentInfo.mFlags, header.mFormatOptions); - final boolean isDeletedGroup = isDeletedGroup(currentInfo.mFlags, + final boolean isDeletedNode = isDeletedPtNode(currentInfo.mFlags, header.mFormatOptions); - if (isMovedGroup) continue; + if (isMovedNode) continue; boolean same = true; for (int p = 0, j = word.offsetByCodePoints(0, wordPos); p < currentInfo.mCharacters.length; @@ -205,30 +205,30 @@ public final class BinaryDictIOUtils { } if (same) { - // found the group matches the word. + // found the PtNode matches the word. if (wordPos + currentInfo.mCharacters.length == wordLen) { - if (currentInfo.mFrequency == CharGroup.NOT_A_TERMINAL - || isDeletedGroup) { + if (currentInfo.mFrequency == PtNode.NOT_A_TERMINAL + || isDeletedNode) { return FormatSpec.NOT_VALID_WORD; } else { - return charGroupPos; + return ptNodePos; } } wordPos += currentInfo.mCharacters.length; if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { return FormatSpec.NOT_VALID_WORD; } - foundNextCharGroup = true; + foundNextPtNode = true; dictBuffer.position(currentInfo.mChildrenAddress); break; } } - // If we found the next char group, it is under the file pointer. + // If we found the next PtNode, it is under the file pointer. // But if not, we are at the end of this node array so we expect to have // a forward link address that we need to consult and possibly resume // search on the next node array in the linked list. - if (foundNextCharGroup) break; + if (foundNextPtNode) break; if (!header.mFormatOptions.mSupportsDynamicUpdate) { return FormatSpec.NOT_VALID_WORD; } @@ -272,7 +272,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictEncoder.getByteSize(value)) { + switch (BinaryDictEncoderUtils.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -286,11 +286,10 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictEncoder.getByteSize(value); + return BinaryDictEncoderUtils.getByteSize(value); } - static void skipCharGroup(final DictBuffer dictBuffer, - final FormatOptions formatOptions) { + static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) { final int flags = dictBuffer.readUnsignedByte(); BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); @@ -299,27 +298,27 @@ public final class BinaryDictIOUtils { if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { final int shortcutsSize = dictBuffer.readUnsignedShort(); dictBuffer.position(dictBuffer.position() + shortcutsSize - - FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE); + - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); } if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { final int bigramFlags = dictBuffer.readUnsignedByte(); - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: dictBuffer.readUnsignedByte(); break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: dictBuffer.readUnsignedShort(); break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: dictBuffer.readUnsignedInt24(); break; } - if ((bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT) == 0) break; + if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - throw new RuntimeException("Too many bigrams in a group."); + if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode."); } } } @@ -360,24 +359,24 @@ public final class BinaryDictIOUtils { size += 3; } } - destination.write((byte)FormatSpec.GROUP_CHARACTERS_TERMINATOR); - size += FormatSpec.GROUP_TERMINATOR_SIZE; + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + size += FormatSpec.PTNODE_TERMINATOR_SIZE; return size; } /** - * Write a char group to an output stream from a CharGroupInfo. - * A char group is an in-memory representation of a node in the patricia trie. - * A char group info is a container for low-level information about how the - * char group is stored in the binary format. + * Write a PtNode to an output stream from a PtNodeInfo. + * A PtNode is an in-memory representation of a node in the patricia trie. + * A PtNode info is a container for low-level information about how the + * PtNode is stored in the binary format. * * @param destination the stream to write. - * @param info the char group info to be written. + * @param info the PtNode info to be written. * @return the size written, in bytes. */ - public static int writeCharGroup(final OutputStream destination, final CharGroupInfo info) + private static int writePtNode(final OutputStream destination, final PtNodeInfo info) throws IOException { - int size = FormatSpec.GROUP_FLAGS_SIZE; + int size = FormatSpec.PTNODE_FLAGS_SIZE; destination.write((byte)info.mFlags); final int parentOffset = info.mParentAddress == FormatSpec.NO_PARENT_ADDRESS ? FormatSpec.NO_PARENT_ADDRESS : info.mParentAddress - info.mOriginalAddress; @@ -392,7 +391,7 @@ public final class BinaryDictIOUtils { } } if (info.mCharacters.length > 1) { - destination.write((byte)FormatSpec.GROUP_CHARACTERS_TERMINATOR); + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); size++; } @@ -402,7 +401,7 @@ public final class BinaryDictIOUtils { } if (DBG) { - MakedictLog.d("writeCharGroup origin=" + info.mOriginalAddress + ", size=" + size + MakedictLog.d("writePtNode origin=" + info.mOriginalAddress + ", size=" + size + ", child=" + info.mChildrenAddress + ", characters =" + new String(info.mCharacters, 0, info.mCharacters.length)); } @@ -413,14 +412,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictEncoder.makeShortcutFlags( + destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -429,28 +428,28 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictEncoder#computeActualNodeArraySize + // in BinaryDictEncoderUtils#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; int bigramFlags = (i < info.mBigrams.size() - 1) - ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0; + ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0; size++; final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); - bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictEncoder.getByteSize(bigramOffset)) { + bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0; + switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; break; case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; break; case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; break; } - bigramFlags |= bigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; + bigramFlags |= bigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; destination.write((byte)bigramFlags); size += writeVariableAddress(destination, Math.abs(bigramOffset)); } @@ -459,22 +458,22 @@ public final class BinaryDictIOUtils { } /** - * Compute the size of the char group. + * Compute the size of the PtNode. */ - static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) { - int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters) + static int computePtNodeSize(final PtNodeInfo info, final FormatOptions formatOptions) { + int size = FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + BinaryDictEncoderUtils.getPtNodeCharactersSize(info.mCharacters) + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - size += FormatSpec.GROUP_FREQUENCY_SIZE; + size += FormatSpec.PTNODE_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { - size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictEncoder.getByteSize(attr.mAddress); + size += FormatSpec.PTNODE_FLAGS_SIZE; + size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); } } return size; @@ -484,14 +483,14 @@ public final class BinaryDictIOUtils { * Write a node array to the stream. * * @param destination the stream to write. - * @param infos an array of CharGroupInfo to be written. + * @param infos an array of PtNodeInfo to be written. * @return the size written, in bytes. * @throws IOException */ - static int writeNodes(final OutputStream destination, final CharGroupInfo[] infos) + static int writeNodes(final OutputStream destination, final PtNodeInfo[] infos) throws IOException { - int size = getGroupCountSize(infos.length); - switch (getGroupCountSize(infos.length)) { + int size = getPtNodeCountSize(infos.length); + switch (getPtNodeCountSize(infos.length)) { case 1: destination.write((byte)infos.length); break; @@ -500,37 +499,13 @@ public final class BinaryDictIOUtils { destination.write((byte)(infos.length & 0xFF)); break; default: - throw new RuntimeException("Invalid group count size."); + throw new RuntimeException("Invalid node count size."); } - for (final CharGroupInfo info : infos) size += writeCharGroup(destination, info); + for (final PtNodeInfo info : infos) size += writePtNode(destination, info); writeSInt24ToStream(destination, FormatSpec.NO_FORWARD_LINK_ADDRESS); return size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } - /** - * Find a word using the Ver3DictDecoder. - * - * @param dictDecoder the dict reader - * @param word the word searched - * @return the found group - * @throws IOException - * @throws UnsupportedFormatException - */ - @UsedForTesting - public static CharGroupInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder, - final String word) throws IOException, UnsupportedFormatException { - int position = getTerminalPosition(dictDecoder, word); - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - if (position != FormatSpec.NOT_VALID_WORD) { - dictBuffer.position(0); - final FileHeader header = dictDecoder.readHeader(); - dictBuffer.position(position); - return BinaryDictDecoderUtils.readCharGroup(dictBuffer, position, - header.mFormatOptions); - } - return null; - } - private static final int HEADER_READING_BUFFER_SIZE = 16384; /** * Convenience method to read the header of a binary file. @@ -541,24 +516,26 @@ public final class BinaryDictIOUtils { * @param offset The offset in the file where to start reading the data. * @param length The length of the data file. */ - public static FileHeader getDictionaryFileHeader( + private static FileHeader getDictionaryFileHeader( final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); - dictDecoder.openDictBuffer(new DictDecoder.DictionaryBufferFactory() { - @Override - public DictBuffer getDictionaryBuffer(File file) - throws FileNotFoundException, IOException { - final FileInputStream inStream = new FileInputStream(file); - try { - inStream.read(buffer); - return new ByteArrayDictBuffer(buffer); - } finally { - inStream.close(); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + new DictDecoder.DictionaryBufferFactory() { + @Override + public DictBuffer getDictionaryBuffer(File file) + throws FileNotFoundException, IOException { + final FileInputStream inStream = new FileInputStream(file); + try { + inStream.skip(offset); + inStream.read(buffer); + return new ByteArrayDictBuffer(buffer); + } finally { + inStream.close(); + } + } } - } - }); + ); return dictDecoder.readHeader(); } @@ -582,11 +559,11 @@ public final class BinaryDictIOUtils { } /** - * Helper method to check whether the group is moved. + * Helper method to check whether the node is moved. */ - public static boolean isMovedGroup(final int flags, final FormatOptions options) { + public static boolean isMovedPtNode(final int flags, final FormatOptions options) { return options.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); + && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); } /** @@ -598,26 +575,26 @@ public final class BinaryDictIOUtils { } /** - * Helper method to check whether the group is deleted. + * Helper method to check whether the node is deleted. */ - public static boolean isDeletedGroup(final int flags, final FormatOptions formatOptions) { + public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { return formatOptions.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); + && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); } /** - * Compute the binary size of the group count - * @param count the group count - * @return the size of the group count, either 1 or 2 bytes. + * Compute the binary size of the node count + * @param count the node count + * @return the size of the node count, either 1 or 2 bytes. */ - public static int getGroupCountSize(final int count) { - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { + public static int getPtNodeCountSize(final int count) { + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= count) { return 1; - } else if (FormatSpec.MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY >= count) { + } else if (FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY >= count) { return 2; } else { throw new RuntimeException("Can't have more than " - + FormatSpec.MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY + " groups in a node (found " + + FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY + " PtNode in a PtNodeArray (found " + count + ")"); } } @@ -625,14 +602,14 @@ public final class BinaryDictIOUtils { static int getChildrenAddressSize(final int optionFlags, final FormatOptions formatOptions) { if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: return 1; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: return 2; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: return 3; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: default: return 0; } diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index a63d9d5ba..11a3f0b3a 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; @@ -28,6 +29,8 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.TreeMap; /** * An interface of binary dictionary decoder. @@ -35,6 +38,62 @@ import java.nio.channels.FileChannel; public interface DictDecoder { public FileHeader readHeader() throws IOException, UnsupportedFormatException; + /** + * Reads PtNode from nodeAddress. + * @param ptNodePos the position of PtNode. + * @param formatOptions the format options. + * @return PtNodeInfo. + */ + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. The optional dict argument is an existing dictionary to + * which words from the buffer should be added. If it is null, a new dictionary is created. + * + * @param dict an optional dictionary to add words to, or null. + * @return the created (or merged) dictionary. + */ + @UsedForTesting + public FusionDictionary readDictionaryBinary(final FusionDictionary dict) + throws FileNotFoundException, IOException, UnsupportedFormatException; + + /** + * Gets the address of the last PtNode of the exact matching word in the dictionary. + * If no match is found, returns NOT_VALID_WORD. + * + * @param word the word we search for. + * @return the address of the terminal node. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + public int getTerminalPosition(final String word) + throws IOException, UnsupportedFormatException; + + /** + * Reads unigrams and bigrams from the binary file. + * Doesn't store a full memory representation of the dictionary. + * + * @param words the map to store the address as a key and the word as a value. + * @param frequencies the map to store the address as a key and the frequency as a value. + * @param bigrams the map to store the address as a key and the list of address as a value. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException; + + // Flags for DictionaryBufferFactory. + public static final int USE_READONLY_BYTEBUFFER = 0x01000000; + public static final int USE_BYTEARRAY = 0x02000000; + public static final int USE_WRITABLE_BYTEBUFFER = 0x04000000; + public static final int MASK_DICTBUFFER = 0x0F000000; + public interface DictionaryBufferFactory { public DictBuffer getDictionaryBuffer(final File file) throws FileNotFoundException, IOException; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java new file mode 100644 index 000000000..89c982e7b --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; + +import java.io.IOException; + +/** + * An interface of binary dictionary encoder. + */ +public interface DictEncoder { + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException; +} diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 99deaa4f9..bf3d19101 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -43,7 +43,7 @@ public final class DynamicBinaryDictIOUtils { } private static int markAsDeleted(final int flags) { - return (flags & (~FormatSpec.MASK_GROUP_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; + return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; } /** @@ -60,7 +60,7 @@ public final class DynamicBinaryDictIOUtils { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); dictBuffer.position(0); final FileHeader header = dictDecoder.readHeader(); - final int wordPosition = BinaryDictIOUtils.getTerminalPosition(dictDecoder, word); + final int wordPosition = dictDecoder.getTerminalPosition(word); if (wordPosition == FormatSpec.NOT_VALID_WORD) return; dictBuffer.position(wordPosition); @@ -70,56 +70,56 @@ public final class DynamicBinaryDictIOUtils { } /** - * Update a parent address in a CharGroup that is referred to by groupOriginAddress. + * Update a parent address in a PtNode that is referred to by ptNodeOriginAddress. * * @param dictBuffer the DictBuffer to write. - * @param groupOriginAddress the address of the group. + * @param ptNodeOriginAddress the address of the PtNode. * @param newParentAddress the absolute address of the parent. * @param formatOptions file format options. */ public static void updateParentAddress(final DictBuffer dictBuffer, - final int groupOriginAddress, final int newParentAddress, + final int ptNodeOriginAddress, final int newParentAddress, final FormatOptions formatOptions) { final int originalPosition = dictBuffer.position(); - dictBuffer.position(groupOriginAddress); + dictBuffer.position(ptNodeOriginAddress); if (!formatOptions.mSupportsDynamicUpdate) { throw new RuntimeException("this file format does not support parent addresses"); } final int flags = dictBuffer.readUnsignedByte(); - if (BinaryDictIOUtils.isMovedGroup(flags, formatOptions)) { - // If the group is moved, the parent address is stored in the destination group. - // We are guaranteed to process the destination group later, so there is no need to + if (BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { + // If the node is moved, the parent address is stored in the destination node. + // We are guaranteed to process the destination node later, so there is no need to // update anything here. dictBuffer.position(originalPosition); return; } if (DBG) { - MakedictLog.d("update parent address flags=" + flags + ", " + groupOriginAddress); + MakedictLog.d("update parent address flags=" + flags + ", " + ptNodeOriginAddress); } - final int parentOffset = newParentAddress - groupOriginAddress; + final int parentOffset = newParentAddress - ptNodeOriginAddress; BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, parentOffset); dictBuffer.position(originalPosition); } /** - * Update parent addresses in a node array stored at nodeOriginAddress. + * Update parent addresses in a node array stored at ptNodeOriginAddress. * * @param dictBuffer the DictBuffer to be modified. - * @param nodeOriginAddress the address of the node array to update. + * @param ptNodeOriginAddress the address of the node array to update. * @param newParentAddress the address to be written. * @param formatOptions file format options. */ public static void updateParentAddresses(final DictBuffer dictBuffer, - final int nodeOriginAddress, final int newParentAddress, + final int ptNodeOriginAddress, final int newParentAddress, final FormatOptions formatOptions) { final int originalPosition = dictBuffer.position(); - dictBuffer.position(nodeOriginAddress); + dictBuffer.position(ptNodeOriginAddress); do { - final int count = BinaryDictDecoderUtils.readCharGroupCount(dictBuffer); + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); for (int i = 0; i < count; ++i) { updateParentAddress(dictBuffer, dictBuffer.position(), newParentAddress, formatOptions); - BinaryDictIOUtils.skipCharGroup(dictBuffer, formatOptions); + BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); } final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); dictBuffer.position(forwardLinkAddress); @@ -129,18 +129,18 @@ public final class DynamicBinaryDictIOUtils { } /** - * Update a children address in a CharGroup that is addressed by groupOriginAddress. + * Update a children address in a PtNode that is addressed by ptNodeOriginAddress. * * @param dictBuffer the DictBuffer to write. - * @param groupOriginAddress the address of the group. + * @param ptNodeOriginAddress the address of the PtNode. * @param newChildrenAddress the absolute address of the child. * @param formatOptions file format options. */ public static void updateChildrenAddress(final DictBuffer dictBuffer, - final int groupOriginAddress, final int newChildrenAddress, + final int ptNodeOriginAddress, final int newChildrenAddress, final FormatOptions formatOptions) { final int originalPosition = dictBuffer.position(); - dictBuffer.position(groupOriginAddress); + dictBuffer.position(ptNodeOriginAddress); final int flags = dictBuffer.readUnsignedByte(); final int parentAddress = BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); @@ -153,21 +153,21 @@ public final class DynamicBinaryDictIOUtils { } /** - * Helper method to move a char group to the tail of the file. + * Helper method to move a PtNode to the tail of the file. */ - private static int moveCharGroup(final OutputStream destination, - final DictBuffer dictBuffer, final CharGroupInfo info, - final int nodeArrayOriginAddress, final int oldGroupAddress, + private static int movePtNode(final OutputStream destination, + final DictBuffer dictBuffer, final PtNodeInfo info, + final int nodeArrayOriginAddress, final int oldNodeAddress, final FormatOptions formatOptions) throws IOException { - updateParentAddress(dictBuffer, oldGroupAddress, dictBuffer.limit() + 1, formatOptions); - dictBuffer.position(oldGroupAddress); + updateParentAddress(dictBuffer, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); + dictBuffer.position(oldNodeAddress); final int currentFlags = dictBuffer.readUnsignedByte(); - dictBuffer.position(oldGroupAddress); + dictBuffer.position(oldNodeAddress); dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); - int size = FormatSpec.GROUP_FLAGS_SIZE; + int size = FormatSpec.PTNODE_FLAGS_SIZE; updateForwardLink(dictBuffer, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); - size += BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { info }); + size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info }); return size; } @@ -178,9 +178,9 @@ public final class DynamicBinaryDictIOUtils { dictBuffer.position(nodeArrayOriginAddress); int jumpCount = 0; while (jumpCount++ < MAX_JUMPS) { - final int count = BinaryDictDecoderUtils.readCharGroupCount(dictBuffer); + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); for (int i = 0; i < count; ++i) { - BinaryDictIOUtils.skipCharGroup(dictBuffer, formatOptions); + BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); } final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { @@ -196,43 +196,43 @@ public final class DynamicBinaryDictIOUtils { } /** - * Move a group that is referred to by oldGroupOrigin to the tail of the file, and set the - * children address to the byte after the group + * Move a PtNode that is referred to by oldPtNodeOrigin to the tail of the file, and set the + * children address to the byte after the PtNode. * * @param fileEndAddress the address of the tail of the file. - * @param codePoints the characters to put inside the group. + * @param codePoints the characters to put inside the PtNode. * @param length how many code points to read from codePoints. - * @param flags the flags for this group. + * @param flags the flags for this PtNode. * @param frequency the frequency of this terminal. - * @param parentAddress the address of the parent group of this group. - * @param shortcutTargets the shortcut targets for this group. - * @param bigrams the bigrams for this group. + * @param parentAddress the address of the parent PtNode of this PtNode. + * @param shortcutTargets the shortcut targets for this PtNode. + * @param bigrams the bigrams for this PtNode. * @param destination the stream representing the tail of the file. * @param dictBuffer the DictBuffer representing the (constant-size) body of the file. - * @param oldNodeArrayOrigin the origin of the old node array this group was a part of. - * @param oldGroupOrigin the old origin where this group used to be stored. + * @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of. + * @param oldPtNodeOrigin the old origin where this PtNode used to be stored. * @param formatOptions format options for this dictionary. * @return the size written, in bytes. * @throws IOException if the file can't be accessed */ - private static int moveGroup(final int fileEndAddress, final int[] codePoints, + private static int movePtNode(final int fileEndAddress, final int[] codePoints, final int length, final int flags, final int frequency, final int parentAddress, final ArrayList<WeightedString> shortcutTargets, final ArrayList<PendingAttribute> bigrams, final OutputStream destination, - final DictBuffer dictBuffer, final int oldNodeArrayOrigin, - final int oldGroupOrigin, final FormatOptions formatOptions) throws IOException { + final DictBuffer dictBuffer, final int oldPtNodeArrayOrigin, + final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException { int size = 0; - final int newGroupOrigin = fileEndAddress + 1; + final int newPtNodeOrigin = fileEndAddress + 1; final int[] writtenCharacters = Arrays.copyOfRange(codePoints, 0, length); - final CharGroupInfo tmpInfo = new CharGroupInfo(newGroupOrigin, -1 /* endAddress */, + final PtNodeInfo tmpInfo = new PtNodeInfo(newPtNodeOrigin, -1 /* endAddress */, flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, shortcutTargets, bigrams); - size = BinaryDictIOUtils.computeGroupSize(tmpInfo, formatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newGroupOrigin, newGroupOrigin + size, + size = BinaryDictIOUtils.computePtNodeSize(tmpInfo, formatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newPtNodeOrigin, newPtNodeOrigin + size, flags, writtenCharacters, frequency, parentAddress, fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, bigrams); - moveCharGroup(destination, dictBuffer, newInfo, oldNodeArrayOrigin, oldGroupOrigin, + movePtNode(destination, dictBuffer, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, formatOptions); return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } @@ -263,7 +263,7 @@ public final class DynamicBinaryDictIOUtils { final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (bigramStrings != null) { for (final WeightedString bigram : bigramStrings) { - int position = BinaryDictIOUtils.getTerminalPosition(dictDecoder, bigram.mWord); + int position = dictDecoder.getTerminalPosition(bigram.mWord); if (position == FormatSpec.NOT_VALID_WORD) { // TODO: figure out what is the correct thing to do here. } else { @@ -288,16 +288,16 @@ public final class DynamicBinaryDictIOUtils { if (wordPos >= wordLen) break; nodeOriginAddress = dictBuffer.position(); int nodeParentAddress = -1; - final int charGroupCount = BinaryDictDecoderUtils.readCharGroupCount(dictBuffer); - boolean foundNextGroup = false; + final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + boolean foundNextNode = false; - for (int i = 0; i < charGroupCount; ++i) { + for (int i = 0; i < ptNodeCount; ++i) { address = dictBuffer.position(); - final CharGroupInfo currentInfo = BinaryDictDecoderUtils.readCharGroup(dictBuffer, - dictBuffer.position(), fileHeader.mFormatOptions); - final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, + final PtNodeInfo currentInfo = dictDecoder.readPtNode(address, fileHeader.mFormatOptions); - if (isMovedGroup) continue; + final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, + fileHeader.mFormatOptions); + if (isMovedNode) continue; nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; boolean matched = true; @@ -314,10 +314,10 @@ public final class DynamicBinaryDictIOUtils { * abc - d - ef */ final int newNodeAddress = dictBuffer.limit(); - final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, false /* isBlackListEntry */, fileHeader.mFormatOptions); - int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, + int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags, frequency, nodeParentAddress, shortcuts, bigrams, destination, dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions); @@ -327,12 +327,12 @@ public final class DynamicBinaryDictIOUtils { updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } - final CharGroupInfo newInfo2 = new CharGroupInfo( + final PtNodeInfo newInfo2 = new PtNodeInfo( newNodeAddress + written + 1, -1 /* endAddress */, currentInfo.mFlags, characters2, currentInfo.mFrequency, newNodeAddress + 1, currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, currentInfo.mBigrams); - BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { newInfo2 }); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo2 }); return; } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { if (p > 0) { @@ -353,12 +353,12 @@ public final class DynamicBinaryDictIOUtils { final int childrenAddress = currentInfo.mChildrenAddress; // move prefix - final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1, + final int prefixFlags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, false /* isTerminal */, 0 /* childrenAddressSize*/, false /* hasShortcut */, false /* hasBigrams */, false /* isNotAWord */, false /* isBlackListEntry */, fileHeader.mFormatOptions); - int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, + int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, destination, dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions); @@ -369,7 +369,7 @@ public final class DynamicBinaryDictIOUtils { updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, newNodeAddress + written + 1, fileHeader.mFormatOptions); } - final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags( + final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags( suffixCharacters.length > 1, (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, 0 /* childrenAddressSize */, @@ -377,26 +377,26 @@ public final class DynamicBinaryDictIOUtils { != 0, (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final CharGroupInfo suffixInfo = new CharGroupInfo( + final PtNodeInfo suffixInfo = new PtNodeInfo( newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, currentInfo.mBigrams); - written += BinaryDictIOUtils.computeGroupSize(suffixInfo, + written += BinaryDictIOUtils.computePtNodeSize(suffixInfo, fileHeader.mFormatOptions) + 1; final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, codePoints.length); - final int flags = BinaryDictEncoder.makeCharGroupFlags( + final int flags = BinaryDictEncoderUtils.makePtNodeFlags( newCharacters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo( + final PtNodeInfo newInfo = new PtNodeInfo( newNodeAddress + written, -1 /* endAddress */, flags, newCharacters, frequency, newNodeAddress + 1, FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); BinaryDictIOUtils.writeNodes(destination, - new CharGroupInfo[] { suffixInfo, newInfo }); + new PtNodeInfo[] { suffixInfo, newInfo }); return; } matched = false; @@ -407,17 +407,17 @@ public final class DynamicBinaryDictIOUtils { if (matched) { if (wordPos + currentInfo.mCharacters.length == wordLen) { // the word exists in the dictionary. - // only update group. + // only update the PtNode. final int newNodeAddress = dictBuffer.limit(); final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, bigrams); - moveCharGroup(destination, dictBuffer, newInfo, nodeOriginAddress, address, + movePtNode(destination, dictBuffer, newInfo, nodeOriginAddress, address, fileHeader.mFormatOptions); return; } @@ -425,7 +425,7 @@ public final class DynamicBinaryDictIOUtils { if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { /* * found the prefix of the word. - * make new node and link to the node from this group. + * make new PtNode and link to the PtNode from this PtNode. * * before * ab - cd @@ -435,28 +435,28 @@ public final class DynamicBinaryDictIOUtils { * after * ab - cd - e */ - final int newNodeAddress = dictBuffer.limit(); - updateChildrenAddress(dictBuffer, address, newNodeAddress, + final int newNodeArrayAddress = dictBuffer.limit(); + updateChildrenAddress(dictBuffer, address, newNodeArrayAddress, fileHeader.mFormatOptions); - final int newGroupAddress = newNodeAddress + 1; + final int newNodeAddress = newNodeArrayAddress + 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final CharGroupInfo newInfo = new CharGroupInfo(newGroupAddress, -1, flags, + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress, -1, flags, characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { newInfo }); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo }); return; } dictBuffer.position(currentInfo.mChildrenAddress); - foundNextGroup = true; + foundNextNode = true; break; } } - if (foundNextGroup) continue; + if (foundNextNode) continue; // reached the end of the array. final int linkAddressPosition = dictBuffer.position(); @@ -485,13 +485,13 @@ public final class DynamicBinaryDictIOUtils { BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1, + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(characters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[]{ newInfo }); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[]{ newInfo }); return; } else { depth--; diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 5e331219c..b8ef57696 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -62,19 +62,19 @@ public final class FormatSpec { /* * Node array (FusionDictionary.PtNodeArray) layout is as follows: * - * g | - * r | the number of groups, 1 or 2 bytes. - * o | 1 byte = bbbbbbbb match - * u | case 1xxxxxxx => xxxxxxx << 8 + next byte - * p | otherwise => bbbbbbbb - * c | - * ount + * n | + * o | the number of PtNodes, 1 or 2 bytes. + * d | 1 byte = bbbbbbbb match + * e | case 1xxxxxxx => xxxxxxx << 8 + next byte + * c | otherwise => bbbbbbbb + * o | + * unt * - * g | - * r | sequence of groups, - * o | the layout of each group is described below. - * u | - * ps + * n | + * o | sequence of PtNodes, + * d | the layout of each PtNode is described below. + * e | + * s * * f | * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) @@ -86,19 +86,19 @@ public final class FormatSpec { * linkaddress */ - /* Node (FusionDictionary.CharGroup) layout is as follows: + /* Node (FusionDictionary.PtNode) layout is as follows: * | IF !SUPPORTS_DYNAMIC_UPDATE - * | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE - * | 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS - * f | 01 = 1 byte : FLAG_GROUP_ADDRESS_TYPE_ONEBYTE - * l | 10 = 2 bytes : FLAG_GROUP_ADDRESS_TYPE_TWOBYTES - * a | 11 = 3 bytes : FLAG_GROUP_ADDRESS_TYPE_THREEBYTES + * | addressType xx : mask with MASK_CHILDREN_ADDRESS_TYPE + * | 2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS + * f | 01 = 1 byte : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE + * l | 10 = 2 bytes : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES + * a | 11 = 3 bytes : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES * g | ELSE - * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED - * | This must be the same as FLAG_GROUP_ADDRESS_TYPE_THREEBYTES - * | 01 = yes : FLAG_IS_MOVED + * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED + * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 01 = yes : FLAG_IS_MOVED * | the new address is stored in the same place as the parent address - * | is deleted? 10 = yes : FLAG_IS_DELETED + * | is deleted? 10 = yes : FLAG_IS_DELETED * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS @@ -116,7 +116,7 @@ public final class FormatSpec { * ddress * * c | IF FLAG_HAS_MULTIPLE_CHARS - * h | char, char, char, char n * (1 or 3 bytes) : use CharGroupInfo for i/o helpers + * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers * a | end 1 byte, = 0 * r | ELSE * s | char 1 or 3 bytes @@ -127,17 +127,22 @@ public final class FormatSpec { * e | frequency 1 byte * q | * - * c | IF 00 = FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = addressType - * h | // nothing - * i | ELSIF 01 = FLAG_GROUP_ADDRESS_TYPE_ONEBYTE == addressType - * l | children address, 1 byte - * d | ELSIF 10 = FLAG_GROUP_ADDRESS_TYPE_TWOBYTES == addressType - * r | children address, 2 bytes - * e | ELSE // 11 = FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = addressType - * n | children address, 3 bytes - * A | END - * d - * dress + * c | IF SUPPORTS_DYNAMIC_UPDATE + * h | children address, 3 bytes + * i | 1 byte = bbbbbbbb match + * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte + * r | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType + * e | // nothing + * n | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType + * A | children address, 1 byte + * d | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType + * d | children address, 2 bytes + * r | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType + * e | children address, 3 bytes + * s | END + * s + * ress * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list @@ -156,33 +161,33 @@ public final class FormatSpec { * characters which should never happen anyway (and still work, but take 3 bytes). * * bigram address list is: - * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT - * | addressSign = 1 bit, : FLAG_ATTRIBUTE_OFFSET_NEGATIVE + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT + * | addressSign = 1 bit, : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE * | 1 = must take -address, 0 = must take +address - * | xx : mask with MASK_ATTRIBUTE_ADDRESS_TYPE - * | addressFormat = 2 bits, 00 = unused : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE - * | 01 = 1 byte : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE - * | 10 = 2 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES - * | 11 = 3 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES - * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY - * <address> | IF (01 == FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE == addressFormat) + * | xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE + * | addressFormat = 2 bits, 00 = unused : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 01 = 1 byte : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES + * | 11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY + * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat) * | read 1 byte, add top 4 bits - * | ELSIF (10 == FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES == addressFormat) + * | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat) * | read 2 bytes, add top 4 bits - * | ELSE // 11 == FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES == addressFormat + * | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat * | read 3 bytes, add top 4 bits * | END - * | if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE) then address = -address - * if (FLAG_ATTRIBUTE_HAS_NEXT) goto bigram_and_shortcut_address_list_is + * | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is * * shortcut string list is: - * <byte size> = GROUP_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. - * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT + * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT * | reserved = 3 bits, must be 0 - * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY * <shortcut> = | string of characters at the char format described above, with the terminator * | used to signal the end of the string. - * if (FLAG_ATTRIBUTE_HAS_NEXT goto flags + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags */ public static final int MAGIC_NUMBER = 0x9BC13AFE; @@ -206,11 +211,11 @@ public final class FormatSpec { static final int FORWARD_LINK_ADDRESS_SIZE = 3; // These flags are used only in the static dictionary. - static final int MASK_GROUP_ADDRESS_TYPE = 0xC0; - static final int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; - static final int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; - static final int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; - static final int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; + static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0; + static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00; + static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40; + static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80; + static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0; static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; @@ -227,32 +232,32 @@ public final class FormatSpec { static final int FLAG_IS_NOT_MOVED = 0x80 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE; static final int FLAG_IS_DELETED = 0x80; - static final int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; - static final int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; - static final int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; - static final int FLAG_ATTRIBUTE_FREQUENCY = 0x0F; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80; + static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40; + static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F; - static final int GROUP_CHARACTERS_TERMINATOR = 0x1F; + static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F; - static final int GROUP_TERMINATOR_SIZE = 1; - static final int GROUP_FLAGS_SIZE = 1; - static final int GROUP_FREQUENCY_SIZE = 1; - static final int GROUP_MAX_ADDRESS_SIZE = 3; - static final int GROUP_ATTRIBUTE_FLAGS_SIZE = 1; - static final int GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; - static final int GROUP_SHORTCUT_LIST_SIZE_SIZE = 2; + static final int PTNODE_TERMINATOR_SIZE = 1; + static final int PTNODE_FLAGS_SIZE = 1; + static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; + static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; static final int INVALID_CHARACTER = -1; - static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127 - static final int MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 - static final int MAX_BIGRAMS_IN_A_GROUP = 10000; + static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 + static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 + static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index fce1c5cdd..3e685a3d7 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -37,15 +37,15 @@ public final class FusionDictionary implements Iterable<Word> { private static int CHARACTER_NOT_FOUND_INDEX = -1; /** - * A node array of the dictionary, containing several CharGroups. + * A node array of the dictionary, containing several PtNodes. * - * A PtNodeArray is but an ordered array of CharGroups, which essentially contain all the + * A PtNodeArray is but an ordered array of PtNodes, which essentially contain all the * real information. * This class also contains fields to cache size and address, to help with binary * generation. */ public static final class PtNodeArray { - ArrayList<CharGroup> mData; + ArrayList<PtNode> mData; // To help with binary generation int mCachedSize = Integer.MIN_VALUE; // mCachedAddressBefore/AfterUpdate are helpers for binary dictionary generation. They @@ -58,9 +58,9 @@ public final class FusionDictionary implements Iterable<Word> { int mCachedParentAddress = 0; public PtNodeArray() { - mData = new ArrayList<CharGroup>(); + mData = new ArrayList<PtNode>(); } - public PtNodeArray(ArrayList<CharGroup> data) { + public PtNodeArray(ArrayList<PtNode> data) { mData = data; } } @@ -93,18 +93,19 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * A group of characters, with a frequency, shortcut targets, bigrams, and children. + * PtNode is a group of characters, with a frequency, shortcut targets, bigrams, and children + * (Pt means Patricia Trie). * - * This is the central class of the in-memory representation. A CharGroup is what can + * This is the central class of the in-memory representation. A PtNode is what can * be seen as a traditional "trie node", except it can hold several characters at the - * same time. A CharGroup essentially represents one or several characters in the middle + * same time. A PtNode essentially represents one or several characters in the middle * of the trie tree; as such, it can be a terminal, and it can have children. - * In this in-memory representation, whether the CharGroup is a terminal or not is represented + * In this in-memory representation, whether the PtNode is a terminal or not is represented * in the frequency, where NOT_A_TERMINAL (= -1) means this is not a terminal and any other * value is the frequency of this terminal. A terminal may have non-null shortcuts and/or * bigrams, but a non-terminal may not. Moreover, children, if present, are null. */ - public static final class CharGroup { + public static final class PtNode { public static final int NOT_A_TERMINAL = -1; final int mChars[]; ArrayList<WeightedString> mShortcutTargets; @@ -119,11 +120,11 @@ public final class FusionDictionary implements Iterable<Word> { // same time. Updating will update the AfterUpdate value, and the code will move them // to BeforeUpdate before the next update pass. // The update process does not need two versions of mCachedSize. - int mCachedSize; // The size, in bytes, of this char group. - int mCachedAddressBeforeUpdate; // The address of this char group (before update) - int mCachedAddressAfterUpdate; // The address of this char group (after update) + int mCachedSize; // The size, in bytes, of this PtNode. + int mCachedAddressBeforeUpdate; // The address of this PtNode (before update) + int mCachedAddressAfterUpdate; // The address of this PtNode (after update) - public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, final boolean isNotAWord, final boolean isBlacklistEntry) { mChars = chars; @@ -135,7 +136,7 @@ public final class FusionDictionary implements Iterable<Word> { mIsBlacklistEntry = isBlacklistEntry; } - public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, final boolean isNotAWord, final boolean isBlacklistEntry, final PtNodeArray children) { @@ -148,7 +149,7 @@ public final class FusionDictionary implements Iterable<Word> { mIsBlacklistEntry = isBlacklistEntry; } - public void addChild(CharGroup n) { + public void addChild(PtNode n) { if (null == mChildren) { mChildren = new PtNodeArray(); } @@ -245,7 +246,7 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Updates the CharGroup with the given properties. Adds the shortcut and bigram lists to + * Updates the PtNode with the given properties. Adds the shortcut and bigram lists to * the existing ones if any. Note: unigram, bigram, and shortcut frequencies are only * updated if they are higher than the existing ones. */ @@ -407,13 +408,13 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Sanity check for a node array. + * Sanity check for a PtNode array. * - * This method checks that all CharGroups in a node array are ordered as expected. + * This method checks that all PtNodes in a node array are ordered as expected. * If they are, nothing happens. If they aren't, an exception is thrown. */ - private void checkStack(PtNodeArray nodeArray) { - ArrayList<CharGroup> stack = nodeArray.mData; + private void checkStack(PtNodeArray ptNodeArray) { + ArrayList<PtNode> stack = ptNodeArray.mData; int lastValue = -1; for (int i = 0; i < stack.size(); ++i) { int currentValue = stack.get(i).mChars[0]; @@ -432,18 +433,18 @@ public final class FusionDictionary implements Iterable<Word> { * @param frequency the bigram frequency */ public void setBigram(final String word1, final String word2, final int frequency) { - CharGroup charGroup = findWordInTree(mRootNodeArray, word1); - if (charGroup != null) { - final CharGroup charGroup2 = findWordInTree(mRootNodeArray, word2); - if (charGroup2 == null) { + PtNode ptNode = findWordInTree(mRootNodeArray, word1); + if (ptNode != null) { + final PtNode ptNode2 = findWordInTree(mRootNodeArray, word2); + if (ptNode2 == null) { add(getCodePoints(word2), 0, null, false /* isNotAWord */, false /* isBlacklistEntry */); - // The chargroup for the first word may have moved by the above insertion, + // The PtNode for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been - // a cutting point until now. In this case, we need to refresh charGroup. - charGroup = findWordInTree(mRootNodeArray, word1); + // a cutting point until now. In this case, we need to refresh ptNode. + ptNode = findWordInTree(mRootNodeArray, word1); } - charGroup.addBigram(word2, frequency); + ptNode.addBigram(word2, frequency); } else { throw new RuntimeException("First word of bigram not found"); } @@ -473,84 +474,83 @@ public final class FusionDictionary implements Iterable<Word> { PtNodeArray currentNodeArray = mRootNodeArray; int charIndex = 0; - CharGroup currentGroup = null; + PtNode currentPtNode = null; int differentCharIndex = 0; // Set by the loop to the index of the char that differs int nodeIndex = findIndexOfChar(mRootNodeArray, word[charIndex]); while (CHARACTER_NOT_FOUND_INDEX != nodeIndex) { - currentGroup = currentNodeArray.mData.get(nodeIndex); - differentCharIndex = compareCharArrays(currentGroup.mChars, word, charIndex); + currentPtNode = currentNodeArray.mData.get(nodeIndex); + differentCharIndex = compareCharArrays(currentPtNode.mChars, word, charIndex); if (ARRAYS_ARE_EQUAL != differentCharIndex - && differentCharIndex < currentGroup.mChars.length) break; - if (null == currentGroup.mChildren) break; - charIndex += currentGroup.mChars.length; + && differentCharIndex < currentPtNode.mChars.length) break; + if (null == currentPtNode.mChildren) break; + charIndex += currentPtNode.mChars.length; if (charIndex >= word.length) break; - currentNodeArray = currentGroup.mChildren; + currentNodeArray = currentPtNode.mChildren; nodeIndex = findIndexOfChar(currentNodeArray, word[charIndex]); } if (CHARACTER_NOT_FOUND_INDEX == nodeIndex) { // No node at this point to accept the word. Create one. final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); - final CharGroup newGroup = new CharGroup( - Arrays.copyOfRange(word, charIndex, word.length), + final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentNodeArray.mData.add(insertionIndex, newGroup); + currentNodeArray.mData.add(insertionIndex, newPtNode); if (DBG) checkStack(currentNodeArray); } else { // There is a word with a common prefix. - if (differentCharIndex == currentGroup.mChars.length) { + if (differentCharIndex == currentPtNode.mChars.length) { if (charIndex + differentCharIndex >= word.length) { // The new word is a prefix of an existing word, but the node on which it - // should end already exists as is. Since the old CharGroup was not a terminal, + // should end already exists as is. Since the old PtNode was not a terminal, // make it one by filling in its frequency and other attributes - currentGroup.update(frequency, shortcutTargets, null, isNotAWord, + currentPtNode.update(frequency, shortcutTargets, null, isNotAWord, isBlacklistEntry); } else { // The new word matches the full old word and extends past it. // We only have to create a new node and add it to the end of this. - final CharGroup newNode = new CharGroup( + final PtNode newNode = new PtNode( Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentGroup.mChildren = new PtNodeArray(); - currentGroup.mChildren.mData.add(newNode); + currentPtNode.mChildren = new PtNodeArray(); + currentPtNode.mChildren.mData.add(newNode); } } else { if (0 == differentCharIndex) { // Exact same word. Update the frequency if higher. This will also add the // new shortcuts to the existing shortcut list if it already exists. - currentGroup.update(frequency, shortcutTargets, null, - currentGroup.mIsNotAWord && isNotAWord, - currentGroup.mIsBlacklistEntry || isBlacklistEntry); + currentPtNode.update(frequency, shortcutTargets, null, + currentPtNode.mIsNotAWord && isNotAWord, + currentPtNode.mIsBlacklistEntry || isBlacklistEntry); } else { // Partial prefix match only. We have to replace the current node with a node // containing the current prefix and create two new ones for the tails. PtNodeArray newChildren = new PtNodeArray(); - final CharGroup newOldWord = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, differentCharIndex, - currentGroup.mChars.length), currentGroup.mShortcutTargets, - currentGroup.mBigrams, currentGroup.mFrequency, - currentGroup.mIsNotAWord, currentGroup.mIsBlacklistEntry, - currentGroup.mChildren); + final PtNode newOldWord = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex, + currentPtNode.mChars.length), currentPtNode.mShortcutTargets, + currentPtNode.mBigrams, currentPtNode.mFrequency, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry, + currentPtNode.mChildren); newChildren.mData.add(newOldWord); - final CharGroup newParent; + final PtNode newParent; if (charIndex + differentCharIndex >= word.length) { - newParent = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, 0, differentCharIndex), + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry, newChildren); } else { - newParent = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, 0, differentCharIndex), + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), null /* shortcutTargets */, null /* bigrams */, -1, false /* isNotAWord */, false /* isBlacklistEntry */, newChildren); - final CharGroup newWord = new CharGroup(Arrays.copyOfRange(word, + final PtNode newWord = new PtNode(Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); final int addIndex = word[charIndex + differentCharIndex] - > currentGroup.mChars[differentCharIndex] ? 1 : 0; + > currentPtNode.mChars[differentCharIndex] ? 1 : 0; newChildren.mData.add(addIndex, newWord); } currentNodeArray.mData.set(nodeIndex, newParent); @@ -589,29 +589,29 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Helper class that compares and sorts two chargroups according to their + * Helper class that compares and sorts two PtNodes according to their * first element only. I repeat: ONLY the first element is considered, the rest * is ignored. * This comparator imposes orderings that are inconsistent with equals. */ - static private final class CharGroupComparator implements java.util.Comparator<CharGroup> { + static private final class PtNodeComparator implements java.util.Comparator<PtNode> { @Override - public int compare(CharGroup c1, CharGroup c2) { - if (c1.mChars[0] == c2.mChars[0]) return 0; - return c1.mChars[0] < c2.mChars[0] ? -1 : 1; + public int compare(PtNode p1, PtNode p2) { + if (p1.mChars[0] == p2.mChars[0]) return 0; + return p1.mChars[0] < p2.mChars[0] ? -1 : 1; } } - final static private CharGroupComparator CHARGROUP_COMPARATOR = new CharGroupComparator(); + final static private PtNodeComparator PTNODE_COMPARATOR = new PtNodeComparator(); /** * Finds the insertion index of a character within a node array. */ private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { - final ArrayList<CharGroup> data = nodeArray.mData; - final CharGroup reference = new CharGroup(new int[] { character }, + final ArrayList<PtNode> data = nodeArray.mData; + final PtNode reference = new PtNode(new int[] { character }, null /* shortcutTargets */, null /* bigrams */, 0, false /* isNotAWord */, false /* isBlacklistEntry */); - int result = Collections.binarySearch(data, reference, CHARGROUP_COMPARATOR); + int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR); return result >= 0 ? result : -result - 1; } @@ -633,35 +633,37 @@ public final class FusionDictionary implements Iterable<Word> { * Helper method to find a word in a given branch. */ @SuppressWarnings("unused") - public static CharGroup findWordInTree(PtNodeArray nodeArray, final String string) { + public static PtNode findWordInTree(PtNodeArray nodeArray, final String string) { int index = 0; final StringBuilder checker = DBG ? new StringBuilder() : null; final int[] codePoints = getCodePoints(string); - CharGroup currentGroup; + PtNode currentPtNode; do { int indexOfGroup = findIndexOfChar(nodeArray, codePoints[index]); if (CHARACTER_NOT_FOUND_INDEX == indexOfGroup) return null; - currentGroup = nodeArray.mData.get(indexOfGroup); + currentPtNode = nodeArray.mData.get(indexOfGroup); - if (codePoints.length - index < currentGroup.mChars.length) return null; + if (codePoints.length - index < currentPtNode.mChars.length) return null; int newIndex = index; - while (newIndex < codePoints.length && newIndex - index < currentGroup.mChars.length) { - if (currentGroup.mChars[newIndex - index] != codePoints[newIndex]) return null; + while (newIndex < codePoints.length && newIndex - index < currentPtNode.mChars.length) { + if (currentPtNode.mChars[newIndex - index] != codePoints[newIndex]) return null; newIndex++; } index = newIndex; - if (DBG) checker.append(new String(currentGroup.mChars, 0, currentGroup.mChars.length)); + if (DBG) { + checker.append(new String(currentPtNode.mChars, 0, currentPtNode.mChars.length)); + } if (index < codePoints.length) { - nodeArray = currentGroup.mChildren; + nodeArray = currentPtNode.mChildren; } } while (null != nodeArray && index < codePoints.length); if (index < codePoints.length) return null; - if (!currentGroup.isTerminal()) return null; + if (!currentPtNode.isTerminal()) return null; if (DBG && !string.equals(checker.toString())) return null; - return currentGroup; + return currentPtNode; } /** @@ -675,18 +677,18 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Recursively count the number of character groups in a given branch of the trie. + * Recursively count the number of PtNodes in a given branch of the trie. * * @param nodeArray the parent node. - * @return the number of char groups in all the branch under this node. + * @return the number of PtNodes in all the branch under this node. */ - public static int countCharGroups(final PtNodeArray nodeArray) { + public static int countPtNodes(final PtNodeArray nodeArray) { final int nodeSize = nodeArray.mData.size(); int size = nodeSize; for (int i = nodeSize - 1; i >= 0; --i) { - CharGroup group = nodeArray.mData.get(i); - if (null != group.mChildren) - size += countCharGroups(group.mChildren); + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countPtNodes(ptNode.mChildren); } return size; } @@ -700,9 +702,9 @@ public final class FusionDictionary implements Iterable<Word> { public static int countNodeArrays(final PtNodeArray nodeArray) { int size = 1; for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { - CharGroup group = nodeArray.mData.get(i); - if (null != group.mChildren) - size += countNodeArrays(group.mChildren); + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countNodeArrays(ptNode.mChildren); } return size; } @@ -713,9 +715,9 @@ public final class FusionDictionary implements Iterable<Word> { private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { if (null == nodeArray) return false; for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { - CharGroup group = nodeArray.mData.get(i); - if (null != group.mBigrams) return true; - if (hasBigramsInternal(group.mChildren)) return true; + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mBigrams) return true; + if (hasBigramsInternal(ptNode.mChildren)) return true; } return false; } @@ -748,8 +750,8 @@ public final class FusionDictionary implements Iterable<Word> { MakedictLog.i("Do not merge tails"); return; -// MakedictLog.i("Merging nodes. Number of nodes : " + countNodes(root)); -// MakedictLog.i("Number of groups : " + countCharGroups(root)); +// MakedictLog.i("Merging PtNodes. Number of PtNodes : " + countPtNodes(root)); +// MakedictLog.i("Number of PtNodes : " + countPtNodes(root)); // // final HashMap<String, ArrayList<PtNodeArray>> repository = // new HashMap<String, ArrayList<PtNodeArray>>(); @@ -771,25 +773,25 @@ public final class FusionDictionary implements Iterable<Word> { // if (a.data.size() != b.data.size()) return false; // final int size = a.data.size(); // for (int i = size - 1; i >= 0; --i) { -// CharGroup aGroup = a.data.get(i); -// CharGroup bGroup = b.data.get(i); -// if (aGroup.frequency != bGroup.frequency) return false; -// if (aGroup.alternates == null && bGroup.alternates != null) return false; -// if (aGroup.alternates != null && !aGroup.equals(bGroup.alternates)) return false; -// if (!Arrays.equals(aGroup.chars, bGroup.chars)) return false; -// if (!isEqual(aGroup.children, bGroup.children)) return false; +// PtNode aPtNode = a.data.get(i); +// PtNode bPtNode = b.data.get(i); +// if (aPtNode.frequency != bPtNode.frequency) return false; +// if (aPtNode.alternates == null && bPtNode.alternates != null) return false; +// if (aPtNode.alternates != null && !aPtNode.equals(bPtNode.alternates)) return false; +// if (!Arrays.equals(aPtNode.chars, bPtNode.chars)) return false; +// if (!isEqual(aPtNode.children, bPtNode.children)) return false; // } // return true; // } // static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner( // final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) { -// final ArrayList<CharGroup> branches = nodeArray.data; +// final ArrayList<PtNode> branches = nodeArray.data; // final int nodeSize = branches.size(); // for (int i = 0; i < nodeSize; ++i) { -// CharGroup group = branches.get(i); -// if (null != group.children) { -// String pseudoHash = getPseudoHash(group.children); +// PtNode ptNode = branches.get(i); +// if (null != ptNode.children) { +// String pseudoHash = getPseudoHash(ptNode.children); // ArrayList<PtNodeArray> similarList = map.get(pseudoHash); // if (null == similarList) { // similarList = new ArrayList<PtNodeArray>(); @@ -797,16 +799,16 @@ public final class FusionDictionary implements Iterable<Word> { // } // boolean merged = false; // for (PtNodeArray similar : similarList) { -// if (isEqual(group.children, similar)) { -// group.children = similar; +// if (isEqual(ptNode.children, similar)) { +// ptNode.children = similar; // merged = true; // break; // } // } // if (!merged) { -// similarList.add(group.children); +// similarList.add(ptNode.children); // } -// mergeTailsInner(map, group.children); +// mergeTailsInner(map, ptNode.children); // } // } // return map; @@ -814,9 +816,9 @@ public final class FusionDictionary implements Iterable<Word> { // private static String getPseudoHash(final PtNodeArray nodeArray) { // StringBuilder s = new StringBuilder(); -// for (CharGroup g : nodeArray.data) { -// s.append(g.frequency); -// for (int ch : g.chars) { +// for (PtNode ptNode : nodeArray.data) { +// s.append(ptNode.frequency); +// for (int ch : ptNode.chars) { // s.append(Character.toChars(ch)); // } // } @@ -830,20 +832,20 @@ public final class FusionDictionary implements Iterable<Word> { */ public static final class DictionaryIterator implements Iterator<Word> { private static final class Position { - public Iterator<CharGroup> pos; + public Iterator<PtNode> pos; public int length; - public Position(ArrayList<CharGroup> groups) { - pos = groups.iterator(); + public Position(ArrayList<PtNode> ptNodes) { + pos = ptNodes.iterator(); length = 0; } } final StringBuilder mCurrentString; final LinkedList<Position> mPositions; - public DictionaryIterator(ArrayList<CharGroup> root) { + public DictionaryIterator(ArrayList<PtNode> ptRoot) { mCurrentString = new StringBuilder(); mPositions = new LinkedList<Position>(); - final Position rootPos = new Position(root); + final Position rootPos = new Position(ptRoot); mPositions.add(rootPos); } @@ -864,20 +866,20 @@ public final class FusionDictionary implements Iterable<Word> { do { if (currentPos.pos.hasNext()) { - final CharGroup currentGroup = currentPos.pos.next(); + final PtNode currentPtNode = currentPos.pos.next(); currentPos.length = mCurrentString.length(); - for (int i : currentGroup.mChars) { + for (int i : currentPtNode.mChars) { mCurrentString.append(Character.toChars(i)); } - if (null != currentGroup.mChildren) { - currentPos = new Position(currentGroup.mChildren.mData); + if (null != currentPtNode.mChildren) { + currentPos = new Position(currentPtNode.mChildren.mData); currentPos.length = mCurrentString.length(); mPositions.addLast(currentPos); } - if (currentGroup.mFrequency >= 0) { - return new Word(mCurrentString.toString(), currentGroup.mFrequency, - currentGroup.mShortcutTargets, currentGroup.mBigrams, - currentGroup.mIsNotAWord, currentGroup.mIsBlacklistEntry); + if (currentPtNode.mFrequency >= 0) { + return new Word(mCurrentString.toString(), currentPtNode.mFrequency, + currentPtNode.mShortcutTargets, currentPtNode.mBigrams, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry); } } else { mPositions.removeLast(); diff --git a/java/src/com/android/inputmethod/latin/makedict/CharGroupInfo.java b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java index b3617443e..188de7a0f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/CharGroupInfo.java +++ b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java @@ -21,9 +21,9 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.util.ArrayList; /** - * Raw char group info straight out of a file. This will contain numbers for addresses. + * Raw PtNode info straight out of a file. This will contain numbers for addresses. */ -public final class CharGroupInfo { +public final class PtNodeInfo { public final int mOriginalAddress; public final int mEndAddress; @@ -35,7 +35,7 @@ public final class CharGroupInfo { public final ArrayList<WeightedString> mShortcutTargets; public final ArrayList<PendingAttribute> mBigrams; - public CharGroupInfo(final int originalAddress, final int endAddress, final int flags, + public PtNodeInfo(final int originalAddress, final int endAddress, final int flags, final int[] characters, final int frequency, final int parentAddress, final int childrenAddress, final ArrayList<WeightedString> shortcutTargets, final ArrayList<PendingAttribute> bigrams) { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java index 7a9323c2f..1a5023ef6 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -21,18 +21,26 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.JniUtils; +import android.util.Log; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.TreeMap; /** * An implementation of DictDecoder for version 3 binary dictionary. */ @UsedForTesting public class Ver3DictDecoder implements DictDecoder { + private static final String TAG = Ver3DictDecoder.class.getSimpleName(); static { JniUtils.loadNativeLibrary(); @@ -70,32 +78,145 @@ public class Ver3DictDecoder implements DictDecoder { } } + private final static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readFrequency(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigrams(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + final int offset = (dictBuffer.readUnsignedByte() << 16) + + dictBuffer.readUnsignedShort(); + bigramAddress += sign * offset; + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return readLength; + } + } + private final File mDictionaryBinaryFile; + private final DictionaryBufferFactory mBufferFactory; private DictBuffer mDictBuffer; public Ver3DictDecoder(final File file) { + this(file, USE_READONLY_BYTEBUFFER); + } + + public Ver3DictDecoder(final File file, final int factoryFlag) { mDictionaryBinaryFile = file; mDictBuffer = null; + + if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { + mBufferFactory = new DictionaryBufferFromByteArrayFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); + } else { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } } - public void openDictBuffer(final DictDecoder.DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - mDictBuffer = factory.getDictionaryBuffer(mDictionaryBinaryFile); + public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { + mDictionaryBinaryFile = file; + mBufferFactory = factory; + } + + public void openDictBuffer() throws FileNotFoundException, IOException { + mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); } - public DictBuffer getDictBuffer() { + /* package */ DictBuffer getDictBuffer() { return mDictBuffer; } @UsedForTesting - public DictBuffer openAndGetDictBuffer(final DictDecoder.DictionaryBufferFactory factory) - throws FileNotFoundException, IOException { - openDictBuffer(factory); + /* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException { + openDictBuffer(); return getDictBuffer(); } @Override public FileHeader readHeader() throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + final int version = HeaderReader.readVersion(mDictBuffer); final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); @@ -116,4 +237,114 @@ public class Ver3DictDecoder implements DictDecoder { 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); return header; } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { + int addressPointer = ptNodePos; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + ++addressPointer; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += 3; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (-1 != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + ++addressPointer; + frequency = PtNodeReader.readFrequency(mDictBuffer); + } else { + frequency = PtNode.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + final ArrayList<WeightedString> shortcutTargets; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + // readShortcut will add shortcuts to shortcutTargets. + shortcutTargets = new ArrayList<WeightedString>(); + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } else { + shortcutTargets = null; + } + + final ArrayList<PendingAttribute> bigrams; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigrams(mDictBuffer, bigrams, addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + MakedictLog.d("too many bigrams in a PtNode."); + } + } else { + bigrams = null; + } + return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } + + @Override + public FusionDictionary readDictionaryBinary(final FusionDictionary dict) + throws FileNotFoundException, IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + try { + return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); + } catch (IOException e) { + Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); + if (!mDictionaryBinaryFile.delete()) { + Log.e(TAG, "Failed to delete the broken dictionary."); + } + throw e; + } catch (UnsupportedFormatException e) { + Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); + if (!mDictionaryBinaryFile.delete()) { + Log.e(TAG, "Failed to delete the broken dictionary."); + } + throw e; + } + } + + @Override + public int getTerminalPosition(String word) throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + return BinaryDictIOUtils.getTerminalPosition(this, word); + } + + @Override + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); + } + } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java new file mode 100644 index 000000000..e81fd4514 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * An implementation of DictEncoder for version 3 binary dictionary. + */ +public class Ver3DictEncoder implements DictEncoder { + + private final File mDictFile; + private OutputStream mOutStream; + + public Ver3DictEncoder(final File dictFile) { + mDictFile = dictFile; + mOutStream = null; + } + + // This constructor is used only by BinaryDictOffdeviceUtilsTests. + // If you want to use this in the production code, you should consider keeping consistency of + // the interface of Ver3DictDecoder by using factory. + public Ver3DictEncoder(final OutputStream outStream) { + mDictFile = null; + mOutStream = outStream; + } + + private void openStream() throws FileNotFoundException { + mOutStream = new FileOutputStream(mDictFile); + } + + private void close() throws IOException { + if (mOutStream != null) { + mOutStream.close(); + mOutStream = null; + } + } + + @Override + public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (mOutStream == null) { + openStream(); + } + BinaryDictEncoderUtils.writeDictionaryBinary(mOutStream, dict, formatOptions); + close(); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java new file mode 100644 index 000000000..7f4f5e74a --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; + +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.AbstractDictionaryWriter; +import com.android.inputmethod.latin.ExpandableDictionary; +import com.android.inputmethod.latin.WordComposer; +import com.android.inputmethod.latin.ExpandableDictionary.NextWord; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + +import java.io.IOException; +import java.util.ArrayList; + +// Currently this class is used to implement dynamic prodiction dictionary. +// TODO: Move to native code. +public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { + private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); + /** Maximum number of pairs. Pruning will start when databases goes above this number. */ + public static final int MAX_HISTORY_BIGRAMS = 10000; + + /** Any pair being typed or picked */ + private static final int FREQUENCY_FOR_TYPED = 2; + + private static final int BINARY_DICT_VERSION = 3; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); + + private final UserHistoryDictionaryBigramList mBigramList = + new UserHistoryDictionaryBigramList(); + private final ExpandableDictionary mExpandableDictionary; + + public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { + super(context, dictType); + mExpandableDictionary = new ExpandableDictionary(dictType); + } + + @Override + public void clear() { + mBigramList.evictAll(); + mExpandableDictionary.clearDictionary(); + } + + /** + * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes + * are done to update the binary dictionary. + */ + @Override + public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, + final boolean isNotAWord) { + mExpandableDictionary.addWord(word, shortcutTarget, frequency); + mBigramList.addBigram(null, word, (byte)frequency); + } + + @Override + public void addBigramWords(final String word0, final String word1, final int frequency, + final boolean isValid, final long lastModifiedTime) { + if (lastModifiedTime > 0) { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(frequency, System.currentTimeMillis(), + lastModifiedTime)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } else { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(isValid)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } + } + + @Override + public void removeBigramWords(final String word0, final String word1) { + if (mBigramList.removeBigram(word0, word1)) { + mExpandableDictionary.removeBigram(word0, word1); + } + } + + @Override + protected void writeDictionary(final DictEncoder dictEncoder) + throws IOException, UnsupportedFormatException { + UserHistoryDictIOUtils.writeDictionary(dictEncoder, + new FrequencyProvider(mBigramList, mExpandableDictionary), mBigramList, + FORMAT_OPTIONS); + } + + private static class FrequencyProvider implements BigramDictionaryInterface { + final private UserHistoryDictionaryBigramList mBigramList; + final private ExpandableDictionary mExpandableDictionary; + + public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, + final ExpandableDictionary expandableDictionary) { + mBigramList = bigramList; + mExpandableDictionary = expandableDictionary; + } + @Override + public int getFrequency(final String word0, final String word1) { + final int freq; + if (word0 == null) { // unigram + freq = FREQUENCY_FOR_TYPED; + } else { // bigram + final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1); + if (nw != null) { + final ForgettingCurveParams forgettingCurveParams = nw.getFcParams(); + final byte prevFc = mBigramList.getBigrams(word0).get(word1); + final byte fc = forgettingCurveParams.getFc(); + final boolean isValid = forgettingCurveParams.isValid(); + if (prevFc > 0 && prevFc == fc) { + freq = fc & 0xFF; + } else if (UserHistoryForgettingCurveUtils. + needsToSave(fc, isValid, mBigramList.size() <= MAX_HISTORY_BIGRAMS)) { + freq = fc & 0xFF; + } else { + // Delete this entry + freq = -1; + } + } else { + // Delete this entry + freq = -1; + } + } + return freq; + } + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + boolean blockOffensiveWords) { + return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo, + blockOffensiveWords); + } + + @Override + public boolean isValidWord(final String word) { + return mExpandableDictionary.isValidWord(word); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index b565b2f9f..be3a9f2f1 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -18,106 +18,88 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; import android.content.SharedPreferences; -import android.os.AsyncTask; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.ExpandableDictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.LatinImeLogger; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.DictDecoder; import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; /** * This class is a base class of a dictionary for the personalized prediction language model. */ -public abstract class DynamicPredictionDictionaryBase extends ExpandableDictionary { - +public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary { private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName(); public static final boolean DBG_SAVE_RESTORE = false; private static final boolean DBG_STRESS_TEST = false; private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; - private static final FormatOptions VERSION3 = new FormatOptions(3, - true /* supportsDynamicUpdate */); - /** Any pair being typed or picked */ - private static final int FREQUENCY_FOR_TYPED = 2; - - /** Maximum number of pairs. Pruning will start when databases goes above this number. */ - private static final int MAX_HISTORY_BIGRAMS = 10000; + public static final int FREQUENCY_FOR_TYPED = 2; /** Locale for which this user history dictionary is storing words */ private final String mLocale; - private final UserHistoryDictionaryBigramList mBigramList = - new UserHistoryDictionaryBigramList(); - private final ReentrantLock mBigramListLock = new ReentrantLock(); + private final String mFileName; + private final SharedPreferences mPrefs; private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = CollectionUtils.newArrayList(); - private final AtomicReference<AsyncTask<Void, Void, Void>> mWaitingTask; - // Should always be false except when we use this class for test @UsedForTesting boolean mIsTest = false; /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale, - final SharedPreferences sp, final String dictionaryType) { - super(context, dictionaryType); + final SharedPreferences sp, final String dictionaryType, final String fileName) { + super(context, locale, dictionaryType, true); mLocale = locale; + mFileName = fileName; mPrefs = sp; - mWaitingTask = new AtomicReference<AsyncTask<Void, Void, Void>>(); if (mLocale != null && mLocale.length() > 1) { - loadDictionary(); + asyncLoadDictionaryToMemory(); + asyncReloadDictionaryIfRequired(); } } @Override public void close() { - flushPendingWrites(); - // Don't close the database as locale changes will require it to be reopened anyway - // Also, the database is written to somewhat frequently, so it needs to be kept alive - // throughout the life of the process. - // mOpenHelper.close(); - // Ignore close because we cache PersonalizationPredictionDictionary for each language. - // See getInstance() above. + // Close only binary dictionary to reuse this dictionary. // super.close(); + closeBinaryDictionary(); + // Flush pending writes. + // TODO: Remove after this class become to use a dynamic binary dictionary. + asyncWriteBinaryDictionary(); + Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); } @Override - protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo) { - // Inhibit suggestions (not predictions) for user history for now. Removing this method - // is enough to use it through the standard ExpandableDictionary way. - return null; + protected boolean hasContentChanged() { + return false; + } + + @Override + protected boolean needsToReloadBeforeWriting() { + return false; } /** * Return whether the passed charsequence is in the dictionary. */ @Override - public synchronized boolean isValidWord(final String word) { - // TODO: figure out what is the correct thing to do here. + public boolean isValidWord(final String word) { + // Words included only in the user history should be treated as not in dictionary words. return false; } @@ -129,74 +111,29 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public int addToPersonalizationPredictionDictionary( - final String word1, final String word2, final boolean isValid) { - if (word2.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || - (word1 != null && word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { - return -1; + public void addToPersonalizationPredictionDictionary( + final String word0, final String word1, final boolean isValid) { + if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || + (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { + return; } - if (mBigramListLock.tryLock()) { - try { - super.addWord( - word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED); - mBigramList.addBigram(null, word2, (byte)FREQUENCY_FOR_TYPED); - // Do not insert a word as a bigram of itself - if (word2.equals(word1)) { - return 0; - } - final int freq; - if (null == word1) { - freq = FREQUENCY_FOR_TYPED; - } else { - freq = super.setBigramAndGetFrequency( - word1, word2, new ForgettingCurveParams(isValid)); - } - mBigramList.addBigram(word1, word2); - return freq; - } finally { - mBigramListLock.unlock(); - } + addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED, + false /* isNotAWord */); + // Do not insert a word as a bigram of itself + if (word1.equals(word0)) { + return; } - return -1; - } - - public boolean cancelAddingUserHistory(final String word1, final String word2) { - if (mBigramListLock.tryLock()) { - try { - if (mBigramList.removeBigram(word1, word2)) { - return super.removeBigram(word1, word2); - } - } finally { - mBigramListLock.unlock(); - } + if (null != word0) { + addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid); } - return false; } - /** - * Schedules a background thread to write any pending words to the database. - */ - private void flushPendingWrites() { - // Create a background thread to write the pending entries - final AsyncTask<Void, Void, Void> old = mWaitingTask.getAndSet(new UpdateBinaryTask( - mBigramList, mLocale, this, mPrefs, getContext()).execute()); - if (old != null) { - old.cancel(false); - } + public void cancelAddingUserHistory(final String word0, final String word1) { + removeBigramDynamically(word0, word1); } @Override - public final void loadDictionaryAsync() { - // This must be run on non-main thread - mBigramListLock.lock(); - try { - loadDictionaryAsyncLocked(); - } finally { - mBigramListLock.unlock(); - } - } - - private void loadDictionaryAsyncLocked() { + protected void loadDictionaryAsync() { final int[] profTotalCount = { 0 }; final String locale = getLocale(); if (DBG_STRESS_TEST) { @@ -208,10 +145,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona } } final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale); - final boolean initializing = last == 0; final long now = System.currentTimeMillis(); - final String fileName = getDictionaryFileName(); - final ExpandableDictionary dictionary = this; + final ExpandableBinaryDictionary dictionary = this; final OnAddWordListener listener = new OnAddWordListener() { @Override public void setUnigram(final String word, final String shortcutTarget, @@ -219,33 +154,30 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona if (DBG_SAVE_RESTORE) { Log.d(TAG, "load unigram: " + word + "," + frequency); } - dictionary.addWord(word, shortcutTarget, frequency); + addWord(word, shortcutTarget, frequency, false /* isNotAWord */); ++profTotalCount[0]; - addToBigramListLocked(null, word, (byte)frequency); } @Override - public void setBigram(final String word1, final String word2, final int frequency) { - if (word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH - && word2.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { + public void setBigram(final String word0, final String word1, final int frequency) { + if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH + && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { if (DBG_SAVE_RESTORE) { - Log.d(TAG, "load bigram: " + word1 + "," + word2 + "," + frequency); + Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); } ++profTotalCount[0]; - dictionary.setBigramAndGetFrequency( - word1, word2, initializing ? new ForgettingCurveParams(true) - : new ForgettingCurveParams(frequency, now, last)); + addBigram(word0, word1, frequency, last); } - addToBigramListLocked(word1, word2, (byte)frequency); } }; // Load the dictionary from binary file - final Ver3DictDecoder reader = new Ver3DictDecoder( - new File(getContext().getFilesDir(), fileName)); + final File dictFile = new File(mContext.getFilesDir(), mFileName); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile, + DictDecoder.USE_BYTEARRAY); try { - reader.openDictBuffer(new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); - UserHistoryDictIOUtils.readDictionaryBinary(reader, listener); + dictDecoder.openDictBuffer(); + UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); } catch (FileNotFoundException e) { // This is an expected condition: we don't have a user history dictionary for this // language yet. It will be created sometime later. @@ -260,150 +192,18 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona } } - protected abstract String getDictionaryFileName(); - protected String getLocale() { return mLocale; } - private void addToBigramListLocked(String word0, String word1, byte fcValue) { - mBigramList.addBigram(word0, word1, fcValue); - } - - /** - * Async task to write pending words to the binarydicts. - */ - private static final class UpdateBinaryTask extends AsyncTask<Void, Void, Void> - implements BigramDictionaryInterface { - private final UserHistoryDictionaryBigramList mBigramList; - private final boolean mAddLevel0Bigrams; - private final String mLocale; - private final DynamicPredictionDictionaryBase mDynamicPredictionDictionary; - private final SharedPreferences mPrefs; - private final Context mContext; - - public UpdateBinaryTask(final UserHistoryDictionaryBigramList pendingWrites, - final String locale, final DynamicPredictionDictionaryBase dict, - final SharedPreferences prefs, final Context context) { - mBigramList = pendingWrites; - mLocale = locale; - mDynamicPredictionDictionary = dict; - mPrefs = prefs; - mContext = context; - mAddLevel0Bigrams = mBigramList.size() <= MAX_HISTORY_BIGRAMS; - } - - @Override - protected Void doInBackground(final Void... v) { - if (isCancelled()) return null; - if (mDynamicPredictionDictionary.mIsTest) { - // If mIsTest == true, wait until the lock is released. - mDynamicPredictionDictionary.mBigramListLock.lock(); - try { - doWriteTaskLocked(); - } finally { - mDynamicPredictionDictionary.mBigramListLock.unlock(); - } - } else if (mDynamicPredictionDictionary.mBigramListLock.tryLock()) { - try { - doWriteTaskLocked(); - } finally { - mDynamicPredictionDictionary.mBigramListLock.unlock(); - } - } - return null; - } - - private void doWriteTaskLocked() { - if (isCancelled()) return; - mDynamicPredictionDictionary.mWaitingTask.compareAndSet(this, null); - - if (DBG_STRESS_TEST) { - try { - Log.w(TAG, "Start stress in closing: " + mLocale); - Thread.sleep(15000); - Log.w(TAG, "End stress in closing"); - } catch (InterruptedException e) { - Log.e(TAG, "In stress test", e); - } - } - - final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0; - final String fileName = - mDynamicPredictionDictionary.getDictionaryFileName(); - final File file = new File(mContext.getFilesDir(), fileName); - FileOutputStream out = null; - - try { - out = new FileOutputStream(file); - UserHistoryDictIOUtils.writeDictionaryBinary(out, this, mBigramList, VERSION3); - out.flush(); - out.close(); - } catch (IOException e) { - Log.e(TAG, "IO Exception while writing file", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // ignore - } - } - } - - // Save the timestamp after we finish writing the binary dictionary. - Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); - if (PROFILE_SAVE_RESTORE) { - final long diff = System.currentTimeMillis() - now; - Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", " + diff + "ms."); - } - } - - @Override - public int getFrequency(final String word1, final String word2) { - final int freq; - if (word1 == null) { // unigram - freq = FREQUENCY_FOR_TYPED; - final byte prevFc = mBigramList.getBigrams(word1).get(word2); - } else { // bigram - final NextWord nw = - mDynamicPredictionDictionary.getBigramWord(word1, word2); - if (nw != null) { - final ForgettingCurveParams fcp = nw.getFcParams(); - final byte prevFc = mBigramList.getBigrams(word1).get(word2); - final byte fc = fcp.getFc(); - final boolean isValid = fcp.isValid(); - if (prevFc > 0 && prevFc == fc) { - freq = fc & 0xFF; - } else if (UserHistoryForgettingCurveUtils. - needsToSave(fc, isValid, mAddLevel0Bigrams)) { - freq = fc & 0xFF; - } else { - // Delete this entry - freq = -1; - } - } else { - // Delete this entry - freq = -1; - } - } - return freq; - } - } - @UsedForTesting /* package for test */ void forceAddWordForTest( - final String word1, final String word2, final boolean isValid) { - mBigramListLock.lock(); - try { - addToPersonalizationPredictionDictionary(word1, word2, isValid); - } finally { - mBigramListLock.unlock(); - } + final String word0, final String word1, final boolean isValid) { + addToPersonalizationPredictionDictionary(word0, word1, isValid); } public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { - session.setPredictionDictionary(mLocale, this); + session.setPredictionDictionary(this); mSessions.add(session); session.onDictionaryReady(); } @@ -414,15 +214,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona public void clearAndFlushDictionary() { // Clear the node structure on memory - clearDictionary(); - mBigramListLock.lock(); - try { - // Clear the bigram list on memory - mBigramList.evictAll(); - } finally { - mBigramListLock.unlock(); - } + clear(); // Then flush the cleared state of the dictionary on disk. - flushPendingWrites(); + asyncWriteBinaryDictionary(); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java index 275ce2fdc..f257165cb 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -18,28 +18,32 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.utils.CollectionUtils; import android.content.Context; +import android.content.SharedPreferences; + +import java.util.ArrayList; /** * This class is a dictionary for the personalized language model that uses binary dictionary. */ public class PersonalizationDictionary extends ExpandableBinaryDictionary { private static final String NAME = "personalization"; - - public static void registerUpdateListener(PersonalizationDictionaryUpdateSession listener) { - // TODO: Implement - } + private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = + CollectionUtils.newArrayList(); /** Locale for which this user history dictionary is storing words */ private final String mLocale; - // Singleton - private PersonalizationDictionary(final Context context, final String locale) { + public PersonalizationDictionary(final Context context, final String locale, + final SharedPreferences prefs) { // TODO: Make isUpdatable true. super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION, false /* isUpdatable */); mLocale = locale; + // TODO: Restore last updated time + loadDictionary(); } @Override @@ -49,15 +53,21 @@ public class PersonalizationDictionary extends ExpandableBinaryDictionary { @Override protected boolean hasContentChanged() { - // TODO: Implement return false; } @Override protected boolean needsToReloadBeforeWriting() { - // TODO: Implement return false; } - // TODO: Implement + public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { + session.setDictionary(this); + mSessions.add(session); + session.onDictionaryReady(); + } + + public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { + mSessions.remove(session); + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java index da59333f5..c1833ff14 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java @@ -31,4 +31,7 @@ public class PersonalizationDictionarySessionRegister { public static void onRemoveData(Context context, String type) { } + + public static void onDestroy(Context context) { + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index 77f0cdbfa..ab3de801c 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -45,22 +45,41 @@ public abstract class PersonalizationDictionaryUpdateSession { } // TODO: Use a dynamic binary dictionary instead + public WeakReference<PersonalizationDictionary> mDictionary; public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary; - public String mLocale; + public final String mSystemLocale; + public PersonalizationDictionaryUpdateSession(String locale) { + mSystemLocale = locale; + } public abstract void onDictionaryReady(); public abstract void onDictionaryClosed(Context context); - public void setPredictionDictionary(String locale, DynamicPredictionDictionaryBase dictionary) { + public void setDictionary(PersonalizationDictionary dictionary) { + mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); + } + + public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) { mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); - mLocale = locale; + } + + protected PersonalizationDictionary getDictionary() { + return mDictionary == null ? null : mDictionary.get(); } protected DynamicPredictionDictionaryBase getPredictionDictionary() { return mPredictionDictionary == null ? null : mPredictionDictionary.get(); } + private void unsetDictionary() { + final PersonalizationDictionary dictionary = getDictionary(); + if (dictionary == null) { + return; + } + dictionary.unRegisterUpdateSession(this); + } + private void unsetPredictionDictionary() { final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { @@ -78,10 +97,12 @@ public abstract class PersonalizationDictionaryUpdateSession { } public void closeSession(Context context) { + unsetDictionary(); unsetPredictionDictionary(); onDictionaryClosed(context); } + // TODO: Support multi locale to add bigram public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, int frequency) { final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); @@ -92,6 +113,7 @@ public abstract class PersonalizationDictionaryUpdateSession { } // Bulk import + // TODO: Support multi locale to add bigram public void addBigramsToPersonalizationDictionary( final ArrayList<PersonalizationLanguageModelParam> lmParams) { final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index b4fd25024..c8deaf90d 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -26,16 +26,20 @@ import android.util.Log; import java.lang.ref.SoftReference; import java.util.concurrent.ConcurrentHashMap; -public class PersonalizationDictionaryHelper { - private static final String TAG = PersonalizationDictionaryHelper.class.getSimpleName(); +public class PersonalizationHelper { + private static final String TAG = PersonalizationHelper.class.getSimpleName(); private static final boolean DEBUG = false; private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>> sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> + sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, SoftReference<PersonalizationPredictionDictionary>> - sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); + sLangPersonalizationPredictionDictCache = + CollectionUtils.newConcurrentHashMap(); public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary( final Context context, final String locale, final SharedPreferences sp) { @@ -48,6 +52,7 @@ public class PersonalizationDictionaryHelper { if (DEBUG) { Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale); } + dict.asyncReloadDictionaryIfRequired(); return dict; } } @@ -60,20 +65,45 @@ public class PersonalizationDictionaryHelper { } public static void registerPersonalizationDictionaryUpdateSession(final Context context, - final PersonalizationDictionaryUpdateSession session) { - final PersonalizationPredictionDictionary dictionary = - getPersonalizationPredictionDictionary(context, - context.getResources().getConfiguration().locale.toString(), + final PersonalizationDictionaryUpdateSession session, String locale) { + final PersonalizationPredictionDictionary predictionDictionary = + getPersonalizationPredictionDictionary(context, locale, + PreferenceManager.getDefaultSharedPreferences(context)); + predictionDictionary.registerUpdateSession(session); + final PersonalizationDictionary dictionary = + getPersonalizationDictionary(context, locale, PreferenceManager.getDefaultSharedPreferences(context)); dictionary.registerUpdateSession(session); } - public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( + public static PersonalizationDictionary getPersonalizationDictionary( final Context context, final String locale, final SharedPreferences sp) { synchronized (sLangPersonalizationDictCache) { if (sLangPersonalizationDictCache.containsKey(locale)) { - final SoftReference<PersonalizationPredictionDictionary> ref = + final SoftReference<PersonalizationDictionary> ref = sLangPersonalizationDictCache.get(locale); + final PersonalizationDictionary dict = ref == null ? null : ref.get(); + if (dict != null) { + if (DEBUG) { + Log.w(TAG, "Use cached PersonalizationDictCache for " + locale); + } + return dict; + } + } + final PersonalizationDictionary dict = + new PersonalizationDictionary(context, locale, sp); + sLangPersonalizationDictCache.put( + locale, new SoftReference<PersonalizationDictionary>(dict)); + return dict; + } + } + + public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( + final Context context, final String locale, final SharedPreferences sp) { + synchronized (sLangPersonalizationPredictionDictCache) { + if (sLangPersonalizationPredictionDictCache.containsKey(locale)) { + final SoftReference<PersonalizationPredictionDictionary> ref = + sLangPersonalizationPredictionDictCache.get(locale); final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -84,7 +114,7 @@ public class PersonalizationDictionaryHelper { } final PersonalizationPredictionDictionary dict = new PersonalizationPredictionDictionary(context, locale, sp); - sLangPersonalizationDictCache.put( + sLangPersonalizationPredictionDictCache.put( locale, new SoftReference<PersonalizationPredictionDictionary>(dict)); return dict; } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java index a038d0ab2..e80953c05 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -27,11 +27,11 @@ public class PersonalizationPredictionDictionary extends DynamicPredictionDictio /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, final SharedPreferences sp) { - super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA); + super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, + getDictionaryFileName(locale)); } - @Override - protected String getDictionaryFileName() { - return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java index 9f289e9ff..6c2c9e26e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java @@ -45,6 +45,7 @@ public final class UserHistoryDictionaryBigramList { /** * Called when the user typed a word. */ + @UsedForTesting public void addBigram(String word1, String word2) { addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE); } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java index 76e48c744..b140c919b 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java @@ -31,11 +31,10 @@ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionary UserHistoryPredictionDictionary.class.getSimpleName(); /* package */ UserHistoryPredictionDictionary(final Context context, final String locale, final SharedPreferences sp) { - super(context, locale, sp, Dictionary.TYPE_USER_HISTORY); + super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); } - @Override - protected String getDictionaryFileName() { - return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java index d585b5c7f..0ebe37782 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java @@ -34,7 +34,7 @@ public final class MoreSuggestionsView extends MoreKeysKeyboardView { private static final String TAG = MoreSuggestionsView.class.getSimpleName(); public MoreSuggestionsView(final Context context, final AttributeSet attrs) { - this(context, attrs, R.attr.moreSuggestionsViewStyle); + this(context, attrs, R.attr.moreKeysKeyboardViewStyle); } public MoreSuggestionsView(final Context context, final AttributeSet attrs, diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index bcf64a8e8..aca249240 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -122,27 +122,15 @@ final class SuggestionStripLayoutHelper { mSuggestionsStripHeight = res.getDimensionPixelSize(R.dimen.suggestions_strip_height); final TypedArray a = context.obtainStyledAttributes(attrs, - R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripViewStyle); + R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripView); mSuggestionStripOption = a.getInt( R.styleable.SuggestionStripView_suggestionStripOption, 0); - final float alphaValidTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaValidTypedWord, 1.0f); - final float alphaTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaTypedWord, 1.0f); - final float alphaAutoCorrect = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaAutoCorrect, 1.0f); - final float alphaSuggested = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); mAlphaObsoleted = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); - mColorValidTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorValidTypedWord, 0), alphaValidTypedWord); - mColorTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorTypedWord, 0), alphaTypedWord); - mColorAutoCorrect = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorAutoCorrect, 0), alphaAutoCorrect); - mColorSuggested = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorSuggested, 0), alphaSuggested); + R.styleable.SuggestionStripView_alphaObsoleted, 1.0f); + mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0); + mColorTypedWord = a.getColor(R.styleable.SuggestionStripView_colorTypedWord, 0); + mColorAutoCorrect = a.getColor(R.styleable.SuggestionStripView_colorAutoCorrect, 0); + mColorSuggested = a.getColor(R.styleable.SuggestionStripView_colorSuggested, 0); mSuggestionsCountInStrip = a.getInt( R.styleable.SuggestionStripView_suggestionsCountInStrip, DEFAULT_SUGGESTIONS_COUNT_IN_STRIP); diff --git a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java index 34eccd65b..021bf0825 100644 --- a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java @@ -27,10 +27,8 @@ import com.android.inputmethod.latin.BinaryDictionaryGetter; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.Locale; @@ -281,13 +279,7 @@ public class DictionaryInfoUtils { } public static FileHeader getDictionaryFileHeaderOrNull(final File file) { - try { - return BinaryDictIOUtils.getDictionaryFileHeader(file, 0, file.length()); - } catch (UnsupportedFormatException e) { - return null; - } catch (IOException e) { - return null; - } + return BinaryDictIOUtils.getDictionaryFileHeaderOrNull(file, 0, file.length()); } private static DictionaryInfo createDictionaryInfoFromFileAddress( diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index 768d68cb3..99788f6f2 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -19,8 +19,8 @@ package com.android.inputmethod.latin.utils; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -30,10 +30,10 @@ import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; /** * Reads and writes Binary files for a UserHistoryDictionary. @@ -57,12 +57,12 @@ public final class UserHistoryDictIOUtils { /** * Writes dictionary to file. */ - public static void writeDictionaryBinary(final OutputStream destination, + public static void writeDictionary(final DictEncoder dictEncoder, final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams, final FormatOptions formatOptions) { final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); try { - BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions); + dictEncoder.writeDictionary(fusionDict, formatOptions); Log.d(TAG, "end writing"); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); @@ -120,12 +120,11 @@ public final class UserHistoryDictIOUtils { */ public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, final OnAddWordListener dict) { - final Map<Integer, String> unigrams = CollectionUtils.newTreeMap(); - final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); - final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap(); + final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap(); + final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap(); try { - BinaryDictIOUtils.readUnigramsAndBigramsBinary(dictDecoder, unigrams, frequencies, - bigrams); + dictDecoder.readUnigramsAndBigramsBinary(unigrams, frequencies, bigrams); } catch (IOException e) { Log.e(TAG, "IO exception while reading file", e); } catch (UnsupportedFormatException e) { @@ -140,10 +139,11 @@ public final class UserHistoryDictIOUtils { * Adds all unigrams and bigrams in maps to OnAddWordListener. */ @UsedForTesting - static void addWordsFromWordMap(final Map<Integer, String> unigrams, - final Map<Integer, Integer> frequencies, - final Map<Integer, ArrayList<PendingAttribute>> bigrams, final OnAddWordListener to) { - for (Map.Entry<Integer, String> entry : unigrams.entrySet()) { + static void addWordsFromWordMap(final TreeMap<Integer, String> unigrams, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams, + final OnAddWordListener to) { + for (Entry<Integer, String> entry : unigrams.entrySet()) { final String word1 = entry.getValue(); final int unigramFrequency = frequencies.get(entry.getKey()); to.setUnigram(word1, null, unigramFrequency); diff --git a/java/src/com/android/inputmethod/research/ResearchLogger.java b/java/src/com/android/inputmethod/research/ResearchLogger.java index 3a3408266..da9c61103 100644 --- a/java/src/com/android/inputmethod/research/ResearchLogger.java +++ b/java/src/com/android/inputmethod/research/ResearchLogger.java @@ -1427,7 +1427,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang kid.navigateNext(), kid.navigatePrevious(), kid.mClobberSettingsKey, isPasswordView, kid.mShortcutKeyEnabled, kid.mHasShortcutKey, kid.mLanguageSwitchKeyEnabled, kid.isMultiLine(), keyboard.mOccupiedWidth, - keyboard.mOccupiedHeight, keyboard.mKeys); + keyboard.mOccupiedHeight, keyboard.getKeys()); } /** |