diff options
74 files changed, 3303 insertions, 3079 deletions
diff --git a/java/res/values-th/donottranslate.xml b/java/res/values-th/donottranslate.xml new file mode 100644 index 000000000..aeeebed15 --- /dev/null +++ b/java/res/values-th/donottranslate.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Whether this language uses spaces --> + <bool name="current_language_has_spaces">false</bool> +</resources> diff --git a/java/res/values/donottranslate.xml b/java/res/values/donottranslate.xml index e352f082b..8983536af 100644 --- a/java/res/values/donottranslate.xml +++ b/java/res/values/donottranslate.xml @@ -18,6 +18,8 @@ */ --> <resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- TODO: these settings depend on the language. They should be put either in the dictionary + header, or in the subtype maybe? --> <!-- Symbols that are suggested between words --> <string name="suggested_punctuations">!,?,\\,,:,;,\",(,),\',-,/,@,_</string> <!-- Symbols that are normally preceded by a space (used to add an auto-space before these) --> @@ -29,6 +31,8 @@ <string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string> <!-- Word connectors --> <string name="symbols_word_connectors">\'-</string> + <!-- Whether this language uses spaces --> + <bool name="current_language_has_spaces">true</bool> <!-- Always show the suggestion strip --> <string name="prefs_suggestion_visibility_show_value">0</string> diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java index 77f323440..8ce126330 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java @@ -172,7 +172,7 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider return null; } final String keyDescription = getKeyDescription(key); - final Rect boundsInParent = key.mHitBox; + final Rect boundsInParent = key.getHitBox(); // Calculate the key's in-screen bounds. mTempBoundsInScreen.set(boundsInParent); @@ -208,8 +208,8 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider * @param key The key to press. */ void simulateKeyPress(final Key key) { - final int x = key.mHitBox.centerX(); - final int y = key.mHitBox.centerY(); + final int x = key.getHitBox().centerX(); + final int y = key.getHitBox().centerY(); final long downTime = SystemClock.uptimeMillis(); final MotionEvent downEvent = MotionEvent.obtain( downTime, downTime, MotionEvent.ACTION_DOWN, x, y, 0); @@ -325,6 +325,6 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider // The key x- and y-coordinates are stable between layout changes. // Generate an identifier by bit-shifting the x-coordinate to the // left-half of the integer and OR'ing with the y-coordinate. - return ((0xFFFF & key.mX) << (Integer.SIZE / 2)) | (0xFFFF & key.mY); + return ((0xFFFF & key.getX()) << (Integer.SIZE / 2)) | (0xFFFF & key.getY()); } } diff --git a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java index 41f5b9a24..58624a2e6 100644 --- a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java +++ b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java @@ -97,7 +97,7 @@ public final class KeyCodeDescriptionMapper { */ public String getDescriptionForKey(final Context context, final Keyboard keyboard, final Key key, final boolean shouldObscure) { - final int code = key.mCode; + final int code = key.getCode(); if (code == Constants.CODE_SWITCH_ALPHA_SYMBOL) { final String description = getDescriptionForSwitchAlphaSymbol(context, keyboard); @@ -116,8 +116,8 @@ public final class KeyCodeDescriptionMapper { return getDescriptionForActionKey(context, keyboard, key); } - if (!TextUtils.isEmpty(key.mLabel)) { - final String label = key.mLabel.toString().trim(); + if (!TextUtils.isEmpty(key.getLabel())) { + final String label = key.getLabel().trim(); // First, attempt to map the label to a pre-defined description. if (mKeyLabelMap.containsKey(label)) { @@ -126,7 +126,7 @@ public final class KeyCodeDescriptionMapper { } // Just attempt to speak the description. - if (key.mCode != Constants.CODE_UNSPECIFIED) { + if (key.getCode() != Constants.CODE_UNSPECIFIED) { return getDescriptionForKeyCode(context, keyboard, key, shouldObscure); } return null; @@ -215,8 +215,8 @@ public final class KeyCodeDescriptionMapper { final int resId; // Always use the label, if available. - if (!TextUtils.isEmpty(key.mLabel)) { - return key.mLabel.toString().trim(); + if (!TextUtils.isEmpty(key.getLabel())) { + return key.getLabel().trim(); } // Otherwise, use the action ID. @@ -267,7 +267,7 @@ public final class KeyCodeDescriptionMapper { */ private String getDescriptionForKeyCode(final Context context, final Keyboard keyboard, final Key key, final boolean shouldObscure) { - final int code = key.mCode; + final int code = key.getCode(); // If the key description should be obscured, now is the time to do it. final boolean isDefinedNonCtrl = Character.isDefined(code) && !Character.isISOControl(code); @@ -280,8 +280,8 @@ public final class KeyCodeDescriptionMapper { if (isDefinedNonCtrl) { return Character.toString((char) code); } - if (!TextUtils.isEmpty(key.mLabel)) { - return key.mLabel; + if (!TextUtils.isEmpty(key.getLabel())) { + return key.getLabel(); } return context.getString(R.string.spoken_description_unknown, code); } diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java index 16c79eb8e..143c6e848 100644 --- a/java/src/com/android/inputmethod/keyboard/Key.java +++ b/java/src/com/android/inputmethod/keyboard/Key.java @@ -58,12 +58,12 @@ public class Key implements Comparable<Key> { /** * The key code (unicode or custom code) that this key generates. */ - public final int mCode; + private final int mCode; /** Label to display */ - public final String mLabel; + private final String mLabel; /** Hint label to display on the key in conjunction with the label */ - public final String mHintLabel; + private final String mHintLabel; /** Flags of the label */ private final int mLabelFlags; private static final int LABEL_FLAGS_ALIGN_LEFT = 0x01; @@ -95,18 +95,18 @@ public class Key implements Comparable<Key> { private final int mIconId; /** Width of the key, not including the gap */ - public final int mWidth; + private final int mWidth; /** Height of the key, not including the gap */ - public final int mHeight; + private final int mHeight; /** X coordinate of the key in the keyboard layout */ - public final int mX; + private final int mX; /** Y coordinate of the key in the keyboard layout */ - public final int mY; + private final int mY; /** Hit bounding box of the key */ - public final Rect mHitBox = new Rect(); + private final Rect mHitBox = new Rect(); /** More keys. It is guaranteed that this is null or an array of one or more elements */ - public final MoreKeySpec[] mMoreKeys; + private final MoreKeySpec[] mMoreKeys; /** More keys column number and flags */ private final int mMoreKeysColumnAndFlags; private static final int MORE_KEYS_COLUMN_MASK = 0x000000ff; @@ -121,7 +121,7 @@ public class Key implements Comparable<Key> { private static final String MORE_KEYS_NO_PANEL_AUTO_MORE_KEY = "!noPanelAutoMoreKey!"; /** Background type that represents different key background visual than normal one. */ - public final int mBackgroundType; + private final int mBackgroundType; public static final int BACKGROUND_TYPE_EMPTY = 0; public static final int BACKGROUND_TYPE_NORMAL = 1; public static final int BACKGROUND_TYPE_FUNCTIONAL = 2; @@ -135,7 +135,7 @@ public class Key implements Comparable<Key> { private static final int ACTION_FLAGS_ALT_CODE_WHILE_TYPING = 0x04; private static final int ACTION_FLAGS_ENABLE_LONG_PRESS = 0x08; - public final KeyVisualAttributes mKeyVisualAttributes; + private final KeyVisualAttributes mKeyVisualAttributes; private final OptionalAttributes mOptionalAttributes; @@ -151,7 +151,7 @@ public class Key implements Comparable<Key> { public final int mVisualInsetsLeft; public final int mVisualInsetsRight; - public OptionalAttributes(final String outputText, final int altCode, + private OptionalAttributes(final String outputText, final int altCode, final int disabledIconId, final int previewIconId, final int visualInsetsLeft, final int visualInsetsRight) { mOutputText = outputText; @@ -161,6 +161,18 @@ public class Key implements Comparable<Key> { mVisualInsetsLeft = visualInsetsLeft; mVisualInsetsRight = visualInsetsRight; } + + public static OptionalAttributes newInstance(final String outputText, final int altCode, + final int disabledIconId, final int previewIconId, + final int visualInsetsLeft, final int visualInsetsRight) { + if (outputText == null && altCode == CODE_UNSPECIFIED + && disabledIconId == ICON_UNDEFINED && previewIconId == ICON_UNDEFINED + && visualInsetsLeft == 0 && visualInsetsRight == 0) { + return null; + } + return new OptionalAttributes(outputText, altCode, disabledIconId, previewIconId, + visualInsetsLeft, visualInsetsRight); + } } private final int mHashCode; @@ -194,12 +206,9 @@ public class Key implements Comparable<Key> { mMoreKeys = null; mMoreKeysColumnAndFlags = 0; mLabel = label; - if (outputText == null) { - mOptionalAttributes = null; - } else { - mOptionalAttributes = new OptionalAttributes(outputText, CODE_UNSPECIFIED, - ICON_UNDEFINED, ICON_UNDEFINED, 0, 0); - } + mOptionalAttributes = OptionalAttributes.newInstance(outputText, CODE_UNSPECIFIED, + ICON_UNDEFINED, ICON_UNDEFINED, + 0 /* visualInsetsLeft */, 0 /* visualInsetsRight */); mCode = code; mEnabled = (code != CODE_UNSPECIFIED); mIconId = iconId; @@ -225,7 +234,7 @@ public class Key implements Comparable<Key> { public Key(final Resources res, final KeyboardParams params, final KeyboardRow row, final XmlPullParser parser) throws XmlPullParserException { final float horizontalGap = isSpacer() ? 0 : params.mHorizontalGap; - final int rowHeight = row.mRowHeight; + final int rowHeight = row.getRowHeight(); mHeight = rowHeight - params.mVerticalGap; final TypedArray keyAttr = res.obtainAttributes(Xml.asAttributeSet(parser), @@ -260,11 +269,11 @@ public class Key implements Comparable<Key> { final int previewIconId = KeySpecParser.getIconId(style.getString(keyAttr, R.styleable.Keyboard_Key_keyIconPreview)); - mLabelFlags = style.getFlag(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags) + mLabelFlags = style.getFlags(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags) | row.getDefaultKeyLabelFlags(); final boolean needsToUpperCase = needsToUpperCase(mLabelFlags, params.mId.mElementId); final Locale locale = params.mId.mLocale; - int actionFlags = style.getFlag(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); + int actionFlags = style.getFlags(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); String[] moreKeys = style.getStringArray(keyAttr, R.styleable.Keyboard_Key_moreKeys); int moreKeysColumn = style.getInt(keyAttr, @@ -360,15 +369,8 @@ public class Key implements Comparable<Key> { KeySpecParser.parseCode(style.getString(keyAttr, R.styleable.Keyboard_Key_altCode), params.mCodesSet, CODE_UNSPECIFIED), needsToUpperCase, locale); - if (outputText == null && altCode == CODE_UNSPECIFIED - && disabledIconId == ICON_UNDEFINED && previewIconId == ICON_UNDEFINED - && visualInsetsLeft == 0 && visualInsetsRight == 0) { - mOptionalAttributes = null; - } else { - mOptionalAttributes = new OptionalAttributes(outputText, altCode, - disabledIconId, previewIconId, - visualInsetsLeft, visualInsetsRight); - } + mOptionalAttributes = OptionalAttributes.newInstance(outputText, altCode, + disabledIconId, previewIconId, visualInsetsLeft, visualInsetsRight); mKeyVisualAttributes = KeyVisualAttributes.newInstance(keyAttr); keyAttr.recycle(); mHashCode = computeHashCode(this); @@ -377,6 +379,35 @@ public class Key implements Comparable<Key> { } } + /** + * Copy constructor. + * + * @param key the original key. + */ + protected Key(final Key key) { + // Final attributes. + mCode = key.mCode; + mLabel = key.mLabel; + mHintLabel = key.mHintLabel; + mLabelFlags = key.mLabelFlags; + mIconId = key.mIconId; + mWidth = key.mWidth; + mHeight = key.mHeight; + mX = key.mX; + mY = key.mY; + mHitBox.set(key.mHitBox); + mMoreKeys = key.mMoreKeys; + mMoreKeysColumnAndFlags = key.mMoreKeysColumnAndFlags; + mBackgroundType = key.mBackgroundType; + mActionFlags = key.mActionFlags; + mKeyVisualAttributes = key.mKeyVisualAttributes; + mOptionalAttributes = key.mOptionalAttributes; + mHashCode = key.mHashCode; + // Key state. + mPressed = key.mPressed; + mEnabled = key.mEnabled; + } + private static boolean needsToUpperCase(final int labelFlags, final int keyboardElementId) { if ((labelFlags & LABEL_FLAGS_PRESERVE_CASE) != 0) return false; switch (keyboardElementId) { @@ -476,6 +507,22 @@ public class Key implements Comparable<Key> { } } + public int getCode() { + return mCode; + } + + public String getLabel() { + return mLabel; + } + + public String getHintLabel() { + return mHintLabel; + } + + public MoreKeySpec[] getMoreKeys() { + return mMoreKeys; + } + public void markAsLeftEdge(final KeyboardParams params) { mHitBox.left = params.mLeftPadding; } @@ -522,6 +569,10 @@ public class Key implements Comparable<Key> { && (mLabelFlags & LABEL_FLAGS_SHIFTED_LETTER_ACTIVATED) == 0; } + public KeyVisualAttributes getVisualAttributes() { + return mKeyVisualAttributes; + } + public final Typeface selectTypeface(final KeyDrawParams params) { // TODO: Handle "bold" here too? if ((mLabelFlags & LABEL_FLAGS_FONT_NORMAL) != 0) { @@ -696,6 +747,22 @@ public class Key implements Comparable<Key> { ? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(mIconId); } + public int getWidth() { + return mWidth; + } + + public int getHeight() { + return mHeight; + } + + public int getX() { + return mX; + } + + public int getY() { + return mY; + } + public final int getDrawX() { final OptionalAttributes attrs = mOptionalAttributes; return (attrs == null) ? mX : mX + attrs.mVisualInsetsLeft; @@ -733,6 +800,10 @@ public class Key implements Comparable<Key> { mEnabled = enabled; } + public Rect getHitBox() { + return mHitBox; + } + /** * Detects if a point falls on this key. * @param x the x-coordinate of the point diff --git a/java/src/com/android/inputmethod/keyboard/KeyDetector.java b/java/src/com/android/inputmethod/keyboard/KeyDetector.java index 17e707f95..befb6fa92 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyDetector.java +++ b/java/src/com/android/inputmethod/keyboard/KeyDetector.java @@ -108,8 +108,9 @@ public class KeyDetector { if (distance > minDistance) { continue; } - // To take care of hitbox overlaps, we compare mCode here too. - if (primaryKey == null || distance < minDistance || key.mCode > primaryKey.mCode) { + // To take care of hitbox overlaps, we compare key's code here too. + if (primaryKey == null || distance < minDistance + || key.getCode() > primaryKey.getCode()) { minDistance = distance; primaryKey = key; } @@ -118,7 +119,7 @@ public class KeyDetector { } public static String printableCode(Key key) { - return key != null ? Constants.printableCode(key.mCode) : "none"; + return key != null ? Constants.printableCode(key.getCode()) : "none"; } public static String printableCodes(int[] codes) { diff --git a/java/src/com/android/inputmethod/keyboard/Keyboard.java b/java/src/com/android/inputmethod/keyboard/Keyboard.java index fefac96fd..c40ddeabe 100644 --- a/java/src/com/android/inputmethod/keyboard/Keyboard.java +++ b/java/src/com/android/inputmethod/keyboard/Keyboard.java @@ -131,7 +131,7 @@ public class Keyboard { } for (final Key key : mKeys) { - if (key.mCode == code) { + if (key.getCode() == code) { mKeyCache.put(code, key); return key; } @@ -148,7 +148,7 @@ public class Keyboard { for (final Key key : mKeys) { if (key == aKey) { - mKeyCache.put(key.mCode, key); + mKeyCache.put(key.getCode(), key); return true; } } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java index bccf8fb15..711de63b3 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java @@ -162,7 +162,8 @@ public final class KeyboardLayoutSet { final KeyboardId id = new KeyboardId(keyboardLayoutSetElementId, mParams); try { return getKeyboard(elementParams, id); - } catch (RuntimeException e) { + } catch (final RuntimeException e) { + Log.e(TAG, "Can't create keyboard: " + id, e); throw new KeyboardLayoutSetException(e, id); } } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java index 28eb58573..d049a861e 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java @@ -265,9 +265,9 @@ public class KeyboardView extends View { mClipRegion.setEmpty(); for (final Key key : mInvalidatedKeys) { if (mKeyboard.hasKey(key)) { - final int x = key.mX + getPaddingLeft(); - final int y = key.mY + getPaddingTop(); - mWorkingRect.set(x, y, x + key.mWidth, y + key.mHeight); + final int x = key.getX() + getPaddingLeft(); + final int y = key.getY() + getPaddingTop(); + mWorkingRect.set(x, y, x + key.getWidth(), y + key.getHeight()); mClipRegion.union(mWorkingRect); } } @@ -310,11 +310,11 @@ public class KeyboardView extends View { private void onDrawKey(final Key key, final Canvas canvas, final Paint paint) { final int keyDrawX = key.getDrawX() + getPaddingLeft(); - final int keyDrawY = key.mY + getPaddingTop(); + final int keyDrawY = key.getY() + getPaddingTop(); canvas.translate(keyDrawX, keyDrawY); final int keyHeight = mKeyboard.mMostCommonKeyHeight - mKeyboard.mVerticalGap; - final KeyVisualAttributes attr = key.mKeyVisualAttributes; + final KeyVisualAttributes attr = key.getVisualAttributes(); final KeyDrawParams params = mKeyDrawParams.mayCloneAndUpdateParams(keyHeight, attr); params.mAnimAlpha = Constants.Color.ALPHA_OPAQUE; @@ -330,7 +330,7 @@ public class KeyboardView extends View { protected void onDrawKeyBackground(final Key key, final Canvas canvas) { final Rect padding = mKeyBackgroundPadding; final int bgWidth = key.getDrawWidth() + padding.left + padding.right; - final int bgHeight = key.mHeight + padding.top + padding.bottom; + final int bgHeight = key.getHeight() + padding.top + padding.bottom; final int bgX = -padding.left; final int bgY = -padding.top; final int[] drawableState = key.getCurrentDrawableState(); @@ -352,7 +352,7 @@ public class KeyboardView extends View { protected void onDrawKeyTopVisuals(final Key key, final Canvas canvas, final Paint paint, final KeyDrawParams params) { final int keyWidth = key.getDrawWidth(); - final int keyHeight = key.mHeight; + final int keyHeight = key.getHeight(); final float centerX = keyWidth * 0.5f; final float centerY = keyHeight * 0.5f; @@ -363,8 +363,8 @@ public class KeyboardView extends View { // Draw key label. final Drawable icon = key.getIcon(mKeyboard.mIconsSet, params.mAnimAlpha); float positionX = centerX; - if (key.mLabel != null) { - final String label = key.mLabel; + final String label = key.getLabel(); + if (label != null) { paint.setTypeface(key.selectTypeface(params)); paint.setTextSize(key.selectTextSize(params)); final float labelCharHeight = TypefaceUtils.getCharHeight( @@ -441,8 +441,8 @@ public class KeyboardView extends View { } // Draw hint label. - if (key.mHintLabel != null) { - final String hintLabel = key.mHintLabel; + final String hintLabel = key.getHintLabel(); + if (hintLabel != null) { paint.setTextSize(key.selectHintTextSize(params)); paint.setColor(key.selectHintTextColor(params)); blendAlpha(paint, params.mAnimAlpha); @@ -481,7 +481,7 @@ public class KeyboardView extends View { } // Draw key icon. - if (key.mLabel == null && icon != null) { + if (label == null && icon != null) { final int iconWidth = Math.min(icon.getIntrinsicWidth(), keyWidth); final int iconHeight = icon.getIntrinsicHeight(); final int iconX, alignX; @@ -505,7 +505,7 @@ public class KeyboardView extends View { } } - if (key.hasPopupHint() && key.mMoreKeys != null) { + if (key.hasPopupHint() && key.getMoreKeys() != null) { drawKeyPopupHint(key, canvas, paint, params); } } @@ -514,7 +514,7 @@ public class KeyboardView extends View { protected void drawKeyPopupHint(final Key key, final Canvas canvas, final Paint paint, final KeyDrawParams params) { final int keyWidth = key.getDrawWidth(); - final int keyHeight = key.mHeight; + final int keyHeight = key.getHeight(); paint.setTypeface(params.mTypeface); paint.setTextSize(params.mHintLetterSize); @@ -602,9 +602,9 @@ public class KeyboardView extends View { if (mInvalidateAllKeys) return; if (key == null) return; mInvalidatedKeys.add(key); - final int x = key.mX + getPaddingLeft(); - final int y = key.mY + getPaddingTop(); - invalidate(x, y, x + key.mWidth, y + key.mHeight); + final int x = key.getX() + getPaddingLeft(); + final int y = key.getY() + getPaddingTop(); + invalidate(x, y, x + key.getWidth(), y + key.getHeight()); } @Override diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java index f3d0eadc8..e4a8f4cb2 100644 --- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java @@ -237,7 +237,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack return; } sendMessageDelayed( - obtainMessage(MSG_REPEAT_KEY, key.mCode, repeatCount, tracker), delay); + obtainMessage(MSG_REPEAT_KEY, key.getCode(), repeatCount, tracker), delay); } public void cancelKeyRepeatTimer() { @@ -298,7 +298,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack final MainKeyboardView keyboardView = getOuterInstance(); // When user hits the space or the enter key, just cancel the while-typing timer. - final int typedCode = typedKey.mCode; + final int typedCode = typedKey.getCode(); if (typedCode == Constants.CODE_SPACE || typedCode == Constants.CODE_ENTER) { if (isTyping) { startWhileTypingFadeinAnimation(keyboardView); @@ -638,7 +638,6 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack mKeyPreviewLingerTimeout = delay; } - private void locatePreviewPlacerView() { if (mPreviewPlacerView.getParent() != null) { return; @@ -807,11 +806,11 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } // The key preview is placed vertically above the top edge of the parent key with an // arbitrary offset. - final int previewY = key.mY - previewHeight + mKeyPreviewOffset + final int previewY = key.getY() - previewHeight + mKeyPreviewOffset + CoordinateUtils.y(mOriginCoords); if (background != null) { - final int hasMoreKeys = (key.mMoreKeys != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL; + final int hasMoreKeys = (key.getMoreKeys() != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL; background.setState(KEY_PREVIEW_BACKGROUND_STATE_TABLE[statePosition][hasMoreKeys]); background.setAlpha(PREVIEW_ALPHA); } @@ -903,7 +902,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } private MoreKeysPanel onCreateMoreKeysPanel(final Key key, final Context context) { - if (key.mMoreKeys == null) { + if (key.getMoreKeys() == null) { return null; } Keyboard moreKeysKeyboard = mMoreKeysKeyboardCache.get(key); @@ -938,7 +937,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } final KeyboardActionListener listener = mKeyboardActionListener; if (key.hasNoPanelAutoMoreKey()) { - final int moreKeyCode = key.mMoreKeys[0].mCode; + final int moreKeyCode = key.getMoreKeys()[0].mCode; tracker.onLongPressed(); listener.onPressKey(moreKeyCode, 0 /* repeatCount */, true /* isSinglePointer */); listener.onCodeInput(moreKeyCode, @@ -946,7 +945,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack listener.onReleaseKey(moreKeyCode, false /* withSliding */); return; } - final int code = key.mCode; + final int code = key.getCode(); if (code == Constants.CODE_SPACE || code == Constants.CODE_LANGUAGE_SWITCH) { // Long pressing the space key invokes IME switcher dialog. if (listener.onCustomRequest(Constants.CUSTOM_CODE_SHOW_INPUT_METHOD_PICKER)) { @@ -972,13 +971,13 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack // keys keyboard is placed at the touch point of the parent key. final int pointX = (mConfigShowMoreKeysKeyboardAtTouchedPoint && !keyPreviewEnabled) ? CoordinateUtils.x(lastCoords) - : key.mX + key.mWidth / 2; + : key.getX() + key.getWidth() / 2; // The more keys keyboard is usually vertically aligned with the top edge of the parent key // (plus vertical gap). If the key preview is enabled, the more keys keyboard is vertically // aligned with the bottom edge of the visible part of the key preview. // {@code mPreviewVisibleOffset} has been set appropriately in // {@link KeyboardView#showKeyPreview(PointerTracker)}. - final int pointY = key.mY + mKeyPreviewDrawParams.mPreviewVisibleOffset; + final int pointY = key.getY() + mKeyPreviewDrawParams.mPreviewVisibleOffset; moreKeysPanel.showMoreKeysPanel(this, this, pointX, pointY, mKeyboardActionListener); tracker.onShowMoreKeysPanel(moreKeysPanel); } @@ -1174,13 +1173,14 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack if (key.altCodeWhileTyping() && key.isEnabled()) { params.mAnimAlpha = mAltCodeKeyWhileTypingAnimAlpha; } - if (key.mCode == Constants.CODE_SPACE) { + final int code = key.getCode(); + if (code == Constants.CODE_SPACE) { drawSpacebar(key, canvas, paint); // Whether space key needs to show the "..." popup hint for special purposes if (key.isLongPressEnabled() && mHasMultipleEnabledIMEsOrSubtypes) { drawKeyPopupHint(key, canvas, paint, params); } - } else if (key.mCode == Constants.CODE_LANGUAGE_SWITCH) { + } else if (code == Constants.CODE_LANGUAGE_SWITCH) { super.onDrawKeyTopVisuals(key, canvas, paint, params); drawKeyPopupHint(key, canvas, paint, params); } else { @@ -1228,8 +1228,8 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } private void drawSpacebar(final Key key, final Canvas canvas, final Paint paint) { - final int width = key.mWidth; - final int height = key.mHeight; + final int width = key.getWidth(); + final int height = key.getHeight(); // If input language are explicitly selected. if (mNeedsToDisplayLanguage) { diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java index 3fd29dcfd..8256d4623 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java @@ -276,6 +276,7 @@ public final class MoreKeysKeyboard extends Keyboard { mParams.mVerticalGap = parentKeyboard.mVerticalGap / 2; mParentKey = parentKey; + final MoreKeySpec[] moreKeys = parentKey.getMoreKeys(); final int width, height; // {@link KeyPreviewDrawParams#mPreviewVisibleWidth} should have been set at // {@link MainKeyboardView#showKeyPreview(PointerTracker}, though there may be @@ -283,7 +284,7 @@ public final class MoreKeysKeyboard extends Keyboard { // zero-division error at // {@link MoreKeysKeyboardParams#setParameters(int,int,int,int,int,int,boolean,int)}. final boolean singleMoreKeyWithPreview = parentKeyboardView.isKeyPreviewPopupEnabled() - && !parentKey.noKeyPreview() && parentKey.mMoreKeys.length == 1 + && !parentKey.noKeyPreview() && moreKeys.length == 1 && keyPreviewDrawParams.mPreviewVisibleWidth > 0; if (singleMoreKeyWithPreview) { // Use pre-computed width and height if this more keys keyboard has only one key to @@ -312,8 +313,8 @@ public final class MoreKeysKeyboard extends Keyboard { mDivider = null; dividerWidth = 0; } - mParams.setParameters(parentKey.mMoreKeys.length, parentKey.getMoreKeysColumn(), - width, height, parentKey.mX + parentKey.mWidth / 2, + mParams.setParameters(moreKeys.length, parentKey.getMoreKeysColumn(), + width, height, parentKey.getX() + parentKey.getWidth() / 2, parentKeyboard.mId.mWidth, parentKey.isFixedColumnOrderMoreKeys(), dividerWidth); } @@ -321,7 +322,7 @@ public final class MoreKeysKeyboard extends Keyboard { private static int getMaxKeyWidth(final Key parentKey, final int minKeyWidth, final float padding, final Paint paint) { int maxWidth = minKeyWidth; - for (final MoreKeySpec spec : parentKey.mMoreKeys) { + for (final MoreKeySpec spec : parentKey.getMoreKeys()) { final String label = spec.mLabel; // If the label is single letter, minKeyWidth is enough to hold the label. if (label != null && StringUtils.codePointCount(label) > 1) { @@ -336,7 +337,7 @@ public final class MoreKeysKeyboard extends Keyboard { public MoreKeysKeyboard build() { final MoreKeysKeyboardParams params = mParams; final int moreKeyFlags = mParentKey.getMoreKeyLabelFlags(); - final MoreKeySpec[] moreKeys = mParentKey.mMoreKeys; + final MoreKeySpec[] moreKeys = mParentKey.getMoreKeys(); for (int n = 0; n < moreKeys.length; n++) { final MoreKeySpec moreKeySpec = moreKeys[n]; final int row = n / params.mNumColumns; diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java index f00f5a99e..973128d36 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java @@ -127,7 +127,7 @@ public class MoreKeysKeyboardView extends KeyboardView implements MoreKeysPanel public void onUpEvent(final int x, final int y, final int pointerId, final long eventTime) { if (mCurrentKey != null && mActivePointerId == pointerId) { updateReleaseKeyGraphics(mCurrentKey); - onCodeInput(mCurrentKey.mCode, x, y); + onCodeInput(mCurrentKey.getCode(), x, y); mCurrentKey = null; } } diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java index 5387ddb6d..d4d0d8718 100644 --- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java +++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java @@ -510,7 +510,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { return false; } if (key.isEnabled()) { - mListener.onPressKey(key.mCode, repeatCount, getActivePointerTrackerCount() == 1); + mListener.onPressKey(key.getCode(), repeatCount, getActivePointerTrackerCount() == 1); final boolean keyboardLayoutHasBeenChanged = mKeyboardLayoutHasBeenChanged; mKeyboardLayoutHasBeenChanged = false; mTimerProxy.startTypingStateTimer(key); @@ -776,7 +776,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (sInGesture || !mGestureStrokeWithPreviewPoints.isStartOfAGesture()) { return; } - if (key == null || !Character.isLetter(key.mCode)) { + if (key == null || !Character.isLetter(key.getCode())) { return; } if (DEBUG_LISTENER) { @@ -1075,8 +1075,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { + " phantom sudden move event (distance=%d) is translated to " + "up[%d,%d,%s]/down[%d,%d,%s] events", mPointerId, getDistance(x, y, lastX, lastY), - lastX, lastY, Constants.printableCode(oldKey.mCode), - x, y, Constants.printableCode(key.mCode))); + lastX, lastY, Constants.printableCode(oldKey.getCode()), + x, y, Constants.printableCode(key.getCode()))); } // TODO: This should be moved to outside of this nested if-clause? if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -1098,8 +1098,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { + " bogus down-move-up event (raidus=%.2f key diagonal) is " + " translated to up[%d,%d,%s]/down[%d,%d,%s] events", mPointerId, radiusRatio, - lastX, lastY, Constants.printableCode(oldKey.mCode), - x, y, Constants.printableCode(key.mCode))); + lastX, lastY, Constants.printableCode(oldKey.getCode()), + x, y, Constants.printableCode(key.getCode()))); } onUpEventInternal(x, y, eventTime); onDownEventInternal(x, y, eventTime); @@ -1107,7 +1107,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { private void processSildeOutFromOldKey(final Key oldKey) { setReleasedKeyGraphics(oldKey); - callListenerOnRelease(oldKey, oldKey.mCode, true /* withSliding */); + callListenerOnRelease(oldKey, oldKey.getCode(), true /* withSliding */); startSlidingKeyInput(oldKey); mTimerProxy.cancelKeyTimers(); } @@ -1263,7 +1263,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (sInGesture) { if (currentKey != null) { - callListenerOnRelease(currentKey, currentKey.mCode, true /* withSliding */); + callListenerOnRelease(currentKey, currentKey.getCode(), true /* withSliding */); } mayEndBatchInput(eventTime); return; @@ -1376,9 +1376,9 @@ public final class PointerTracker implements PointerTrackerQueue.Element { // doesn't have its more keys. (e.g. spacebar, globe key) // We always need to start the long press timer if the key has its more keys regardless of // whether or not we are in the sliding input mode. - if (mIsInSlidingKeyInput && key.mMoreKeys == null) return; + if (mIsInSlidingKeyInput && key.getMoreKeys() == null) return; final int delay; - switch (key.mCode) { + switch (key.getCode()) { case Constants.CODE_SHIFT: delay = sParams.mLongPressShiftLockTimeout; break; @@ -1401,7 +1401,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { return; } - final int code = key.mCode; + final int code = key.getCode(); callListenerOnCodeInput(key, code, x, y, eventTime); callListenerOnRelease(key, code, false /* withSliding */); } @@ -1412,14 +1412,14 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (!key.isRepeatable()) return; // Don't start key repeat when we are in sliding input mode. if (mIsInSlidingKeyInput) return; - detectAndSendKey(key, key.mX, key.mY, SystemClock.uptimeMillis()); + detectAndSendKey(key, key.getX(), key.getY(), SystemClock.uptimeMillis()); final int startRepeatCount = 1; mTimerProxy.startKeyRepeatTimer(this, startRepeatCount, sParams.mKeyRepeatStartTimeout); } public void onKeyRepeat(final int code, final int repeatCount) { final Key key = getKey(); - if (key == null || key.mCode != code) { + if (key == null || key.getCode() != code) { return; } final int nextRepeatCount = repeatCount + 1; diff --git a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java index 9b0a33cec..c0c02f10a 100644 --- a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java +++ b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java @@ -96,7 +96,7 @@ public class ProximityInfo { private static boolean needsProximityInfo(final Key key) { // Don't include special keys into ProximityInfo. - return key.mCode >= Constants.CODE_SPACE; + return key.getCode() >= Constants.CODE_SPACE; } private static int getProximityInfoKeysCount(final Key[] keys) { @@ -122,7 +122,7 @@ public class ProximityInfo { if (!needsProximityInfo(neighborKey)) { continue; } - proximityCharsArray[infoIndex] = neighborKey.mCode; + proximityCharsArray[infoIndex] = neighborKey.getCode(); infoIndex++; } } @@ -159,11 +159,11 @@ public class ProximityInfo { if (!needsProximityInfo(key)) { continue; } - keyXCoordinates[infoIndex] = key.mX; - keyYCoordinates[infoIndex] = key.mY; - keyWidths[infoIndex] = key.mWidth; - keyHeights[infoIndex] = key.mHeight; - keyCharCodes[infoIndex] = key.mCode; + keyXCoordinates[infoIndex] = key.getX(); + keyYCoordinates[infoIndex] = key.getY(); + keyWidths[infoIndex] = key.getWidth(); + keyHeights[infoIndex] = key.getHeight(); + keyCharCodes[infoIndex] = key.getCode(); infoIndex++; } @@ -183,7 +183,7 @@ public class ProximityInfo { if (!needsProximityInfo(key)) { continue; } - final Rect hitBox = key.mHitBox; + final Rect hitBox = key.getHitBox(); sweetSpotCenterXs[infoIndex] = hitBox.exactCenterX(); sweetSpotCenterYs[infoIndex] = hitBox.exactCenterY(); sweetSpotRadii[infoIndex] = defaultRadius; @@ -204,7 +204,7 @@ public class ProximityInfo { " [%2d] row=%d x/y/r=%7.2f/%7.2f/%5.2f %s code=%s", infoIndex, row, sweetSpotCenterXs[infoIndex], sweetSpotCenterYs[infoIndex], sweetSpotRadii[infoIndex], (row < rows ? "correct" : "default"), - Constants.printableCode(key.mCode))); + Constants.printableCode(key.getCode()))); } infoIndex++; } @@ -322,19 +322,21 @@ y |---+---+---+---+-v-+-|-+---+---+---+---+---| | thresholdBase and get have to align this on the center of the key. Hence, we don't need a separate value for bottomPixelWithinThreshold and call this yEnd right away. */ - final int topPixelWithinThreshold = key.mY - threshold; + final int keyX = key.getX(); + final int keyY = key.getY(); + final int topPixelWithinThreshold = keyY - threshold; final int yDeltaToGrid = topPixelWithinThreshold % mCellHeight; final int yMiddleOfTopCell = topPixelWithinThreshold - yDeltaToGrid + halfCellHeight; final int yStart = Math.max(halfCellHeight, yMiddleOfTopCell + (yDeltaToGrid <= halfCellHeight ? 0 : mCellHeight)); - final int yEnd = Math.min(fullGridHeight, key.mY + key.mHeight + threshold); + final int yEnd = Math.min(fullGridHeight, keyY + key.getHeight() + threshold); - final int leftPixelWithinThreshold = key.mX - threshold; + final int leftPixelWithinThreshold = keyX - threshold; final int xDeltaToGrid = leftPixelWithinThreshold % mCellWidth; final int xMiddleOfLeftCell = leftPixelWithinThreshold - xDeltaToGrid + halfCellWidth; final int xStart = Math.max(halfCellWidth, xMiddleOfLeftCell + (xDeltaToGrid <= halfCellWidth ? 0 : mCellWidth)); - final int xEnd = Math.min(fullGridWidth, key.mX + key.mWidth + threshold); + final int xEnd = Math.min(fullGridWidth, keyX + key.getWidth() + threshold); int baseIndexOfCurrentRow = (yStart / mCellHeight) * mGridWidth + (xStart / mCellWidth); for (int centerY = yStart; centerY <= yEnd; centerY += mCellHeight) { @@ -372,7 +374,7 @@ y |---+---+---+---+-v-+-|-+---+---+---+---+---| | thresholdBase and get if (index >= destLength) { break; } - final int code = key.mCode; + final int code = key.getCode(); if (code <= Constants.CODE_SPACE) { break; } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java b/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java index f65056948..e6a674334 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java @@ -24,7 +24,7 @@ public abstract class KeyStyle { public abstract String[] getStringArray(TypedArray a, int index); public abstract String getString(TypedArray a, int index); public abstract int getInt(TypedArray a, int index, int defaultValue); - public abstract int getFlag(TypedArray a, int index); + public abstract int getFlags(TypedArray a, int index); protected KeyStyle(final KeyboardTextsSet textsSet) { mTextsSet = textsSet; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java index 6aab3e7b3..b21ea3f71 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java @@ -66,7 +66,7 @@ public final class KeyStylesSet { } @Override - public int getFlag(final TypedArray a, final int index) { + public int getFlags(final TypedArray a, final int index) { return a.getInt(index, 0); } } @@ -123,14 +123,11 @@ public final class KeyStylesSet { } @Override - public int getFlag(final TypedArray a, final int index) { - int flags = a.getInt(index, 0); - final Object value = mStyleAttributes.get(index); - if (value != null) { - flags |= (Integer)value; - } + public int getFlags(final TypedArray a, final int index) { + final Integer value = (Integer)mStyleAttributes.get(index); + final int flags = a.getInt(index, (value != null) ? value : 0); final KeyStyle parentStyle = mStyles.get(mParentStyleName); - return flags | parentStyle.getFlag(a, index); + return flags | parentStyle.getFlags(a, index); } public void readKeyAttributes(final TypedArray keyAttr) { @@ -142,13 +139,13 @@ public final class KeyStylesSet { readString(keyAttr, R.styleable.Keyboard_Key_keyHintLabel); readStringArray(keyAttr, R.styleable.Keyboard_Key_moreKeys); readStringArray(keyAttr, R.styleable.Keyboard_Key_additionalMoreKeys); - readFlag(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags); + readFlags(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags); readString(keyAttr, R.styleable.Keyboard_Key_keyIcon); readString(keyAttr, R.styleable.Keyboard_Key_keyIconDisabled); readString(keyAttr, R.styleable.Keyboard_Key_keyIconPreview); readInt(keyAttr, R.styleable.Keyboard_Key_maxMoreKeysColumn); readInt(keyAttr, R.styleable.Keyboard_Key_backgroundType); - readFlag(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); + readFlags(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); } private void readString(final TypedArray a, final int index) { @@ -163,7 +160,7 @@ public final class KeyStylesSet { } } - private void readFlag(final TypedArray a, final int index) { + private void readFlags(final TypedArray a, final int index) { if (a.hasValue(index)) { final Integer value = (Integer)mStyleAttributes.get(index); mStyleAttributes.put(index, a.getInt(index, 0) | (value != null ? value : 0)); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java index b34d7c45f..65eb1712a 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java @@ -218,20 +218,18 @@ public class KeyboardBuilder<KP extends KeyboardParams> { parseKeyboardAttributes(parser); startKeyboard(); parseKeyboardContent(parser, false); - break; - } else { - throw new XmlParseUtils.IllegalStartTag(parser, tag, TAG_KEYBOARD); + return; } + throw new XmlParseUtils.IllegalStartTag(parser, tag, TAG_KEYBOARD); } } } private void parseKeyboardAttributes(final XmlPullParser parser) { + final AttributeSet attr = Xml.asAttributeSet(parser); final TypedArray keyboardAttr = mContext.obtainStyledAttributes( - Xml.asAttributeSet(parser), R.styleable.Keyboard, R.attr.keyboardStyle, - R.style.Keyboard); - final TypedArray keyAttr = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_Key); + attr, R.styleable.Keyboard, R.attr.keyboardStyle, R.style.Keyboard); + final TypedArray keyAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key); try { final KeyboardParams params = mParams; final int height = params.mId.mHeight; @@ -328,31 +326,30 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (DEBUG) endTag("</%s>", tag); if (TAG_KEYBOARD.equals(tag)) { endKeyboard(); - break; - } else if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) - || TAG_MERGE.equals(tag)) { - break; - } else { - throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); + return; + } + if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) || TAG_MERGE.equals(tag)) { + return; } + throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); } } } private KeyboardRow parseRowAttributes(final XmlPullParser parser) throws XmlPullParserException { - final TypedArray a = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard); + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray keyboardAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard); try { - if (a.hasValue(R.styleable.Keyboard_horizontalGap)) { + if (keyboardAttr.hasValue(R.styleable.Keyboard_horizontalGap)) { throw new XmlParseUtils.IllegalAttribute(parser, TAG_ROW, "horizontalGap"); } - if (a.hasValue(R.styleable.Keyboard_verticalGap)) { + if (keyboardAttr.hasValue(R.styleable.Keyboard_verticalGap)) { throw new XmlParseUtils.IllegalAttribute(parser, TAG_ROW, "verticalGap"); } return new KeyboardRow(mResources, mParams, parser, mCurrentY); } finally { - a.recycle(); + keyboardAttr.recycle(); } } @@ -382,13 +379,12 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (!skip) { endRow(row); } - break; - } else if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) - || TAG_MERGE.equals(tag)) { - break; - } else { - throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); + return; } + if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) || TAG_MERGE.equals(tag)) { + return; + } + throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); } } } @@ -397,19 +393,16 @@ public class KeyboardBuilder<KP extends KeyboardParams> { throws XmlPullParserException, IOException { if (skip) { XmlParseUtils.checkEndTag(TAG_KEY, parser); - if (DEBUG) { - startEndTag("<%s /> skipped", TAG_KEY); - } - } else { - final Key key = new Key(mResources, mParams, row, parser); - if (DEBUG) { - startEndTag("<%s%s %s moreKeys=%s />", TAG_KEY, - (key.isEnabled() ? "" : " disabled"), key, - Arrays.toString(key.mMoreKeys)); - } - XmlParseUtils.checkEndTag(TAG_KEY, parser); - endKey(key); + if (DEBUG) startEndTag("<%s /> skipped", TAG_KEY); + return; } + final Key key = new Key(mResources, mParams, row, parser); + if (DEBUG) { + startEndTag("<%s%s %s moreKeys=%s />", TAG_KEY, (key.isEnabled() ? "" : " disabled"), + key, Arrays.toString(key.getMoreKeys())); + } + XmlParseUtils.checkEndTag(TAG_KEY, parser); + endKey(key); } private void parseSpacer(final XmlPullParser parser, final KeyboardRow row, final boolean skip) @@ -417,12 +410,12 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (skip) { XmlParseUtils.checkEndTag(TAG_SPACER, parser); if (DEBUG) startEndTag("<%s /> skipped", TAG_SPACER); - } else { - final Key.Spacer spacer = new Key.Spacer(mResources, mParams, row, parser); - if (DEBUG) startEndTag("<%s />", TAG_SPACER); - XmlParseUtils.checkEndTag(TAG_SPACER, parser); - endKey(spacer); + return; } + final Key.Spacer spacer = new Key.Spacer(mResources, mParams, row, parser); + if (DEBUG) startEndTag("<%s />", TAG_SPACER); + XmlParseUtils.checkEndTag(TAG_SPACER, parser); + endKey(spacer); } private void parseIncludeKeyboardContent(final XmlPullParser parser, final boolean skip) @@ -440,66 +433,63 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (skip) { XmlParseUtils.checkEndTag(TAG_INCLUDE, parser); if (DEBUG) startEndTag("</%s> skipped", TAG_INCLUDE); - } else { - final AttributeSet attr = Xml.asAttributeSet(parser); - final TypedArray keyboardAttr = mResources.obtainAttributes(attr, - R.styleable.Keyboard_Include); - final TypedArray keyAttr = mResources.obtainAttributes(attr, - R.styleable.Keyboard_Key); - int keyboardLayout = 0; - float savedDefaultKeyWidth = 0; - int savedDefaultKeyLabelFlags = 0; - int savedDefaultBackgroundType = Key.BACKGROUND_TYPE_NORMAL; - try { - XmlParseUtils.checkAttributeExists(keyboardAttr, - R.styleable.Keyboard_Include_keyboardLayout, "keyboardLayout", - TAG_INCLUDE, parser); - keyboardLayout = keyboardAttr.getResourceId( - R.styleable.Keyboard_Include_keyboardLayout, 0); - if (row != null) { - if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { - // Override current x coordinate. - row.setXPos(row.getKeyX(keyAttr)); - } - // TODO: Remove this if-clause and do the same as backgroundType below. - savedDefaultKeyWidth = row.getDefaultKeyWidth(); - if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyWidth)) { - // Override default key width. - row.setDefaultKeyWidth(row.getKeyWidth(keyAttr)); - } - savedDefaultKeyLabelFlags = row.getDefaultKeyLabelFlags(); - // Bitwise-or default keyLabelFlag if exists. - row.setDefaultKeyLabelFlags(keyAttr.getInt( - R.styleable.Keyboard_Key_keyLabelFlags, 0) - | savedDefaultKeyLabelFlags); - savedDefaultBackgroundType = row.getDefaultBackgroundType(); - // Override default backgroundType if exists. - row.setDefaultBackgroundType(keyAttr.getInt( - R.styleable.Keyboard_Key_backgroundType, - savedDefaultBackgroundType)); + return; + } + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray keyboardAttr = mResources.obtainAttributes( + attr, R.styleable.Keyboard_Include); + final TypedArray keyAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key); + int keyboardLayout = 0; + float savedDefaultKeyWidth = 0; + int savedDefaultKeyLabelFlags = 0; + int savedDefaultBackgroundType = Key.BACKGROUND_TYPE_NORMAL; + try { + XmlParseUtils.checkAttributeExists( + keyboardAttr, R.styleable.Keyboard_Include_keyboardLayout, "keyboardLayout", + TAG_INCLUDE, parser); + keyboardLayout = keyboardAttr.getResourceId( + R.styleable.Keyboard_Include_keyboardLayout, 0); + if (row != null) { + if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { + // Override current x coordinate. + row.setXPos(row.getKeyX(keyAttr)); + } + // TODO: Remove this if-clause and do the same as backgroundType below. + savedDefaultKeyWidth = row.getDefaultKeyWidth(); + if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyWidth)) { + // Override default key width. + row.setDefaultKeyWidth(row.getKeyWidth(keyAttr)); } - } finally { - keyboardAttr.recycle(); - keyAttr.recycle(); + savedDefaultKeyLabelFlags = row.getDefaultKeyLabelFlags(); + // Bitwise-or default keyLabelFlag if exists. + row.setDefaultKeyLabelFlags(keyAttr.getInt( + R.styleable.Keyboard_Key_keyLabelFlags, 0) | savedDefaultKeyLabelFlags); + savedDefaultBackgroundType = row.getDefaultBackgroundType(); + // Override default backgroundType if exists. + row.setDefaultBackgroundType(keyAttr.getInt( + R.styleable.Keyboard_Key_backgroundType, savedDefaultBackgroundType)); } + } finally { + keyboardAttr.recycle(); + keyAttr.recycle(); + } - XmlParseUtils.checkEndTag(TAG_INCLUDE, parser); - if (DEBUG) { - startEndTag("<%s keyboardLayout=%s />",TAG_INCLUDE, - mResources.getResourceEntryName(keyboardLayout)); - } - final XmlResourceParser parserForInclude = mResources.getXml(keyboardLayout); - try { - parseMerge(parserForInclude, row, skip); - } finally { - if (row != null) { - // Restore default keyWidth, keyLabelFlags, and backgroundType. - row.setDefaultKeyWidth(savedDefaultKeyWidth); - row.setDefaultKeyLabelFlags(savedDefaultKeyLabelFlags); - row.setDefaultBackgroundType(savedDefaultBackgroundType); - } - parserForInclude.close(); + XmlParseUtils.checkEndTag(TAG_INCLUDE, parser); + if (DEBUG) { + startEndTag("<%s keyboardLayout=%s />",TAG_INCLUDE, + mResources.getResourceEntryName(keyboardLayout)); + } + final XmlResourceParser parserForInclude = mResources.getXml(keyboardLayout); + try { + parseMerge(parserForInclude, row, skip); + } finally { + if (row != null) { + // Restore default keyWidth, keyLabelFlags, and backgroundType. + row.setDefaultKeyWidth(savedDefaultKeyWidth); + row.setDefaultKeyLabelFlags(savedDefaultKeyLabelFlags); + row.setDefaultBackgroundType(savedDefaultBackgroundType); } + parserForInclude.close(); } } @@ -516,11 +506,10 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } else { parseRowContent(parser, row, skip); } - break; - } else { - throw new XmlParseUtils.ParseException( - "Included keyboard layout must have <merge> root element", parser); + return; } + throw new XmlParseUtils.ParseException( + "Included keyboard layout must have <merge> root element", parser); } } } @@ -554,10 +543,9 @@ public class KeyboardBuilder<KP extends KeyboardParams> { final String tag = parser.getName(); if (TAG_SWITCH.equals(tag)) { if (DEBUG) endTag("</%s>", TAG_SWITCH); - break; - } else { - throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_SWITCH); + return; } + throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_SWITCH); } } } @@ -580,40 +568,40 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (id == null) { return true; } - final TypedArray a = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_Case); + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray caseAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Case); try { - final boolean keyboardLayoutSetElementMatched = matchTypedValue(a, + final boolean keyboardLayoutSetElementMatched = matchTypedValue(caseAttr, R.styleable.Keyboard_Case_keyboardLayoutSetElement, id.mElementId, KeyboardId.elementIdToName(id.mElementId)); - final boolean modeMatched = matchTypedValue(a, + final boolean modeMatched = matchTypedValue(caseAttr, R.styleable.Keyboard_Case_mode, id.mMode, KeyboardId.modeName(id.mMode)); - final boolean navigateNextMatched = matchBoolean(a, + final boolean navigateNextMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_navigateNext, id.navigateNext()); - final boolean navigatePreviousMatched = matchBoolean(a, + final boolean navigatePreviousMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_navigatePrevious, id.navigatePrevious()); - final boolean passwordInputMatched = matchBoolean(a, + final boolean passwordInputMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_passwordInput, id.passwordInput()); - final boolean clobberSettingsKeyMatched = matchBoolean(a, + final boolean clobberSettingsKeyMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_clobberSettingsKey, id.mClobberSettingsKey); - final boolean shortcutKeyEnabledMatched = matchBoolean(a, + final boolean shortcutKeyEnabledMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_shortcutKeyEnabled, id.mShortcutKeyEnabled); - final boolean shortcutKeyOnSymbolsMatched = matchBoolean(a, + final boolean shortcutKeyOnSymbolsMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_shortcutKeyOnSymbols, id.mShortcutKeyOnSymbols); - final boolean hasShortcutKeyMatched = matchBoolean(a, + final boolean hasShortcutKeyMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_hasShortcutKey, id.mHasShortcutKey); - final boolean languageSwitchKeyEnabledMatched = matchBoolean(a, + final boolean languageSwitchKeyEnabledMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_languageSwitchKeyEnabled, id.mLanguageSwitchKeyEnabled); - final boolean isMultiLineMatched = matchBoolean(a, + final boolean isMultiLineMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_isMultiLine, id.isMultiLine()); - final boolean imeActionMatched = matchInteger(a, + final boolean imeActionMatched = matchInteger(caseAttr, R.styleable.Keyboard_Case_imeAction, id.imeAction()); - final boolean localeCodeMatched = matchString(a, + final boolean localeCodeMatched = matchString(caseAttr, R.styleable.Keyboard_Case_localeCode, id.mLocale.toString()); - final boolean languageCodeMatched = matchString(a, + final boolean languageCodeMatched = matchString(caseAttr, R.styleable.Keyboard_Case_languageCode, id.mLocale.getLanguage()); - final boolean countryCodeMatched = matchString(a, + final boolean countryCodeMatched = matchString(caseAttr, R.styleable.Keyboard_Case_countryCode, id.mLocale.getCountry()); final boolean selected = keyboardLayoutSetElementMatched && modeMatched && navigateNextMatched && navigatePreviousMatched && passwordInputMatched @@ -624,42 +612,42 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (DEBUG) { startTag("<%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s>%s", TAG_CASE, - textAttr(a.getString( + textAttr(caseAttr.getString( R.styleable.Keyboard_Case_keyboardLayoutSetElement), "keyboardLayoutSetElement"), - textAttr(a.getString(R.styleable.Keyboard_Case_mode), "mode"), - textAttr(a.getString(R.styleable.Keyboard_Case_imeAction), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_mode), "mode"), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_imeAction), "imeAction"), - booleanAttr(a, R.styleable.Keyboard_Case_navigateNext, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_navigateNext, "navigateNext"), - booleanAttr(a, R.styleable.Keyboard_Case_navigatePrevious, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_navigatePrevious, "navigatePrevious"), - booleanAttr(a, R.styleable.Keyboard_Case_clobberSettingsKey, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_clobberSettingsKey, "clobberSettingsKey"), - booleanAttr(a, R.styleable.Keyboard_Case_passwordInput, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_passwordInput, "passwordInput"), - booleanAttr(a, R.styleable.Keyboard_Case_shortcutKeyEnabled, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_shortcutKeyEnabled, "shortcutKeyEnabled"), - booleanAttr(a, R.styleable.Keyboard_Case_shortcutKeyOnSymbols, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_shortcutKeyOnSymbols, "shortcutKeyOnSymbols"), - booleanAttr(a, R.styleable.Keyboard_Case_hasShortcutKey, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_hasShortcutKey, "hasShortcutKey"), - booleanAttr(a, R.styleable.Keyboard_Case_languageSwitchKeyEnabled, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_languageSwitchKeyEnabled, "languageSwitchKeyEnabled"), - booleanAttr(a, R.styleable.Keyboard_Case_isMultiLine, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_isMultiLine, "isMultiLine"), - textAttr(a.getString(R.styleable.Keyboard_Case_localeCode), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_localeCode), "localeCode"), - textAttr(a.getString(R.styleable.Keyboard_Case_languageCode), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_languageCode), "languageCode"), - textAttr(a.getString(R.styleable.Keyboard_Case_countryCode), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_countryCode), "countryCode"), selected ? "" : " skipped"); } return selected; } finally { - a.recycle(); + caseAttr.recycle(); } } @@ -692,7 +680,8 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } if (ResourceUtils.isIntegerValue(v)) { return intValue == a.getInt(index, 0); - } else if (ResourceUtils.isStringValue(v)) { + } + if (ResourceUtils.isStringValue(v)) { return StringUtils.containsInArray(strValue, a.getString(index).split("\\|")); } return false; @@ -711,10 +700,10 @@ public class KeyboardBuilder<KP extends KeyboardParams> { private void parseKeyStyle(final XmlPullParser parser, final boolean skip) throws XmlPullParserException, IOException { - TypedArray keyStyleAttr = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_KeyStyle); - TypedArray keyAttrs = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_Key); + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray keyStyleAttr = mResources.obtainAttributes( + attr, R.styleable.Keyboard_KeyStyle); + final TypedArray keyAttrs = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key); try { if (!keyStyleAttr.hasValue(R.styleable.Keyboard_KeyStyle_styleName)) { throw new XmlParseUtils.ParseException("<" + TAG_KEY_STYLE @@ -756,7 +745,7 @@ public class KeyboardBuilder<KP extends KeyboardParams> { mRightEdgeKey = null; } addEdgeSpace(mParams.mRightPadding, row); - mCurrentY += row.mRowHeight; + mCurrentY += row.getRowHeight(); mCurrentRow = null; mTopEdge = false; } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java index a57b83ac0..d32bb7581 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java @@ -85,7 +85,7 @@ public class KeyboardParams { public void onAddKey(final Key newKey) { final Key key = (mKeysCache != null) ? mKeysCache.get(newKey) : newKey; final boolean isSpacer = key.isSpacer(); - if (isSpacer && key.mWidth == 0) { + if (isSpacer && key.getWidth() == 0) { // Ignore zero width {@link Spacer}. return; } @@ -94,7 +94,7 @@ public class KeyboardParams { return; } updateHistogram(key); - if (key.mCode == Constants.CODE_SHIFT) { + if (key.getCode() == Constants.CODE_SHIFT) { mShiftKeys.add(key); } if (key.altCodeWhileTyping()) { @@ -125,14 +125,14 @@ public class KeyboardParams { } private void updateHistogram(final Key key) { - final int height = key.mHeight + mVerticalGap; + final int height = key.getHeight() + mVerticalGap; final int heightCount = updateHistogramCounter(mHeightHistogram, height); if (heightCount > mMaxHeightCount) { mMaxHeightCount = heightCount; mMostCommonKeyHeight = height; } - final int width = key.mWidth + mHorizontalGap; + final int width = key.getWidth() + mHorizontalGap; final int widthCount = updateHistogramCounter(mWidthHistogram, width); if (widthCount > mMaxWidthCount) { mMaxWidthCount = widthCount; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java index 5fe84a704..23c0b91e6 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java @@ -38,10 +38,10 @@ public final class KeyboardRow { private static final int KEYWIDTH_FILL_RIGHT = -1; private final KeyboardParams mParams; + /** The height of this row. */ + private final int mRowHeight; /** Default width of a key in this row. */ private float mDefaultKeyWidth; - /** Default height of a key in this row. */ - public final int mRowHeight; /** Default keyLabelFlags in this row. */ private int mDefaultKeyLabelFlags; /** Default backgroundType for this row */ @@ -64,16 +64,19 @@ public final class KeyboardRow { R.styleable.Keyboard_Key); mDefaultKeyWidth = keyAttr.getFraction(R.styleable.Keyboard_Key_keyWidth, params.mBaseWidth, params.mBaseWidth, params.mDefaultKeyWidth); + mDefaultKeyLabelFlags = keyAttr.getInt(R.styleable.Keyboard_Key_keyLabelFlags, 0); mDefaultBackgroundType = keyAttr.getInt(R.styleable.Keyboard_Key_backgroundType, Key.BACKGROUND_TYPE_NORMAL); keyAttr.recycle(); - // TODO: Initialize this with <Row> attribute as backgroundType is done. - mDefaultKeyLabelFlags = 0; mCurrentY = y; mCurrentX = 0.0f; } + public int getRowHeight() { + return mRowHeight; + } + public float getDefaultKeyWidth() { return mDefaultKeyWidth; } @@ -111,22 +114,21 @@ public final class KeyboardRow { } public float getKeyX(final TypedArray keyAttr) { - if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { - final float keyXPos = keyAttr.getFraction(R.styleable.Keyboard_Key_keyXPos, - mParams.mBaseWidth, mParams.mBaseWidth, 0); - if (keyXPos < 0) { - // If keyXPos is negative, the actual x-coordinate will be - // keyboardWidth + keyXPos. - // keyXPos shouldn't be less than mCurrentX because drawable area for this - // key starts at mCurrentX. Or, this key will overlaps the adjacent key on - // its left hand side. - final int keyboardRightEdge = mParams.mOccupiedWidth - mParams.mRightPadding; - return Math.max(keyXPos + keyboardRightEdge, mCurrentX); - } else { - return keyXPos + mParams.mLeftPadding; - } + if (keyAttr == null || !keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { + return mCurrentX; } - return mCurrentX; + final float keyXPos = keyAttr.getFraction(R.styleable.Keyboard_Key_keyXPos, + mParams.mBaseWidth, mParams.mBaseWidth, 0); + if (keyXPos >= 0) { + return keyXPos + mParams.mLeftPadding; + } + // If keyXPos is negative, the actual x-coordinate will be + // keyboardWidth + keyXPos. + // keyXPos shouldn't be less than mCurrentX because drawable area for this + // key starts at mCurrentX. Or, this key will overlaps the adjacent key on + // its left hand side. + final int keyboardRightEdge = mParams.mOccupiedWidth - mParams.mRightPadding; + return Math.max(keyXPos + keyboardRightEdge, mCurrentX); } public float getKeyWidth(final TypedArray keyAttr) { @@ -134,6 +136,9 @@ public final class KeyboardRow { } public float getKeyWidth(final TypedArray keyAttr, final float keyXPos) { + if (keyAttr == null) { + return mDefaultKeyWidth; + } final int widthType = ResourceUtils.getEnumValue(keyAttr, R.styleable.Keyboard_Key_keyWidth, KEYWIDTH_NOT_ENUM); switch (widthType) { diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index d0a4afd50..7e497e9b9 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,7 +21,7 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; @@ -231,8 +231,8 @@ final public class BinaryDictionaryGetter { try { // Read the version of the file inStream = new FileInputStream(f); - final BinaryDictInputOutput.ByteBufferWrapper buffer = - new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( + final BinaryDictDecoder.ByteBufferWrapper buffer = + new BinaryDictDecoder.ByteBufferWrapper(inStream.getChannel().map( FileChannel.MapMode.READ_ONLY, 0, f.length())); final int magic = buffer.readInt(); if (magic != FormatSpec.MAGIC_NUMBER) { @@ -241,7 +241,7 @@ final public class BinaryDictionaryGetter { final int formatVersion = buffer.readInt(); final int headerSize = buffer.readInt(); final HashMap<String, String> options = CollectionUtils.newHashMap(); - BinaryDictInputOutput.populateOptions(buffer, headerSize, options); + BinaryDictDecoder.populateOptions(buffer, headerSize, options); final String version = options.get(VERSION_KEY); if (null == version) { diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index a3c6071c5..1ececd5c1 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -20,10 +20,10 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -51,7 +51,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override public void clear() { final HashMap<String, String> attributes = CollectionUtils.newHashMap(); - mFusionDictionary = new FusionDictionary(new Node(), + mFusionDictionary = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(attributes, false, false)); } @@ -87,7 +87,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override protected void writeBinaryDictionary(final FileOutputStream out) throws IOException, UnsupportedFormatException { - BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + BinaryDictEncoder.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); } @Override diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index b3e930145..37256770a 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -88,6 +88,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Controls access to the local binary dictionary for this instance. */ private final DictionaryController mLocalDictionaryController = new DictionaryController(); + /* A extension for a binary dictionary file. */ + public static final String DICT_FILE_EXTENSION = ".dict"; + /** * Abstract method for loading the unigrams and bigrams of a given dictionary in a background * thread. @@ -146,7 +149,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } protected static String getFilenameWithLocale(final String name, final String localeStr) { - return name + "." + localeStr + ".dict"; + return name + "." + localeStr + DICT_FILE_EXTENSION; } /** diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index ffe317161..ee7478ca2 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1948,7 +1948,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } } - if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) { + if (currentSettings.isSuggestionsRequested(mDisplayOrientation) + && currentSettings.mCurrentLanguageHasSpaces) { restartSuggestionsOnWordBeforeCursorIfAtEndOfWord(); } // We just removed a character. We need to update the auto-caps state. @@ -1977,6 +1978,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private void handleCharacter(final int primaryCode, final int x, final int y, final int spaceState) { + // TODO: refactor this method to stop flipping isComposingWord around all the time, and + // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter + // which has the same name as other handle* methods but is not the same. boolean isComposingWord = mWordComposer.isComposingWord(); // TODO: remove isWordConnector() and use isUsuallyFollowedBySpace() instead. @@ -1996,12 +2000,20 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen resetEntireInputState(mLastSelectionStart); isComposingWord = false; } - // NOTE: isCursorTouchingWord() is a blocking IPC call, so it often takes several - // dozen milliseconds. Avoid calling it as much as possible, since we are on the UI - // thread here. - if (!isComposingWord && currentSettings.isWordCodePoint(primaryCode) + // We want to find out whether to start composing a new word with this character. If so, + // we need to reset the composing state and switch isComposingWord. The order of the + // tests is important for good performance. + // We only start composing if we're not already composing. + if (!isComposingWord + // We only start composing if this is a word code point. Essentially that means it's a + // a letter or a word connector. + && currentSettings.isWordCodePoint(primaryCode) + // We never go into composing state if suggestions are not requested. && currentSettings.isSuggestionsRequested(mDisplayOrientation) && - !mConnection.isCursorTouchingWord(currentSettings)) { + // In languages with spaces, we only start composing a word when we are not already + // touching a word. In languages without spaces, the above conditions are sufficient. + (!mConnection.isCursorTouchingWord(currentSettings) + || !currentSettings.mCurrentLanguageHasSpaces)) { // Reset entirely the composing state anyway, then start composing a new word unless // the character is a single quote. The idea here is, single quote is not a // separator and it should be treated as a normal character, except in the first @@ -2089,16 +2101,20 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean handleSeparator(final int primaryCode, final int x, final int y, final int spaceState) { boolean didAutoCorrect = false; + final SettingsValues currentSettings = mSettings.getCurrent(); + // We avoid sending spaces in languages without spaces if we were composing. + final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == primaryCode + && !currentSettings.mCurrentLanguageHasSpaces && mWordComposer.isComposingWord(); if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the separator at the current cursor position. resetEntireInputState(mLastSelectionStart); } - final SettingsValues currentSettings = mSettings.getCurrent(); - if (mWordComposer.isComposingWord()) { + if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing if (currentSettings.mCorrectionEnabled) { - // TODO: maybe cache Strings in an <String> sparse array or something - commitCurrentAutoCorrection(new String(new int[]{primaryCode}, 0, 1)); + final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR + : new String(new int[] { primaryCode }, 0, 1); + commitCurrentAutoCorrection(separator); didAutoCorrect = true; } else { commitTyped(new String(new int[]{primaryCode}, 0, 1)); @@ -2115,7 +2131,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_handleSeparator(primaryCode, mWordComposer.isComposingWord()); } - sendKeyCodePoint(primaryCode); + + if (!shouldAvoidSendingCode) { + sendKeyCodePoint(primaryCode); + } if (Constants.CODE_SPACE == primaryCode) { if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) { @@ -2260,11 +2279,17 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Get the word on which we should search the bigrams. If we are composing a word, it's // whatever is *before* the half-committed word in the buffer, hence 2; if we aren't, we // should just skip whitespace if any, so 1. - // TODO: this is slow (2-way IPC) - we should probably cache this instead. final SettingsValues currentSettings = mSettings.getCurrent(); - final String prevWord = - mConnection.getNthPreviousWord(currentSettings.mWordSeparators, - mWordComposer.isComposingWord() ? 2 : 1); + final String prevWord; + if (currentSettings.mCurrentLanguageHasSpaces) { + // If we are typing in a language with spaces we can just look up the previous + // word from textview. + prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, + mWordComposer.isComposingWord() ? 2 : 1); + } else { + prevWord = LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null + : mLastComposedWord.mCommittedWord; + } return suggest.getSuggestedWords(mWordComposer, prevWord, keyboard.getProximityInfo(), currentSettings.mBlockPotentiallyOffensive, currentSettings.mCorrectionEnabled, sessionId); @@ -2534,6 +2559,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // recorrection. This is a temporary, stopgap measure that will be removed later. // TODO: remove this. if (mAppWorkAroundsUtils.isBrokenByRecorrection()) return; + // Recorrection is not supported in languages without spaces because we don't know + // how to segment them yet. + if (!mSettings.getCurrent().mCurrentLanguageHasSpaces) return; // If the cursor is not touching a word, or if there is a selection, return right away. if (mLastSelectionStart != mLastSelectionEnd) return; // If we don't know the cursor location, return. @@ -2656,7 +2684,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) { mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord); } - mConnection.commitText(originallyTypedWord + mLastComposedWord.mSeparatorString, 1); + final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString; + if (mSettings.getCurrent().mCurrentLanguageHasSpaces) { + // For languages with spaces, we revert to the typed string, but the cursor is still + // after the separator so we don't resume suggestions. If the user wants to correct + // the word, they have to press backspace again. + mConnection.commitText(stringToCommit, 1); + } else { + // For languages without spaces, we revert the typed string but the cursor is flush + // with the typed word, so we need to resume suggestions right away. + mWordComposer.setComposingWord(stringToCommit, mKeyboardSwitcher.getKeyboard()); + mConnection.setComposingText(stringToCommit, 1); + } if (mSettings.isInternal()) { LatinImeLoggerUtils.onSeparator(mLastComposedWord.mSeparatorString, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); @@ -2674,7 +2713,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This essentially inserts a space, and that's it. public void promotePhantomSpace() { - if (mSettings.getCurrent().shouldInsertSpacesAutomatically() + final SettingsValues currentSettings = mSettings.getCurrent(); + if (currentSettings.shouldInsertSpacesAutomatically() + && currentSettings.mCurrentLanguageHasSpaces && !mConnection.textBeforeCursorLooksLikeURL()) { if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_promotePhantomSpace(); @@ -2887,6 +2928,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return mSuggest.hasMainDictionary(); } + // DO NOT USE THIS for any other purpose than testing. This can break the keyboard badly. + @UsedForTesting + /* package for test */ void replaceMainDictionaryForTest(final Locale locale) { + mSuggest.resetMainDict(this, locale, null); + } + public void debugDumpStateAndCrashWithException(final String context) { final StringBuilder s = new StringBuilder(mAppWorkAroundsUtils.toString()); s.append("\nAttributes : ").append(mSettings.getCurrent().mInputAttributes) diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index a09ca605c..039dadc66 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -272,8 +272,8 @@ public final class WordComposer { final int x, y; final Key key; if (keyboard != null && (key = keyboard.getKey(codePoint)) != null) { - x = key.mX + key.mWidth / 2; - y = key.mY + key.mHeight / 2; + x = key.getX() + key.getWidth() / 2; + y = key.getY() + key.getHeight() / 2; } else { x = Constants.NOT_A_COORDINATE; y = Constants.NOT_A_COORDINATE; diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java new file mode 100644 index 000000000..5e3d6d22d --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoder.java @@ -0,0 +1,795 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.makedict.decoder.HeaderReaderInterface; +import com.android.inputmethod.latin.utils.JniUtils; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * Decodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + */ +public final class BinaryDictDecoder { + + private static final boolean DBG = MakedictLog.DBG; + + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + + private BinaryDictDecoder() { + // This utility class is not publicly instantiable. + } + + private static final int MAX_JUMPS = 12; + + @UsedForTesting + public interface FusionDictionaryBufferInterface { + public int readUnsignedByte(); + public int readUnsignedShort(); + public int readUnsignedInt24(); + public int readInt(); + public int position(); + public void position(int newPosition); + public void put(final byte b); + public int limit(); + @UsedForTesting + public int capacity(); + } + + public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface { + private ByteBuffer mBuffer; + + public ByteBufferWrapper(final ByteBuffer buffer) { + mBuffer = buffer; + } + + @Override + public int readUnsignedByte() { + return mBuffer.get() & 0xFF; + } + + @Override + public int readUnsignedShort() { + return mBuffer.getShort() & 0xFFFF; + } + + @Override + public int readUnsignedInt24() { + final int retval = readUnsignedByte(); + return (retval << 16) + readUnsignedShort(); + } + + @Override + public int readInt() { + return mBuffer.getInt(); + } + + @Override + public int position() { + return mBuffer.position(); + } + + @Override + public void position(int newPos) { + mBuffer.position(newPos); + } + + @Override + public void put(final byte b) { + mBuffer.put(b); + } + + @Override + public int limit() { + return mBuffer.limit(); + } + + @Override + public int capacity() { + return mBuffer.capacity(); + } + } + + /** + * A class grouping utility function for our specific character encoding. + */ + static final class CharEncoding { + private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; + private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; + + /** + * Helper method to find out whether this code fits on one byte + */ + private static boolean fitsOnOneByte(final int character) { + return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE + && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; + } + + /** + * Compute the size of a character given its character code. + * + * Char format is: + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + * + * @param character the character code. + * @return the size in binary encoded-form, either 1 or 3 bytes. + */ + static int getCharSize(final int character) { + // See char encoding in FusionDictionary.java + if (fitsOnOneByte(character)) return 1; + if (FormatSpec.INVALID_CHARACTER == character) return 1; + return 3; + } + + /** + * Compute the byte size of a character array. + */ + static int getCharArraySize(final int[] chars) { + int size = 0; + for (int character : chars) size += getCharSize(character); + return size; + } + + /** + * Writes a char array to a byte buffer. + * + * @param codePoints the code point array to write. + * @param buffer the byte buffer to write to. + * @param index the index in buffer to write the character array to. + * @return the index after the last character. + */ + static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { + for (int codePoint : codePoints) { + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + return index; + } + + /** + * Writes a string with our character format to a byte buffer. + * + * This will also write the terminator byte. + * + * @param buffer the byte buffer to write to. + * @param origin the offset to write from. + * @param word the string to write. + * @return the size written, in bytes. + */ + static int writeString(final byte[] buffer, final int origin, + final String word) { + final int length = word.length(); + int index = origin; + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; + return index - origin; + } + + /** + * Writes a string with our character format to a ByteArrayOutputStream. + * + * This will also write the terminator byte. + * + * @param buffer the ByteArrayOutputStream to write to. + * @param word the string to write. + */ + static void writeString(final ByteArrayOutputStream buffer, final String word) { + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer.write((byte) codePoint); + } else { + buffer.write((byte) (0xFF & (codePoint >> 16))); + buffer.write((byte) (0xFF & (codePoint >> 8))); + buffer.write((byte) (0xFF & codePoint)); + } + } + buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); + } + + /** + * Reads a string from a buffer. This is the converse of the above method. + */ + static String readString(final FusionDictionaryBufferInterface buffer) { + final StringBuilder s = new StringBuilder(); + int character = readChar(buffer); + while (character != FormatSpec.INVALID_CHARACTER) { + s.appendCodePoint(character); + character = readChar(buffer); + } + return s.toString(); + } + + /** + * Reads a character from the buffer. + * + * This follows the character format documented earlier in this source file. + * + * @param buffer the buffer, positioned over an encoded character. + * @return the character code. + */ + static int readChar(final FusionDictionaryBufferInterface buffer) { + int character = buffer.readUnsignedByte(); + if (!fitsOnOneByte(character)) { + if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { + return FormatSpec.INVALID_CHARACTER; + } + character <<= 16; + character += buffer.readUnsignedShort(); + } + return character; + } + } + + // Input methods: Read a binary dictionary to memory. + // readDictionaryBinary is the public entry point for them. + + static int readChildrenAddress(final FusionDictionaryBufferInterface buffer, + final int optionFlags, final FormatOptions options) { + if (options.mSupportsDynamicUpdate) { + final int address = buffer.readUnsignedInt24(); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + if ((address & FormatSpec.MSB24) != 0) { + return -(address & FormatSpec.SINT24_MAX); + } else { + return address; + } + } + int address; + switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + return buffer.readUnsignedByte(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + return buffer.readUnsignedShort(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + return buffer.readUnsignedInt24(); + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + + static int readParentAddress(final FusionDictionaryBufferInterface buffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int parentAddress = buffer.readUnsignedInt24(); + final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (parentAddress & FormatSpec.SINT24_MAX); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; + public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer, + final int originalGroupAddress, final FormatOptions options) { + int addressPointer = originalGroupAddress; + final int flags = buffer.readUnsignedByte(); + ++addressPointer; + + final int parentAddress = readParentAddress(buffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += 3; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(buffer); + addressPointer += CharEncoding.getCharSize(character); + while (-1 != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of CHARACTER_BUFFER. + CHARACTER_BUFFER[index++] = character; + character = CharEncoding.readChar(buffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); + } else { + final int character = CharEncoding.readChar(buffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + ++addressPointer; + frequency = buffer.readUnsignedByte(); + } else { + frequency = CharGroup.NOT_A_TERMINAL; + } + int childrenAddress = readChildrenAddress(buffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + ArrayList<WeightedString> shortcutTargets = null; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + final int pointerBefore = buffer.position(); + shortcutTargets = new ArrayList<WeightedString>(); + buffer.readUnsignedShort(); // Skip the size + while (true) { + final int targetFlags = buffer.readUnsignedByte(); + final String word = CharEncoding.readString(buffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + addressPointer += buffer.position() - pointerBefore; + } + ArrayList<PendingAttribute> bigrams = null; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + final int bigramFlags = buffer.readUnsignedByte(); + ++addressPointer; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = addressPointer; + switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * buffer.readUnsignedByte(); + addressPointer += 1; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * buffer.readUnsignedShort(); + addressPointer += 2; + break; + case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: + final int offset = (buffer.readUnsignedByte() << 16) + + buffer.readUnsignedShort(); + bigramAddress += sign * offset; + addressPointer += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; + } + if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { + MakedictLog.d("too many bigrams in a group."); + } + } + return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } + + /** + * Reads and returns the char group count out of a buffer and forwards the pointer. + */ + public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) { + final int msb = buffer.readUnsignedByte(); + if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { + return msb; + } else { + return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) + + buffer.readUnsignedByte(); + } + } + + /** + * Finds, as a string, the word at the address passed as an argument. + * + * @param buffer the buffer to read from. + * @param headerSize the size of the header. + * @param address the address to seek. + * @param formatOptions file format options. + * @return the word with its frequency, as a weighted string. + */ + /* package for tests */ static WeightedString getWordAtAddress( + final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, + final FormatOptions formatOptions) { + final WeightedString result; + final int originalPointer = buffer.position(); + buffer.position(address); + + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions); + } else { + result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address, + formatOptions); + } + + buffer.position(originalPointer); + return result; + } + + @SuppressWarnings("unused") + private static WeightedString getWordAtAddressWithParentAddress( + final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, + final FormatOptions options) { + int currentAddress = address; + int frequency = Integer.MIN_VALUE; + final StringBuilder builder = new StringBuilder(); + // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH + for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { + CharGroupInfo currentInfo; + int loopCounter = 0; + do { + buffer.position(currentAddress + headerSize); + currentInfo = readCharGroup(buffer, currentAddress, options); + if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { + currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + } + if (DBG && loopCounter++ > MAX_JUMPS) { + MakedictLog.d("Too many jumps - probably a bug"); + } + } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)); + if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; + builder.insert(0, + new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); + if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; + currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + } + return new WeightedString(builder.toString(), frequency); + } + + private static WeightedString getWordAtAddressWithoutParentAddress( + final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, + final FormatOptions options) { + buffer.position(headerSize); + final int count = readCharGroupCount(buffer); + int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); + final StringBuilder builder = new StringBuilder(); + WeightedString result = null; + + CharGroupInfo last = null; + for (int i = count - 1; i >= 0; --i) { + CharGroupInfo info = readCharGroup(buffer, groupOffset, options); + groupOffset = info.mEndAddress; + if (info.mOriginalAddress == address) { + builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); + result = new WeightedString(builder.toString(), info.mFrequency); + break; // and return + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + if (info.mChildrenAddress > address) { + if (null == last) continue; + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + buffer.position(last.mChildrenAddress + headerSize); + i = readCharGroupCount(buffer); + groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); + last = null; + continue; + } + last = info; + } + if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + buffer.position(last.mChildrenAddress + headerSize); + i = readCharGroupCount(buffer); + groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); + last = null; + continue; + } + } + return result; + } + + /** + * Reads a single node array from a buffer. + * + * This methods reads the file at the current position. A node array is fully expected to start + * at the current position. + * This will recursively read other node arrays into the structure, populating the reverse + * maps on the fly and using them to keep track of already read nodes. + * + * @param buffer the buffer, correctly positioned at the start of a node array. + * @param headerSize the size, in bytes, of the file header. + * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. + * @param reverseGroupMap a mapping from addresses to already read character groups. + * @param options file format options. + * @return the read node array with all his children already read. + */ + private static PtNodeArray readNodeArray(final FusionDictionaryBufferInterface buffer, + final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, + final Map<Integer, CharGroup> reverseGroupMap, final FormatOptions options) + throws IOException { + final ArrayList<CharGroup> nodeArrayContents = new ArrayList<CharGroup>(); + final int nodeArrayOrigin = buffer.position() - headerSize; + + do { // Scan the linked-list node. + final int nodeArrayHeadPosition = buffer.position() - headerSize; + final int count = readCharGroupCount(buffer); + int groupOffset = nodeArrayHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); + for (int i = count; i > 0; --i) { // Scan the array of CharGroup. + CharGroupInfo info = readCharGroup(buffer, groupOffset, options); + if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; + ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; + ArrayList<WeightedString> bigrams = null; + if (null != info.mBigrams) { + bigrams = new ArrayList<WeightedString>(); + for (PendingAttribute bigram : info.mBigrams) { + final WeightedString word = getWordAtAddress( + buffer, headerSize, bigram.mAddress, options); + final int reconstructedFrequency = + reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); + bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); + } + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); + if (null == children) { + final int currentPosition = buffer.position(); + buffer.position(info.mChildrenAddress + headerSize); + children = readNodeArray( + buffer, headerSize, reverseNodeArrayMap, reverseGroupMap, options); + buffer.position(currentPosition); + } + nodeArrayContents.add( + new CharGroup(info.mCharacters, shortcutTargets, bigrams, + info.mFrequency, + 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), + 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); + } else { + nodeArrayContents.add( + new CharGroup(info.mCharacters, shortcutTargets, bigrams, + info.mFrequency, + 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), + 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); + } + groupOffset = info.mEndAddress; + } + + // reach the end of the array. + if (options.mSupportsDynamicUpdate) { + final int nextAddress = buffer.readUnsignedInt24(); + if (nextAddress >= 0 && nextAddress < buffer.limit()) { + buffer.position(nextAddress); + } else { + break; + } + } + } while (options.mSupportsDynamicUpdate && + buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); + + final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOrigin; + nodeArray.mCachedAddressAfterUpdate = nodeArrayOrigin; + reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); + return nodeArray; + } + + /** + * Helper function to get the binary format version from the header. + * @throws IOException + */ + private static int getFormatVersion(final FusionDictionaryBufferInterface buffer) + throws IOException { + final int magic = buffer.readInt(); + if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort(); + return FormatSpec.NOT_A_VERSION_NUMBER; + } + + /** + * Helper function to get and validate the binary format version. + * @throws UnsupportedFormatException + * @throws IOException + */ + static int checkFormatVersion(final FusionDictionaryBufferInterface buffer) + throws IOException, UnsupportedFormatException { + final int version = getFormatVersion(buffer); + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("This file has version " + version + + ", but this implementation does not support versions above " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + return version; + } + + /** + * Reads a header from a buffer. + * @param headerReader the header reader + * @throws IOException + * @throws UnsupportedFormatException + */ + public static FileHeader readHeader(final HeaderReaderInterface headerReader) + throws IOException, UnsupportedFormatException { + final int version = headerReader.readVersion(); + final int optionsFlags = headerReader.readOptionFlags(); + + final int headerSize = headerReader.readHeaderSize(); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = headerReader.readAttributes(headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + /** + * Reads options from a buffer and populate a map with their contents. + * + * The buffer is read at the current position, so the caller must take care the pointer + * is in the right place before calling this. + */ + public static void populateOptions(final FusionDictionaryBufferInterface buffer, + final int headerSize, final HashMap<String, String> options) { + while (buffer.position() < headerSize) { + final String key = CharEncoding.readString(buffer); + final String value = CharEncoding.readString(buffer); + options.put(key, value); + } + } + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. The optional dict argument is an existing dictionary to + * which words from the buffer should be added. If it is null, a new dictionary is created. + * + * @param reader the reader. + * @param dict an optional dictionary to add words to, or null. + * @return the created (or merged) dictionary. + */ + @UsedForTesting + public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader, + final FusionDictionary dict) throws FileNotFoundException, IOException, + UnsupportedFormatException { + + // if the buffer has not been opened, open the buffer with bytebuffer. + if (reader.getBuffer() == null) reader.openBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + if (reader.getBuffer() == null) { + MakedictLog.e("Cannot open the buffer"); + } + + // Read header + final FileHeader fileHeader = readHeader(reader); + + Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); + Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); + final PtNodeArray root = readNodeArray(reader.getBuffer(), fileHeader.mHeaderSize, + reverseNodeArrayMapping, reverseGroupMapping, fileHeader.mFormatOptions); + + FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); + if (null != dict) { + for (final Word w : dict) { + if (w.mIsBlacklistEntry) { + newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); + } else { + newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); + } + } + for (final Word w : dict) { + // By construction a binary dictionary may not have bigrams pointing to + // words that are not also registered as unigrams so we don't have to avoid + // them explicitly here. + for (final WeightedString bigram : w.mBigrams) { + newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); + } + } + } + + return newDict; + } + + /** + * Helper method to pass a file name instead of a File object to isBinaryDictionary. + */ + public static boolean isBinaryDictionary(final String filename) { + final File file = new File(filename); + return isBinaryDictionary(file); + } + + /** + * Basic test to find out whether the file is a binary dictionary or not. + * + * Concretely this only tests the magic number. + * + * @param file The file to test. + * @return true if it's a binary dictionary, false otherwise + */ + public static boolean isBinaryDictionary(final File file) { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final ByteBuffer buffer = inStream.getChannel().map( + FileChannel.MapMode.READ_ONLY, 0, file.length()); + final int version = getFormatVersion(new ByteBufferWrapper(buffer)); + return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION + && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } catch (FileNotFoundException e) { + return false; + } catch (IOException e) { + return false; + } finally { + if (inStream != null) { + try { + inStream.close(); + } catch (IOException e) { + // do nothing + } + } + } + } + + /** + * Calculate bigram frequency from compressed value + * + * @param unigramFrequency + * @param bigramFrequency compressed frequency + * @return approximate bigram frequency + */ + public static int reconstructBigramFrequency(final int unigramFrequency, + final int bigramFrequency) { + final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); + return (int)resultFreqFloat; + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java new file mode 100644 index 000000000..d9005b926 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoder.java @@ -0,0 +1,1006 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * Encodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + */ +public class BinaryDictEncoder { + + private static final boolean DBG = MakedictLog.DBG; + + private BinaryDictEncoder() { + // This utility class is not publicly instantiable. + } + + // Arbitrary limit to how much passes we consider address size compression should + // terminate in. At the time of this writing, our largest dictionary completes + // compression in five passes. + // If the number of passes exceeds this number, makedict bails with an exception on + // suspicion that a bug might be causing an infinite loop. + private static final int MAX_PASSES = 24; + + /** + * Compute the binary size of the character array. + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param characters the character array + * @return the size of the char array, including the terminator if any + */ + static int getGroupCharactersSize(final int[] characters) { + int size = CharEncoding.getCharArraySize(characters); + if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the binary size of the character array in a group + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param group the group + * @return the size of the char array, including the terminator if any + */ + private static int getGroupCharactersSize(final CharGroup group) { + return getGroupCharactersSize(group.mChars); + } + + /** + * Compute the binary size of the group count for a node array. + * @param nodeArray the nodeArray + * @return the size of the group count, either 1 or 2 bytes. + */ + private static int getGroupCountSize(final PtNodeArray nodeArray) { + return BinaryDictIOUtils.getGroupCountSize(nodeArray.mData.size()); + } + + /** + * Compute the size of a shortcut in bytes. + */ + private static int getShortcutSize(final WeightedString shortcut) { + int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE; + final String word = shortcut.mWord; + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + size += CharEncoding.getCharSize(codePoint); + } + size += FormatSpec.GROUP_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the size of a shortcut list in bytes. + * + * This is known in advance and does not change according to position in the file + * like address lists do. + */ + static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { + if (null == shortcutList) return 0; + int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; + for (final WeightedString shortcut : shortcutList) { + size += getShortcutSize(shortcut); + } + return size; + } + + /** + * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. + * + * @param group the CharGroup to compute the size of. + * @param options file format options. + * @return the maximum size of the group. + */ + private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) { + int size = getGroupHeaderSize(group, options); + // If terminal, one byte for the frequency + if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE; + size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address + size += getShortcutListSize(group.mShortcutTargets); + if (null != group.mBigrams) { + size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE + + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) + * group.mBigrams.size(); + } + return size; + } + + /** + * Compute the maximum size of each node of a node array, assuming 3-byte addresses for + * everything, and caches it in the `mCachedSize' member of the nodes; deduce the size of + * the containing node array, and cache it it its 'mCachedSize' member. + * + * @param nodeArray the node array to compute the maximum size of. + * @param options file format options. + */ + private static void calculateNodeArrayMaximumSize(final PtNodeArray nodeArray, + final FormatOptions options) { + int size = getGroupCountSize(nodeArray); + for (CharGroup g : nodeArray.mData) { + final int groupSize = getCharGroupMaximumSize(g, options); + g.mCachedSize = groupSize; + size += groupSize; + } + if (options.mSupportsDynamicUpdate) { + size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + nodeArray.mCachedSize = size; + } + + /** + * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. + * + * @param group the group of which to compute the size of the header + * @param options file format options. + */ + private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + getGroupCharactersSize(group); + } else { + return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group); + } + } + + /** + * Compute the size, in bytes, that an address will occupy. + * + * This can be used either for children addresses (which are always positive) or for + * attribute, which may be positive or negative but + * store their sign bit separately. + * + * @param address the address + * @return the byte size. + */ + static int getByteSize(final int address) { + assert(address <= FormatSpec.UINT24_MAX); + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + return 0; + } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) { + return 1; + } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) { + return 2; + } else { + return 3; + } + } + + // End utility methods + + // This method is responsible for finding a nice ordering of the nodes that favors run-time + // cache performance and dictionary size. + /* package for tests */ static ArrayList<PtNodeArray> flattenTree( + final PtNodeArray rootNodeArray) { + final int treeSize = FusionDictionary.countCharGroups(rootNodeArray); + MakedictLog.i("Counted nodes : " + treeSize); + final ArrayList<PtNodeArray> flatTree = new ArrayList<PtNodeArray>(treeSize); + return flattenTreeInner(flatTree, rootNodeArray); + } + + private static ArrayList<PtNodeArray> flattenTreeInner(final ArrayList<PtNodeArray> list, + final PtNodeArray nodeArray) { + // Removing the node is necessary if the tails are merged, because we would then + // add the same node several times when we only want it once. A number of places in + // the code also depends on any node being only once in the list. + // Merging tails can only be done if there are no attributes. Searching for attributes + // in LatinIME code depends on a total breadth-first ordering, which merging tails + // breaks. If there are no attributes, it should be fine (and reduce the file size) + // to merge tails, and removing the node from the list would be necessary. However, + // we don't merge tails because breaking the breadth-first ordering would result in + // extreme overhead at bigram lookup time (it would make the search function O(n) instead + // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty + // high). + // If no nodes are ever merged, we can't have the same node twice in the list, hence + // searching for duplicates in unnecessary. It is also very performance consuming, + // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making + // this simple list.remove operation O(n*n) overall. On Android this overhead is very + // high. + // For future reference, the code to remove duplicate is a simple : list.remove(node); + list.add(nodeArray); + final ArrayList<CharGroup> branches = nodeArray.mData; + final int nodeSize = branches.size(); + for (CharGroup group : branches) { + if (null != group.mChildren) flattenTreeInner(list, group.mChildren); + } + return list; + } + + /** + * Get the offset from a position inside a current node array to a target node array, during + * update. + * + * If the current node array is before the target node array, the target node array has not + * been updated yet, so we should return the offset from the old position of the current node + * array to the old position of the target node array. If on the other hand the target is + * before the current node array, it already has been updated, so we should return the offset + * from the new position in the current node array to the new position in the target node + * array. + * + * @param currentNodeArray node array containing the CharGroup where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetNodeArray the target node array to get the offset to + * @return the offset to the target node array + */ + private static int getOffsetToTargetNodeArrayDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNodeArray targetNodeArray) { + final boolean isTargetBeforeCurrent = (targetNodeArray.mCachedAddressBeforeUpdate + < currentNodeArray.mCachedAddressBeforeUpdate); + if (isTargetBeforeCurrent) { + return targetNodeArray.mCachedAddressAfterUpdate + - (currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray); + } else { + return targetNodeArray.mCachedAddressBeforeUpdate + - (currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray); + } + } + + /** + * Get the offset from a position inside a current node array to a target CharGroup, during + * update. + * + * @param currentNodeArray node array containing the CharGroup where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetCharGroup the target CharGroup to get the offset to + * @return the offset to the target CharGroup + */ + // TODO: is there any way to factorize this method with the one above? + private static int getOffsetToTargetCharGroupDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final CharGroup targetCharGroup) { + final int oldOffsetBasePoint = currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray; + final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate + < oldOffsetBasePoint); + // If the target is before the current node array, then its address has already been + // updated. We can use the AfterUpdate member, and compare it to our own member after + // update. Otherwise, the AfterUpdate member is not updated yet, so we need to use the + // BeforeUpdate member, and of course we have to compare this to our own address before + // update. + if (isTargetBeforeCurrent) { + final int newOffsetBasePoint = currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray; + return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint; + } else { + return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint; + } + } + + /** + * Computes the actual node array size, based on the cached addresses of the children nodes. + * + * Each node array stores its tentative address. During dictionary address computing, these + * are not final, but they can be used to compute the node array size (the node array size + * depends on the address of the children because the number of bytes necessary to store an + * address depends on its numeric value. The return value indicates whether the node array + * contents (as in, any of the addresses stored in the cache fields) have changed with + * respect to their previous value. + * + * @param nodeArray the node array to compute the size of. + * @param dict the dictionary in which the word/attributes are to be found. + * @param formatOptions file format options. + * @return false if none of the cached addresses inside the node array changed, true otherwise. + */ + private static boolean computeActualNodeArraySize(final PtNodeArray nodeArray, + final FusionDictionary dict, final FormatOptions formatOptions) { + boolean changed = false; + int size = getGroupCountSize(nodeArray); + for (CharGroup group : nodeArray.mData) { + group.mCachedAddressAfterUpdate = nodeArray.mCachedAddressAfterUpdate + size; + if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) { + changed = true; + } + int groupSize = getGroupHeaderSize(group, formatOptions); + if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE; + if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) { + groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + } else if (null != group.mChildren) { + if (formatOptions.mSupportsDynamicUpdate) { + groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + } else { + groupSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(nodeArray, + groupSize + size, group.mChildren)); + } + } + groupSize += getShortcutListSize(group.mShortcutTargets); + if (null != group.mBigrams) { + for (WeightedString bigram : group.mBigrams) { + final int offset = getOffsetToTargetCharGroupDuringUpdate(nodeArray, + groupSize + size + FormatSpec.GROUP_FLAGS_SIZE, + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); + groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE; + } + } + group.mCachedSize = groupSize; + size += groupSize; + } + if (formatOptions.mSupportsDynamicUpdate) { + size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + if (nodeArray.mCachedSize != size) { + nodeArray.mCachedSize = size; + changed = true; + } + return changed; + } + + /** + * Initializes the cached addresses of node arrays and their containing nodes from their size. + * + * @param flatNodes the list of node arrays. + * @param formatOptions file format options. + * @return the byte size of the entire stack. + */ + private static int initializeNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes, + final FormatOptions formatOptions) { + int nodeArrayOffset = 0; + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset; + int groupCountSize = getGroupCountSize(nodeArray); + int groupOffset = 0; + for (final CharGroup g : nodeArray.mData) { + g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate = + groupCountSize + nodeArrayOffset + groupOffset; + groupOffset += g.mCachedSize; + } + final int nodeSize = groupCountSize + groupOffset + + (formatOptions.mSupportsDynamicUpdate + ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); + nodeArrayOffset += nodeArray.mCachedSize; + } + return nodeArrayOffset; + } + + /** + * Updates the cached addresses of node arrays after recomputing their new positions. + * + * @param flatNodes the list of node arrays. + */ + private static void updateNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArray.mCachedAddressAfterUpdate; + for (final CharGroup g : nodeArray.mData) { + g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate; + } + } + } + + /** + * Compute the cached parent addresses after all has been updated. + * + * The parent addresses are used by some binary formats at write-to-disk time. Not all formats + * need them. In particular, version 2 does not need them, and version 3 does. + * + * @param flatNodes the flat array of node arrays to fill in + */ + private static void computeParentAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + for (final CharGroup group : nodeArray.mData) { + if (null != group.mChildren) { + // Assign my address to children's parent address + // Here BeforeUpdate and AfterUpdate addresses have the same value, so it + // does not matter which we use. + group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate + - group.mChildren.mCachedAddressAfterUpdate; + } + } + } + } + + /** + * Compute the addresses and sizes of an ordered list of node arrays. + * + * This method takes a list of node arrays and will update their cached address and size + * values so that they can be written into a file. It determines the smallest size each of the + * nodes arrays can be given the addresses of its children and attributes, and store that into + * each node. + * The order of the node is given by the order of the array. This method makes no effort + * to find a good order; it only mechanically computes the size this order results in. + * + * @param dict the dictionary + * @param flatNodes the ordered list of nodes arrays + * @param formatOptions file format options. + * @return the same array it was passed. The nodes have been updated for address and size. + */ + private static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, + final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) { + // First get the worst possible sizes and offsets + for (final PtNodeArray n : flatNodes) calculateNodeArrayMaximumSize(n, formatOptions); + final int offset = initializeNodeArraysCachedAddresses(flatNodes, formatOptions); + + MakedictLog.i("Compressing the array addresses. Original size : " + offset); + MakedictLog.i("(Recursively seen size : " + offset + ")"); + + int passes = 0; + boolean changesDone = false; + do { + changesDone = false; + int nodeArrayStartOffset = 0; + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressAfterUpdate = nodeArrayStartOffset; + final int oldNodeArraySize = nodeArray.mCachedSize; + final boolean changed = computeActualNodeArraySize(nodeArray, dict, formatOptions); + final int newNodeArraySize = nodeArray.mCachedSize; + if (oldNodeArraySize < newNodeArraySize) { + throw new RuntimeException("Increased size ?!"); + } + nodeArrayStartOffset += newNodeArraySize; + changesDone |= changed; + } + updateNodeArraysCachedAddresses(flatNodes); + ++passes; + if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); + } while (changesDone); + + if (formatOptions.mSupportsDynamicUpdate) { + computeParentAddresses(flatNodes); + } + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + MakedictLog.i("Compression complete in " + passes + " passes."); + MakedictLog.i("After address compression : " + + (lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize)); + + return flatNodes; + } + + /** + * Sanity-checking method. + * + * This method checks a list of node arrays for juxtaposition, that is, it will do + * nothing if each node array's cached address is actually the previous node array's address + * plus the previous node's size. + * If this is not the case, it will throw an exception. + * + * @param arrays the list of node arrays to check + */ + private static void checkFlatNodeArrayList(final ArrayList<PtNodeArray> arrays) { + int offset = 0; + int index = 0; + for (final PtNodeArray nodeArray : arrays) { + // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter + // which we use. + if (nodeArray.mCachedAddressAfterUpdate != offset) { + throw new RuntimeException("Wrong address for node " + index + + " : expected " + offset + ", got " + nodeArray.mCachedAddressAfterUpdate); + } + ++index; + offset += nodeArray.mCachedSize; + } + } + + /** + * Helper method to write a variable-size address to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param address the address to write. + * @return the size in bytes the address actually took. + */ + private static int writeVariableAddress(final byte[] buffer, int index, final int address) { + switch (getByteSize(address)) { + case 1: + buffer[index++] = (byte)address; + return 1; + case 2: + buffer[index++] = (byte)(0xFF & (address >> 8)); + buffer[index++] = (byte)(0xFF & address); + return 2; + case 3: + buffer[index++] = (byte)(0xFF & (address >> 16)); + buffer[index++] = (byte)(0xFF & (address >> 8)); + buffer[index++] = (byte)(0xFF & address); + return 3; + case 0: + return 0; + default: + throw new RuntimeException("Address " + address + " has a strange size"); + } + } + + /** + * Helper method to write a variable-size signed address to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param address the address to write. + * @return the size in bytes the address actually took. + */ + private static int writeVariableSignedAddress(final byte[] buffer, int index, + final int address) { + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absAddress = Math.abs(address); + buffer[index++] = + (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16))); + buffer[index++] = (byte)(0xFF & (absAddress >> 8)); + buffer[index++] = (byte)(0xFF & absAddress); + } + return 3; + } + + /** + * Makes the flag value for a char group. + * + * @param hasMultipleChars whether the group has multiple chars. + * @param isTerminal whether the group is terminal. + * @param childrenAddressSize the size of a children address. + * @param hasShortcuts whether the group has shortcuts. + * @param hasBigrams whether the group has bigrams. + * @param isNotAWord whether the group is not a word. + * @param isBlackListEntry whether the group is a blacklist entry. + * @param formatOptions file format options. + * @return the flags + */ + static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal, + final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, + final boolean isNotAWord, final boolean isBlackListEntry, + final FormatOptions formatOptions) { + byte flags = 0; + if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; + if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; + if (formatOptions.mSupportsDynamicUpdate) { + flags |= FormatSpec.FLAG_IS_NOT_MOVED; + } else if (true) { + switch (childrenAddressSize) { + case 1: + flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; + break; + case 0: + flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS; + break; + default: + throw new RuntimeException("Node with a strange address"); + } + } + if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; + if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; + if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; + if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; + return flags; + } + + private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress, + final int childrenOffset, final FormatOptions formatOptions) { + return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0, + getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null, + group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions); + } + + /** + * Makes the flag value for a bigram. + * + * @param more whether there are more bigrams after this one. + * @param offset the offset of the bigram. + * @param bigramFrequency the frequency of the bigram, 0..255. + * @param unigramFrequency the unigram frequency of the same word, 0..255. + * @param word the second bigram, for debugging purposes + * @return the flags + */ + private static final int makeBigramFlags(final boolean more, final int offset, + int bigramFrequency, final int unigramFrequency, final String word) { + int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) + + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0); + switch (getByteSize(offset)) { + case 1: + bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + break; + default: + throw new RuntimeException("Strange offset size"); + } + if (unigramFrequency > bigramFrequency) { + MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word + + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " + + word + " is " + unigramFrequency); + bigramFrequency = unigramFrequency; + } + // We compute the difference between 255 (which means probability = 1) and the + // unigram score. We split this into a number of discrete steps. + // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 + // represents an increase of 16 steps: a value of 15 will be interpreted as the median + // value of the 16th step. In all justice, if the bigram frequency is low enough to be + // rounded below the first step (which means it is less than half a step higher than the + // unigram frequency) then the unigram frequency itself is the best approximation of the + // bigram freq that we could possibly supply, hence we should *not* include this bigram + // in the file at all. + // until this is done, we'll write 0 and slightly overestimate this case. + // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step + // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to + // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the + // step size. Then we compute the start of the first step (the one where value 0 starts) + // by adding half-a-step to the unigramFrequency. From there, we compute the integer + // number of steps to the bigramFrequency. One last thing: we want our steps to include + // their lower bound and exclude their higher bound so we need to have the first step + // start at exactly 1 unit higher than floor(unigramFreq + half a step). + // Note : to reconstruct the score, the dictionary reader will need to divide + // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, + // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best + // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the + // step pointed by the discretized frequency. + final float stepSize = + (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); + final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); + // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 + // here. The best approximation would be the unigram freq itself, so we should not + // include this bigram in the dictionary. For now, register as 0, and live with the + // small over-estimation that we get in this case. TODO: actually remove this bigram + // if discretizedFrequency < 0. + final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; + bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; + return bigramFlags; + } + + /** + * Makes the 2-byte value for options flags. + */ + private static final int makeOptionsValue(final FusionDictionary dictionary, + final FormatOptions formatOptions) { + final DictionaryOptions options = dictionary.mOptions; + final boolean hasBigrams = dictionary.hasBigrams(); + return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) + + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) + + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) + + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); + } + + /** + * Makes the flag value for a shortcut. + * + * @param more whether there are more attributes after this one. + * @param frequency the frequency of the attribute, 0..15 + * @return the flags + */ + static final int makeShortcutFlags(final boolean more, final int frequency) { + return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) + + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY); + } + + private static final int writeParentAddress(final byte[] buffer, final int index, + final int address, final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + if (address == FormatSpec.NO_PARENT_ADDRESS) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absAddress = Math.abs(address); + assert(absAddress <= FormatSpec.SINT24_MAX); + buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0) + | ((absAddress >> 16) & 0xFF)); + buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); + buffer[index + 2] = (byte)(absAddress & 0xFF); + } + return index + 3; + } else { + return index; + } + } + + /** + * Write a node array to memory. The node array is expected to have its final position cached. + * + * @param dict the dictionary the node array is a part of (for relative offsets). + * @param buffer the memory buffer to write to. + * @param nodeArray the node array to write. + * @param formatOptions file format options. + * @return the address of the END of the node. + */ + @SuppressWarnings("unused") + private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, + final PtNodeArray nodeArray, final FormatOptions formatOptions) { + // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup + int index = nodeArray.mCachedAddressAfterUpdate; + + final int groupCount = nodeArray.mData.size(); + final int countSize = getGroupCountSize(nodeArray); + final int parentAddress = nodeArray.mCachedParentAddress; + if (1 == countSize) { + buffer[index++] = (byte)groupCount; + } else if (2 == countSize) { + // We need to signal 2-byte size by setting the top bit of the MSB to 1, so + // we | 0x80 to do this. + buffer[index++] = (byte)((groupCount >> 8) | 0x80); + buffer[index++] = (byte)(groupCount & 0xFF); + } else { + throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); + } + int groupAddress = index; + for (int i = 0; i < groupCount; ++i) { + final CharGroup group = nodeArray.mData.get(i); + if (index != group.mCachedAddressAfterUpdate) { + throw new RuntimeException("Bug: write index is not the same as the cached address " + + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate); + } + groupAddress += getGroupHeaderSize(group, formatOptions); + // Sanity checks. + if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { + throw new RuntimeException("A node has a frequency > " + + FormatSpec.MAX_TERMINAL_FREQUENCY + + " : " + group.mFrequency); + } + if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE; + final int childrenOffset = null == group.mChildren + ? FormatSpec.NO_CHILDREN_ADDRESS + : group.mChildren.mCachedAddressAfterUpdate - groupAddress; + buffer[index++] = + makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions); + + if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { + index = writeParentAddress(buffer, index, parentAddress, formatOptions); + } else { + index = writeParentAddress(buffer, index, parentAddress + + (nodeArray.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate), + formatOptions); + } + + index = CharEncoding.writeCharArray(group.mChars, buffer, index); + if (group.hasSeveralChars()) { + buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; + } + if (group.mFrequency >= 0) { + buffer[index++] = (byte) group.mFrequency; + } + + final int shift; + if (formatOptions.mSupportsDynamicUpdate) { + shift = writeVariableSignedAddress(buffer, index, childrenOffset); + } else { + shift = writeVariableAddress(buffer, index, childrenOffset); + } + index += shift; + groupAddress += shift; + + // Write shortcuts + if (null != group.mShortcutTargets) { + final int indexOfShortcutByteSize = index; + index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; + groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + ++groupAddress; + int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), + target.mFrequency); + buffer[index++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); + index += shortcutShift; + groupAddress += shortcutShift; + } + final int shortcutByteSize = index - indexOfShortcutByteSize; + if (shortcutByteSize > 0xFFFF) { + throw new RuntimeException("Shortcut list too large"); + } + buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); + buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); + } + // Write bigrams + if (null != group.mBigrams) { + final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final CharGroup target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + ++groupAddress; + final int offset = addressOfBigram - groupAddress; + int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, + bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + buffer[index++] = (byte)bigramFlags; + final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); + index += bigramShift; + groupAddress += bigramShift; + } + } + + } + if (formatOptions.mSupportsDynamicUpdate) { + buffer[index] = buffer[index + 1] = buffer[index + 2] + = FormatSpec.NO_FORWARD_LINK_ADDRESS; + index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + if (index != nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize) { + throw new RuntimeException( + "Not the same size : written " + (index - nodeArray.mCachedAddressAfterUpdate) + + " bytes from a node that should have " + nodeArray.mCachedSize + " bytes"); + } + return index; + } + + /** + * Dumps a collection of useful statistics about a list of node arrays. + * + * This prints purely informative stuff, like the total estimated file size, the + * number of node arrays, of character groups, the repartition of each address size, etc + * + * @param nodeArrays the list of node arrays. + */ + private static void showStatistics(ArrayList<PtNodeArray> nodeArrays) { + int firstTerminalAddress = Integer.MAX_VALUE; + int lastTerminalAddress = Integer.MIN_VALUE; + int size = 0; + int charGroups = 0; + int maxGroups = 0; + int maxRuns = 0; + for (final PtNodeArray nodeArray : nodeArrays) { + if (maxGroups < nodeArray.mData.size()) maxGroups = nodeArray.mData.size(); + for (final CharGroup cg : nodeArray.mData) { + ++charGroups; + if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length; + if (cg.mFrequency >= 0) { + if (nodeArray.mCachedAddressAfterUpdate < firstTerminalAddress) + firstTerminalAddress = nodeArray.mCachedAddressAfterUpdate; + if (nodeArray.mCachedAddressAfterUpdate > lastTerminalAddress) + lastTerminalAddress = nodeArray.mCachedAddressAfterUpdate; + } + } + if (nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize > size) { + size = nodeArray.mCachedAddressAfterUpdate + nodeArray.mCachedSize; + } + } + final int[] groupCounts = new int[maxGroups + 1]; + final int[] runCounts = new int[maxRuns + 1]; + for (final PtNodeArray nodeArray : nodeArrays) { + ++groupCounts[nodeArray.mData.size()]; + for (final CharGroup cg : nodeArray.mData) { + ++runCounts[cg.mChars.length]; + } + } + + MakedictLog.i("Statistics:\n" + + " total file size " + size + "\n" + + " " + nodeArrays.size() + " node arrays\n" + + " " + charGroups + " groups (" + ((float)charGroups / nodeArrays.size()) + + " groups per node)\n" + + " first terminal at " + firstTerminalAddress + "\n" + + " last terminal at " + lastTerminalAddress + "\n" + + " Group stats : max = " + maxGroups); + for (int i = 0; i < groupCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + groupCounts[i]); + } + MakedictLog.i(" Character run stats : max = " + maxRuns); + for (int i = 0; i < runCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + runCounts[i]); + } + } + + /** + * Dumps a FusionDictionary to a file. + * + * This is the public entry point to write a dictionary to a file. + * + * @param destination the stream to write the binary data to. + * @param dict the dictionary to write. + * @param formatOptions file format options. + */ + public static void writeDictionaryBinary(final OutputStream destination, + final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + + // Addresses are limited to 3 bytes, but since addresses can be relative to each node + // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding + // the order of the node arrays becomes a quite complicated problem, because though the + // dictionary itself does not have a size limit, each node array must still be within 16MB + // of all its children and parents. As long as this is ensured, the dictionary file may + // grow to any size. + + final int version = formatOptions.mVersion; + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("Requested file format version " + version + + ", but this implementation only supports versions " + + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + + ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); + + // The magic number in big-endian order. + // Magic number for all versions. + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8))); + headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER)); + // Dictionary version. + headerBuffer.write((byte) (0xFF & (version >> 8))); + headerBuffer.write((byte) (0xFF & version)); + + // Options flags + final int options = makeOptionsValue(dict, formatOptions); + headerBuffer.write((byte) (0xFF & (options >> 8))); + headerBuffer.write((byte) (0xFF & options)); + final int headerSizeOffset = headerBuffer.size(); + // Placeholder to be written later with header size. + for (int i = 0; i < 4; ++i) { + headerBuffer.write(0); + } + // Write out the options. + for (final String key : dict.mOptions.mAttributes.keySet()) { + final String value = dict.mOptions.mAttributes.get(key); + CharEncoding.writeString(headerBuffer, key); + CharEncoding.writeString(headerBuffer, value); + } + final int size = headerBuffer.size(); + final byte[] bytes = headerBuffer.toByteArray(); + // Write out the header size. + bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); + bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); + bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); + bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); + destination.write(bytes); + + headerBuffer.close(); + + // Leave the choice of the optimal node order to the flattenTree function. + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = flattenTree(dict.mRootNodeArray); + + MakedictLog.i("Computing addresses..."); + computeAddresses(dict, flatNodes, formatOptions); + MakedictLog.i("Checking array..."); + if (DBG) checkFlatNodeArrayList(flatNodes); + + // Create a buffer that matches the final dictionary size. + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + final byte[] buffer = new byte[bufferSize]; + int index = 0; + + MakedictLog.i("Writing file..."); + int dataEndOffset = 0; + for (PtNodeArray nodeArray : flatNodes) { + dataEndOffset = writePlacedNode(dict, buffer, nodeArray, formatOptions); + } + + if (DBG) showStatistics(flatNodes); + + destination.write(buffer, 0, dataEndOffset); + + destination.close(); + MakedictLog.i("Done"); + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 2e6c4b2f8..e5735aaca 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -18,19 +18,19 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.ByteArrayWrapper; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Iterator; import java.util.Map; @@ -59,7 +59,7 @@ public final class BinaryDictIOUtils { } /** - * Tours all node without recursive call. + * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner( final FusionDictionaryBufferInterface buffer, final int headerSize, @@ -86,7 +86,7 @@ public final class BinaryDictIOUtils { if (index != p.mLength) index = p.mLength; if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) { - p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer); + p.mNumOfCharGroup = BinaryDictDecoder.readCharGroupCount(buffer); p.mAddress += getGroupCountSize(p.mNumOfCharGroup); p.mPosition = 0; } @@ -94,7 +94,7 @@ public final class BinaryDictIOUtils { stack.pop(); continue; } - CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer, + CharGroupInfo info = BinaryDictDecoder.readCharGroup(buffer, p.mAddress - headerSize, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; @@ -116,7 +116,7 @@ public final class BinaryDictIOUtils { if (formatOptions.mSupportsDynamicUpdate) { final int forwardLinkAddress = buffer.readUnsignedInt24(); if (forwardLinkAddress != FormatSpec.NO_FORWARD_LINK_ADDRESS) { - // the node has a forward link. + // The node array has a forward link. p.mNumOfCharGroup = Position.NOT_READ_GROUPCOUNT; p.mAddress = forwardLinkAddress; } else { @@ -126,7 +126,7 @@ public final class BinaryDictIOUtils { stack.pop(); } } else { - // the node has more groups. + // The node array has more groups. p.mAddress = buffer.position(); } @@ -139,22 +139,22 @@ public final class BinaryDictIOUtils { /** * Reads unigrams and bigrams from the binary file. - * Doesn't make the memory representation of the dictionary. + * Doesn't store a full memory representation of the dictionary. * - * @param reader the reader. + * @param dictReader the dict reader. * @param words the map to store the address as a key and the word as a value. * @param frequencies the map to store the address as a key and the frequency as a value. * @param bigrams the map to store the address as a key and the list of address as a value. - * @throws IOException - * @throws UnsupportedFormatException + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader, + public static void readUnigramsAndBigramsBinary(final BinaryDictReader dictReader, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header - final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer()); - readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words, + final FileHeader header = BinaryDictDecoder.readHeader(dictReader); + readUnigramsAndBigramsBinaryInner(dictReader.getBuffer(), header.mHeaderSize, words, frequencies, bigrams, header.mFormatOptions); } @@ -162,30 +162,31 @@ public final class BinaryDictIOUtils { * Gets the address of the last CharGroup of the exact matching word in the dictionary. * If no match is found, returns NOT_VALID_WORD. * - * @param buffer the buffer to read. + * @param dictReader the dict reader. * @param word the word we search for. * @return the address of the terminal node. - * @throws IOException - * @throws UnsupportedFormatException + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final FusionDictionaryBufferInterface buffer, + public static int getTerminalPosition(final BinaryDictReader dictReader, final String word) throws IOException, UnsupportedFormatException { + final FusionDictionaryBufferInterface buffer = dictReader.getBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; if (buffer.position() != 0) buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); + final FileHeader header = BinaryDictDecoder.readHeader(dictReader); int wordPos = 0; final int wordLen = word.codePointCount(0, word.length()); for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; do { - final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); + final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer); boolean foundNextCharGroup = false; for (int i = 0; i < charGroupCount; ++i) { final int charGroupPos = buffer.position(); - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, + final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer, buffer.position(), header.mFormatOptions); final boolean isMovedGroup = isMovedGroup(currentInfo.mFlags, header.mFormatOptions); @@ -224,9 +225,9 @@ public final class BinaryDictIOUtils { } // If we found the next char group, it is under the file pointer. - // But if not, we are at the end of this node so we expect to have + // But if not, we are at the end of this node array so we expect to have // a forward link address that we need to consult and possibly resume - // search on the next node in the linked list. + // search on the next node array in the linked list. if (foundNextCharGroup) break; if (!header.mFormatOptions.mSupportsDynamicUpdate) { return FormatSpec.NOT_VALID_WORD; @@ -271,7 +272,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictInputOutput.getByteSize(value)) { + switch (BinaryDictEncoder.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -285,15 +286,15 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictInputOutput.getByteSize(value); + return BinaryDictEncoder.getByteSize(value); } static void skipCharGroup(final FusionDictionaryBufferInterface buffer, final FormatOptions formatOptions) { final int flags = buffer.readUnsignedByte(); - BinaryDictInputOutput.readParentAddress(buffer, formatOptions); + BinaryDictDecoder.readParentAddress(buffer, formatOptions); skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions); + BinaryDictDecoder.readChildrenAddress(buffer, flags, formatOptions); if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { final int shortcutsSize = buffer.readUnsignedShort(); @@ -365,9 +366,10 @@ public final class BinaryDictIOUtils { } /** - * Write a char group to an output stream. - * A char group is an in-memory representation of a node in trie. - * A char group info is an on-disk representation of a node. + * Write a char group to an output stream from a CharGroupInfo. + * A char group is an in-memory representation of a node in the patricia trie. + * A char group info is a container for low-level information about how the + * char group is stored in the binary format. * * @param destination the stream to write. * @param info the char group info to be written. @@ -411,14 +413,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictInputOutput.makeShortcutFlags( + destination.write((byte)BinaryDictEncoder.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -427,7 +429,7 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictionaryInputOutput#computeActualNodeSize + // in BinaryDictEncoder#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; @@ -437,7 +439,7 @@ public final class BinaryDictIOUtils { final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictInputOutput.getByteSize(bigramOffset)) { + switch (BinaryDictEncoder.getByteSize(bigramOffset)) { case 1: bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; break; @@ -461,32 +463,32 @@ public final class BinaryDictIOUtils { */ static int computeGroupSize(final CharGroupInfo info, final FormatOptions formatOptions) { int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters) - + BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions); + + BinaryDictEncoder.getGroupCharactersSize(info.mCharacters) + + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { size += FormatSpec.GROUP_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoder.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictInputOutput.getByteSize(attr.mAddress); + size += BinaryDictEncoder.getByteSize(attr.mAddress); } } return size; } /** - * Write a node to the stream. + * Write a node array to the stream. * * @param destination the stream to write. - * @param infos groups to be written. + * @param infos an array of CharGroupInfo to be written. * @return the size written, in bytes. * @throws IOException */ - static int writeNode(final OutputStream destination, final CharGroupInfo[] infos) + static int writeNodes(final OutputStream destination, final CharGroupInfo[] infos) throws IOException { int size = getGroupCountSize(infos.length); switch (getGroupCountSize(infos.length)) { @@ -506,23 +508,24 @@ public final class BinaryDictIOUtils { } /** - * Find a word from the buffer. + * Find a word using the BinaryDictReader. * - * @param buffer the buffer representing the body of the dictionary file. + * @param dictReader the dict reader * @param word the word searched * @return the found group * @throws IOException * @throws UnsupportedFormatException */ @UsedForTesting - public static CharGroupInfo findWordFromBuffer(final FusionDictionaryBufferInterface buffer, + public static CharGroupInfo findWordByBinaryDictReader(final BinaryDictReader dictReader, final String word) throws IOException, UnsupportedFormatException { - int position = getTerminalPosition(buffer, word); + int position = getTerminalPosition(dictReader, word); + final FusionDictionaryBufferInterface buffer = dictReader.getBuffer(); if (position != FormatSpec.NOT_VALID_WORD) { buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); + final FileHeader header = BinaryDictDecoder.readHeader(dictReader); buffer.position(position); - return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions); + return BinaryDictDecoder.readCharGroup(buffer, position, header.mFormatOptions); } return null; } @@ -541,16 +544,21 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final FileInputStream inStream = new FileInputStream(file); - try { - inStream.read(buffer); - final BinaryDictInputOutput.ByteBufferWrapper wrapper = - new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, offset, length)); - return BinaryDictInputOutput.readHeader(wrapper); - } finally { - inStream.close(); - } + final BinaryDictReader dictReader = new BinaryDictReader(file); + dictReader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFactory() { + @Override + public FusionDictionaryBufferInterface getFusionDictionaryBuffer(File file) + throws FileNotFoundException, IOException { + final FileInputStream inStream = new FileInputStream(file); + try { + inStream.read(buffer); + return new ByteArrayWrapper(buffer); + } finally { + inStream.close(); + } + } + }); + return BinaryDictDecoder.readHeader(dictReader); } public static FileHeader getDictionaryFileHeaderOrNull(final File file, final long offset, @@ -604,12 +612,28 @@ public final class BinaryDictIOUtils { public static int getGroupCountSize(final int count) { if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { return 1; - } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) { + } else if (FormatSpec.MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY >= count) { return 2; } else { throw new RuntimeException("Can't have more than " - + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count - + ")"); + + FormatSpec.MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY + " groups in a node (found " + + count + ")"); + } + } + + static int getChildrenAddressSize(final int optionFlags, + final FormatOptions formatOptions) { + if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: + return 1; + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: + return 2; + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: + return 3; + case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: + default: + return 0; } } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java deleted file mode 100644 index a33cddb49..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ /dev/null @@ -1,1783 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.utils.JniUtils; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - -/** - * Reads and writes XML files for a FusionDictionary. - * - * All the methods in this class are static. - */ -public final class BinaryDictInputOutput { - - private static final boolean DBG = MakedictLog.DBG; - - static { - JniUtils.loadNativeLibrary(); - } - - // TODO: implement something sensical instead of just a phony method - private static native int doNothing(); - - // Arbitrary limit to how much passes we consider address size compression should - // terminate in. At the time of this writing, our largest dictionary completes - // compression in five passes. - // If the number of passes exceeds this number, makedict bails with an exception on - // suspicion that a bug might be causing an infinite loop. - private static final int MAX_PASSES = 24; - private static final int MAX_JUMPS = 12; - - @UsedForTesting - public interface FusionDictionaryBufferInterface { - public int readUnsignedByte(); - public int readUnsignedShort(); - public int readUnsignedInt24(); - public int readInt(); - public int position(); - public void position(int newPosition); - public void put(final byte b); - public int limit(); - @UsedForTesting - public int capacity(); - } - - public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface { - private ByteBuffer mBuffer; - - public ByteBufferWrapper(final ByteBuffer buffer) { - mBuffer = buffer; - } - - @Override - public int readUnsignedByte() { - return mBuffer.get() & 0xFF; - } - - @Override - public int readUnsignedShort() { - return mBuffer.getShort() & 0xFFFF; - } - - @Override - public int readUnsignedInt24() { - final int retval = readUnsignedByte(); - return (retval << 16) + readUnsignedShort(); - } - - @Override - public int readInt() { - return mBuffer.getInt(); - } - - @Override - public int position() { - return mBuffer.position(); - } - - @Override - public void position(int newPos) { - mBuffer.position(newPos); - } - - @Override - public void put(final byte b) { - mBuffer.put(b); - } - - @Override - public int limit() { - return mBuffer.limit(); - } - - @Override - public int capacity() { - return mBuffer.capacity(); - } - } - - /** - * A class grouping utility function for our specific character encoding. - */ - static final class CharEncoding { - private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; - private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; - - /** - * Helper method to find out whether this code fits on one byte - */ - private static boolean fitsOnOneByte(final int character) { - return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE - && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; - } - - /** - * Compute the size of a character given its character code. - * - * Char format is: - * 1 byte = bbbbbbbb match - * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte - * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because - * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with - * 00011111 would be outside unicode. - * else: iso-latin-1 code - * This allows for the whole unicode range to be encoded, including chars outside of - * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control - * characters which should never happen anyway (and still work, but take 3 bytes). - * - * @param character the character code. - * @return the size in binary encoded-form, either 1 or 3 bytes. - */ - static int getCharSize(final int character) { - // See char encoding in FusionDictionary.java - if (fitsOnOneByte(character)) return 1; - if (FormatSpec.INVALID_CHARACTER == character) return 1; - return 3; - } - - /** - * Compute the byte size of a character array. - */ - private static int getCharArraySize(final int[] chars) { - int size = 0; - for (int character : chars) size += getCharSize(character); - return size; - } - - /** - * Writes a char array to a byte buffer. - * - * @param codePoints the code point array to write. - * @param buffer the byte buffer to write to. - * @param index the index in buffer to write the character array to. - * @return the index after the last character. - */ - private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { - for (int codePoint : codePoints) { - if (1 == getCharSize(codePoint)) { - buffer[index++] = (byte)codePoint; - } else { - buffer[index++] = (byte)(0xFF & (codePoint >> 16)); - buffer[index++] = (byte)(0xFF & (codePoint >> 8)); - buffer[index++] = (byte)(0xFF & codePoint); - } - } - return index; - } - - /** - * Writes a string with our character format to a byte buffer. - * - * This will also write the terminator byte. - * - * @param buffer the byte buffer to write to. - * @param origin the offset to write from. - * @param word the string to write. - * @return the size written, in bytes. - */ - private static int writeString(final byte[] buffer, final int origin, - final String word) { - final int length = word.length(); - int index = origin; - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer[index++] = (byte)codePoint; - } else { - buffer[index++] = (byte)(0xFF & (codePoint >> 16)); - buffer[index++] = (byte)(0xFF & (codePoint >> 8)); - buffer[index++] = (byte)(0xFF & codePoint); - } - } - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; - return index - origin; - } - - /** - * Writes a string with our character format to a ByteArrayOutputStream. - * - * This will also write the terminator byte. - * - * @param buffer the ByteArrayOutputStream to write to. - * @param word the string to write. - */ - private static void writeString(final ByteArrayOutputStream buffer, final String word) { - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer.write((byte) codePoint); - } else { - buffer.write((byte) (0xFF & (codePoint >> 16))); - buffer.write((byte) (0xFF & (codePoint >> 8))); - buffer.write((byte) (0xFF & codePoint)); - } - } - buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); - } - - /** - * Reads a string from a buffer. This is the converse of the above method. - */ - private static String readString(final FusionDictionaryBufferInterface buffer) { - final StringBuilder s = new StringBuilder(); - int character = readChar(buffer); - while (character != FormatSpec.INVALID_CHARACTER) { - s.appendCodePoint(character); - character = readChar(buffer); - } - return s.toString(); - } - - /** - * Reads a character from the buffer. - * - * This follows the character format documented earlier in this source file. - * - * @param buffer the buffer, positioned over an encoded character. - * @return the character code. - */ - static int readChar(final FusionDictionaryBufferInterface buffer) { - int character = buffer.readUnsignedByte(); - if (!fitsOnOneByte(character)) { - if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { - return FormatSpec.INVALID_CHARACTER; - } - character <<= 16; - character += buffer.readUnsignedShort(); - } - return character; - } - } - - /** - * Compute the binary size of the character array. - * - * If only one character, this is the size of this character. If many, it's the sum of their - * sizes + 1 byte for the terminator. - * - * @param characters the character array - * @return the size of the char array, including the terminator if any - */ - static int getGroupCharactersSize(final int[] characters) { - int size = CharEncoding.getCharArraySize(characters); - if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the binary size of the character array in a group - * - * If only one character, this is the size of this character. If many, it's the sum of their - * sizes + 1 byte for the terminator. - * - * @param group the group - * @return the size of the char array, including the terminator if any - */ - private static int getGroupCharactersSize(final CharGroup group) { - return getGroupCharactersSize(group.mChars); - } - - /** - * Compute the binary size of the group count for a node - * @param node the node - * @return the size of the group count, either 1 or 2 bytes. - */ - private static int getGroupCountSize(final Node node) { - return BinaryDictIOUtils.getGroupCountSize(node.mData.size()); - } - - /** - * Compute the size of a shortcut in bytes. - */ - private static int getShortcutSize(final WeightedString shortcut) { - int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE; - final String word = shortcut.mWord; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - size += CharEncoding.getCharSize(codePoint); - } - size += FormatSpec.GROUP_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the size of a shortcut list in bytes. - * - * This is known in advance and does not change according to position in the file - * like address lists do. - */ - static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { - if (null == shortcutList) return 0; - int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - for (final WeightedString shortcut : shortcutList) { - size += getShortcutSize(shortcut); - } - return size; - } - - /** - * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. - * - * @param group the CharGroup to compute the size of. - * @param options file format options. - * @return the maximum size of the group. - */ - private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) { - int size = getGroupHeaderSize(group, options); - // If terminal, one byte for the frequency - if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE; - size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address - size += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE - + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) - * group.mBigrams.size(); - } - return size; - } - - /** - * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches - * it in the 'actualSize' member of the node. - * - * @param node the node to compute the maximum size of. - * @param options file format options. - */ - private static void calculateNodeMaximumSize(final Node node, final FormatOptions options) { - int size = getGroupCountSize(node); - for (CharGroup g : node.mData) { - final int groupSize = getCharGroupMaximumSize(g, options); - g.mCachedSize = groupSize; - size += groupSize; - } - if (options.mSupportsDynamicUpdate) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - node.mCachedSize = size; - } - - /** - * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. - * - * @param group the group of which to compute the size of the header - * @param options file format options. - */ - private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + getGroupCharactersSize(group); - } else { - return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group); - } - } - - /** - * Compute the size, in bytes, that an address will occupy. - * - * This can be used either for children addresses (which are always positive) or for - * attribute, which may be positive or negative but - * store their sign bit separately. - * - * @param address the address - * @return the byte size. - */ - static int getByteSize(final int address) { - assert(address <= FormatSpec.UINT24_MAX); - if (!BinaryDictIOUtils.hasChildrenAddress(address)) { - return 0; - } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) { - return 1; - } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) { - return 2; - } else { - return 3; - } - } - - // End utility methods. - - // This method is responsible for finding a nice ordering of the nodes that favors run-time - // cache performance and dictionary size. - /* package for tests */ static ArrayList<Node> flattenTree(final Node root) { - final int treeSize = FusionDictionary.countCharGroups(root); - MakedictLog.i("Counted nodes : " + treeSize); - final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize); - return flattenTreeInner(flatTree, root); - } - - private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) { - // Removing the node is necessary if the tails are merged, because we would then - // add the same node several times when we only want it once. A number of places in - // the code also depends on any node being only once in the list. - // Merging tails can only be done if there are no attributes. Searching for attributes - // in LatinIME code depends on a total breadth-first ordering, which merging tails - // breaks. If there are no attributes, it should be fine (and reduce the file size) - // to merge tails, and removing the node from the list would be necessary. However, - // we don't merge tails because breaking the breadth-first ordering would result in - // extreme overhead at bigram lookup time (it would make the search function O(n) instead - // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty - // high). - // If no nodes are ever merged, we can't have the same node twice in the list, hence - // searching for duplicates in unnecessary. It is also very performance consuming, - // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making - // this simple list.remove operation O(n*n) overall. On Android this overhead is very - // high. - // For future reference, the code to remove duplicate is a simple : list.remove(node); - list.add(node); - final ArrayList<CharGroup> branches = node.mData; - final int nodeSize = branches.size(); - for (CharGroup group : branches) { - if (null != group.mChildren) flattenTreeInner(list, group.mChildren); - } - return list; - } - - /** - * Get the offset from a position inside a current node to a target node, during update. - * - * If the current node is before the target node, the target node has not been updated yet, - * so we should return the offset from the old position of the current node to the old position - * of the target node. If on the other hand the target is before the current node, it already - * has been updated, so we should return the offset from the new position in the current node - * to the new position in the target node. - * @param currentNode the node containing the CharGroup where the offset will be written - * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode - * @param targetNode the target node to get the offset to - * @return the offset to the target node - */ - private static int getOffsetToTargetNodeDuringUpdate(final Node currentNode, - final int offsetFromStartOfCurrentNode, final Node targetNode) { - final boolean isTargetBeforeCurrent = (targetNode.mCachedAddressBeforeUpdate - < currentNode.mCachedAddressBeforeUpdate); - if (isTargetBeforeCurrent) { - return targetNode.mCachedAddressAfterUpdate - - (currentNode.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNode); - } else { - return targetNode.mCachedAddressBeforeUpdate - - (currentNode.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNode); - } - } - - /** - * Get the offset from a position inside a current node to a target CharGroup, during update. - * @param currentNode the node containing the CharGroup where the offset will be written - * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode - * @param targetCharGroup the target CharGroup to get the offset to - * @return the offset to the target CharGroup - */ - // TODO: is there any way to factorize this method with the one above? - private static int getOffsetToTargetCharGroupDuringUpdate(final Node currentNode, - final int offsetFromStartOfCurrentNode, final CharGroup targetCharGroup) { - final int oldOffsetBasePoint = currentNode.mCachedAddressBeforeUpdate - + offsetFromStartOfCurrentNode; - final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate - < oldOffsetBasePoint); - // If the target is before the current node, then its address has already been updated. - // We can use the AfterUpdate member, and compare it to our own member after update. - // Otherwise, the AfterUpdate member is not updated yet, so we need to use the BeforeUpdate - // member, and of course we have to compare this to our own address before update. - if (isTargetBeforeCurrent) { - final int newOffsetBasePoint = currentNode.mCachedAddressAfterUpdate - + offsetFromStartOfCurrentNode; - return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint; - } else { - return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint; - } - } - - /** - * Computes the actual node size, based on the cached addresses of the children nodes. - * - * Each node stores its tentative address. During dictionary address computing, these - * are not final, but they can be used to compute the node size (the node size depends - * on the address of the children because the number of bytes necessary to store an - * address depends on its numeric value. The return value indicates whether the node - * contents (as in, any of the addresses stored in the cache fields) have changed with - * respect to their previous value. - * - * @param node the node to compute the size of. - * @param dict the dictionary in which the word/attributes are to be found. - * @param formatOptions file format options. - * @return false if none of the cached addresses inside the node changed, true otherwise. - */ - private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict, - final FormatOptions formatOptions) { - boolean changed = false; - int size = getGroupCountSize(node); - for (CharGroup group : node.mData) { - group.mCachedAddressAfterUpdate = node.mCachedAddressAfterUpdate + size; - if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) { - changed = true; - } - int groupSize = getGroupHeaderSize(group, formatOptions); - if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE; - if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else if (null != group.mChildren) { - if (formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else { - groupSize += getByteSize(getOffsetToTargetNodeDuringUpdate(node, - groupSize + size, group.mChildren)); - } - } - groupSize += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - for (WeightedString bigram : group.mBigrams) { - final int offset = getOffsetToTargetCharGroupDuringUpdate(node, - groupSize + size + FormatSpec.GROUP_FLAGS_SIZE, - FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord)); - groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE; - } - } - group.mCachedSize = groupSize; - size += groupSize; - } - if (formatOptions.mSupportsDynamicUpdate) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - if (node.mCachedSize != size) { - node.mCachedSize = size; - changed = true; - } - return changed; - } - - /** - * Initializes the cached addresses of nodes from their size. - * - * @param flatNodes the array of nodes. - * @param formatOptions file format options. - * @return the byte size of the entire stack. - */ - private static int initializeNodesCachedAddresses(final ArrayList<Node> flatNodes, - final FormatOptions formatOptions) { - int nodeOffset = 0; - for (final Node n : flatNodes) { - n.mCachedAddressBeforeUpdate = nodeOffset; - int groupCountSize = getGroupCountSize(n); - int groupOffset = 0; - for (final CharGroup g : n.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate = - groupCountSize + nodeOffset + groupOffset; - groupOffset += g.mCachedSize; - } - final int nodeSize = groupCountSize + groupOffset - + (formatOptions.mSupportsDynamicUpdate - ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); - nodeOffset += n.mCachedSize; - } - return nodeOffset; - } - - /** - * Updates the cached addresses of nodes after recomputing their new positions. - * - * @param flatNodes the array of nodes. - */ - private static void updateNodeCachedAddresses(final ArrayList<Node> flatNodes) { - for (final Node n : flatNodes) { - n.mCachedAddressBeforeUpdate = n.mCachedAddressAfterUpdate; - for (final CharGroup g : n.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate; - } - } - } - - /** - * Compute the cached parent addresses after all has been updated. - * - * The parent addresses are used by some binary formats at write-to-disk time. Not all formats - * need them. In particular, version 2 does not need them, and version 3 does. - * - * @param flatNodes the flat array of nodes to fill in - */ - private static void computeParentAddresses(final ArrayList<Node> flatNodes) { - for (final Node node : flatNodes) { - for (final CharGroup group : node.mData) { - if (null != group.mChildren) { - // Assign my address to children's parent address - // Here BeforeUpdate and AfterUpdate addresses have the same value, so it - // does not matter which we use. - group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate - - group.mChildren.mCachedAddressAfterUpdate; - } - } - } - } - - /** - * Compute the addresses and sizes of an ordered node array. - * - * This method takes a node array and will update its cached address and size values - * so that they can be written into a file. It determines the smallest size each of the - * nodes can be given the addresses of its children and attributes, and store that into - * each node. - * The order of the node is given by the order of the array. This method makes no effort - * to find a good order; it only mechanically computes the size this order results in. - * - * @param dict the dictionary - * @param flatNodes the ordered array of nodes - * @param formatOptions file format options. - * @return the same array it was passed. The nodes have been updated for address and size. - */ - private static ArrayList<Node> computeAddresses(final FusionDictionary dict, - final ArrayList<Node> flatNodes, final FormatOptions formatOptions) { - // First get the worst possible sizes and offsets - for (final Node n : flatNodes) calculateNodeMaximumSize(n, formatOptions); - final int offset = initializeNodesCachedAddresses(flatNodes, formatOptions); - - MakedictLog.i("Compressing the array addresses. Original size : " + offset); - MakedictLog.i("(Recursively seen size : " + offset + ")"); - - int passes = 0; - boolean changesDone = false; - do { - changesDone = false; - int nodeStartOffset = 0; - for (final Node n : flatNodes) { - n.mCachedAddressAfterUpdate = nodeStartOffset; - final int oldNodeSize = n.mCachedSize; - final boolean changed = computeActualNodeSize(n, dict, formatOptions); - final int newNodeSize = n.mCachedSize; - if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!"); - nodeStartOffset += newNodeSize; - changesDone |= changed; - } - updateNodeCachedAddresses(flatNodes); - ++passes; - if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); - } while (changesDone); - - if (formatOptions.mSupportsDynamicUpdate) { - computeParentAddresses(flatNodes); - } - final Node lastNode = flatNodes.get(flatNodes.size() - 1); - MakedictLog.i("Compression complete in " + passes + " passes."); - MakedictLog.i("After address compression : " - + (lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize)); - - return flatNodes; - } - - /** - * Sanity-checking method. - * - * This method checks an array of node for juxtaposition, that is, it will do - * nothing if each node's cached address is actually the previous node's address - * plus the previous node's size. - * If this is not the case, it will throw an exception. - * - * @param array the array node to check - */ - private static void checkFlatNodeArray(final ArrayList<Node> array) { - int offset = 0; - int index = 0; - for (final Node n : array) { - // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter - // which we use. - if (n.mCachedAddressAfterUpdate != offset) { - throw new RuntimeException("Wrong address for node " + index - + " : expected " + offset + ", got " + n.mCachedAddressAfterUpdate); - } - ++index; - offset += n.mCachedSize; - } - } - - /** - * Helper method to write a variable-size address to a file. - * - * @param buffer the buffer to write to. - * @param index the index in the buffer to write the address to. - * @param address the address to write. - * @return the size in bytes the address actually took. - */ - private static int writeVariableAddress(final byte[] buffer, int index, final int address) { - switch (getByteSize(address)) { - case 1: - buffer[index++] = (byte)address; - return 1; - case 2: - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); - return 2; - case 3: - buffer[index++] = (byte)(0xFF & (address >> 16)); - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); - return 3; - case 0: - return 0; - default: - throw new RuntimeException("Address " + address + " has a strange size"); - } - } - - /** - * Helper method to write a variable-size signed address to a file. - * - * @param buffer the buffer to write to. - * @param index the index in the buffer to write the address to. - * @param address the address to write. - * @return the size in bytes the address actually took. - */ - private static int writeVariableSignedAddress(final byte[] buffer, int index, - final int address) { - if (!BinaryDictIOUtils.hasChildrenAddress(address)) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - buffer[index++] = - (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16))); - buffer[index++] = (byte)(0xFF & (absAddress >> 8)); - buffer[index++] = (byte)(0xFF & absAddress); - } - return 3; - } - - /** - * Makes the flag value for a char group. - * - * @param hasMultipleChars whether the group has multiple chars. - * @param isTerminal whether the group is terminal. - * @param childrenAddressSize the size of a children address. - * @param hasShortcuts whether the group has shortcuts. - * @param hasBigrams whether the group has bigrams. - * @param isNotAWord whether the group is not a word. - * @param isBlackListEntry whether the group is a blacklist entry. - * @param formatOptions file format options. - * @return the flags - */ - static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal, - final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, - final boolean isNotAWord, final boolean isBlackListEntry, - final FormatOptions formatOptions) { - byte flags = 0; - if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; - if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.mSupportsDynamicUpdate) { - flags |= FormatSpec.FLAG_IS_NOT_MOVED; - } else if (true) { - switch (childrenAddressSize) { - case 1: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; - break; - case 0: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS; - break; - default: - throw new RuntimeException("Node with a strange address"); - } - } - if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; - if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; - if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; - if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; - return flags; - } - - private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress, - final int childrenOffset, final FormatOptions formatOptions) { - return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0, - getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null, - group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions); - } - - /** - * Makes the flag value for a bigram. - * - * @param more whether there are more bigrams after this one. - * @param offset the offset of the bigram. - * @param bigramFrequency the frequency of the bigram, 0..255. - * @param unigramFrequency the unigram frequency of the same word, 0..255. - * @param word the second bigram, for debugging purposes - * @return the flags - */ - private static final int makeBigramFlags(final boolean more, final int offset, - int bigramFrequency, final int unigramFrequency, final String word) { - int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0); - switch (getByteSize(offset)) { - case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; - break; - default: - throw new RuntimeException("Strange offset size"); - } - if (unigramFrequency > bigramFrequency) { - MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word - + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " - + word + " is " + unigramFrequency); - bigramFrequency = unigramFrequency; - } - // We compute the difference between 255 (which means probability = 1) and the - // unigram score. We split this into a number of discrete steps. - // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 - // represents an increase of 16 steps: a value of 15 will be interpreted as the median - // value of the 16th step. In all justice, if the bigram frequency is low enough to be - // rounded below the first step (which means it is less than half a step higher than the - // unigram frequency) then the unigram frequency itself is the best approximation of the - // bigram freq that we could possibly supply, hence we should *not* include this bigram - // in the file at all. - // until this is done, we'll write 0 and slightly overestimate this case. - // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step - // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to - // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the - // step size. Then we compute the start of the first step (the one where value 0 starts) - // by adding half-a-step to the unigramFrequency. From there, we compute the integer - // number of steps to the bigramFrequency. One last thing: we want our steps to include - // their lower bound and exclude their higher bound so we need to have the first step - // start at exactly 1 unit higher than floor(unigramFreq + half a step). - // Note : to reconstruct the score, the dictionary reader will need to divide - // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, - // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best - // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the - // step pointed by the discretized frequency. - final float stepSize = - (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) - / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); - final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); - final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); - // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 - // here. The best approximation would be the unigram freq itself, so we should not - // include this bigram in the dictionary. For now, register as 0, and live with the - // small over-estimation that we get in this case. TODO: actually remove this bigram - // if discretizedFrequency < 0. - final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; - bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; - return bigramFlags; - } - - /** - * Makes the 2-byte value for options flags. - */ - private static final int makeOptionsValue(final FusionDictionary dictionary, - final FormatOptions formatOptions) { - final DictionaryOptions options = dictionary.mOptions; - final boolean hasBigrams = dictionary.hasBigrams(); - return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) - + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) - + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) - + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); - } - - /** - * Makes the flag value for a shortcut. - * - * @param more whether there are more attributes after this one. - * @param frequency the frequency of the attribute, 0..15 - * @return the flags - */ - static final int makeShortcutFlags(final boolean more, final int frequency) { - return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY); - } - - private static final int writeParentAddress(final byte[] buffer, final int index, - final int address, final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - if (address == FormatSpec.NO_PARENT_ADDRESS) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - assert(absAddress <= FormatSpec.SINT24_MAX); - buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0) - | ((absAddress >> 16) & 0xFF)); - buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); - buffer[index + 2] = (byte)(absAddress & 0xFF); - } - return index + 3; - } else { - return index; - } - } - - /** - * Write a node to memory. The node is expected to have its final position cached. - * - * This can be an empty map, but the more is inside the faster the lookups will be. It can - * be carried on as long as nodes do not move. - * - * @param dict the dictionary the node is a part of (for relative offsets). - * @param buffer the memory buffer to write to. - * @param node the node to write. - * @param formatOptions file format options. - * @return the address of the END of the node. - */ - @SuppressWarnings("unused") - private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, - final Node node, final FormatOptions formatOptions) { - // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup - int index = node.mCachedAddressAfterUpdate; - - final int groupCount = node.mData.size(); - final int countSize = getGroupCountSize(node); - final int parentAddress = node.mCachedParentAddress; - if (1 == countSize) { - buffer[index++] = (byte)groupCount; - } else if (2 == countSize) { - // We need to signal 2-byte size by setting the top bit of the MSB to 1, so - // we | 0x80 to do this. - buffer[index++] = (byte)((groupCount >> 8) | 0x80); - buffer[index++] = (byte)(groupCount & 0xFF); - } else { - throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); - } - int groupAddress = index; - for (int i = 0; i < groupCount; ++i) { - final CharGroup group = node.mData.get(i); - if (index != group.mCachedAddressAfterUpdate) { - throw new RuntimeException("Bug: write index is not the same as the cached address " - + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate); - } - groupAddress += getGroupHeaderSize(group, formatOptions); - // Sanity checks. - if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { - throw new RuntimeException("A node has a frequency > " - + FormatSpec.MAX_TERMINAL_FREQUENCY - + " : " + group.mFrequency); - } - if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE; - final int childrenOffset = null == group.mChildren - ? FormatSpec.NO_CHILDREN_ADDRESS - : group.mChildren.mCachedAddressAfterUpdate - groupAddress; - buffer[index++] = - makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions); - - if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { - index = writeParentAddress(buffer, index, parentAddress, formatOptions); - } else { - index = writeParentAddress(buffer, index, parentAddress - + (node.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate), - formatOptions); - } - - index = CharEncoding.writeCharArray(group.mChars, buffer, index); - if (group.hasSeveralChars()) { - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; - } - if (group.mFrequency >= 0) { - buffer[index++] = (byte) group.mFrequency; - } - - final int shift; - if (formatOptions.mSupportsDynamicUpdate) { - shift = writeVariableSignedAddress(buffer, index, childrenOffset); - } else { - shift = writeVariableAddress(buffer, index, childrenOffset); - } - index += shift; - groupAddress += shift; - - // Write shortcuts - if (null != group.mShortcutTargets) { - final int indexOfShortcutByteSize = index; - index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - ++groupAddress; - int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), - target.mFrequency); - buffer[index++] = (byte)shortcutFlags; - final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); - index += shortcutShift; - groupAddress += shortcutShift; - } - final int shortcutByteSize = index - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { - throw new RuntimeException("Shortcut list too large"); - } - buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); - buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); - } - // Write bigrams - if (null != group.mBigrams) { - final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); - while (bigramIterator.hasNext()) { - final WeightedString bigram = bigramIterator.next(); - final CharGroup target = - FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord); - final int addressOfBigram = target.mCachedAddressAfterUpdate; - final int unigramFrequencyForThisWord = target.mFrequency; - ++groupAddress; - final int offset = addressOfBigram - groupAddress; - int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, - bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); - buffer[index++] = (byte)bigramFlags; - final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); - index += bigramShift; - groupAddress += bigramShift; - } - } - - } - if (formatOptions.mSupportsDynamicUpdate) { - buffer[index] = buffer[index + 1] = buffer[index + 2] - = FormatSpec.NO_FORWARD_LINK_ADDRESS; - index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - if (index != node.mCachedAddressAfterUpdate + node.mCachedSize) throw new RuntimeException( - "Not the same size : written " - + (index - node.mCachedAddressAfterUpdate) + " bytes from a node that should have " - + node.mCachedSize + " bytes"); - return index; - } - - /** - * Dumps a collection of useful statistics about a node array. - * - * This prints purely informative stuff, like the total estimated file size, the - * number of nodes, of character groups, the repartition of each address size, etc - * - * @param nodes the node array. - */ - private static void showStatistics(ArrayList<Node> nodes) { - int firstTerminalAddress = Integer.MAX_VALUE; - int lastTerminalAddress = Integer.MIN_VALUE; - int size = 0; - int charGroups = 0; - int maxGroups = 0; - int maxRuns = 0; - for (final Node n : nodes) { - if (maxGroups < n.mData.size()) maxGroups = n.mData.size(); - for (final CharGroup cg : n.mData) { - ++charGroups; - if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length; - if (cg.mFrequency >= 0) { - if (n.mCachedAddressAfterUpdate < firstTerminalAddress) - firstTerminalAddress = n.mCachedAddressAfterUpdate; - if (n.mCachedAddressAfterUpdate > lastTerminalAddress) - lastTerminalAddress = n.mCachedAddressAfterUpdate; - } - } - if (n.mCachedAddressAfterUpdate + n.mCachedSize > size) { - size = n.mCachedAddressAfterUpdate + n.mCachedSize; - } - } - final int[] groupCounts = new int[maxGroups + 1]; - final int[] runCounts = new int[maxRuns + 1]; - for (final Node n : nodes) { - ++groupCounts[n.mData.size()]; - for (final CharGroup cg : n.mData) { - ++runCounts[cg.mChars.length]; - } - } - - MakedictLog.i("Statistics:\n" - + " total file size " + size + "\n" - + " " + nodes.size() + " nodes\n" - + " " + charGroups + " groups (" + ((float)charGroups / nodes.size()) - + " groups per node)\n" - + " first terminal at " + firstTerminalAddress + "\n" - + " last terminal at " + lastTerminalAddress + "\n" - + " Group stats : max = " + maxGroups); - for (int i = 0; i < groupCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + groupCounts[i]); - } - MakedictLog.i(" Character run stats : max = " + maxRuns); - for (int i = 0; i < runCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + runCounts[i]); - } - } - - /** - * Dumps a FusionDictionary to a file. - * - * This is the public entry point to write a dictionary to a file. - * - * @param destination the stream to write the binary data to. - * @param dict the dictionary to write. - * @param formatOptions file format options. - */ - public static void writeDictionaryBinary(final OutputStream destination, - final FusionDictionary dict, final FormatOptions formatOptions) - throws IOException, UnsupportedFormatException { - - // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the - // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order - // of the nodes becomes a quite complicated problem, because though the dictionary itself - // does not have a size limit, each node must still be within 16MB of all its children and - // parents. As long as this is ensured, the dictionary file may grow to any size. - - final int version = formatOptions.mVersion; - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("Requested file format version " + version - + ", but this implementation only supports versions " - + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - - ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); - - // The magic number in big-endian order. - // Magic number for all versions. - headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24))); - headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16))); - headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8))); - headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER)); - // Dictionary version. - headerBuffer.write((byte) (0xFF & (version >> 8))); - headerBuffer.write((byte) (0xFF & version)); - - // Options flags - final int options = makeOptionsValue(dict, formatOptions); - headerBuffer.write((byte) (0xFF & (options >> 8))); - headerBuffer.write((byte) (0xFF & options)); - final int headerSizeOffset = headerBuffer.size(); - // Placeholder to be written later with header size. - for (int i = 0; i < 4; ++i) { - headerBuffer.write(0); - } - // Write out the options. - for (final String key : dict.mOptions.mAttributes.keySet()) { - final String value = dict.mOptions.mAttributes.get(key); - CharEncoding.writeString(headerBuffer, key); - CharEncoding.writeString(headerBuffer, value); - } - final int size = headerBuffer.size(); - final byte[] bytes = headerBuffer.toByteArray(); - // Write out the header size. - bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); - bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); - bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); - bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); - destination.write(bytes); - - headerBuffer.close(); - - // Leave the choice of the optimal node order to the flattenTree function. - MakedictLog.i("Flattening the tree..."); - ArrayList<Node> flatNodes = flattenTree(dict.mRoot); - - MakedictLog.i("Computing addresses..."); - computeAddresses(dict, flatNodes, formatOptions); - MakedictLog.i("Checking array..."); - if (DBG) checkFlatNodeArray(flatNodes); - - // Create a buffer that matches the final dictionary size. - final Node lastNode = flatNodes.get(flatNodes.size() - 1); - final int bufferSize = lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize; - final byte[] buffer = new byte[bufferSize]; - int index = 0; - - MakedictLog.i("Writing file..."); - int dataEndOffset = 0; - for (Node n : flatNodes) { - dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions); - } - - if (DBG) showStatistics(flatNodes); - - destination.write(buffer, 0, dataEndOffset); - - destination.close(); - MakedictLog.i("Done"); - } - - - // Input methods: Read a binary dictionary to memory. - // readDictionaryBinary is the public entry point for them. - - static int getChildrenAddressSize(final int optionFlags, - final FormatOptions formatOptions) { - if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - return 1; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - return 2; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - return 3; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: - default: - return 0; - } - } - - static int readChildrenAddress(final FusionDictionaryBufferInterface buffer, - final int optionFlags, final FormatOptions options) { - if (options.mSupportsDynamicUpdate) { - final int address = buffer.readUnsignedInt24(); - if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; - if ((address & FormatSpec.MSB24) != 0) { - return -(address & FormatSpec.SINT24_MAX); - } else { - return address; - } - } - int address; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - return buffer.readUnsignedByte(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - return buffer.readUnsignedShort(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - return buffer.readUnsignedInt24(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - - static int readParentAddress(final FusionDictionaryBufferInterface buffer, - final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - final int parentAddress = buffer.readUnsignedInt24(); - final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; - return sign * (parentAddress & FormatSpec.SINT24_MAX); - } else { - return FormatSpec.NO_PARENT_ADDRESS; - } - } - - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = buffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(buffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = buffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(buffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = buffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - buffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = buffer.readUnsignedByte(); - final String word = CharEncoding.readString(buffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += buffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = buffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * buffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * buffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (buffer.readUnsignedByte() << 16) - + buffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - - /** - * Reads and returns the char group count out of a buffer and forwards the pointer. - */ - public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) { - final int msb = buffer.readUnsignedByte(); - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { - return msb; - } else { - return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) - + buffer.readUnsignedByte(); - } - } - - // The word cache here is a stopgap bandaid to help the catastrophic performance - // of this method. Since it performs direct, unbuffered random access to the file and - // may be called hundreds of thousands of times, the resulting performance is not - // reasonable without some kind of cache. Thus: - private static TreeMap<Integer, WeightedString> wordCache = - new TreeMap<Integer, WeightedString>(); - /** - * Finds, as a string, the word at the address passed as an argument. - * - * @param buffer the buffer to read from. - * @param headerSize the size of the header. - * @param address the address to seek. - * @param formatOptions file format options. - * @return the word with its frequency, as a weighted string. - */ - /* package for tests */ static WeightedString getWordAtAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions formatOptions) { - final WeightedString cachedString = wordCache.get(address); - if (null != cachedString) return cachedString; - - final WeightedString result; - final int originalPointer = buffer.position(); - buffer.position(address); - - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions); - } else { - result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address, - formatOptions); - } - - wordCache.put(address, result); - buffer.position(originalPointer); - return result; - } - - // TODO: static!? This will behave erratically when used in multi-threaded code. - // We need to fix this - private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; - @SuppressWarnings("unused") - private static WeightedString getWordAtAddressWithParentAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions options) { - int currentAddress = address; - int index = FormatSpec.MAX_WORD_LENGTH - 1; - int frequency = Integer.MIN_VALUE; - // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH - for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { - CharGroupInfo currentInfo; - int loopCounter = 0; - do { - buffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(buffer, currentAddress, options); - if (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)) { - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - if (DBG && loopCounter++ > MAX_JUMPS) { - MakedictLog.d("Too many jumps - probably a bug"); - } - } while (BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, options)); - if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; - for (int i = 0; i < currentInfo.mCharacters.length; ++i) { - sGetWordBuffer[index--] = - currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1]; - } - if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - - return new WeightedString( - new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1), - frequency); - } - - private static WeightedString getWordAtAddressWithoutParentAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions options) { - buffer.position(headerSize); - final int count = readCharGroupCount(buffer); - int groupOffset = BinaryDictIOUtils.getGroupCountSize(count); - final StringBuilder builder = new StringBuilder(); - WeightedString result = null; - - CharGroupInfo last = null; - for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - groupOffset = info.mEndAddress; - if (info.mOriginalAddress == address) { - builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); - result = new WeightedString(builder.toString(), info.mFrequency); - break; // and return - } - if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { - if (info.mChildrenAddress > address) { - if (null == last) continue; - builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - buffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); - last = null; - continue; - } - last = info; - } - if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { - builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - buffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + BinaryDictIOUtils.getGroupCountSize(i); - last = null; - continue; - } - } - return result; - } - - /** - * Reads a single node from a buffer. - * - * This methods reads the file at the current position. A node is fully expected to start at - * the current position. - * This will recursively read other nodes into the structure, populating the reverse - * maps on the fly and using them to keep track of already read nodes. - * - * @param buffer the buffer, correctly positioned at the start of a node. - * @param headerSize the size, in bytes, of the file header. - * @param reverseNodeMap a mapping from addresses to already read nodes. - * @param reverseGroupMap a mapping from addresses to already read character groups. - * @param options file format options. - * @return the read node with all his children already read. - */ - private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize, - final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap, - final FormatOptions options) - throws IOException { - final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>(); - final int nodeOrigin = buffer.position() - headerSize; - - do { // Scan the linked-list node. - final int nodeHeadPosition = buffer.position() - headerSize; - final int count = readCharGroupCount(buffer); - int groupOffset = nodeHeadPosition + BinaryDictIOUtils.getGroupCountSize(count); - for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - if (BinaryDictIOUtils.isMovedGroup(info.mFlags, options)) continue; - ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; - ArrayList<WeightedString> bigrams = null; - if (null != info.mBigrams) { - bigrams = new ArrayList<WeightedString>(); - for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - buffer, headerSize, bigram.mAddress, options); - final int reconstructedFrequency = - reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); - bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); - } - } - if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { - Node children = reverseNodeMap.get(info.mChildrenAddress); - if (null == children) { - final int currentPosition = buffer.position(); - buffer.position(info.mChildrenAddress + headerSize); - children = readNode( - buffer, headerSize, reverseNodeMap, reverseGroupMap, options); - buffer.position(currentPosition); - } - nodeContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, - info.mFrequency, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); - } else { - nodeContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, - info.mFrequency, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); - } - groupOffset = info.mEndAddress; - } - - // reach the end of the array. - if (options.mSupportsDynamicUpdate) { - final int nextAddress = buffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < buffer.limit()) { - buffer.position(nextAddress); - } else { - break; - } - } - } while (options.mSupportsDynamicUpdate && - buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); - - final Node node = new Node(nodeContents); - node.mCachedAddressBeforeUpdate = nodeOrigin; - node.mCachedAddressAfterUpdate = nodeOrigin; - reverseNodeMap.put(node.mCachedAddressAfterUpdate, node); - return node; - } - - /** - * Helper function to get the binary format version from the header. - * @throws IOException - */ - private static int getFormatVersion(final FusionDictionaryBufferInterface buffer) - throws IOException { - final int magic = buffer.readInt(); - if (FormatSpec.MAGIC_NUMBER == magic) return buffer.readUnsignedShort(); - return FormatSpec.NOT_A_VERSION_NUMBER; - } - - /** - * Helper function to get and validate the binary format version. - * @throws UnsupportedFormatException - * @throws IOException - */ - private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer) - throws IOException, UnsupportedFormatException { - final int version = getFormatVersion(buffer); - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("This file has version " + version - + ", but this implementation does not support versions above " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - return version; - } - - /** - * Reads a header from a buffer. - * @param buffer the buffer to read. - * @throws IOException - * @throws UnsupportedFormatException - */ - public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer) - throws IOException, UnsupportedFormatException { - final int version = checkFormatVersion(buffer); - final int optionsFlags = buffer.readUnsignedShort(); - - final HashMap<String, String> attributes = new HashMap<String, String>(); - final int headerSize; - headerSize = buffer.readInt(); - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - populateOptions(buffer, headerSize, attributes); - buffer.position(headerSize); - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } - - /** - * Reads options from a buffer and populate a map with their contents. - * - * The buffer is read at the current position, so the caller must take care the pointer - * is in the right place before calling this. - */ - public static void populateOptions(final FusionDictionaryBufferInterface buffer, - final int headerSize, final HashMap<String, String> options) { - while (buffer.position() < headerSize) { - final String key = CharEncoding.readString(buffer); - final String value = CharEncoding.readString(buffer); - options.put(key, value); - } - } - - /** - * Reads a buffer and returns the memory representation of the dictionary. - * - * This high-level method takes a buffer and reads its contents, populating a - * FusionDictionary structure. The optional dict argument is an existing dictionary to - * which words from the buffer should be added. If it is null, a new dictionary is created. - * - * @param reader the reader. - * @param dict an optional dictionary to add words to, or null. - * @return the created (or merged) dictionary. - */ - @UsedForTesting - public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader, - final FusionDictionary dict) throws FileNotFoundException, IOException, - UnsupportedFormatException { - // clear cache - wordCache.clear(); - - // if the buffer has not been opened, open the buffer with bytebuffer. - if (reader.getBuffer() == null) reader.openBuffer( - new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); - if (reader.getBuffer() == null) { - MakedictLog.e("Cannot open the buffer"); - } - - // Read header - final FileHeader header = readHeader(reader.getBuffer()); - - Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>(); - Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping, - reverseGroupMapping, header.mFormatOptions); - - FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions); - if (null != dict) { - for (final Word w : dict) { - if (w.mIsBlacklistEntry) { - newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); - } else { - newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); - } - } - for (final Word w : dict) { - // By construction a binary dictionary may not have bigrams pointing to - // words that are not also registered as unigrams so we don't have to avoid - // them explicitly here. - for (final WeightedString bigram : w.mBigrams) { - newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); - } - } - } - - return newDict; - } - - /** - * Helper method to pass a file name instead of a File object to isBinaryDictionary. - */ - public static boolean isBinaryDictionary(final String filename) { - final File file = new File(filename); - return isBinaryDictionary(file); - } - - /** - * Basic test to find out whether the file is a binary dictionary or not. - * - * Concretely this only tests the magic number. - * - * @param file The file to test. - * @return true if it's a binary dictionary, false otherwise - */ - public static boolean isBinaryDictionary(final File file) { - FileInputStream inStream = null; - try { - inStream = new FileInputStream(file); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - final int version = getFormatVersion(new ByteBufferWrapper(buffer)); - return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION - && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } catch (FileNotFoundException e) { - return false; - } catch (IOException e) { - return false; - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } - } - } - - /** - * Calculate bigram frequency from compressed value - * - * @see #makeBigramFlags - * - * @param unigramFrequency - * @param bigramFrequency compressed frequency - * @return approximate bigram frequency - */ - public static int reconstructBigramFrequency(final int unigramFrequency, - final int bigramFrequency) { - final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) - / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); - final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); - return (int)resultFreqFloat; - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java index a4a7ce458..6d3b31a28 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java @@ -17,7 +17,9 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.decoder.HeaderReaderInterface; import com.android.inputmethod.latin.utils.ByteArrayWrapper; import java.io.File; @@ -27,8 +29,9 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.util.HashMap; -public class BinaryDictReader { +public class BinaryDictReader implements HeaderReaderInterface { public interface FusionDictionaryBufferFactory { public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file) @@ -55,7 +58,7 @@ public class BinaryDictReader { } } if (buffer != null) { - return new BinaryDictInputOutput.ByteBufferWrapper(buffer); + return new BinaryDictDecoder.ByteBufferWrapper(buffer); } return null; } @@ -103,7 +106,7 @@ public class BinaryDictReader { } } if (buffer != null) { - return new BinaryDictInputOutput.ByteBufferWrapper(buffer); + return new BinaryDictDecoder.ByteBufferWrapper(buffer); } return null; } @@ -133,4 +136,34 @@ public class BinaryDictReader { openBuffer(factory); return getBuffer(); } + + // The implementation of HeaderReaderInterface + @Override + public int readVersion() throws IOException, UnsupportedFormatException { + return BinaryDictDecoder.checkFormatVersion(mFusionDictionaryBuffer); + } + + @Override + public int readOptionFlags() { + return mFusionDictionaryBuffer.readUnsignedShort(); + } + + @Override + public int readHeaderSize() { + return mFusionDictionaryBuffer.readInt(); + } + + @Override + public HashMap<String, String> readAttributes(final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (mFusionDictionaryBuffer.position() < headerSize) { + // We can avoid infinite loop here since mFusionDictonary.position() is always increased + // by calling CharEncoding.readString. + final String key = CharEncoding.readString(mFusionDictionaryBuffer); + final String value = CharEncoding.readString(mFusionDictionaryBuffer); + attributes.put(key, value); + } + mFusionDictionaryBuffer.position(headerSize); + return attributes; + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index 5b10912ea..584b793fb 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -49,17 +49,18 @@ public final class DynamicBinaryDictIOUtils { /** * Delete the word from the binary file. * - * @param buffer the buffer to write. + * @param dictReader the dict reader. * @param word the word we delete * @throws IOException * @throws UnsupportedFormatException */ @UsedForTesting - public static void deleteWord(final FusionDictionaryBufferInterface buffer, - final String word) throws IOException, UnsupportedFormatException { + public static void deleteWord(final BinaryDictReader dictReader, final String word) + throws IOException, UnsupportedFormatException { + final FusionDictionaryBufferInterface buffer = dictReader.getBuffer(); buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - final int wordPosition = BinaryDictIOUtils.getTerminalPosition(buffer, word); + final FileHeader header = BinaryDictDecoder.readHeader(dictReader); + final int wordPosition = BinaryDictIOUtils.getTerminalPosition(dictReader, word); if (wordPosition == FormatSpec.NOT_VALID_WORD) return; buffer.position(wordPosition); @@ -86,7 +87,7 @@ public final class DynamicBinaryDictIOUtils { } final int flags = buffer.readUnsignedByte(); if (BinaryDictIOUtils.isMovedGroup(flags, formatOptions)) { - // if the group is moved, the parent address is stored in the destination group. + // If the group is moved, the parent address is stored in the destination group. // We are guaranteed to process the destination group later, so there is no need to // update anything here. buffer.position(originalPosition); @@ -101,10 +102,10 @@ public final class DynamicBinaryDictIOUtils { } /** - * Update parent addresses in a Node that is referred to by nodeOriginAddress. + * Update parent addresses in a node array stored at nodeOriginAddress. * * @param buffer the buffer to be modified. - * @param nodeOriginAddress the address of a modified Node. + * @param nodeOriginAddress the address of the node array to update. * @param newParentAddress the address to be written. * @param formatOptions file format options. */ @@ -114,7 +115,7 @@ public final class DynamicBinaryDictIOUtils { final int originalPosition = buffer.position(); buffer.position(nodeOriginAddress); do { - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); + final int count = BinaryDictDecoder.readCharGroupCount(buffer); for (int i = 0; i < count; ++i) { updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions); BinaryDictIOUtils.skipCharGroup(buffer, formatOptions); @@ -140,7 +141,7 @@ public final class DynamicBinaryDictIOUtils { final int originalPosition = buffer.position(); buffer.position(groupOriginAddress); final int flags = buffer.readUnsignedByte(); - final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions); + final int parentAddress = BinaryDictDecoder.readParentAddress(buffer, formatOptions); BinaryDictIOUtils.skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS @@ -154,7 +155,7 @@ public final class DynamicBinaryDictIOUtils { */ private static int moveCharGroup(final OutputStream destination, final FusionDictionaryBufferInterface buffer, final CharGroupInfo info, - final int nodeOriginAddress, final int oldGroupAddress, + final int nodeArrayOriginAddress, final int oldGroupAddress, final FormatOptions formatOptions) throws IOException { updateParentAddress(buffer, oldGroupAddress, buffer.limit() + 1, formatOptions); buffer.position(oldGroupAddress); @@ -163,23 +164,24 @@ public final class DynamicBinaryDictIOUtils { buffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); int size = FormatSpec.GROUP_FLAGS_SIZE; - updateForwardLink(buffer, nodeOriginAddress, buffer.limit(), formatOptions); - size += BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[] { info }); + updateForwardLink(buffer, nodeArrayOriginAddress, buffer.limit(), formatOptions); + size += BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { info }); return size; } + @SuppressWarnings("unused") private static void updateForwardLink(final FusionDictionaryBufferInterface buffer, - final int nodeOriginAddress, final int newNodeAddress, + final int nodeArrayOriginAddress, final int newNodeArrayAddress, final FormatOptions formatOptions) { - buffer.position(nodeOriginAddress); + buffer.position(nodeArrayOriginAddress); int jumpCount = 0; while (jumpCount++ < MAX_JUMPS) { - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); + final int count = BinaryDictDecoder.readCharGroupCount(buffer); for (int i = 0; i < count; ++i) BinaryDictIOUtils.skipCharGroup(buffer, formatOptions); final int forwardLinkAddress = buffer.readUnsignedInt24(); if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { buffer.position(buffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE); - BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress); + BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeArrayAddress); return; } buffer.position(forwardLinkAddress); @@ -190,71 +192,73 @@ public final class DynamicBinaryDictIOUtils { } /** - * Move a group that is referred to by oldGroupOrigin to the tail of the file. - * And set the children address to the byte after the group. + * Move a group that is referred to by oldGroupOrigin to the tail of the file, and set the + * children address to the byte after the group * - * @param nodeOrigin the address of the tail of the file. - * @param characters - * @param length - * @param flags - * @param frequency - * @param parentAddress - * @param shortcutTargets - * @param bigrams + * @param fileEndAddress the address of the tail of the file. + * @param codePoints the characters to put inside the group. + * @param length how many code points to read from codePoints. + * @param flags the flags for this group. + * @param frequency the frequency of this terminal. + * @param parentAddress the address of the parent group of this group. + * @param shortcutTargets the shortcut targets for this group. + * @param bigrams the bigrams for this group. * @param destination the stream representing the tail of the file. * @param buffer the buffer representing the (constant-size) body of the file. - * @param oldNodeOrigin - * @param oldGroupOrigin - * @param formatOptions + * @param oldNodeArrayOrigin the origin of the old node array this group was a part of. + * @param oldGroupOrigin the old origin where this group used to be stored. + * @param formatOptions format options for this dictionary. * @return the size written, in bytes. - * @throws IOException + * @throws IOException if the file can't be accessed */ - private static int moveGroup(final int nodeOrigin, final int[] characters, final int length, - final int flags, final int frequency, final int parentAddress, + private static int moveGroup(final int fileEndAddress, final int[] codePoints, + final int length, final int flags, final int frequency, final int parentAddress, final ArrayList<WeightedString> shortcutTargets, final ArrayList<PendingAttribute> bigrams, final OutputStream destination, - final FusionDictionaryBufferInterface buffer, final int oldNodeOrigin, + final FusionDictionaryBufferInterface buffer, final int oldNodeArrayOrigin, final int oldGroupOrigin, final FormatOptions formatOptions) throws IOException { int size = 0; - final int newGroupOrigin = nodeOrigin + 1; - final int[] writtenCharacters = Arrays.copyOfRange(characters, 0, length); + final int newGroupOrigin = fileEndAddress + 1; + final int[] writtenCharacters = Arrays.copyOfRange(codePoints, 0, length); final CharGroupInfo tmpInfo = new CharGroupInfo(newGroupOrigin, -1 /* endAddress */, flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, shortcutTargets, bigrams); size = BinaryDictIOUtils.computeGroupSize(tmpInfo, formatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newGroupOrigin, newGroupOrigin + size, flags, writtenCharacters, frequency, parentAddress, - nodeOrigin + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, + fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, bigrams); - moveCharGroup(destination, buffer, newInfo, oldNodeOrigin, oldGroupOrigin, formatOptions); + moveCharGroup(destination, buffer, newInfo, oldNodeArrayOrigin, oldGroupOrigin, + formatOptions); return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } /** * Insert a word into a binary dictionary. * - * @param buffer - * @param destination - * @param word - * @param frequency - * @param bigramStrings - * @param shortcuts - * @throws IOException - * @throws UnsupportedFormatException + * @param dictReader the dict reader. + * @param destination a stream to the underlying file, with the pointer at the end of the file. + * @param word the word to insert. + * @param frequency the frequency of the new word. + * @param bigramStrings bigram list, or null if none. + * @param shortcuts shortcut list, or null if none. + * @param isBlackListEntry whether this should be a blacklist entry. + * @throws IOException if the file can't be accessed. + * @throws UnsupportedFormatException if the existing dictionary is in an unexpected format. */ // TODO: Support batch insertion. // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. @UsedForTesting - public static void insertWord(final FusionDictionaryBufferInterface buffer, - final OutputStream destination, final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, + public static void insertWord(final BinaryDictReader dictReader, final OutputStream destination, + final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, final boolean isBlackListEntry) throws IOException, UnsupportedFormatException { final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); + final FusionDictionaryBufferInterface buffer = dictReader.getBuffer(); if (bigramStrings != null) { for (final WeightedString bigram : bigramStrings) { - int position = BinaryDictIOUtils.getTerminalPosition(buffer, bigram.mWord); + int position = BinaryDictIOUtils.getTerminalPosition(dictReader, bigram.mWord); if (position == FormatSpec.NOT_VALID_WORD) { // TODO: figure out what is the correct thing to do here. } else { @@ -269,7 +273,7 @@ public final class DynamicBinaryDictIOUtils { // find the insert position of the word. if (buffer.position() != 0) buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); + final FileHeader fileHeader = BinaryDictDecoder.readHeader(dictReader); int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position(); final int[] codePoints = FusionDictionary.getCodePoints(word); @@ -279,15 +283,15 @@ public final class DynamicBinaryDictIOUtils { if (wordPos >= wordLen) break; nodeOriginAddress = buffer.position(); int nodeParentAddress = -1; - final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); + final int charGroupCount = BinaryDictDecoder.readCharGroupCount(buffer); boolean foundNextGroup = false; for (int i = 0; i < charGroupCount; ++i) { address = buffer.position(); - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, - buffer.position(), header.mFormatOptions); + final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer, + buffer.position(), fileHeader.mFormatOptions); final boolean isMovedGroup = BinaryDictIOUtils.isMovedGroup(currentInfo.mFlags, - header.mFormatOptions); + fileHeader.mFormatOptions); if (isMovedGroup) continue; nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; @@ -305,25 +309,25 @@ public final class DynamicBinaryDictIOUtils { * abc - d - ef */ final int newNodeAddress = buffer.limit(); - final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, + final int flags = BinaryDictEncoder.makeCharGroupFlags(p > 1, isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, - false /* isBlackListEntry */, header.mFormatOptions); + false /* isBlackListEntry */, fileHeader.mFormatOptions); int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, frequency, nodeParentAddress, shortcuts, bigrams, destination, - buffer, nodeOriginAddress, address, header.mFormatOptions); + buffer, nodeOriginAddress, address, fileHeader.mFormatOptions); final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, currentInfo.mCharacters.length); if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { updateParentAddresses(buffer, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, header.mFormatOptions); + newNodeAddress + written + 1, fileHeader.mFormatOptions); } final CharGroupInfo newInfo2 = new CharGroupInfo( newNodeAddress + written + 1, -1 /* endAddress */, currentInfo.mFlags, characters2, currentInfo.mFrequency, newNodeAddress + 1, currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, currentInfo.mBigrams); - BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[] { newInfo2 }); + BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { newInfo2 }); return; } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { if (p > 0) { @@ -344,49 +348,49 @@ public final class DynamicBinaryDictIOUtils { final int childrenAddress = currentInfo.mChildrenAddress; // move prefix - final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, + final int prefixFlags = BinaryDictEncoder.makeCharGroupFlags(p > 1, false /* isTerminal */, 0 /* childrenAddressSize*/, false /* hasShortcut */, false /* hasBigrams */, false /* isNotAWord */, false /* isBlackListEntry */, - header.mFormatOptions); + fileHeader.mFormatOptions); int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, destination, buffer, nodeOriginAddress, address, - header.mFormatOptions); + fileHeader.mFormatOptions); final int[] suffixCharacters = Arrays.copyOfRange( currentInfo.mCharacters, p, currentInfo.mCharacters.length); if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { updateParentAddresses(buffer, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, header.mFormatOptions); + newNodeAddress + written + 1, fileHeader.mFormatOptions); } - final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags( + final int suffixFlags = BinaryDictEncoder.makeCharGroupFlags( suffixCharacters.length > 1, (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, 0 /* childrenAddressSize */, (currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0, (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, - isNotAWord, isBlackListEntry, header.mFormatOptions); + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo suffixInfo = new CharGroupInfo( newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, currentInfo.mBigrams); written += BinaryDictIOUtils.computeGroupSize(suffixInfo, - header.mFormatOptions) + 1; + fileHeader.mFormatOptions) + 1; final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, codePoints.length); - final int flags = BinaryDictInputOutput.makeCharGroupFlags( + final int flags = BinaryDictEncoder.makeCharGroupFlags( newCharacters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo( newNodeAddress + written, -1 /* endAddress */, flags, newCharacters, frequency, newNodeAddress + 1, FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNode(destination, + BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { suffixInfo, newInfo }); return; } @@ -401,15 +405,15 @@ public final class DynamicBinaryDictIOUtils { // only update group. final int newNodeAddress = buffer.limit(); final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, bigrams); moveCharGroup(destination, buffer, newInfo, nodeOriginAddress, address, - header.mFormatOptions); + fileHeader.mFormatOptions); return; } wordPos += currentInfo.mCharacters.length; @@ -428,17 +432,17 @@ public final class DynamicBinaryDictIOUtils { */ final int newNodeAddress = buffer.limit(); updateChildrenAddress(buffer, address, newNodeAddress, - header.mFormatOptions); + fileHeader.mFormatOptions); final int newGroupAddress = newNodeAddress + 1; final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, + final int flags = BinaryDictEncoder.makeCharGroupFlags(hasMultipleChars, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); final CharGroupInfo newInfo = new CharGroupInfo(newGroupAddress, -1, flags, characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[] { newInfo }); + BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[] { newInfo }); return; } buffer.position(currentInfo.mChildrenAddress); @@ -476,13 +480,13 @@ public final class DynamicBinaryDictIOUtils { BinaryDictIOUtils.writeSInt24ToBuffer(buffer, newNodeAddress); final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1, + final int flags = BinaryDictEncoder.makeCharGroupFlags(characters.length > 1, isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - BinaryDictIOUtils.writeNode(destination, new CharGroupInfo[]{ newInfo }); + BinaryDictIOUtils.writeNodes(destination, new CharGroupInfo[]{ newInfo }); return; } else { depth--; diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 9af66ed4c..5e331219c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -60,7 +60,7 @@ public final class FormatSpec { */ /* - * Array of Node(FusionDictionary.Node) layout is as follows: + * Node array (FusionDictionary.PtNodeArray) layout is as follows: * * g | * r | the number of groups, 1 or 2 bytes. @@ -86,7 +86,7 @@ public final class FormatSpec { * linkaddress */ - /* Node(CharGroup) layout is as follows: + /* Node (FusionDictionary.CharGroup) layout is as follows: * | IF !SUPPORTS_DYNAMIC_UPDATE * | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE * | 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS @@ -251,7 +251,7 @@ public final class FormatSpec { static final int INVALID_CHARACTER = -1; static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127 - static final int MAX_CHARGROUPS_IN_A_NODE = 0x7FFF; // 32767 + static final int MAX_CHARGROUPS_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 static final int MAX_BIGRAMS_IN_A_GROUP = 10000; static final int MAX_TERMINAL_FREQUENCY = 255; diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 118dc22b8..fce1c5cdd 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -37,14 +37,14 @@ public final class FusionDictionary implements Iterable<Word> { private static int CHARACTER_NOT_FOUND_INDEX = -1; /** - * A node of the dictionary, containing several CharGroups. + * A node array of the dictionary, containing several CharGroups. * - * A node is but an ordered array of CharGroups, which essentially contain all the + * A PtNodeArray is but an ordered array of CharGroups, which essentially contain all the * real information. * This class also contains fields to cache size and address, to help with binary * generation. */ - public static final class Node { + public static final class PtNodeArray { ArrayList<CharGroup> mData; // To help with binary generation int mCachedSize = Integer.MIN_VALUE; @@ -57,10 +57,10 @@ public final class FusionDictionary implements Iterable<Word> { int mCachedAddressAfterUpdate = Integer.MIN_VALUE; int mCachedParentAddress = 0; - public Node() { + public PtNodeArray() { mData = new ArrayList<CharGroup>(); } - public Node(ArrayList<CharGroup> data) { + public PtNodeArray(ArrayList<CharGroup> data) { mData = data; } } @@ -98,7 +98,7 @@ public final class FusionDictionary implements Iterable<Word> { * This is the central class of the in-memory representation. A CharGroup is what can * be seen as a traditional "trie node", except it can hold several characters at the * same time. A CharGroup essentially represents one or several characters in the middle - * of the trie trie; as such, it can be a terminal, and it can have children. + * of the trie tree; as such, it can be a terminal, and it can have children. * In this in-memory representation, whether the CharGroup is a terminal or not is represented * in the frequency, where NOT_A_TERMINAL (= -1) means this is not a terminal and any other * value is the frequency of this terminal. A terminal may have non-null shortcuts and/or @@ -110,7 +110,7 @@ public final class FusionDictionary implements Iterable<Word> { ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal. - Node mChildren; + PtNodeArray mChildren; boolean mIsNotAWord; // Only a shortcut boolean mIsBlacklistEntry; // mCachedSize and mCachedAddressBefore/AfterUpdate are helpers for binary dictionary @@ -137,7 +137,8 @@ public final class FusionDictionary implements Iterable<Word> { public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, - final boolean isNotAWord, final boolean isBlacklistEntry, final Node children) { + final boolean isNotAWord, final boolean isBlacklistEntry, + final PtNodeArray children) { mChars = chars; mFrequency = frequency; mShortcutTargets = shortcutTargets; @@ -149,7 +150,7 @@ public final class FusionDictionary implements Iterable<Word> { public void addChild(CharGroup n) { if (null == mChildren) { - mChildren = new Node(); + mChildren = new PtNodeArray(); } mChildren.mData.add(n); } @@ -344,10 +345,10 @@ public final class FusionDictionary implements Iterable<Word> { } public final DictionaryOptions mOptions; - public final Node mRoot; + public final PtNodeArray mRootNodeArray; - public FusionDictionary(final Node root, final DictionaryOptions options) { - mRoot = root; + public FusionDictionary(final PtNodeArray rootNodeArray, final DictionaryOptions options) { + mRootNodeArray = rootNodeArray; mOptions = options; } @@ -406,13 +407,13 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Sanity check for a node. + * Sanity check for a node array. * - * This method checks that all CharGroups in a node are ordered as expected. + * This method checks that all CharGroups in a node array are ordered as expected. * If they are, nothing happens. If they aren't, an exception is thrown. */ - private void checkStack(Node node) { - ArrayList<CharGroup> stack = node.mData; + private void checkStack(PtNodeArray nodeArray) { + ArrayList<CharGroup> stack = nodeArray.mData; int lastValue = -1; for (int i = 0; i < stack.size(); ++i) { int currentValue = stack.get(i).mChars[0]; @@ -431,16 +432,16 @@ public final class FusionDictionary implements Iterable<Word> { * @param frequency the bigram frequency */ public void setBigram(final String word1, final String word2, final int frequency) { - CharGroup charGroup = findWordInTree(mRoot, word1); + CharGroup charGroup = findWordInTree(mRootNodeArray, word1); if (charGroup != null) { - final CharGroup charGroup2 = findWordInTree(mRoot, word2); + final CharGroup charGroup2 = findWordInTree(mRootNodeArray, word2); if (charGroup2 == null) { add(getCodePoints(word2), 0, null, false /* isNotAWord */, false /* isBlacklistEntry */); // The chargroup for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been // a cutting point until now. In this case, we need to refresh charGroup. - charGroup = findWordInTree(mRoot, word1); + charGroup = findWordInTree(mRootNodeArray, word1); } charGroup.addBigram(word2, frequency); } else { @@ -469,38 +470,38 @@ public final class FusionDictionary implements Iterable<Word> { return; } - Node currentNode = mRoot; + PtNodeArray currentNodeArray = mRootNodeArray; int charIndex = 0; CharGroup currentGroup = null; int differentCharIndex = 0; // Set by the loop to the index of the char that differs - int nodeIndex = findIndexOfChar(mRoot, word[charIndex]); + int nodeIndex = findIndexOfChar(mRootNodeArray, word[charIndex]); while (CHARACTER_NOT_FOUND_INDEX != nodeIndex) { - currentGroup = currentNode.mData.get(nodeIndex); - differentCharIndex = compareArrays(currentGroup.mChars, word, charIndex); + currentGroup = currentNodeArray.mData.get(nodeIndex); + differentCharIndex = compareCharArrays(currentGroup.mChars, word, charIndex); if (ARRAYS_ARE_EQUAL != differentCharIndex && differentCharIndex < currentGroup.mChars.length) break; if (null == currentGroup.mChildren) break; charIndex += currentGroup.mChars.length; if (charIndex >= word.length) break; - currentNode = currentGroup.mChildren; - nodeIndex = findIndexOfChar(currentNode, word[charIndex]); + currentNodeArray = currentGroup.mChildren; + nodeIndex = findIndexOfChar(currentNodeArray, word[charIndex]); } if (CHARACTER_NOT_FOUND_INDEX == nodeIndex) { // No node at this point to accept the word. Create one. - final int insertionIndex = findInsertionIndex(currentNode, word[charIndex]); + final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); final CharGroup newGroup = new CharGroup( Arrays.copyOfRange(word, charIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentNode.mData.add(insertionIndex, newGroup); - if (DBG) checkStack(currentNode); + currentNodeArray.mData.add(insertionIndex, newGroup); + if (DBG) checkStack(currentNodeArray); } else { // There is a word with a common prefix. if (differentCharIndex == currentGroup.mChars.length) { if (charIndex + differentCharIndex >= word.length) { // The new word is a prefix of an existing word, but the node on which it - // should end already exists as is. Since the old CharNode was not a terminal, + // should end already exists as is. Since the old CharGroup was not a terminal, // make it one by filling in its frequency and other attributes currentGroup.update(frequency, shortcutTargets, null, isNotAWord, isBlacklistEntry); @@ -511,7 +512,7 @@ public final class FusionDictionary implements Iterable<Word> { Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentGroup.mChildren = new Node(); + currentGroup.mChildren = new PtNodeArray(); currentGroup.mChildren.mData.add(newNode); } } else { @@ -524,7 +525,7 @@ public final class FusionDictionary implements Iterable<Word> { } else { // Partial prefix match only. We have to replace the current node with a node // containing the current prefix and create two new ones for the tails. - Node newChildren = new Node(); + PtNodeArray newChildren = new PtNodeArray(); final CharGroup newOldWord = new CharGroup( Arrays.copyOfRange(currentGroup.mChars, differentCharIndex, currentGroup.mChars.length), currentGroup.mShortcutTargets, @@ -552,9 +553,9 @@ public final class FusionDictionary implements Iterable<Word> { > currentGroup.mChars[differentCharIndex] ? 1 : 0; newChildren.mData.add(addIndex, newWord); } - currentNode.mData.set(nodeIndex, newParent); + currentNodeArray.mData.set(nodeIndex, newParent); } - if (DBG) checkStack(currentNode); + if (DBG) checkStack(currentNodeArray); } } } @@ -576,7 +577,7 @@ public final class FusionDictionary implements Iterable<Word> { * @param dstOffset the offset in the right-hand side string. * @return the index at which the strings differ, or ARRAYS_ARE_EQUAL = 0 if they don't. */ - private static int compareArrays(final int[] src, final int[] dst, int dstOffset) { + private static int compareCharArrays(final int[] src, final int[] dst, int dstOffset) { // We do NOT test the first char, because we come from a method that already // tested it. for (int i = 1; i < src.length; ++i) { @@ -603,10 +604,10 @@ public final class FusionDictionary implements Iterable<Word> { final static private CharGroupComparator CHARGROUP_COMPARATOR = new CharGroupComparator(); /** - * Finds the insertion index of a character within a node. + * Finds the insertion index of a character within a node array. */ - private static int findInsertionIndex(final Node node, int character) { - final ArrayList<CharGroup> data = node.mData; + private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { + final ArrayList<CharGroup> data = nodeArray.mData; final CharGroup reference = new CharGroup(new int[] { character }, null /* shortcutTargets */, null /* bigrams */, 0, false /* isNotAWord */, false /* isBlacklistEntry */); @@ -615,16 +616,16 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Find the index of a char in a node, if it exists. + * Find the index of a char in a node array, if it exists. * - * @param node the node to search in. + * @param nodeArray the node array to search in. * @param character the character to search for. * @return the position of the character if it's there, or CHARACTER_NOT_FOUND_INDEX = -1 else. */ - private static int findIndexOfChar(final Node node, int character) { - final int insertionIndex = findInsertionIndex(node, character); - if (node.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; - return character == node.mData.get(insertionIndex).mChars[0] ? insertionIndex + private static int findIndexOfChar(final PtNodeArray nodeArray, int character) { + final int insertionIndex = findInsertionIndex(nodeArray, character); + if (nodeArray.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; + return character == nodeArray.mData.get(insertionIndex).mChars[0] ? insertionIndex : CHARACTER_NOT_FOUND_INDEX; } @@ -632,16 +633,16 @@ public final class FusionDictionary implements Iterable<Word> { * Helper method to find a word in a given branch. */ @SuppressWarnings("unused") - public static CharGroup findWordInTree(Node node, final String string) { + public static CharGroup findWordInTree(PtNodeArray nodeArray, final String string) { int index = 0; final StringBuilder checker = DBG ? new StringBuilder() : null; final int[] codePoints = getCodePoints(string); CharGroup currentGroup; do { - int indexOfGroup = findIndexOfChar(node, codePoints[index]); + int indexOfGroup = findIndexOfChar(nodeArray, codePoints[index]); if (CHARACTER_NOT_FOUND_INDEX == indexOfGroup) return null; - currentGroup = node.mData.get(indexOfGroup); + currentGroup = nodeArray.mData.get(indexOfGroup); if (codePoints.length - index < currentGroup.mChars.length) return null; int newIndex = index; @@ -653,9 +654,9 @@ public final class FusionDictionary implements Iterable<Word> { if (DBG) checker.append(new String(currentGroup.mChars, 0, currentGroup.mChars.length)); if (index < codePoints.length) { - node = currentGroup.mChildren; + nodeArray = currentGroup.mChildren; } - } while (null != node && index < codePoints.length); + } while (null != nodeArray && index < codePoints.length); if (index < codePoints.length) return null; if (!currentGroup.isTerminal()) return null; @@ -670,20 +671,20 @@ public final class FusionDictionary implements Iterable<Word> { if (null == s || "".equals(s)) { throw new RuntimeException("Can't search for a null or empty string"); } - return null != findWordInTree(mRoot, s); + return null != findWordInTree(mRootNodeArray, s); } /** * Recursively count the number of character groups in a given branch of the trie. * - * @param node the parent node. + * @param nodeArray the parent node. * @return the number of char groups in all the branch under this node. */ - public static int countCharGroups(final Node node) { - final int nodeSize = node.mData.size(); + public static int countCharGroups(final PtNodeArray nodeArray) { + final int nodeSize = nodeArray.mData.size(); int size = nodeSize; for (int i = nodeSize - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); + CharGroup group = nodeArray.mData.get(i); if (null != group.mChildren) size += countCharGroups(group.mChildren); } @@ -693,15 +694,15 @@ public final class FusionDictionary implements Iterable<Word> { /** * Recursively count the number of nodes in a given branch of the trie. * - * @param node the node to count. + * @param nodeArray the node array to count. * @return the number of nodes in this branch. */ - public static int countNodes(final Node node) { + public static int countNodeArrays(final PtNodeArray nodeArray) { int size = 1; - for (int i = node.mData.size() - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + CharGroup group = nodeArray.mData.get(i); if (null != group.mChildren) - size += countNodes(group.mChildren); + size += countNodeArrays(group.mChildren); } return size; } @@ -709,10 +710,10 @@ public final class FusionDictionary implements Iterable<Word> { // Recursively find out whether there are any bigrams. // This can be pretty expensive especially if there aren't any (we return as soon // as we find one, so it's much cheaper if there are bigrams) - private static boolean hasBigramsInternal(final Node node) { - if (null == node) return false; - for (int i = node.mData.size() - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); + private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { + if (null == nodeArray) return false; + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + CharGroup group = nodeArray.mData.get(i); if (null != group.mBigrams) return true; if (hasBigramsInternal(group.mChildren)) return true; } @@ -729,7 +730,7 @@ public final class FusionDictionary implements Iterable<Word> { // find a more efficient way of doing this, without compromising too much on memory // and ease of use. public boolean hasBigrams() { - return hasBigramsInternal(mRoot); + return hasBigramsInternal(mRootNodeArray); } // Historically, the tails of the words were going to be merged to save space. @@ -750,13 +751,13 @@ public final class FusionDictionary implements Iterable<Word> { // MakedictLog.i("Merging nodes. Number of nodes : " + countNodes(root)); // MakedictLog.i("Number of groups : " + countCharGroups(root)); // -// final HashMap<String, ArrayList<Node>> repository = -// new HashMap<String, ArrayList<Node>>(); +// final HashMap<String, ArrayList<PtNodeArray>> repository = +// new HashMap<String, ArrayList<PtNodeArray>>(); // mergeTailsInner(repository, root); // // MakedictLog.i("Number of different pseudohashes : " + repository.size()); // int size = 0; -// for (ArrayList<Node> a : repository.values()) { +// for (ArrayList<PtNodeArray> a : repository.values()) { // size += a.size(); // } // MakedictLog.i("Number of nodes after merge : " + (1 + size)); @@ -764,7 +765,7 @@ public final class FusionDictionary implements Iterable<Word> { } // The following methods are used by the deactivated mergeTails() -// private static boolean isEqual(Node a, Node b) { +// private static boolean isEqual(PtNodeArray a, PtNodeArray b) { // if (null == a && null == b) return true; // if (null == a || null == b) return false; // if (a.data.size() != b.data.size()) return false; @@ -781,21 +782,21 @@ public final class FusionDictionary implements Iterable<Word> { // return true; // } -// static private HashMap<String, ArrayList<Node>> mergeTailsInner( -// final HashMap<String, ArrayList<Node>> map, final Node node) { -// final ArrayList<CharGroup> branches = node.data; +// static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner( +// final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) { +// final ArrayList<CharGroup> branches = nodeArray.data; // final int nodeSize = branches.size(); // for (int i = 0; i < nodeSize; ++i) { // CharGroup group = branches.get(i); // if (null != group.children) { // String pseudoHash = getPseudoHash(group.children); -// ArrayList<Node> similarList = map.get(pseudoHash); +// ArrayList<PtNodeArray> similarList = map.get(pseudoHash); // if (null == similarList) { -// similarList = new ArrayList<Node>(); +// similarList = new ArrayList<PtNodeArray>(); // map.put(pseudoHash, similarList); // } // boolean merged = false; -// for (Node similar : similarList) { +// for (PtNodeArray similar : similarList) { // if (isEqual(group.children, similar)) { // group.children = similar; // merged = true; @@ -811,9 +812,9 @@ public final class FusionDictionary implements Iterable<Word> { // return map; // } -// private static String getPseudoHash(final Node node) { +// private static String getPseudoHash(final PtNodeArray nodeArray) { // StringBuilder s = new StringBuilder(); -// for (CharGroup g : node.data) { +// for (CharGroup g : nodeArray.data) { // s.append(g.frequency); // for (int ch : g.chars) { // s.append(Character.toChars(ch)); @@ -901,6 +902,6 @@ public final class FusionDictionary implements Iterable<Word> { */ @Override public Iterator<Word> iterator() { - return new DictionaryIterator(mRoot.mData); + return new DictionaryIterator(mRootNodeArray.mData); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReaderInterface.java b/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReaderInterface.java new file mode 100644 index 000000000..7cddef2a4 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/decoder/HeaderReaderInterface.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict.decoder; + +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; + +import java.io.IOException; +import java.util.HashMap; + +/** + * An interface to read a binary dictionary file header. + */ +public interface HeaderReaderInterface { + public int readVersion() throws IOException, UnsupportedFormatException; + public int readOptionFlags(); + public int readHeaderSize(); + public HashMap<String, String> readAttributes(final int headerSize); +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 065e00e4a..525d3cd11 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -43,6 +43,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; /** @@ -75,6 +76,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = CollectionUtils.newArrayList(); + private final AtomicReference<AsyncTask<Void, Void, Void>> mWaitingTask; + // Should always be false except when we use this class for test @UsedForTesting boolean mIsTest = false; @@ -83,6 +86,7 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona super(context, dictionaryType); mLocale = locale; mPrefs = sp; + mWaitingTask = new AtomicReference<AsyncTask<Void, Void, Void>>(); if (mLocale != null && mLocale.length() > 1) { loadDictionary(); } @@ -174,7 +178,11 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona */ private void flushPendingWrites() { // Create a background thread to write the pending entries - new UpdateBinaryTask(mBigramList, mLocale, this, mPrefs, getContext()).execute(); + final AsyncTask<Void, Void, Void> old = mWaitingTask.getAndSet(new UpdateBinaryTask( + mBigramList, mLocale, this, mPrefs, getContext()).execute()); + if (old != null) { + old.cancel(false); + } } @Override @@ -287,6 +295,7 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona @Override protected Void doInBackground(final Void... v) { + if (isCancelled()) return null; if (mDynamicPredictionDictionary.mIsTest) { // If mIsTest == true, wait until the lock is released. mDynamicPredictionDictionary.mBigramListLock.lock(); @@ -306,6 +315,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona } private void doWriteTaskLocked() { + if (isCancelled()) return; + mDynamicPredictionDictionary.mWaitingTask.compareAndSet(this, null); + if (DBG_STRESS_TEST) { try { Log.w(TAG, "Start stress in closing: " + mLocale); diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index a755f90d5..858aa32df 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -48,6 +48,8 @@ public abstract class PersonalizationDictionaryUpdateSession { public abstract void onDictionaryReady(); + public abstract void onDictionaryClosed(); + public void setPredictionDictionary(String locale, DynamicPredictionDictionaryBase dictionary) { mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); mLocale = locale; @@ -68,6 +70,7 @@ public abstract class PersonalizationDictionaryUpdateSession { public void closeSession() { unsetPredictionDictionary(); + onDictionaryClosed(); } public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java index 955bd2762..a038d0ab2 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; import android.content.SharedPreferences; @@ -31,6 +32,6 @@ public class PersonalizationPredictionDictionary extends DynamicPredictionDictio @Override protected String getDictionaryFileName() { - return NAME + "." + getLocale() + ".dict"; + return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java index d11784454..76e48c744 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; import android.content.SharedPreferences; @@ -26,7 +27,8 @@ import android.content.SharedPreferences; * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. */ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase { - private static final String NAME = UserHistoryPredictionDictionary.class.getSimpleName(); + /* package for tests */ static final String NAME = + UserHistoryPredictionDictionary.class.getSimpleName(); /* package */ UserHistoryPredictionDictionary(final Context context, final String locale, final SharedPreferences sp) { super(context, locale, sp, Dictionary.TYPE_USER_HISTORY); @@ -34,6 +36,6 @@ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionary @Override protected String getDictionaryFileName() { - return NAME + "." + getLocale() + ".dict"; + return NAME + "." + getLocale() + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java index d432087d3..20b675257 100644 --- a/java/src/com/android/inputmethod/latin/settings/Settings.java +++ b/java/src/com/android/inputmethod/latin/settings/Settings.java @@ -27,10 +27,10 @@ import com.android.inputmethod.latin.AudioAndHapticFeedbackManager; import com.android.inputmethod.latin.InputAttributes; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils; -import com.android.inputmethod.latin.utils.DebugLogUtils; import com.android.inputmethod.latin.utils.LocaleUtils; import com.android.inputmethod.latin.utils.ResourceUtils; import com.android.inputmethod.latin.utils.RunInLocale; +import com.android.inputmethod.latin.utils.StringUtils; import java.util.HashMap; import java.util.Locale; @@ -90,6 +90,8 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang private static final String PREF_SUPPRESS_LANGUAGE_SWITCH_KEY = "pref_suppress_language_switch_key"; + private static final String PREF_LAST_USED_PERSONALIZATION_TOKEN = + "pref_last_used_personalization_token"; public static final String PREF_SEND_FEEDBACK = "send_feedback"; public static final String PREF_ABOUT_KEYBOARD = "about_keyboard"; @@ -343,4 +345,14 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang return prefs.getBoolean( DebugSettings.PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false); } + + public void writeLastUsedPersonalizationToken(byte[] token) { + final String tokenStr = StringUtils.byteArrayToHexString(token); + mPrefs.edit().putString(PREF_LAST_USED_PERSONALIZATION_TOKEN, tokenStr).apply(); + } + + public byte[] readLastUsedPersonalizationToken() { + final String tokenStr = mPrefs.getString(PREF_LAST_USED_PERSONALIZATION_TOKEN, null); + return StringUtils.hexStringToByteArray(tokenStr); + } } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 195f9f8ef..a0b744dd2 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -57,6 +57,7 @@ public final class SettingsValues { public final SuggestedWords mSuggestPuncList; public final String mWordSeparators; public final CharSequence mHintToSaveText; + public final boolean mCurrentLanguageHasSpaces; // From preferences, in the same order as xml/prefs.xml: public final boolean mAutoCap; @@ -118,6 +119,7 @@ public final class SettingsValues { mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec); mWordSeparators = res.getString(R.string.symbols_word_separators); mHintToSaveText = res.getText(R.string.hint_add_to_dictionary); + mCurrentLanguageHasSpaces = res.getBoolean(R.bool.current_language_has_spaces); // Store the input attributes if (null == inputAttributes) { @@ -186,6 +188,7 @@ public final class SettingsValues { mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec); mWordSeparators = "&\t \n()[]{}*&<>+=|.,;:!?/_\""; mHintToSaveText = "Touch again to save"; + mCurrentLanguageHasSpaces = true; mInputAttributes = new InputAttributes(null, false /* isFullscreenMode */); mAutoCap = true; mVibrateOn = true; diff --git a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java b/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java index 1bb27aa2b..d93b61451 100644 --- a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java +++ b/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java @@ -16,7 +16,7 @@ package com.android.inputmethod.latin.utils; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; /** * This class provides an implementation for the FusionDictionary buffer interface that is backed diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index 0b4838cfc..be4184093 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -357,4 +357,37 @@ public final class StringUtils { } return true; } + + @UsedForTesting + public static String byteArrayToHexString(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return ""; + } + final StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } + + /** + * Convert hex string to byte array. The string length must be an even number. + */ + @UsedForTesting + public static byte[] hexStringToByteArray(String hexString) { + if (TextUtils.isEmpty(hexString)) { + return null; + } + final int N = hexString.length(); + if (N % 2 != 0) { + throw new NumberFormatException("Input hex string length must be an even number." + + " Length = " + N); + } + final byte[] bytes = new byte[N / 2]; + for (int i = 0; i < N; i += 2) { + bytes[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4) + + Character.digit(hexString.charAt(i + 1), 16)); + } + return bytes; + } } diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index a0ad27cfb..cd03b3822 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -19,12 +19,13 @@ package com.android.inputmethod.latin.utils; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoder; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; @@ -62,7 +63,7 @@ public final class UserHistoryDictIOUtils { final FormatOptions formatOptions) { final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); try { - BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions); + BinaryDictEncoder.writeDictionaryBinary(destination, fusionDict, formatOptions); Log.d(TAG, "end writing"); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); @@ -77,7 +78,7 @@ public final class UserHistoryDictIOUtils { @UsedForTesting static FusionDictionary constructFusionDictionary( final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams) { - final FusionDictionary fusionDict = new FusionDictionary(new Node(), + final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, false)); int profTotal = 0; @@ -101,7 +102,7 @@ public final class UserHistoryDictIOUtils { if (word1 == null) { // unigram fusionDict.add(word2, freq, null, false /* isNotAWord */); } else { // bigram - if (FusionDictionary.findWordInTree(fusionDict.mRoot, word1) == null) { + if (FusionDictionary.findWordInTree(fusionDict.mRootNodeArray, word1) == null) { fusionDict.add(word1, 2, null, false /* isNotAWord */); } fusionDict.setBigram(word1, word2, freq); @@ -156,7 +157,7 @@ public final class UserHistoryDictIOUtils { continue; } to.setBigram(word1, word2, - BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency, + BinaryDictDecoder.reconstructBigramFrequency(unigramFrequency, attr.mFrequency)); } } diff --git a/java/src/com/android/inputmethod/research/JsonUtils.java b/java/src/com/android/inputmethod/research/JsonUtils.java index 63d524df7..977f843e9 100644 --- a/java/src/com/android/inputmethod/research/JsonUtils.java +++ b/java/src/com/android/inputmethod/research/JsonUtils.java @@ -75,12 +75,12 @@ import java.util.Map; private static void writeJson(final Key key, final JsonWriter jsonWriter) throws IOException { jsonWriter.beginObject(); - jsonWriter.name("code").value(key.mCode); + jsonWriter.name("code").value(key.getCode()); jsonWriter.name("altCode").value(key.getAltCode()); - jsonWriter.name("x").value(key.mX); - jsonWriter.name("y").value(key.mY); - jsonWriter.name("w").value(key.mWidth); - jsonWriter.name("h").value(key.mHeight); + jsonWriter.name("x").value(key.getX()); + jsonWriter.name("y").value(key.getY()); + jsonWriter.name("w").value(key.getWidth()); + jsonWriter.name("h").value(key.getHeight()); jsonWriter.endObject(); } diff --git a/native/jni/Android.mk b/native/jni/Android.mk index 8d51a2f26..9e7407abd 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -43,7 +43,7 @@ LATIN_IME_JNI_SRC_FILES := \ com_android_inputmethod_keyboard_ProximityInfo.cpp \ com_android_inputmethod_latin_BinaryDictionary.cpp \ com_android_inputmethod_latin_DicTraverseSession.cpp \ - com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \ + com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \ jni_common.cpp LATIN_IME_CORE_SRC_FILES := \ diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp index f78883c2d..457b226b6 100644 --- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp +++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp @@ -14,16 +14,16 @@ * limitations under the License. */ -#define LOG_TAG "LatinIME: jni: BinaryDictInputOutput" +#define LOG_TAG "LatinIME: jni: BinaryDictDecoder" -#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h" +#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h" #include "defines.h" #include "jni.h" #include "jni_common.h" namespace latinime { -static int latinime_BinaryDictInputOutput_doNothing(JNIEnv *env, jclass clazz) { +static int latinime_BinaryDictDecoder_doNothing(JNIEnv *env, jclass clazz) { // This is a phony method for test - it does nothing. It just returns some value // unlikely to be in memory by chance for testing purposes. // TODO: remove this method. @@ -35,13 +35,13 @@ static const JNINativeMethod sMethods[] = { // TODO: remove this entry when we have one useful method in here const_cast<char *>("doNothing"), const_cast<char *>("()I"), - reinterpret_cast<void *>(latinime_BinaryDictInputOutput_doNothing) + reinterpret_cast<void *>(latinime_BinaryDictDecoder_doNothing) }, }; -int register_BinaryDictInputOutput(JNIEnv *env) { +int register_BinaryDictDecoder(JNIEnv *env) { const char *const kClassPathName = - "com/android/inputmethod/latin/makedict/BinaryDictInputOutput"; + "com/android/inputmethod/latin/makedict/BinaryDictDecoder"; return registerNativeMethods(env, kClassPathName, sMethods, NELEMS(sMethods)); } } // namespace latinime diff --git a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h index e622ed4ba..7f3cb67e6 100644 --- a/native/jni/com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h +++ b/native/jni/com_android_inputmethod_latin_makedict_BinaryDictDecoder.h @@ -14,12 +14,12 @@ * limitations under the License. */ -#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H -#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H +#ifndef _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H +#define _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H #include "jni.h" namespace latinime { -int register_BinaryDictInputOutput(JNIEnv *env); +int register_BinaryDictDecoder(JNIEnv *env); } // namespace latinime -#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTINPUTOUTPUT_H +#endif // _COM_ANDROID_INPUTMETHOD_LATIN_MAKEDICT_BINARYDICTDECODER_H diff --git a/native/jni/jni_common.cpp b/native/jni/jni_common.cpp index 733e15f73..d44be6705 100644 --- a/native/jni/jni_common.cpp +++ b/native/jni/jni_common.cpp @@ -23,7 +23,7 @@ #include "com_android_inputmethod_latin_BinaryDictionary.h" #include "com_android_inputmethod_latin_DicTraverseSession.h" #endif -#include "com_android_inputmethod_latin_makedict_BinaryDictInputOutput.h" +#include "com_android_inputmethod_latin_makedict_BinaryDictDecoder.h" #include "defines.h" /* @@ -55,8 +55,8 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { return -1; } #endif - if (!latinime::register_BinaryDictInputOutput(env)) { - AKLOGE("ERROR: BinaryDictInputOutput native registration failed"); + if (!latinime::register_BinaryDictDecoder(env)) { + AKLOGE("ERROR: BinaryDictDecoder native registration failed"); return -1; } /* success -- return valid version number */ diff --git a/native/jni/src/suggest/core/dictionary/probability_utils.h b/native/jni/src/suggest/core/dictionary/probability_utils.h index f450087d8..219213574 100644 --- a/native/jni/src/suggest/core/dictionary/probability_utils.h +++ b/native/jni/src/suggest/core/dictionary/probability_utils.h @@ -41,7 +41,7 @@ class ProbabilityUtils { // the unigram probability to be the median value of the 17th step from the top. A value of // 0 for the bigram probability represents the middle of the 16th step from the top, // while a value of 15 represents the middle of the top step. - // See makedict.BinaryDictInputOutput for details. + // See makedict.BinaryDictDecoder for details. const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability) / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY); return unigramProbability diff --git a/native/jni/src/suggest/policyimpl/dictionary/binary_format.h b/native/jni/src/suggest/policyimpl/dictionary/binary_format.h deleted file mode 100644 index 23f4c7fec..000000000 --- a/native/jni/src/suggest/policyimpl/dictionary/binary_format.h +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef LATINIME_BINARY_FORMAT_H -#define LATINIME_BINARY_FORMAT_H - -#include <stdint.h> - -#include "suggest/core/dictionary/probability_utils.h" -#include "utils/char_utils.h" - -namespace latinime { - -class BinaryFormat { - public: - // Mask and flags for children address type selection. - static const int MASK_GROUP_ADDRESS_TYPE = 0xC0; - - // Flag for single/multiple char group - static const int FLAG_HAS_MULTIPLE_CHARS = 0x20; - - // Flag for terminal groups - static const int FLAG_IS_TERMINAL = 0x10; - - // Flag for shortcut targets presence - static const int FLAG_HAS_SHORTCUT_TARGETS = 0x08; - // Flag for bigram presence - static const int FLAG_HAS_BIGRAMS = 0x04; - // Flag for non-words (typically, shortcut only entries) - static const int FLAG_IS_NOT_A_WORD = 0x02; - // Flag for blacklist - static const int FLAG_IS_BLACKLISTED = 0x01; - - // Attribute (bigram/shortcut) related flags: - // Flag for presence of more attributes - static const int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; - // Flag for sign of offset. If this flag is set, the offset value must be negated. - static const int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; - - // Mask for attribute probability, stored on 4 bits inside the flags byte. - static const int MASK_ATTRIBUTE_PROBABILITY = 0x0F; - - // Mask and flags for attribute address type selection. - static const int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; - - static int getGroupCountAndForwardPointer(const uint8_t *const dict, int *pos); - static uint8_t getFlagsAndForwardPointer(const uint8_t *const dict, int *pos); - static int getCodePointAndForwardPointer(const uint8_t *const dict, int *pos); - static int readProbabilityWithoutMovingPointer(const uint8_t *const dict, const int pos); - static int skipOtherCharacters(const uint8_t *const dict, const int pos); - static int skipChildrenPosition(const uint8_t flags, const int pos); - static int skipProbability(const uint8_t flags, const int pos); - static int skipShortcuts(const uint8_t *const dict, const uint8_t flags, const int pos); - static int skipChildrenPosAndAttributes(const uint8_t *const dict, const uint8_t flags, - const int pos); - static int readChildrenPosition(const uint8_t *const dict, const uint8_t flags, const int pos); - static bool hasChildrenInFlags(const uint8_t flags); - static int getTerminalPosition(const uint8_t *const root, const int *const inWord, - const int length, const bool forceLowerCaseSearch); - static int getCodePointsAndProbabilityAndReturnCodePointCount( - const uint8_t *const root, const int nodePos, const int maxCodePointCount, - int *const outCodePoints, int *const outUnigramProbability); - - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryFormat); - - static const int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; - static const int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; - static const int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; - static const int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; - static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; - static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; - static const int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; - - static const int CHARACTER_ARRAY_TERMINATOR_SIZE = 1; - static const int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; - static const int CHARACTER_ARRAY_TERMINATOR = 0x1F; - static const int MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE = 2; - static const int NO_FLAGS = 0; - static int skipAllAttributes(const uint8_t *const dict, const uint8_t flags, const int pos); - static int skipBigrams(const uint8_t *const dict, const uint8_t flags, const int pos); -}; - -AK_FORCE_INLINE int BinaryFormat::getGroupCountAndForwardPointer(const uint8_t *const dict, - int *pos) { - const int msb = dict[(*pos)++]; - if (msb < 0x80) return msb; - return ((msb & 0x7F) << 8) | dict[(*pos)++]; -} - -inline uint8_t BinaryFormat::getFlagsAndForwardPointer(const uint8_t *const dict, int *pos) { - return dict[(*pos)++]; -} - -AK_FORCE_INLINE int BinaryFormat::getCodePointAndForwardPointer(const uint8_t *const dict, - int *pos) { - const int origin = *pos; - const int codePoint = dict[origin]; - if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { - if (codePoint == CHARACTER_ARRAY_TERMINATOR) { - *pos = origin + 1; - return NOT_A_CODE_POINT; - } else { - *pos = origin + 3; - const int char_1 = codePoint << 16; - const int char_2 = char_1 + (dict[origin + 1] << 8); - return char_2 + dict[origin + 2]; - } - } else { - *pos = origin + 1; - return codePoint; - } -} - -inline int BinaryFormat::readProbabilityWithoutMovingPointer(const uint8_t *const dict, - const int pos) { - return dict[pos]; -} - -AK_FORCE_INLINE int BinaryFormat::skipOtherCharacters(const uint8_t *const dict, const int pos) { - int currentPos = pos; - int character = dict[currentPos++]; - while (CHARACTER_ARRAY_TERMINATOR != character) { - if (character < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { - currentPos += MULTIPLE_BYTE_CHARACTER_ADDITIONAL_SIZE; - } - character = dict[currentPos++]; - } - return currentPos; -} - -static inline int attributeAddressSize(const uint8_t flags) { - static const int ATTRIBUTE_ADDRESS_SHIFT = 4; - return (flags & BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) >> ATTRIBUTE_ADDRESS_SHIFT; - /* Note: this is a value-dependant optimization of what may probably be - more readably written this way: - switch (flags * BinaryFormat::MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: return 1; - case FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: return 2; - case FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTE: return 3; - default: return 0; - } - */ -} - -static AK_FORCE_INLINE int skipExistingBigrams(const uint8_t *const dict, const int pos) { - int currentPos = pos; - uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(dict, ¤tPos); - while (flags & BinaryFormat::FLAG_ATTRIBUTE_HAS_NEXT) { - currentPos += attributeAddressSize(flags); - flags = BinaryFormat::getFlagsAndForwardPointer(dict, ¤tPos); - } - currentPos += attributeAddressSize(flags); - return currentPos; -} - -static inline int childrenAddressSize(const uint8_t flags) { - static const int CHILDREN_ADDRESS_SHIFT = 6; - return (BinaryFormat::MASK_GROUP_ADDRESS_TYPE & flags) >> CHILDREN_ADDRESS_SHIFT; - /* See the note in attributeAddressSize. The same applies here */ -} - -static AK_FORCE_INLINE int shortcutByteSize(const uint8_t *const dict, const int pos) { - return (static_cast<int>(dict[pos] << 8)) + (dict[pos + 1]); -} - -inline int BinaryFormat::skipChildrenPosition(const uint8_t flags, const int pos) { - return pos + childrenAddressSize(flags); -} - -inline int BinaryFormat::skipProbability(const uint8_t flags, const int pos) { - return FLAG_IS_TERMINAL & flags ? pos + 1 : pos; -} - -AK_FORCE_INLINE int BinaryFormat::skipShortcuts(const uint8_t *const dict, const uint8_t flags, - const int pos) { - if (FLAG_HAS_SHORTCUT_TARGETS & flags) { - return pos + shortcutByteSize(dict, pos); - } else { - return pos; - } -} - -AK_FORCE_INLINE int BinaryFormat::skipBigrams(const uint8_t *const dict, const uint8_t flags, - const int pos) { - if (FLAG_HAS_BIGRAMS & flags) { - return skipExistingBigrams(dict, pos); - } else { - return pos; - } -} - -AK_FORCE_INLINE int BinaryFormat::skipAllAttributes(const uint8_t *const dict, const uint8_t flags, - const int pos) { - // This function skips all attributes: shortcuts and bigrams. - int newPos = pos; - newPos = skipShortcuts(dict, flags, newPos); - newPos = skipBigrams(dict, flags, newPos); - return newPos; -} - -AK_FORCE_INLINE int BinaryFormat::skipChildrenPosAndAttributes(const uint8_t *const dict, - const uint8_t flags, const int pos) { - int currentPos = pos; - currentPos = skipChildrenPosition(flags, currentPos); - currentPos = skipAllAttributes(dict, flags, currentPos); - return currentPos; -} - -AK_FORCE_INLINE int BinaryFormat::readChildrenPosition(const uint8_t *const dict, - const uint8_t flags, const int pos) { - int offset = 0; - switch (MASK_GROUP_ADDRESS_TYPE & flags) { - case FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - offset = dict[pos]; - break; - case FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - offset = dict[pos] << 8; - offset += dict[pos + 1]; - break; - case FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - offset = dict[pos] << 16; - offset += dict[pos + 1] << 8; - offset += dict[pos + 2]; - break; - default: - // If we come here, it means we asked for the children of a word with - // no children. - return -1; - } - return pos + offset; -} - -inline bool BinaryFormat::hasChildrenInFlags(const uint8_t flags) { - return (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != (MASK_GROUP_ADDRESS_TYPE & flags)); -} - -// This function gets the byte position of the last chargroup of the exact matching word in the -// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS. -AK_FORCE_INLINE int BinaryFormat::getTerminalPosition(const uint8_t *const root, - const int *const inWord, const int length, const bool forceLowerCaseSearch) { - int pos = 0; - int wordPos = 0; - - while (true) { - // If we already traversed the tree further than the word is long, there means - // there was no match (or we would have found it). - if (wordPos >= length) return NOT_A_VALID_WORD_POS; - int charGroupCount = BinaryFormat::getGroupCountAndForwardPointer(root, &pos); - const int wChar = forceLowerCaseSearch - ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos]; - while (true) { - // If there are no more character groups in this node, it means we could not - // find a matching character for this depth, therefore there is no match. - if (0 >= charGroupCount) return NOT_A_VALID_WORD_POS; - const int charGroupPos = pos; - const uint8_t flags = BinaryFormat::getFlagsAndForwardPointer(root, &pos); - int character = BinaryFormat::getCodePointAndForwardPointer(root, &pos); - if (character == wChar) { - // This is the correct node. Only one character group may start with the same - // char within a node, so either we found our match in this node, or there is - // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the - // characters in this character group indeed does match. - if (FLAG_HAS_MULTIPLE_CHARS & flags) { - character = BinaryFormat::getCodePointAndForwardPointer(root, &pos); - while (NOT_A_CODE_POINT != character) { - ++wordPos; - // If we shoot the length of the word we search for, or if we find a single - // character that does not match, as explained above, it means the word is - // not in the dictionary (by virtue of this chargroup being the only one to - // match the word on the first character, but not matching the whole word). - if (wordPos >= length) return NOT_A_VALID_WORD_POS; - if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS; - character = BinaryFormat::getCodePointAndForwardPointer(root, &pos); - } - } - // If we come here we know that so far, we do match. Either we are on a terminal - // and we match the length, in which case we found it, or we traverse children. - // If we don't match the length AND don't have children, then a word in the - // dictionary fully matches a prefix of the searched word but not the full word. - ++wordPos; - if (FLAG_IS_TERMINAL & flags) { - if (wordPos == length) { - return charGroupPos; - } - pos = BinaryFormat::skipProbability(FLAG_IS_TERMINAL, pos); - } - if (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS == (MASK_GROUP_ADDRESS_TYPE & flags)) { - return NOT_A_VALID_WORD_POS; - } - // We have children and we are still shorter than the word we are searching for, so - // we need to traverse children. Put the pointer on the children position, and - // break - pos = BinaryFormat::readChildrenPosition(root, flags, pos); - break; - } else { - // This chargroup does not match, so skip the remaining part and go to the next. - if (FLAG_HAS_MULTIPLE_CHARS & flags) { - pos = BinaryFormat::skipOtherCharacters(root, pos); - } - pos = BinaryFormat::skipProbability(flags, pos); - pos = BinaryFormat::skipChildrenPosAndAttributes(root, flags, pos); - } - --charGroupCount; - } - } -} - -// This function searches for a terminal in the dictionary by its address. -// Due to the fact that words are ordered in the dictionary in a strict breadth-first order, -// it is possible to check for this with advantageous complexity. For each node, we search -// for groups with children and compare the children address with the address we look for. -// When we shoot the address we look for, it means the word we look for is in the children -// of the previous group. The only tricky part is the fact that if we arrive at the end of a -// node with the last group's children address still less than what we are searching for, we -// must descend the last group's children (for example, if the word we are searching for starts -// with a z, it's the last group of the root node, so all children addresses will be smaller -// than the address we look for, and we have to descend the z node). -/* Parameters : - * root: the dictionary buffer - * address: the byte position of the last chargroup of the word we are searching for (this is - * what is stored as the "bigram address" in each bigram) - * outword: an array to write the found word, with MAX_WORD_LENGTH size. - * outUnigramProbability: a pointer to an int to write the probability into. - * Return value : the length of the word, of 0 if the word was not found. - */ -AK_FORCE_INLINE int BinaryFormat::getCodePointsAndProbabilityAndReturnCodePointCount( - const uint8_t *const root, const int nodePos, const int maxCodePointCount, - int *const outCodePoints, int *const outUnigramProbability) { - int pos = 0; - int wordPos = 0; - - // One iteration of the outer loop iterates through nodes. As stated above, we will only - // traverse nodes that are actually a part of the terminal we are searching, so each time - // we enter this loop we are one depth level further than last time. - // The only reason we count nodes is because we want to reduce the probability of infinite - // looping in case there is a bug. Since we know there is an upper bound to the depth we are - // supposed to traverse, it does not hurt to count iterations. - for (int loopCount = maxCodePointCount; loopCount > 0; --loopCount) { - int lastCandidateGroupPos = 0; - // Let's loop through char groups in this node searching for either the terminal - // or one of its ascendants. - for (int charGroupCount = getGroupCountAndForwardPointer(root, &pos); charGroupCount > 0; - --charGroupCount) { - const int startPos = pos; - const uint8_t flags = getFlagsAndForwardPointer(root, &pos); - const int character = getCodePointAndForwardPointer(root, &pos); - if (nodePos == startPos) { - // We found the address. Copy the rest of the word in the buffer and return - // the length. - outCodePoints[wordPos] = character; - if (FLAG_HAS_MULTIPLE_CHARS & flags) { - int nextChar = getCodePointAndForwardPointer(root, &pos); - // We count chars in order to avoid infinite loops if the file is broken or - // if there is some other bug - int charCount = maxCodePointCount; - while (NOT_A_CODE_POINT != nextChar && --charCount > 0) { - outCodePoints[++wordPos] = nextChar; - nextChar = getCodePointAndForwardPointer(root, &pos); - } - } - *outUnigramProbability = readProbabilityWithoutMovingPointer(root, pos); - return ++wordPos; - } - // We need to skip past this char group, so skip any remaining chars after the - // first and possibly the probability. - if (FLAG_HAS_MULTIPLE_CHARS & flags) { - pos = skipOtherCharacters(root, pos); - } - pos = skipProbability(flags, pos); - - // The fact that this group has children is very important. Since we already know - // that this group does not match, if it has no children we know it is irrelevant - // to what we are searching for. - const bool hasChildren = (FLAG_GROUP_ADDRESS_TYPE_NOADDRESS != - (MASK_GROUP_ADDRESS_TYPE & flags)); - // We will write in `found' whether we have passed the children address we are - // searching for. For example if we search for "beer", the children of b are less - // than the address we are searching for and the children of c are greater. When we - // come here for c, we realize this is too big, and that we should descend b. - bool found; - if (hasChildren) { - // Here comes the tricky part. First, read the children position. - const int childrenPos = readChildrenPosition(root, flags, pos); - if (childrenPos > nodePos) { - // If the children pos is greater than address, it means the previous chargroup, - // which address is stored in lastCandidateGroupPos, was the right one. - found = true; - } else if (1 >= charGroupCount) { - // However if we are on the LAST group of this node, and we have NOT shot the - // address we should descend THIS node. So we trick the lastCandidateGroupPos - // so that we will descend this node, not the previous one. - lastCandidateGroupPos = startPos; - found = true; - } else { - // Else, we should continue looking. - found = false; - } - } else { - // Even if we don't have children here, we could still be on the last group of this - // node. If this is the case, we should descend the last group that had children, - // and their address is already in lastCandidateGroup. - found = (1 >= charGroupCount); - } - - if (found) { - // Okay, we found the group we should descend. Its address is in - // the lastCandidateGroupPos variable, so we just re-read it. - if (0 != lastCandidateGroupPos) { - const uint8_t lastFlags = - getFlagsAndForwardPointer(root, &lastCandidateGroupPos); - const int lastChar = - getCodePointAndForwardPointer(root, &lastCandidateGroupPos); - // We copy all the characters in this group to the buffer - outCodePoints[wordPos] = lastChar; - if (FLAG_HAS_MULTIPLE_CHARS & lastFlags) { - int nextChar = getCodePointAndForwardPointer(root, &lastCandidateGroupPos); - int charCount = maxCodePointCount; - while (-1 != nextChar && --charCount > 0) { - outCodePoints[++wordPos] = nextChar; - nextChar = getCodePointAndForwardPointer(root, &lastCandidateGroupPos); - } - } - ++wordPos; - // Now we only need to branch to the children address. Skip the probability if - // it's there, read pos, and break to resume the search at pos. - lastCandidateGroupPos = skipProbability(lastFlags, lastCandidateGroupPos); - pos = readChildrenPosition(root, lastFlags, lastCandidateGroupPos); - break; - } else { - // Here is a little tricky part: we come here if we found out that all children - // addresses in this group are bigger than the address we are searching for. - // Should we conclude the word is not in the dictionary? No! It could still be - // one of the remaining chargroups in this node, so we have to keep looking in - // this node until we find it (or we realize it's not there either, in which - // case it's actually not in the dictionary). Pass the end of this group, ready - // to start the next one. - pos = skipChildrenPosAndAttributes(root, flags, pos); - } - } else { - // If we did not find it, we should record the last children address for the next - // iteration. - if (hasChildren) lastCandidateGroupPos = startPos; - // Now skip the end of this group (children pos and the attributes if any) so that - // our pos is after the end of this char group, at the start of the next one. - pos = skipChildrenPosAndAttributes(root, flags, pos); - } - - } - } - // If we have looked through all the chargroups and found no match, the address is - // not the address of a terminal in this dictionary. - return 0; -} - -} // namespace latinime -#endif // LATINIME_BINARY_FORMAT_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp index 3e664a29b..15eb0674d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp @@ -20,7 +20,6 @@ #include "defines.h" #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_vector.h" -#include "suggest/policyimpl/dictionary/binary_format.h" #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" namespace latinime { @@ -38,17 +37,273 @@ void PatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, } } +// This retrieves code points and the probability of the word by its terminal position. +// Due to the fact that words are ordered in the dictionary in a strict breadth-first order, +// it is possible to check for this with advantageous complexity. For each node, we search +// for groups with children and compare the children position with the position we look for. +// When we shoot the position we look for, it means the word we look for is in the children +// of the previous group. The only tricky part is the fact that if we arrive at the end of a +// node with the last group's children position still less than what we are searching for, we +// must descend the last group's children (for example, if the word we are searching for starts +// with a z, it's the last group of the root node, so all children addresses will be smaller +// than the position we look for, and we have to descend the z node). +/* Parameters : + * nodePos: the byte position of the terminal chargroup of the word we are searching for (this is + * what is stored as the "bigram position" in each bigram) + * outCodePoints: an array to write the found word, with MAX_WORD_LENGTH size. + * outUnigramProbability: a pointer to an int to write the probability into. + * Return value : the code point count, of 0 if the word was not found. + */ +// TODO: Split this function to be more readable int PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( const int nodePos, const int maxCodePointCount, int *const outCodePoints, int *const outUnigramProbability) const { - return BinaryFormat::getCodePointsAndProbabilityAndReturnCodePointCount(mDictRoot, nodePos, - maxCodePointCount, outCodePoints, outUnigramProbability); + int pos = getRootPosition(); + int wordPos = 0; + // One iteration of the outer loop iterates through nodes. As stated above, we will only + // traverse nodes that are actually a part of the terminal we are searching, so each time + // we enter this loop we are one depth level further than last time. + // The only reason we count nodes is because we want to reduce the probability of infinite + // looping in case there is a bug. Since we know there is an upper bound to the depth we are + // supposed to traverse, it does not hurt to count iterations. + for (int loopCount = maxCodePointCount; loopCount > 0; --loopCount) { + int lastCandidateGroupPos = 0; + // Let's loop through char groups in this node searching for either the terminal + // or one of its ascendants. + for (int charGroupCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition( + mDictRoot, &pos); charGroupCount > 0; --charGroupCount) { + const int startPos = pos; + const PatriciaTrieReadingUtils::NodeFlags flags = + PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos); + const int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &pos); + if (nodePos == startPos) { + // We found the position. Copy the rest of the code points in the buffer and return + // the length. + outCodePoints[wordPos] = character; + if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) { + int nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &pos); + // We count code points in order to avoid infinite loops if the file is broken + // or if there is some other bug + int charCount = maxCodePointCount; + while (NOT_A_CODE_POINT != nextChar && --charCount > 0) { + outCodePoints[++wordPos] = nextChar; + nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &pos); + } + } + *outUnigramProbability = + PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, + &pos); + return ++wordPos; + } + // We need to skip past this char group, so skip any remaining code points after the + // first and possibly the probability. + if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) { + PatriciaTrieReadingUtils::skipCharacters(mDictRoot, flags, MAX_WORD_LENGTH, &pos); + } + if (PatriciaTrieReadingUtils::isTerminal(flags)) { + PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos); + } + // The fact that this group has children is very important. Since we already know + // that this group does not match, if it has no children we know it is irrelevant + // to what we are searching for. + const bool hasChildren = PatriciaTrieReadingUtils::hasChildrenInFlags(flags); + // We will write in `found' whether we have passed the children position we are + // searching for. For example if we search for "beer", the children of b are less + // than the address we are searching for and the children of c are greater. When we + // come here for c, we realize this is too big, and that we should descend b. + bool found; + if (hasChildren) { + int currentPos = pos; + // Here comes the tricky part. First, read the children position. + const int childrenPos = PatriciaTrieReadingUtils + ::readChildrenPositionAndAdvancePosition(mDictRoot, flags, ¤tPos); + if (childrenPos > nodePos) { + // If the children pos is greater than the position, it means the previous + // chargroup, which position is stored in lastCandidateGroupPos, was the right + // one. + found = true; + } else if (1 >= charGroupCount) { + // However if we are on the LAST group of this node, and we have NOT shot the + // position we should descend THIS node. So we trick the lastCandidateGroupPos + // so that we will descend this node, not the previous one. + lastCandidateGroupPos = startPos; + found = true; + } else { + // Else, we should continue looking. + found = false; + } + } else { + // Even if we don't have children here, we could still be on the last group of this + // node. If this is the case, we should descend the last group that had children, + // and their position is already in lastCandidateGroup. + found = (1 >= charGroupCount); + } + + if (found) { + // Okay, we found the group we should descend. Its position is in + // the lastCandidateGroupPos variable, so we just re-read it. + if (0 != lastCandidateGroupPos) { + const PatriciaTrieReadingUtils::NodeFlags lastFlags = + PatriciaTrieReadingUtils::getFlagsAndAdvancePosition( + mDictRoot, &lastCandidateGroupPos); + const int lastChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &lastCandidateGroupPos); + // We copy all the characters in this group to the buffer + outCodePoints[wordPos] = lastChar; + if (PatriciaTrieReadingUtils::hasMultipleChars(lastFlags)) { + int nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &lastCandidateGroupPos); + int charCount = maxCodePointCount; + while (-1 != nextChar && --charCount > 0) { + outCodePoints[++wordPos] = nextChar; + nextChar = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &lastCandidateGroupPos); + } + } + ++wordPos; + // Now we only need to branch to the children address. Skip the probability if + // it's there, read pos, and break to resume the search at pos. + if (PatriciaTrieReadingUtils::isTerminal(lastFlags)) { + PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, + &lastCandidateGroupPos); + } + pos = PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( + mDictRoot, lastFlags, &lastCandidateGroupPos); + break; + } else { + // Here is a little tricky part: we come here if we found out that all children + // addresses in this group are bigger than the address we are searching for. + // Should we conclude the word is not in the dictionary? No! It could still be + // one of the remaining chargroups in this node, so we have to keep looking in + // this node until we find it (or we realize it's not there either, in which + // case it's actually not in the dictionary). Pass the end of this group, ready + // to start the next one. + if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) { + PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( + mDictRoot, flags, &pos); + } + if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) { + mShortcutListPolicy.skipAllShortcuts(&pos); + } + if (PatriciaTrieReadingUtils::hasBigrams(flags)) { + mBigramListPolicy.skipAllBigrams(&pos); + } + } + } else { + // If we did not find it, we should record the last children address for the next + // iteration. + if (hasChildren) lastCandidateGroupPos = startPos; + // Now skip the end of this group (children pos and the attributes if any) so that + // our pos is after the end of this char group, at the start of the next one. + if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) { + PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( + mDictRoot, flags, &pos); + } + if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) { + mShortcutListPolicy.skipAllShortcuts(&pos); + } + if (PatriciaTrieReadingUtils::hasBigrams(flags)) { + mBigramListPolicy.skipAllBigrams(&pos); + } + } + + } + } + // If we have looked through all the chargroups and found no match, the nodePos is + // not the position of a terminal in this dictionary. + return 0; } +// This function gets the position of the terminal node of the exact matching word in the +// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS. int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) const { - return BinaryFormat::getTerminalPosition(mDictRoot, inWord, - length, forceLowerCaseSearch); + int pos = getRootPosition(); + int wordPos = 0; + + while (true) { + // If we already traversed the tree further than the word is long, there means + // there was no match (or we would have found it). + if (wordPos >= length) return NOT_A_VALID_WORD_POS; + int charGroupCount = PatriciaTrieReadingUtils::getGroupCountAndAdvancePosition(mDictRoot, + &pos); + const int wChar = forceLowerCaseSearch + ? CharUtils::toLowerCase(inWord[wordPos]) : inWord[wordPos]; + while (true) { + // If there are no more character groups in this node, it means we could not + // find a matching character for this depth, therefore there is no match. + if (0 >= charGroupCount) return NOT_A_VALID_WORD_POS; + const int charGroupPos = pos; + const PatriciaTrieReadingUtils::NodeFlags flags = + PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos); + int character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot, + &pos); + if (character == wChar) { + // This is the correct node. Only one character group may start with the same + // char within a node, so either we found our match in this node, or there is + // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the + // characters in this character group indeed does match. + if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) { + character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot, + &pos); + while (NOT_A_CODE_POINT != character) { + ++wordPos; + // If we shoot the length of the word we search for, or if we find a single + // character that does not match, as explained above, it means the word is + // not in the dictionary (by virtue of this chargroup being the only one to + // match the word on the first character, but not matching the whole word). + if (wordPos >= length) return NOT_A_VALID_WORD_POS; + if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS; + character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( + mDictRoot, &pos); + } + } + // If we come here we know that so far, we do match. Either we are on a terminal + // and we match the length, in which case we found it, or we traverse children. + // If we don't match the length AND don't have children, then a word in the + // dictionary fully matches a prefix of the searched word but not the full word. + ++wordPos; + if (PatriciaTrieReadingUtils::isTerminal(flags)) { + if (wordPos == length) { + return charGroupPos; + } + PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos); + } + if (!PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) { + return NOT_A_VALID_WORD_POS; + } + // We have children and we are still shorter than the word we are searching for, so + // we need to traverse children. Put the pointer on the children position, and + // break + pos = PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot, + flags, &pos); + break; + } else { + // This chargroup does not match, so skip the remaining part and go to the next. + if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) { + PatriciaTrieReadingUtils::skipCharacters(mDictRoot, flags, MAX_WORD_LENGTH, + &pos); + } + if (PatriciaTrieReadingUtils::isTerminal(flags)) { + PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos); + } + if (PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) { + PatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(mDictRoot, + flags, &pos); + } + if (PatriciaTrieReadingUtils::hasShortcutTargets(flags)) { + mShortcutListPolicy.skipAllShortcuts(&pos); + } + if (PatriciaTrieReadingUtils::hasBigrams(flags)) { + mBigramListPolicy.skipAllBigrams(&pos); + } + } + --charGroupCount; + } + } } int PatriciaTriePolicy::getUnigramProbability(const int nodePos) const { diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h index e3aabb084..08add03c6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmaped_buffer.h @@ -48,13 +48,13 @@ class MmapedBuffer { close(fd); return 0; } - uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + bufOffset; + uint8_t *const buffer = static_cast<uint8_t *>(mmapedBuffer) + offset; if (!buffer) { AKLOGE("DICT: buffer is null"); close(fd); return 0; } - return new MmapedBuffer(buffer, adjSize, fd, adjOffset, isUpdatable); + return new MmapedBuffer(buffer, adjSize, fd, offset, isUpdatable); } ~MmapedBuffer() { diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index 65dfd2dde..cadd0f8f3 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -20,7 +20,7 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import java.util.HashMap; @@ -30,21 +30,21 @@ import java.util.HashMap; @SmallTest public class FusionDictionaryTests extends AndroidTestCase { public void testFindWordInTree() { - FusionDictionary dict = new FusionDictionary(new Node(), + FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); dict.add("abc", 10, null, false /* isNotAWord */); - assertNull(FusionDictionary.findWordInTree(dict.mRoot, "aaa")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "abc")); + assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc")); dict.add("aa", 10, null, false /* isNotAWord */); - assertNull(FusionDictionary.findWordInTree(dict.mRoot, "aaa")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "aa")); + assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa")); dict.add("babcd", 10, null, false /* isNotAWord */); dict.add("bacde", 10, null, false /* isNotAWord */); - assertNull(FusionDictionary.findWordInTree(dict.mRoot, "ba")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "babcd")); - assertNotNull(FusionDictionary.findWordInTree(dict.mRoot, "bacde")); + assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd")); + assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "bacde")); } } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java index d27a7a903..6cc4befae 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin; import android.test.suitebuilder.annotation.LargeTest; +import android.view.inputmethod.BaseInputConnection; @LargeTest public class InputLogicTests extends InputTestsBase { @@ -290,5 +291,19 @@ public class InputLogicTests extends InputTestsBase { } assertEquals("delete whole composing word", "", mEditText.getText().toString()); } + + public void testResumeSuggestionOnBackspace() { + final String WORD_TO_TYPE = "and this "; + type(WORD_TO_TYPE); + assertEquals("resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + type(Constants.CODE_DELETE); + assertEquals("resume suggestion on backspace", 4, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("resume suggestion on backspace", 8, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } // TODO: Add some tests for non-BMP characters } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java new file mode 100644 index 000000000..0f0ebafb9 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/InputLogicTestsLanguageWithoutSpaces.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.suitebuilder.annotation.LargeTest; +import android.view.inputmethod.BaseInputConnection; + +import com.android.inputmethod.latin.suggestions.SuggestionStripView; + +@LargeTest +public class InputLogicTestsLanguageWithoutSpaces extends InputTestsBase { + public void testAutoCorrectForLanguageWithoutSpaces() { + final String STRING_TO_TYPE = "tgis is"; + final String EXPECTED_RESULT = "thisis"; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(STRING_TO_TYPE); + assertEquals("simple auto-correct for language without spaces", EXPECTED_RESULT, + mEditText.getText().toString()); + } + + public void testRevertAutoCorrectForLanguageWithoutSpaces() { + final String STRING_TO_TYPE = "tgis "; + final String EXPECTED_INTERMEDIATE_RESULT = "this"; + final String EXPECTED_FINAL_RESULT = "tgis"; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(STRING_TO_TYPE); + assertEquals("simple auto-correct for language without spaces", + EXPECTED_INTERMEDIATE_RESULT, mEditText.getText().toString()); + type(Constants.CODE_DELETE); + assertEquals("simple auto-correct for language without spaces", + EXPECTED_FINAL_RESULT, mEditText.getText().toString()); + // Check we are back to composing the word + assertEquals("don't resume suggestion on backspace", 0, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("don't resume suggestion on backspace", 4, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testDontResumeSuggestionOnBackspace() { + final String WORD_TO_TYPE = "and this "; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(WORD_TO_TYPE); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + type(" "); + type(Constants.CODE_DELETE); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("don't resume suggestion on backspace", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testStartComposingInsideText() { + final String WORD_TO_TYPE = "abcdefgh "; + final int typedLength = WORD_TO_TYPE.length() - 1; // -1 because space gets eaten + final int CURSOR_POS = 4; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(WORD_TO_TYPE); + mLatinIME.onUpdateSelection(0, 0, typedLength, typedLength, -1, -1); + mInputConnection.setSelection(CURSOR_POS, CURSOR_POS); + mLatinIME.onUpdateSelection(typedLength, typedLength, + CURSOR_POS, CURSOR_POS, -1, -1); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + assertEquals("start composing inside text", -1, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("start composing inside text", -1, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + type("xxxx"); + assertEquals("start composing inside text", 4, + BaseInputConnection.getComposingSpanStart(mEditText.getText())); + assertEquals("start composing inside text", 8, + BaseInputConnection.getComposingSpanEnd(mEditText.getText())); + } + + public void testPredictions() { + final String WORD_TO_TYPE = "Barack "; + changeKeyboardLocaleAndDictLocale("th", "en_US"); + type(WORD_TO_TYPE); + sleep(DELAY_TO_WAIT_FOR_PREDICTIONS); + runMessages(); + // Make sure there is no space + assertEquals("predictions in lang without spaces", "Barack", + mEditText.getText().toString()); + // Test the first prediction is displayed + assertEquals("predictions in lang without spaces", "Obama", + mLatinIME.getFirstSuggestedWord()); + } +} diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java index ead134b73..480570e62 100644 --- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java +++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java @@ -46,6 +46,8 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { // The message that sets the underline is posted with a 100 ms delay protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 200; + // The message that sets predictions is posted with a 100 ms delay + protected static final int DELAY_TO_WAIT_FOR_PREDICTIONS = 200; protected LatinIME mLatinIME; protected Keyboard mKeyboard; @@ -207,8 +209,8 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { //mLatinIME.onPressKey(codePoint, 0 /* repeatCount */, true /* isSinglePointer */); final Key key = mKeyboard.getKey(codePoint); if (key != null) { - final int x = key.mX + key.mWidth / 2; - final int y = key.mY + key.mHeight / 2; + final int x = key.getX() + key.getWidth() / 2; + final int y = key.getY() + key.getHeight() / 2; mLatinIME.onCodeInput(codePoint, x, y); return; } @@ -233,9 +235,6 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { --remainingAttempts; } } - if (!mLatinIME.hasMainDictionary()) { - throw new RuntimeException("Can't initialize the main dictionary"); - } } protected void changeLanguage(final String locale) { @@ -247,6 +246,16 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { waitForDictionaryToBeLoaded(); } + protected void changeKeyboardLocaleAndDictLocale(final String keyboardLocale, + final String dictLocale) { + changeLanguage(keyboardLocale); + if (!keyboardLocale.equals(dictLocale)) { + mLatinIME.replaceMainDictionaryForTest( + LocaleUtils.constructLocaleFromString(dictLocale)); + } + waitForDictionaryToBeLoaded(); + } + protected void pickSuggestionManually(final int index, final String suggestion) { mLatinIME.pickSuggestionManually(index, new SuggestedWordInfo(suggestion, 1, SuggestedWordInfo.KIND_CORRECTION, "main")); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 7bfd6032a..be468c19b 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -22,10 +22,10 @@ import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; import android.util.SparseArray; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -44,11 +44,11 @@ import java.util.Random; import java.util.Set; /** - * Unit tests for BinaryDictInputOutput + * Unit tests for BinaryDictDecoder and BinaryDictEncoder. */ @LargeTest -public class BinaryDictIOTests extends AndroidTestCase { - private static final String TAG = BinaryDictIOTests.class.getSimpleName(); +public class BinaryDictDecoderEncoderTests extends AndroidTestCase { + private static final String TAG = BinaryDictDecoderEncoderTests.class.getSimpleName(); private static final int DEFAULT_MAX_UNIGRAMS = 100; private static final int DEFAULT_CODE_POINT_SET_SIZE = 50; private static final int UNIGRAM_FREQ = 10; @@ -71,11 +71,13 @@ public class BinaryDictIOTests extends AndroidTestCase { private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); - public BinaryDictIOTests() { + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + + public BinaryDictDecoderEncoderTests() { this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); } - public BinaryDictIOTests(final long seed, final int maxUnigrams) { + public BinaryDictDecoderEncoderTests(final long seed, final int maxUnigrams) { super(); Log.e(TAG, "Testing dictionary: seed is " + seed); final Random random = new Random(seed); @@ -206,7 +208,7 @@ public class BinaryDictIOTests extends AndroidTestCase { // If you need to dump the dict to a textual file, uncomment the line below and the // function above // dumpToCombinedFileForDebug(file, "/tmp/foo"); - BinaryDictInputOutput.writeDictionaryBinary(out, dict, formatOptions); + BinaryDictEncoder.writeDictionaryBinary(out, dict, formatOptions); diff = System.currentTimeMillis() - now; out.flush(); @@ -226,7 +228,7 @@ public class BinaryDictIOTests extends AndroidTestCase { // check unigram for (final String word : words) { - final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, word); + final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); assertNotNull(cg); } @@ -234,7 +236,8 @@ public class BinaryDictIOTests extends AndroidTestCase { for (int i = 0; i < bigrams.size(); ++i) { final int w1 = bigrams.keyAt(i); for (final int w2 : bigrams.valueAt(i)) { - final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, words.get(w1)); + final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray, + words.get(w1)); assertNotNull(words.get(w1) + "," + words.get(w2), cg.getBigram(words.get(w2))); } } @@ -242,7 +245,8 @@ public class BinaryDictIOTests extends AndroidTestCase { // check shortcut if (shortcutMap != null) { for (final Map.Entry<String, List<String>> entry : shortcutMap.entrySet()) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, entry.getKey()); + final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray, + entry.getKey()); for (final String word : entry.getValue()) { assertNotNull("shortcut not found: " + entry.getKey() + ", " + word, group.getShortcut(word)); @@ -272,7 +276,7 @@ public class BinaryDictIOTests extends AndroidTestCase { getBuffer(reader, bufferType); assertNotNull(reader.getBuffer()); now = System.currentTimeMillis(); - dict = BinaryDictInputOutput.readDictionaryBinary(reader, null); + dict = BinaryDictDecoder.readDictionaryBinary(reader, null); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException while reading dictionary", e); @@ -291,13 +295,14 @@ public class BinaryDictIOTests extends AndroidTestCase { final String message) { File file = null; try { - file = File.createTempFile("runReadAndWrite", ".dict", getContext().getCacheDir()); + file = File.createTempFile("runReadAndWrite", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { Log.e(TAG, "IOException", e); } assertNotNull(file); - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); @@ -383,7 +388,7 @@ public class BinaryDictIOTests extends AndroidTestCase { } actBigrams.get(word1).add(word2); - final int bigramFreq = BinaryDictInputOutput.reconstructBigramFrequency( + final int bigramFreq = BinaryDictDecoder.reconstructBigramFrequency( unigramFreq, attr.mFrequency); assertTrue(Math.abs(bigramFreq - BIGRAM_FREQ) < TOLERANCE_OF_BIGRAM_FREQ); } @@ -433,14 +438,15 @@ public class BinaryDictIOTests extends AndroidTestCase { final FormatSpec.FormatOptions formatOptions, final String message) { File file = null; try { - file = File.createTempFile("runReadUnigrams", ".dict", getContext().getCacheDir()); + file = File.createTempFile("runReadUnigrams", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { Log.e(TAG, "IOException", e); } assertNotNull(file); // making the dictionary from lists of words. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( new HashMap<String, String>(), false, false)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); @@ -491,31 +497,31 @@ public class BinaryDictIOTests extends AndroidTestCase { } // Tests for getTerminalPosition - private String getWordFromBinary(final FusionDictionaryBufferInterface buffer, - final int address) { + private String getWordFromBinary(final BinaryDictReader dictReader, final int address) { + final FusionDictionaryBufferInterface buffer = dictReader.getBuffer(); if (buffer.position() != 0) buffer.position(0); - FileHeader header = null; + FileHeader fileHeader = null; try { - header = BinaryDictInputOutput.readHeader(buffer); + fileHeader = BinaryDictDecoder.readHeader(dictReader); } catch (IOException e) { return null; } catch (UnsupportedFormatException e) { return null; } - if (header == null) return null; - return BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize, - address - header.mHeaderSize, header.mFormatOptions).mWord; + if (fileHeader == null) return null; + return BinaryDictDecoder.getWordAtAddress(buffer, fileHeader.mHeaderSize, + address - fileHeader.mHeaderSize, fileHeader.mFormatOptions).mWord; } - private long runGetTerminalPosition(final FusionDictionaryBufferInterface buffer, - final String word, int index, boolean contained) { + private long runGetTerminalPosition(final BinaryDictReader reader, final String word, int index, + boolean contained) { final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; long diff = -1; int position = -1; try { final long now = System.nanoTime(); - position = BinaryDictIOUtils.getTerminalPosition(buffer, word); + position = BinaryDictIOUtils.getTerminalPosition(reader, word); diff = System.nanoTime() - now; } catch (IOException e) { Log.e(TAG, "IOException while getTerminalPosition", e); @@ -524,50 +530,48 @@ public class BinaryDictIOTests extends AndroidTestCase { } assertEquals(FormatSpec.NOT_VALID_WORD != position, contained); - if (contained) assertEquals(getWordFromBinary(buffer, position), word); + if (contained) assertEquals(getWordFromBinary(reader, position), word); return diff; } public void testGetTerminalPosition() { File file = null; try { - file = File.createTempFile("testGetTerminalPosition", ".dict", + file = File.createTempFile("testGetTerminalPosition", TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); } catch (IOException e) { // do nothing } assertNotNull(file); - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( new HashMap<String, String>(), false, false)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); final BinaryDictReader reader = new BinaryDictReader(file); - FusionDictionaryBufferInterface buffer = null; try { - buffer = reader.openAndGetBuffer( - new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); + reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", buffer); + assertNotNull("Can't get the buffer", reader.getBuffer()); try { // too long word final String longWord = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, longWord)); + BinaryDictIOUtils.getTerminalPosition(reader, longWord)); // null assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, null)); + BinaryDictIOUtils.getTerminalPosition(reader, null)); // empty string assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, "")); + BinaryDictIOUtils.getTerminalPosition(reader, "")); } catch (IOException e) { } catch (UnsupportedFormatException e) { } @@ -575,7 +579,7 @@ public class BinaryDictIOTests extends AndroidTestCase { // Test a word that is contained within the dictionary. long sum = 0; for (int i = 0; i < sWords.size(); ++i) { - final long time = runGetTerminalPosition(buffer, sWords.get(i), i, true); + final long time = runGetTerminalPosition(reader, sWords.get(i), i, true); sum += time == -1 ? 0 : time; } Log.d(TAG, "per a search : " + (((double)sum) / sWords.size() / 1000000)); @@ -586,48 +590,48 @@ public class BinaryDictIOTests extends AndroidTestCase { for (int i = 0; i < 1000; ++i) { final String word = generateWord(random, codePointSet); if (sWords.indexOf(word) != -1) continue; - runGetTerminalPosition(buffer, word, i, false); + runGetTerminalPosition(reader, word, i, false); } } public void testDeleteWord() { File file = null; try { - file = File.createTempFile("testDeleteWord", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testDeleteWord", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { // do nothing } assertNotNull(file); - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( new HashMap<String, String>(), false, false)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); final BinaryDictReader reader = new BinaryDictReader(file); - FusionDictionaryBufferInterface buffer = null; try { - buffer = reader.openAndGetBuffer( + reader.openBuffer( new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory()); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", buffer); + assertNotNull("Can't get the buffer", reader.getBuffer()); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0))); - DynamicBinaryDictIOUtils.deleteWord(buffer, sWords.get(0)); + BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(0))); + DynamicBinaryDictIOUtils.deleteWord(reader, sWords.get(0)); assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(0))); + BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(0))); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5))); - DynamicBinaryDictIOUtils.deleteWord(buffer, sWords.get(5)); + BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(5))); + DynamicBinaryDictIOUtils.deleteWord(reader, sWords.get(5)); assertEquals(FormatSpec.NOT_VALID_WORD, - BinaryDictIOUtils.getTerminalPosition(buffer, sWords.get(5))); + BinaryDictIOUtils.getTerminalPosition(reader, sWords.get(5))); } catch (IOException e) { } catch (UnsupportedFormatException e) { } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index e75950703..bcf2c3187 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -21,10 +21,11 @@ import android.test.MoreAsserts; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.ByteBufferWrapper; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictReader. + FusionDictionaryBufferFromWritableByteBufferFactory; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -33,8 +34,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.Random; @@ -49,6 +48,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { public static final int DEFAULT_MAX_UNIGRAMS = 1500; private final int mMaxUnigrams; + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", @@ -114,10 +115,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private static void printNode(final FusionDictionaryBufferInterface buffer, final FormatSpec.FormatOptions formatOptions) { Log.d(TAG, "Node at " + buffer.position()); - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); + final int count = BinaryDictDecoder.readCharGroupCount(buffer); Log.d(TAG, " charGroupCount = " + count); for (int i = 0; i < count; ++i) { - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, + final CharGroupInfo currentInfo = BinaryDictDecoder.readCharGroup(buffer, buffer.position(), formatOptions); printCharGroup(currentInfo); } @@ -127,22 +128,24 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } } - private static void printBinaryFile(final FusionDictionaryBufferInterface buffer) + private static void printBinaryFile(final BinaryDictReader dictReader) throws IOException, UnsupportedFormatException { - FileHeader header = BinaryDictInputOutput.readHeader(buffer); + final FileHeader fileHeader = BinaryDictDecoder.readHeader(dictReader); + final FusionDictionaryBufferInterface buffer = dictReader.getBuffer(); while (buffer.position() < buffer.limit()) { - printNode(buffer, header.mFormatOptions); + printNode(buffer, fileHeader.mFormatOptions); } } private int getWordPosition(final File file, final String word) { int position = FormatSpec.NOT_VALID_WORD; + final BinaryDictReader dictReader = new BinaryDictReader(file); FileInputStream inStream = null; try { inStream = new FileInputStream(file); - final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( - inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); - position = BinaryDictIOUtils.getTerminalPosition(buffer, word); + dictReader.openBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + position = BinaryDictIOUtils.getTerminalPosition(dictReader, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { } finally { @@ -158,23 +161,14 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } private CharGroupInfo findWordFromFile(final File file, final String word) { - FileInputStream inStream = null; + final BinaryDictReader dictReader = new BinaryDictReader(file); CharGroupInfo info = null; try { - inStream = new FileInputStream(file); - final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( - inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); - info = BinaryDictIOUtils.findWordFromBuffer(buffer, word); + dictReader.openBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + info = BinaryDictIOUtils.findWordByBinaryDictReader(dictReader, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } } return info; } @@ -183,42 +177,33 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList<WeightedString> bigrams, final ArrayList<WeightedString> shortcuts) { - RandomAccessFile raFile = null; + final BinaryDictReader dictReader = new BinaryDictReader(file); BufferedOutputStream outStream = null; - FusionDictionaryBufferInterface buffer = null; long amountOfTime = -1; try { - raFile = new RandomAccessFile(file, "rw"); - buffer = new ByteBufferWrapper(raFile.getChannel().map( - FileChannel.MapMode.READ_WRITE, 0, file.length())); + dictReader.openBuffer(new FusionDictionaryBufferFromWritableByteBufferFactory()); outStream = new BufferedOutputStream(new FileOutputStream(file, true)); if (!exist) { assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } final long now = System.nanoTime(); - DynamicBinaryDictIOUtils.insertWord(buffer, outStream, word, frequency, bigrams, + DynamicBinaryDictIOUtils.insertWord(dictReader, outStream, word, frequency, bigrams, shortcuts, false, false); amountOfTime = System.nanoTime() - now; outStream.flush(); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); outStream.close(); - raFile.close(); } catch (IOException e) { + Log.e(TAG, "Raised an IOException while inserting a word", e); } catch (UnsupportedFormatException e) { + Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e); } finally { if (outStream != null) { try { outStream.close(); } catch (IOException e) { - // do nothing - } - } - if (raFile != null) { - try { - raFile.close(); - } catch (IOException e) { - // do nothing + Log.e(TAG, "Failed to close the output stream", e); } } } @@ -226,64 +211,49 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } private void deleteWord(final File file, final String word) { - RandomAccessFile raFile = null; - FusionDictionaryBufferInterface buffer = null; + final BinaryDictReader dictReader = new BinaryDictReader(file); try { - raFile = new RandomAccessFile(file, "rw"); - buffer = new ByteBufferWrapper(raFile.getChannel().map( - FileChannel.MapMode.READ_WRITE, 0, file.length())); - DynamicBinaryDictIOUtils.deleteWord(buffer, word); + dictReader.openBuffer(new FusionDictionaryBufferFromWritableByteBufferFactory()); + DynamicBinaryDictIOUtils.deleteWord(dictReader, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { - } finally { - if (raFile != null) { - try { - raFile.close(); - } catch (IOException e) { - // do nothing - } - } } } private void checkReverseLookup(final File file, final String word, final int position) { - FileInputStream inStream = null; + final BinaryDictReader dictReader = new BinaryDictReader(file); try { - inStream = new FileInputStream(file); - final FusionDictionaryBufferInterface buffer = new ByteBufferWrapper( - inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, file.length())); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - assertEquals(word, BinaryDictInputOutput.getWordAtAddress(buffer, header.mHeaderSize, - position - header.mHeaderSize, header.mFormatOptions).mWord); + final FusionDictionaryBufferInterface buffer = dictReader.openAndGetBuffer( + new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); + final FileHeader fileHeader = BinaryDictDecoder.readHeader(dictReader); + assertEquals(word, + BinaryDictDecoder.getWordAtAddress(dictReader.getBuffer(), + fileHeader.mHeaderSize, position - fileHeader.mHeaderSize, + fileHeader.mFormatOptions).mWord); } catch (IOException e) { + Log.e(TAG, "Raised an IOException while looking up a word", e); } catch (UnsupportedFormatException e) { - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } + Log.e(TAG, "Raised an UnsupportedFormatException error while looking up a word", e); } } public void testInsertWord() { File file = null; try { - file = File.createTempFile("testInsertWord", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { fail("IOException while creating temporary file: " + e); } // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); dict.add("abcd", 10, null, false); try { final FileOutputStream out = new FileOutputStream(file); - BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); + BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); out.close(); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); @@ -321,21 +291,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { public void testInsertWordWithBigrams() { File file = null; try { - file = File.createTempFile("testInsertWordWithBigrams", ".dict", + file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); } catch (IOException e) { fail("IOException while creating temporary file: " + e); } // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); dict.add("abcd", 10, null, false); dict.add("efgh", 15, null, false); try { final FileOutputStream out = new FileOutputStream(file); - BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); + BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); out.close(); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); @@ -359,20 +329,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { public void testRandomWords() { File file = null; try { - file = File.createTempFile("testRandomWord", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { } assertNotNull(file); // set an initial dictionary. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, false)); dict.add("initial", 10, null, false); try { final FileOutputStream out = new FileOutputStream(file); - BinaryDictInputOutput.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); + BinaryDictEncoder.writeDictionaryBinary(out, dict, FORMAT_OPTIONS); out.close(); } catch (IOException e) { assertTrue(false); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java index a46e5831b..1c6de50f0 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictReaderTests.java @@ -16,7 +16,7 @@ package com.android.inputmethod.latin.makedict; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder.FusionDictionaryBufferInterface; import com.android.inputmethod.latin.makedict.BinaryDictReader.FusionDictionaryBufferFactory; import com.android.inputmethod.latin.makedict.BinaryDictReader. FusionDictionaryBufferFromByteArrayFactory; diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index b3e2ee0ff..304b792a0 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -22,6 +22,7 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; @@ -29,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.Set; +import java.util.concurrent.TimeUnit; /** * Unit tests for UserHistoryDictionary @@ -43,6 +45,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" }; + private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000; + @Override public void setUp() { mPrefs = PreferenceManager.getDefaultSharedPreferences(getContext()); @@ -78,43 +82,43 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } } + private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords, + final Random random) { + final List<String> words = generateWords(numberOfWords, random); + final UserHistoryPredictionDictionary dict = + PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix /* locale */, mPrefs); + // Add random words to the user history dictionary. + addToDict(dict, words); + // write to file. + dict.close(); + } + public void testRandomWords() { File dictFile = null; - try { - Log.d(TAG, "This test can be used for profiling."); - Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); - final int numberOfWords = 1000; - final Random random = new Random(123456); - List<String> words = generateWords(numberOfWords, random); - - final String locale = "testRandomWords"; - final String fileName = "UserHistoryDictionary." + locale + ".dict"; - dictFile = new File(getContext().getFilesDir(), fileName); - final UserHistoryPredictionDictionary dict = - PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary( - getContext(), locale, mPrefs); - dict.mIsTest = true; - - addToDict(dict, words); + Log.d(TAG, "This test can be used for profiling."); + Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); + final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final int numberOfWords = 1000; + final Random random = new Random(123456); + try { + addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random); + } finally { try { - Log.d(TAG, "waiting for adding the word ..."); - Thread.sleep(2000); + Log.d(TAG, "waiting for writing ..."); + Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } - // write to file - dict.close(); + final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + dictFile = new File(getContext().getFilesDir(), fileName); - try { - Log.d(TAG, "waiting for writing ..."); - Thread.sleep(5000); - } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: " + e); - } - } finally { if (dictFile != null) { + assertTrue(dictFile.exists()); + assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); dictFile.delete(); } } @@ -122,49 +126,46 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { public void testStressTestForSwitchingLanguagesAndAddingWords() { final int numberOfLanguages = 2; - final int numberOfLanguageSwitching = 100; - final int numberOfWordsIntertedForEachLanguageSwitch = 100; + final int numberOfLanguageSwitching = 80; + final int numberOfWordsInsertedForEachLanguageSwitch = 100; final File dictFiles[] = new File[numberOfLanguages]; try { final Random random = new Random(123456); - // Create locales for this test. - String locales[] = new String[numberOfLanguages]; + // Create filename suffixes for this test. + String testFilenameSuffixes[] = new String[numberOfLanguages]; for (int i = 0; i < numberOfLanguages; i++) { - locales[i] = "testSwitchingLanguages" + i; - final String fileName = "UserHistoryDictionary." + locales[i] + ".dict"; + testFilenameSuffixes[i] = "testSwitchingLanguages" + i; + final String fileName = UserHistoryPredictionDictionary.NAME + "." + + testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFiles[i] = new File(getContext().getFilesDir(), fileName); } - final long now = System.currentTimeMillis(); + final long start = System.currentTimeMillis(); for (int i = 0; i < numberOfLanguageSwitching; i++) { final int index = i % numberOfLanguages; - // Switch languages to locales[index]. - final UserHistoryPredictionDictionary dict = - PersonalizationDictionaryHelper.getUserHistoryPredictionDictionary( - getContext(), locales[index], mPrefs); - final List<String> words = generateWords( - numberOfWordsIntertedForEachLanguageSwitch, random); - // Add random words to the user history dictionary. - addToDict(dict, words); - // write to file - dict.close(); + // Switch languages to testFilenameSuffixes[index]. + addAndWriteRandomWords(testFilenameSuffixes[index], + numberOfWordsInsertedForEachLanguageSwitch, random); } final long end = System.currentTimeMillis(); Log.d(TAG, "testStressTestForSwitchingLanguageAndAddingWords took " - + (end - now) + " ms"); + + (end - start) + " ms"); + } finally { + Log.d(TAG, "waiting for writing ..."); try { Log.d(TAG, "waiting for writing ..."); - Thread.sleep(5000); + Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } - } finally { for (final File file : dictFiles) { if (file != null) { + assertTrue(file.exists()); + assertTrue(file.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); file.delete(); } } diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java index 175e511b0..c6fa943fe 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java @@ -256,4 +256,16 @@ public class StringUtilsTests extends AndroidTestCase { // code for now True is acceptable. assertTrue(StringUtils.lastPartLooksLikeURL(".abc/def")); } + + public void testHexStringUtils() { + final byte[] bytes = new byte[] { (byte)0x01, (byte)0x11, (byte)0x22, (byte)0x33, + (byte)0x55, (byte)0x88, (byte)0xEE }; + final String bytesStr = StringUtils.byteArrayToHexString(bytes); + final byte[] bytes2 = StringUtils.hexStringToByteArray(bytesStr); + for (int i = 0; i < bytes.length; ++i) { + assertTrue(bytes[i] == bytes2[i]); + } + final String bytesStr2 = StringUtils.byteArrayToHexString(bytes2); + assertTrue(bytesStr.equals(bytesStr2)); + } } diff --git a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java index baebda2ed..856b2dbda 100644 --- a/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtilsTests.java @@ -214,7 +214,7 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(FR_CA)); assertEquals("de ", "Allemand", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(DE)); - assertEquals("zz ", "Alphabet (QWERTY)", + assertEquals("zz ", "Alphabet latin (QWERTY)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ)); return null; } @@ -236,7 +236,7 @@ public class SubtypeLocaleUtilsTests extends AndroidTestCase { SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(EN_UK_DVORAK)); assertEquals("es_US colemak","Espagnol (États-Unis) (Colemak)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ES_US_COLEMAK)); - assertEquals("zz pc", "Alphabet (PC)", + assertEquals("zz pc", "Alphabet latin (PC)", SubtypeLocaleUtils.getSubtypeDisplayNameInSystemLocale(ZZ_PC)); return null; } diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java index ce62bf21a..8f5bec8a4 100644 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java @@ -49,6 +49,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase private static final int BIGRAM_FREQUENCY = 100; private static final ArrayList<String> NOT_HAVE_BIGRAM = new ArrayList<String>(); private static final FormatSpec.FormatOptions FORMAT_OPTIONS = new FormatSpec.FormatOptions(2); + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; /** * Return same frequency for all words and bigrams @@ -86,7 +87,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase private void checkWordInFusionDict(final FusionDictionary dict, final String word, final ArrayList<String> expectedBigrams) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word); + final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); assertNotNull(group); assertTrue(group.isTerminal()); @@ -177,7 +178,8 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase File file = null; try { - file = File.createTempFile("testReadAndWrite", ".dict", getContext().getCacheDir()); + file = File.createTempFile("testReadAndWrite", TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); } catch (IOException e) { Log.d(TAG, "IOException while creating a temporary file", e); } diff --git a/tools/dicttool/NativeLib.mk b/tools/dicttool/NativeLib.mk index eeae003fb..84e54a931 100644 --- a/tools/dicttool/NativeLib.mk +++ b/tools/dicttool/NativeLib.mk @@ -34,7 +34,7 @@ LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(LATINIME_NATIVE_SRC_DIR) # Used in jni_common.cpp to avoid registering useless methods. LATIN_IME_JNI_SRC_FILES := \ - com_android_inputmethod_latin_makedict_BinaryDictInputOutput.cpp \ + com_android_inputmethod_latin_makedict_BinaryDictDecoder.cpp \ jni_common.cpp LATIN_IME_CORE_SRC_FILES := diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index d0b460af0..4ddfb405f 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -16,7 +16,7 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder; import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; @@ -31,8 +31,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.ArrayList; import javax.xml.parsers.ParserConfigurationException; @@ -99,7 +97,7 @@ public final class BinaryDictOffdeviceUtils { // over and over, ending in a stack overflow. Hence we limit the depth at which we try // decoding the file. if (depth > MAX_DECODE_DEPTH) return null; - if (BinaryDictInputOutput.isBinaryDictionary(src)) { + if (BinaryDictDecoder.isBinaryDictionary(src)) { spec.mFile = src; return spec; } @@ -194,7 +192,7 @@ public final class BinaryDictOffdeviceUtils { System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); } - return BinaryDictInputOutput.readDictionaryBinary(reader, null); + return BinaryDictDecoder.readDictionaryBinary(reader, null); } } } catch (IOException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java index 092ee767f..4b6716936 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java @@ -19,7 +19,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -117,7 +117,7 @@ public class CombinedInputOutput { final boolean processLigatures = FRENCH_LIGATURE_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); attributes.remove(OPTIONS_TAG); - final FusionDictionary dict = new FusionDictionary(new Node(), new DictionaryOptions( + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions( attributes, processUmlauts, processLigatures)); String line; diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index 9bce988ac..09052c937 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -16,7 +16,8 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoder; import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; @@ -30,8 +31,6 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.Arrays; import java.util.LinkedList; @@ -177,7 +176,7 @@ public class DictionaryMaker { inputUnigramXml = filename; } else if (CombinedInputOutput.isCombinedDictionary(filename)) { inputCombined = filename; - } else if (BinaryDictInputOutput.isBinaryDictionary(filename)) { + } else if (BinaryDictDecoder.isBinaryDictionary(filename)) { inputBinary = filename; } else { throw new IllegalArgumentException( @@ -199,7 +198,7 @@ public class DictionaryMaker { } } else { if (null == inputBinary && null == inputUnigramXml) { - if (BinaryDictInputOutput.isBinaryDictionary(arg)) { + if (BinaryDictDecoder.isBinaryDictionary(arg)) { inputBinary = arg; } else if (CombinedInputOutput.isCombinedDictionary(arg)) { inputCombined = arg; @@ -269,7 +268,7 @@ public class DictionaryMaker { final File file = new File(binaryFilename); final BinaryDictReader reader = new BinaryDictReader(file); reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); - return BinaryDictInputOutput.readDictionaryBinary(reader, null); + return BinaryDictDecoder.readDictionaryBinary(reader, null); } /** @@ -358,7 +357,7 @@ public class DictionaryMaker { throws FileNotFoundException, IOException, UnsupportedFormatException { final File outputFile = new File(outputFilename); final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version); - BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, + BinaryDictEncoder.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, formatOptions); } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java index 5c3e87e10..d790d0652 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java @@ -121,7 +121,8 @@ public class Diff extends Dicttool.Command { private static void diffWords(final FusionDictionary dict0, final FusionDictionary dict1) { boolean hasDifferences = false; for (final Word word0 : dict0) { - final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRoot, word0.mWord); + final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray, + word0.mWord); if (null == word1) { // This word is not in dict1 System.out.println("Deleted: " + word0.mWord + " " + word0.mFrequency); @@ -150,7 +151,8 @@ public class Diff extends Dicttool.Command { } } for (final Word word1 : dict1) { - final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRoot, word1.mWord); + final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray, + word1.mWord); if (null == word0) { // This word is not in dict0 System.out.println("Added: " + word1.mWord + " " + word1.mFrequency); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java index f2894544f..fa8c5f776 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java @@ -65,7 +65,7 @@ public class Info extends Dicttool.Command { private static void showWordInfo(final FusionDictionary dict, final String word, final boolean plumbing) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word); + final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); if (null == group) { System.out.println(word + " is not in the dictionary"); return; diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java index 972b6e7e6..9174238da 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java @@ -16,9 +16,9 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictIOTests; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests; import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest; import com.android.inputmethod.latin.makedict.FusionDictionaryTest; import java.lang.reflect.Constructor; @@ -37,9 +37,9 @@ public class Test extends Dicttool.Command { private static final Class<?>[] sClassesToTest = { BinaryDictOffdeviceUtilsTests.class, FusionDictionaryTest.class, - BinaryDictInputOutputTest.class, - BinaryDictIOUtilsTests.class, - BinaryDictIOTests.class + BinaryDictDecoderEncoderTests.class, + BinaryDictEncoderFlattenTreeTests.class, + BinaryDictIOUtilsTests.class }; private ArrayList<Method> mAllTestMethods = new ArrayList<Method>(); private ArrayList<String> mUsedTestMethods = new ArrayList<String>(); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java index 1fd2cba7a..4e99bf979 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -124,8 +124,8 @@ public class XmlDictInputOutput { GERMAN_UMLAUT_PROCESSING_OPTION.equals(optionsString); final boolean processLigatures = FRENCH_LIGATURE_PROCESSING_OPTION.equals(optionsString); - mDictionary = new FusionDictionary(new Node(), new DictionaryOptions(attributes, - processUmlauts, processLigatures)); + mDictionary = new FusionDictionary(new PtNodeArray(), + new DictionaryOptions(attributes, processUmlauts, processLigatures)); } else { mState = UNKNOWN; } diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index fb1cc8493..3bda77fe9 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -16,36 +16,33 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoder; +import com.android.inputmethod.latin.makedict.BinaryDictEncoder; import com.android.inputmethod.latin.makedict.BinaryDictReader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import junit.framework.TestCase; import java.io.File; import java.io.BufferedOutputStream; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; import java.util.HashMap; /** - * Unit tests for BinaryDictOffdeviceUtilsTests + * Unit tests for BinaryDictOffdeviceUtils */ public class BinaryDictOffdeviceUtilsTests extends TestCase { private static final int TEST_FREQ = 37; // Some arbitrary value unlikely to happen by chance public void testGetRawDictWorks() throws IOException, UnsupportedFormatException { // Create a thrice-compressed dictionary file. - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions(new HashMap<String, String>(), false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); dict.add("foo", TEST_FREQ, null, false /* isNotAWord */); @@ -61,7 +58,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { Compress.getCompressedStream( new BufferedOutputStream(new FileOutputStream(dst))))); - BinaryDictInputOutput.writeDictionaryBinary(out, dict, new FormatOptions(2, false)); + BinaryDictEncoder.writeDictionaryBinary(out, dict, new FormatOptions(2, false)); // Test for an actually compressed dictionary and its contents final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec = @@ -72,10 +69,11 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); final BinaryDictReader reader = new BinaryDictReader(decodeSpec.mFile); reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory()); - final FusionDictionary resultDict = BinaryDictInputOutput.readDictionaryBinary(reader, + final FusionDictionary resultDict = BinaryDictDecoder.readDictionaryBinary(reader, null /* dict : an optional dictionary to add words to, or null */); assertEquals("Dictionary can't be read back correctly", - resultDict.findWordInTree(resultDict.mRoot, "foo").getFrequency(), TEST_FREQ); + FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(), + TEST_FREQ); } public void testGetRawDictFails() throws IOException { diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java index 096902879..fe6738303 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictInputOutputTest.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java @@ -17,7 +17,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import junit.framework.TestCase; @@ -25,13 +25,13 @@ import java.util.ArrayList; import java.util.HashMap; /** - * Unit tests for BinaryDictInputOutput. + * Unit tests for BinaryDictEncoder.flattenTree(). */ -public class BinaryDictInputOutputTest extends TestCase { +public class BinaryDictEncoderFlattenTreeTests extends TestCase { // Test the flattened array contains the expected number of nodes, and // that it does not contain any duplicates. public void testFlattenNodes() { - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions(new HashMap<String, String>(), false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); dict.add("foo", 1, null, false /* isNotAWord */); @@ -39,10 +39,10 @@ public class BinaryDictInputOutputTest extends TestCase { dict.add("ftb", 1, null, false /* isNotAWord */); dict.add("bar", 1, null, false /* isNotAWord */); dict.add("fool", 1, null, false /* isNotAWord */); - final ArrayList<Node> result = BinaryDictInputOutput.flattenTree(dict.mRoot); + final ArrayList<PtNodeArray> result = BinaryDictEncoder.flattenTree(dict.mRootNodeArray); assertEquals(4, result.size()); while (!result.isEmpty()) { - final Node n = result.remove(0); + final PtNodeArray n = result.remove(0); assertFalse("Flattened array contained the same node twice", result.contains(n)); } } diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java index 76071133d..22c0ceb4c 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java @@ -19,7 +19,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.Word; import junit.framework.TestCase; @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Random; /** - * Unit tests for BinaryDictInputOutput. + * Unit tests for FusionDictionary. */ public class FusionDictionaryTest extends TestCase { private static final ArrayList<String> sWords = new ArrayList<String>(); @@ -72,7 +72,7 @@ public class FusionDictionaryTest extends TestCase { assertNotNull(dict); for (final String word : words) { if (--limit < 0) return; - final CharGroup cg = FusionDictionary.findWordInTree(dict.mRoot, word); + final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); assertNotNull(cg); } } @@ -95,7 +95,7 @@ public class FusionDictionaryTest extends TestCase { // Test the flattened array contains the expected number of nodes, and // that it does not contain any duplicates. public void testFusion() { - final FusionDictionary dict = new FusionDictionary(new Node(), + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions(new HashMap<String, String>(), false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */)); final long time = System.currentTimeMillis(); diff --git a/tools/dicttool/tests/etc/test-dicttool.sh b/tools/dicttool/tests/etc/test-dicttool.sh index 092120769..5eb44fc85 100755 --- a/tools/dicttool/tests/etc/test-dicttool.sh +++ b/tools/dicttool/tests/etc/test-dicttool.sh @@ -24,5 +24,5 @@ fi find out -name "dicttool_aosp*" -exec rm -rf {} \; > /dev/null 2>&1 mmm -j8 external/junit DICTTOOL_UNITTEST=true mmm -j8 packages/inputmethods/LatinIME/tools/dicttool -java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest +java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests java -classpath ${ANDROID_HOST_OUT}/framework/junit.jar:${ANDROID_HOST_OUT}/framework/dicttool_aosp.jar junit.textui.TestRunner com.android.inputmethod.latin.dicttool.BinaryDictOffdeviceUtilsTests |