diff options
Diffstat (limited to 'java/src')
91 files changed, 6492 insertions, 4089 deletions
diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java index 77f323440..7639432aa 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java @@ -157,7 +157,7 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider // Add the virtual children of the root View. final Keyboard keyboard = mKeyboardView.getKeyboard(); - final Key[] keys = keyboard.mKeys; + final Key[] keys = keyboard.getKeys(); for (Key key : keys) { final int childVirtualViewId = generateVirtualViewIdForKey(key); rootInfo.addChild(mKeyboardView, childVirtualViewId); @@ -172,7 +172,7 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider return null; } final String keyDescription = getKeyDescription(key); - final Rect boundsInParent = key.mHitBox; + final Rect boundsInParent = key.getHitBox(); // Calculate the key's in-screen bounds. mTempBoundsInScreen.set(boundsInParent); @@ -208,8 +208,8 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider * @param key The key to press. */ void simulateKeyPress(final Key key) { - final int x = key.mHitBox.centerX(); - final int y = key.mHitBox.centerY(); + final int x = key.getHitBox().centerX(); + final int y = key.getHitBox().centerY(); final long downTime = SystemClock.uptimeMillis(); final MotionEvent downEvent = MotionEvent.obtain( downTime, downTime, MotionEvent.ACTION_DOWN, x, y, 0); @@ -300,7 +300,7 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider } mVirtualViewIdToKey.clear(); - final Key[] keys = keyboard.mKeys; + final Key[] keys = keyboard.getKeys(); for (Key key : keys) { final int virtualViewId = generateVirtualViewIdForKey(key); mVirtualViewIdToKey.put(virtualViewId, key); @@ -325,6 +325,6 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider // The key x- and y-coordinates are stable between layout changes. // Generate an identifier by bit-shifting the x-coordinate to the // left-half of the integer and OR'ing with the y-coordinate. - return ((0xFFFF & key.mX) << (Integer.SIZE / 2)) | (0xFFFF & key.mY); + return ((0xFFFF & key.getX()) << (Integer.SIZE / 2)) | (0xFFFF & key.getY()); } } diff --git a/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java b/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java index 73896dfd3..b3bb767af 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibleKeyboardViewProxy.java @@ -357,7 +357,6 @@ public final class AccessibleKeyboardViewProxy extends AccessibilityDelegateComp break; case KeyboardId.ELEMENT_ALPHABET_AUTOMATIC_SHIFTED: case KeyboardId.ELEMENT_ALPHABET_MANUAL_SHIFTED: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: text = context.getText(R.string.spoken_description_shiftmode_on); break; default: @@ -389,7 +388,6 @@ public final class AccessibleKeyboardViewProxy extends AccessibilityDelegateComp resId = R.string.spoken_description_mode_alpha; break; case KeyboardId.ELEMENT_SYMBOLS: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_mode_symbol; break; case KeyboardId.ELEMENT_PHONE: diff --git a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java index 41f5b9a24..085ca93d5 100644 --- a/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java +++ b/java/src/com/android/inputmethod/accessibility/KeyCodeDescriptionMapper.java @@ -97,7 +97,7 @@ public final class KeyCodeDescriptionMapper { */ public String getDescriptionForKey(final Context context, final Keyboard keyboard, final Key key, final boolean shouldObscure) { - final int code = key.mCode; + final int code = key.getCode(); if (code == Constants.CODE_SWITCH_ALPHA_SYMBOL) { final String description = getDescriptionForSwitchAlphaSymbol(context, keyboard); @@ -116,8 +116,8 @@ public final class KeyCodeDescriptionMapper { return getDescriptionForActionKey(context, keyboard, key); } - if (!TextUtils.isEmpty(key.mLabel)) { - final String label = key.mLabel.toString().trim(); + if (!TextUtils.isEmpty(key.getLabel())) { + final String label = key.getLabel().trim(); // First, attempt to map the label to a pre-defined description. if (mKeyLabelMap.containsKey(label)) { @@ -126,7 +126,7 @@ public final class KeyCodeDescriptionMapper { } // Just attempt to speak the description. - if (key.mCode != Constants.CODE_UNSPECIFIED) { + if (key.getCode() != Constants.CODE_UNSPECIFIED) { return getDescriptionForKeyCode(context, keyboard, key, shouldObscure); } return null; @@ -156,7 +156,6 @@ public final class KeyCodeDescriptionMapper { resId = R.string.spoken_description_to_symbol; break; case KeyboardId.ELEMENT_SYMBOLS: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_to_alpha; break; case KeyboardId.ELEMENT_PHONE: @@ -191,7 +190,6 @@ public final class KeyCodeDescriptionMapper { break; case KeyboardId.ELEMENT_ALPHABET_AUTOMATIC_SHIFTED: case KeyboardId.ELEMENT_ALPHABET_MANUAL_SHIFTED: - case KeyboardId.ELEMENT_SYMBOLS_SHIFTED: resId = R.string.spoken_description_shift_shifted; break; default: @@ -215,8 +213,8 @@ public final class KeyCodeDescriptionMapper { final int resId; // Always use the label, if available. - if (!TextUtils.isEmpty(key.mLabel)) { - return key.mLabel.toString().trim(); + if (!TextUtils.isEmpty(key.getLabel())) { + return key.getLabel().trim(); } // Otherwise, use the action ID. @@ -267,7 +265,7 @@ public final class KeyCodeDescriptionMapper { */ private String getDescriptionForKeyCode(final Context context, final Keyboard keyboard, final Key key, final boolean shouldObscure) { - final int code = key.mCode; + final int code = key.getCode(); // If the key description should be obscured, now is the time to do it. final boolean isDefinedNonCtrl = Character.isDefined(code) && !Character.isISOControl(code); @@ -280,8 +278,8 @@ public final class KeyCodeDescriptionMapper { if (isDefinedNonCtrl) { return Character.toString((char) code); } - if (!TextUtils.isEmpty(key.mLabel)) { - return key.mLabel; + if (!TextUtils.isEmpty(key.getLabel())) { + return key.getLabel(); } return context.getString(R.string.spoken_description_unknown, code); } diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java new file mode 100644 index 000000000..25ff8d0ce --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java @@ -0,0 +1,370 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard; + +import static com.android.inputmethod.latin.Constants.NOT_A_COORDINATE; + +import android.content.Context; +import android.content.res.ColorStateList; +import android.content.res.Resources; +import android.content.res.TypedArray; +import android.support.v4.view.PagerAdapter; +import android.support.v4.view.ViewPager; +import android.util.AttributeSet; +import android.util.SparseArray; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.ImageView; +import android.widget.LinearLayout; +import android.widget.TabHost; +import android.widget.TabHost.OnTabChangeListener; +import android.widget.TextView; + +import com.android.inputmethod.keyboard.internal.RecentsKeyboard; +import com.android.inputmethod.keyboard.internal.ScrollKeyboardView; +import com.android.inputmethod.keyboard.internal.ScrollViewWithNotifier; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.SubtypeSwitcher; +import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.ResourceUtils; + +import java.util.HashMap; + +/** + * View class to implement Emoji keyboards. + * The Emoji keyboard consists of group of views {@link R.layout#emoji_keyboard_view}. + * <ol> + * <li> Emoji category tabs. + * <li> Delete button. + * <li> Emoji keyboard pages that can be scrolled by swiping horizontally or by selecting a tab. + * <li> Back to main keyboard button and enter button. + * </ol> + * Because of the above reasons, this class doesn't extend {@link KeyboardView}. + */ +public final class EmojiKeyboardView extends LinearLayout implements OnTabChangeListener, + ViewPager.OnPageChangeListener, View.OnClickListener, + ScrollKeyboardView.OnKeyClickListener { + private final int mKeyBackgroundId; + private final ColorStateList mTabLabelColor; + private final EmojiKeyboardAdapter mEmojiKeyboardAdapter; + + private TabHost mTabHost; + private ViewPager mEmojiPager; + + private KeyboardActionListener mKeyboardActionListener = KeyboardActionListener.EMPTY_LISTENER; + + private int mCurrentCategory = CATEGORY_UNSPECIFIED; + private static final int CATEGORY_UNSPECIFIED = -1; + private static final int CATEGORY_RECENTS = 0; + private static final int CATEGORY_PEOPLE = 1; + private static final int CATEGORY_OBJECTS = 2; + private static final int CATEGORY_NATURE = 3; + private static final int CATEGORY_PLACES = 4; + private static final int CATEGORY_SYMBOLS = 5; + private static final int CATEGORY_EMOTICONS = 6; + private static final HashMap<String, Integer> sCategoryNameToIdMap = + CollectionUtils.newHashMap(); + private static final String[] sCategoryName = { + "recents", "people", "objects", "nature", "places", "symbols", "emoticons" + }; + private static final int[] sCategoryIcon = new int[] { + R.drawable.ic_emoji_recent_light, + R.drawable.ic_emoji_people_light, + R.drawable.ic_emoji_objects_light, + R.drawable.ic_emoji_nature_light, + R.drawable.ic_emoji_places_light, + R.drawable.ic_emoji_symbols_light, + 0 + }; + private static final String[] sCategoryLabel = { + null, null, null, null, null, null, + ":-)" + }; + private static final int[] sCategoryElementId = { + KeyboardId.ELEMENT_EMOJI_RECENTS, + KeyboardId.ELEMENT_EMOJI_CATEGORY1, + KeyboardId.ELEMENT_EMOJI_CATEGORY2, + KeyboardId.ELEMENT_EMOJI_CATEGORY3, + KeyboardId.ELEMENT_EMOJI_CATEGORY4, + KeyboardId.ELEMENT_EMOJI_CATEGORY5, + KeyboardId.ELEMENT_EMOJI_CATEGORY6, + }; + + public EmojiKeyboardView(final Context context, final AttributeSet attrs) { + this(context, attrs, R.attr.emojiKeyboardViewStyle); + } + + public EmojiKeyboardView(final Context context, final AttributeSet attrs, final int defStyle) { + super(context, attrs, defStyle); + final TypedArray keyboardViewAttr = context.obtainStyledAttributes(attrs, + R.styleable.KeyboardView, defStyle, R.style.KeyboardView); + mKeyBackgroundId = keyboardViewAttr.getResourceId( + R.styleable.KeyboardView_keyBackground, 0); + keyboardViewAttr.recycle(); + final TypedArray emojiKeyboardViewAttr = context.obtainStyledAttributes(attrs, + R.styleable.EmojiKeyboardView, defStyle, R.style.EmojiKeyboardView); + mTabLabelColor = emojiKeyboardViewAttr.getColorStateList( + R.styleable.EmojiKeyboardView_emojiTabLabelColor); + emojiKeyboardViewAttr.recycle(); + final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( + context, null /* editorInfo */); + final Resources res = context.getResources(); + builder.setSubtype(SubtypeSwitcher.getInstance().getEmojiSubtype()); + // TODO: Make Keyboard height variable. + builder.setKeyboardGeometry(ResourceUtils.getDefaultKeyboardWidth(res), + (int)(ResourceUtils.getDefaultKeyboardHeight(res) + - res.getDimension(R.dimen.suggestions_strip_height))); + builder.setOptions(false, false, false /* lanuageSwitchKeyEnabled */); + final KeyboardLayoutSet layoutSet = builder.build(); + mEmojiKeyboardAdapter = new EmojiKeyboardAdapter(layoutSet, this); + // TODO: Save/restore recent keys from/to preferences. + } + + @Override + protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + final Resources res = getContext().getResources(); + // The main keyboard expands to the entire this {@link KeyboardView}. + final int width = ResourceUtils.getDefaultKeyboardWidth(res) + + getPaddingLeft() + getPaddingRight(); + final int height = ResourceUtils.getDefaultKeyboardHeight(res) + + res.getDimensionPixelSize(R.dimen.suggestions_strip_height) + + getPaddingTop() + getPaddingBottom(); + setMeasuredDimension(width, height); + } + + private void addTab(final TabHost host, final int category) { + final String tabId = sCategoryName[category]; + sCategoryNameToIdMap.put(tabId, category); + final TabHost.TabSpec tspec = host.newTabSpec(tabId); + tspec.setContent(R.id.emoji_keyboard_dummy); + if (sCategoryIcon[category] != 0) { + final ImageView iconView = (ImageView)LayoutInflater.from(getContext()).inflate( + R.layout.emoji_keyboard_tab_icon, null); + iconView.setImageResource(sCategoryIcon[category]); + tspec.setIndicator(iconView); + } + if (sCategoryLabel[category] != null) { + final TextView textView = (TextView)LayoutInflater.from(getContext()).inflate( + R.layout.emoji_keyboard_tab_label, null); + textView.setText(sCategoryLabel[category]); + textView.setTextColor(mTabLabelColor); + textView.setBackgroundResource(mKeyBackgroundId); + tspec.setIndicator(textView); + } + host.addTab(tspec); + } + + @Override + protected void onFinishInflate() { + mTabHost = (TabHost)findViewById(R.id.emoji_category_tabhost); + mTabHost.setup(); + addTab(mTabHost, CATEGORY_RECENTS); + addTab(mTabHost, CATEGORY_PEOPLE); + addTab(mTabHost, CATEGORY_OBJECTS); + addTab(mTabHost, CATEGORY_NATURE); + addTab(mTabHost, CATEGORY_PLACES); + addTab(mTabHost, CATEGORY_SYMBOLS); + addTab(mTabHost, CATEGORY_EMOTICONS); + mTabHost.setOnTabChangedListener(this); + mTabHost.getTabWidget().setStripEnabled(true); + + mEmojiPager = (ViewPager)findViewById(R.id.emoji_keyboard_pager); + mEmojiPager.setAdapter(mEmojiKeyboardAdapter); + mEmojiPager.setOnPageChangeListener(this); + mEmojiPager.setOffscreenPageLimit(0); + final ViewGroup.LayoutParams lp = mEmojiPager.getLayoutParams(); + final Resources res = getResources(); + lp.height = ResourceUtils.getDefaultKeyboardHeight(res) + - res.getDimensionPixelSize(R.dimen.suggestions_strip_height); + mEmojiPager.setLayoutParams(lp); + + // TODO: Record current category. + final int category = CATEGORY_PEOPLE; + setCurrentCategory(category, true /* force */); + + // TODO: Implement auto repeat, using View.OnTouchListener? + final View deleteKey = findViewById(R.id.emoji_keyboard_delete); + deleteKey.setBackgroundResource(mKeyBackgroundId); + deleteKey.setTag(Constants.CODE_DELETE); + deleteKey.setOnClickListener(this); + final View alphabetKey = findViewById(R.id.emoji_keyboard_alphabet); + alphabetKey.setBackgroundResource(mKeyBackgroundId); + alphabetKey.setTag(Constants.CODE_SWITCH_ALPHA_SYMBOL); + alphabetKey.setOnClickListener(this); + final View sendKey = findViewById(R.id.emoji_keyboard_send); + sendKey.setBackgroundResource(mKeyBackgroundId); + sendKey.setTag(Constants.CODE_ENTER); + sendKey.setOnClickListener(this); + } + + @Override + public void onTabChanged(final String tabId) { + final int category = sCategoryNameToIdMap.get(tabId); + setCurrentCategory(category, false /* force */); + } + + + @Override + public void onPageSelected(final int position) { + setCurrentCategory(position, false /* force */); + } + + @Override + public void onPageScrollStateChanged(final int state) { + // Ignore this message. Only want the actual page selected. + } + + @Override + public void onPageScrolled(final int position, final float positionOffset, + final int positionOffsetPixels) { + // Ignore this message. Only want the actual page selected. + } + + @Override + public void onClick(final View v) { + if (v.getTag() instanceof Integer) { + final int code = (Integer)v.getTag(); + registerCode(code); + return; + } + } + + private void registerCode(final int code) { + mKeyboardActionListener.onPressKey(code, 0 /* repeatCount */, true /* isSinglePointer */); + mKeyboardActionListener.onCodeInput(code, NOT_A_COORDINATE, NOT_A_COORDINATE); + mKeyboardActionListener.onReleaseKey(code, false /* withSliding */); + } + + @Override + public void onKeyClick(final Key key) { + mEmojiKeyboardAdapter.addRecentKey(key); + final int code = key.getCode(); + if (code == Constants.CODE_OUTPUT_TEXT) { + mKeyboardActionListener.onTextInput(key.getOutputText()); + return; + } + registerCode(code); + } + + public void setHardwareAcceleratedDrawingEnabled(final boolean enabled) { + // TODO: + } + + public void setKeyboardActionListener(final KeyboardActionListener listener) { + mKeyboardActionListener = listener; + } + + private void setCurrentCategory(final int category, final boolean force) { + if (mCurrentCategory == category && !force) { + return; + } + + mCurrentCategory = category; + if (force || mEmojiPager.getCurrentItem() != category) { + mEmojiPager.setCurrentItem(category, true /* smoothScroll */); + } + if (force || mTabHost.getCurrentTab() != category) { + mTabHost.setCurrentTab(category); + } + // TODO: Record current category + } + + private static class EmojiKeyboardAdapter extends PagerAdapter { + private final ScrollKeyboardView.OnKeyClickListener mListener; + private final KeyboardLayoutSet mLayoutSet; + private final RecentsKeyboard mRecentsKeyboard; + private final SparseArray<ScrollKeyboardView> mActiveKeyboardView = + CollectionUtils.newSparseArray(); + private int mActivePosition = CATEGORY_UNSPECIFIED; + + public EmojiKeyboardAdapter(final KeyboardLayoutSet layoutSet, + final ScrollKeyboardView.OnKeyClickListener listener) { + mListener = listener; + mLayoutSet = layoutSet; + mRecentsKeyboard = new RecentsKeyboard( + layoutSet.getKeyboard(KeyboardId.ELEMENT_EMOJI_RECENTS)); + } + + public void addRecentKey(final Key key) { + if (mActivePosition == CATEGORY_RECENTS) { + return; + } + mRecentsKeyboard.addRecentKey(key); + final KeyboardView recentKeyboardView = mActiveKeyboardView.get(CATEGORY_RECENTS); + if (recentKeyboardView != null) { + recentKeyboardView.invalidateAllKeys(); + } + } + + @Override + public int getCount() { + return sCategoryName.length; + } + + @Override + public void setPrimaryItem(final View container, final int position, final Object object) { + if (mActivePosition == position) { + return; + } + final ScrollKeyboardView oldKeyboardView = mActiveKeyboardView.get(mActivePosition); + if (oldKeyboardView != null) { + oldKeyboardView.releaseCurrentKey(); + oldKeyboardView.deallocateMemory(); + } + mActivePosition = position; + } + + @Override + public Object instantiateItem(final ViewGroup container, final int position) { + final int elementId = sCategoryElementId[position]; + final Keyboard keyboard = (elementId == KeyboardId.ELEMENT_EMOJI_RECENTS) + ? mRecentsKeyboard : mLayoutSet.getKeyboard(elementId); + final LayoutInflater inflater = LayoutInflater.from(container.getContext()); + final View view = inflater.inflate( + R.layout.emoji_keyboard_page, container, false /* attachToRoot */); + final ScrollKeyboardView keyboardView = (ScrollKeyboardView)view.findViewById( + R.id.emoji_keyboard_page); + keyboardView.setKeyboard(keyboard); + keyboardView.setOnKeyClickListener(mListener); + final ScrollViewWithNotifier scrollView = (ScrollViewWithNotifier)view.findViewById( + R.id.emoji_keyboard_scroller); + keyboardView.setScrollView(scrollView); + container.addView(view); + mActiveKeyboardView.put(position, keyboardView); + return view; + } + + @Override + public boolean isViewFromObject(final View view, final Object object) { + return view == object; + } + + @Override + public void destroyItem(final ViewGroup container, final int position, + final Object object) { + final ScrollKeyboardView keyboardView = mActiveKeyboardView.get(position); + if (keyboardView != null) { + keyboardView.deallocateMemory(); + mActiveKeyboardView.remove(position); + } + container.removeView(keyboardView); + } + } +} diff --git a/java/src/com/android/inputmethod/keyboard/Key.java b/java/src/com/android/inputmethod/keyboard/Key.java index 61805286d..3ea68806b 100644 --- a/java/src/com/android/inputmethod/keyboard/Key.java +++ b/java/src/com/android/inputmethod/keyboard/Key.java @@ -58,12 +58,12 @@ public class Key implements Comparable<Key> { /** * The key code (unicode or custom code) that this key generates. */ - public final int mCode; + private final int mCode; /** Label to display */ - public final String mLabel; + private final String mLabel; /** Hint label to display on the key in conjunction with the label */ - public final String mHintLabel; + private final String mHintLabel; /** Flags of the label */ private final int mLabelFlags; private static final int LABEL_FLAGS_ALIGN_LEFT = 0x01; @@ -95,18 +95,18 @@ public class Key implements Comparable<Key> { private final int mIconId; /** Width of the key, not including the gap */ - public final int mWidth; + private final int mWidth; /** Height of the key, not including the gap */ - public final int mHeight; + private final int mHeight; /** X coordinate of the key in the keyboard layout */ - public final int mX; + private final int mX; /** Y coordinate of the key in the keyboard layout */ - public final int mY; + private final int mY; /** Hit bounding box of the key */ - public final Rect mHitBox = new Rect(); + private final Rect mHitBox = new Rect(); /** More keys. It is guaranteed that this is null or an array of one or more elements */ - public final MoreKeySpec[] mMoreKeys; + private final MoreKeySpec[] mMoreKeys; /** More keys column number and flags */ private final int mMoreKeysColumnAndFlags; private static final int MORE_KEYS_COLUMN_MASK = 0x000000ff; @@ -121,12 +121,13 @@ public class Key implements Comparable<Key> { private static final String MORE_KEYS_NO_PANEL_AUTO_MORE_KEY = "!noPanelAutoMoreKey!"; /** Background type that represents different key background visual than normal one. */ - public final int mBackgroundType; - public static final int BACKGROUND_TYPE_NORMAL = 0; - public static final int BACKGROUND_TYPE_FUNCTIONAL = 1; - public static final int BACKGROUND_TYPE_ACTION = 2; - public static final int BACKGROUND_TYPE_STICKY_OFF = 3; - public static final int BACKGROUND_TYPE_STICKY_ON = 4; + private final int mBackgroundType; + public static final int BACKGROUND_TYPE_EMPTY = 0; + public static final int BACKGROUND_TYPE_NORMAL = 1; + public static final int BACKGROUND_TYPE_FUNCTIONAL = 2; + public static final int BACKGROUND_TYPE_ACTION = 3; + public static final int BACKGROUND_TYPE_STICKY_OFF = 4; + public static final int BACKGROUND_TYPE_STICKY_ON = 5; private final int mActionFlags; private static final int ACTION_FLAGS_IS_REPEATABLE = 0x01; @@ -134,7 +135,7 @@ public class Key implements Comparable<Key> { private static final int ACTION_FLAGS_ALT_CODE_WHILE_TYPING = 0x04; private static final int ACTION_FLAGS_ENABLE_LONG_PRESS = 0x08; - public final KeyVisualAttributes mKeyVisualAttributes; + private final KeyVisualAttributes mKeyVisualAttributes; private final OptionalAttributes mOptionalAttributes; @@ -150,7 +151,7 @@ public class Key implements Comparable<Key> { public final int mVisualInsetsLeft; public final int mVisualInsetsRight; - public OptionalAttributes(final String outputText, final int altCode, + private OptionalAttributes(final String outputText, final int altCode, final int disabledIconId, final int previewIconId, final int visualInsetsLeft, final int visualInsetsRight) { mOutputText = outputText; @@ -160,6 +161,18 @@ public class Key implements Comparable<Key> { mVisualInsetsLeft = visualInsetsLeft; mVisualInsetsRight = visualInsetsRight; } + + public static OptionalAttributes newInstance(final String outputText, final int altCode, + final int disabledIconId, final int previewIconId, + final int visualInsetsLeft, final int visualInsetsRight) { + if (outputText == null && altCode == CODE_UNSPECIFIED + && disabledIconId == ICON_UNDEFINED && previewIconId == ICON_UNDEFINED + && visualInsetsLeft == 0 && visualInsetsRight == 0) { + return null; + } + return new OptionalAttributes(outputText, altCode, disabledIconId, previewIconId, + visualInsetsLeft, visualInsetsRight); + } } private final int mHashCode; @@ -175,7 +188,7 @@ public class Key implements Comparable<Key> { public Key(final KeyboardParams params, final MoreKeySpec moreKeySpec, final int x, final int y, final int width, final int height, final int labelFlags) { this(params, moreKeySpec.mLabel, null, moreKeySpec.mIconId, moreKeySpec.mCode, - moreKeySpec.mOutputText, x, y, width, height, labelFlags); + moreKeySpec.mOutputText, x, y, width, height, labelFlags, BACKGROUND_TYPE_NORMAL); } /** @@ -183,22 +196,19 @@ public class Key implements Comparable<Key> { */ public Key(final KeyboardParams params, final String label, final String hintLabel, final int iconId, final int code, final String outputText, final int x, final int y, - final int width, final int height, final int labelFlags) { + final int width, final int height, final int labelFlags, final int backgroundType) { mHeight = height - params.mVerticalGap; mWidth = width - params.mHorizontalGap; mHintLabel = hintLabel; mLabelFlags = labelFlags; - mBackgroundType = BACKGROUND_TYPE_NORMAL; + mBackgroundType = backgroundType; mActionFlags = 0; mMoreKeys = null; mMoreKeysColumnAndFlags = 0; mLabel = label; - if (outputText == null) { - mOptionalAttributes = null; - } else { - mOptionalAttributes = new OptionalAttributes(outputText, CODE_UNSPECIFIED, - ICON_UNDEFINED, ICON_UNDEFINED, 0, 0); - } + mOptionalAttributes = OptionalAttributes.newInstance(outputText, CODE_UNSPECIFIED, + ICON_UNDEFINED, ICON_UNDEFINED, + 0 /* visualInsetsLeft */, 0 /* visualInsetsRight */); mCode = code; mEnabled = (code != CODE_UNSPECIFIED); mIconId = iconId; @@ -224,7 +234,7 @@ public class Key implements Comparable<Key> { public Key(final Resources res, final KeyboardParams params, final KeyboardRow row, final XmlPullParser parser) throws XmlPullParserException { final float horizontalGap = isSpacer() ? 0 : params.mHorizontalGap; - final int rowHeight = row.mRowHeight; + final int rowHeight = row.getRowHeight(); mHeight = rowHeight - params.mVerticalGap; final TypedArray keyAttr = res.obtainAttributes(Xml.asAttributeSet(parser), @@ -259,11 +269,11 @@ public class Key implements Comparable<Key> { final int previewIconId = KeySpecParser.getIconId(style.getString(keyAttr, R.styleable.Keyboard_Key_keyIconPreview)); - mLabelFlags = style.getFlag(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags) + mLabelFlags = style.getFlags(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags) | row.getDefaultKeyLabelFlags(); final boolean needsToUpperCase = needsToUpperCase(mLabelFlags, params.mId.mElementId); final Locale locale = params.mId.mLocale; - int actionFlags = style.getFlag(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); + int actionFlags = style.getFlags(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); String[] moreKeys = style.getStringArray(keyAttr, R.styleable.Keyboard_Key_moreKeys); int moreKeysColumn = style.getInt(keyAttr, @@ -359,15 +369,8 @@ public class Key implements Comparable<Key> { KeySpecParser.parseCode(style.getString(keyAttr, R.styleable.Keyboard_Key_altCode), params.mCodesSet, CODE_UNSPECIFIED), needsToUpperCase, locale); - if (outputText == null && altCode == CODE_UNSPECIFIED - && disabledIconId == ICON_UNDEFINED && previewIconId == ICON_UNDEFINED - && visualInsetsLeft == 0 && visualInsetsRight == 0) { - mOptionalAttributes = null; - } else { - mOptionalAttributes = new OptionalAttributes(outputText, altCode, - disabledIconId, previewIconId, - visualInsetsLeft, visualInsetsRight); - } + mOptionalAttributes = OptionalAttributes.newInstance(outputText, altCode, + disabledIconId, previewIconId, visualInsetsLeft, visualInsetsRight); mKeyVisualAttributes = KeyVisualAttributes.newInstance(keyAttr); keyAttr.recycle(); mHashCode = computeHashCode(this); @@ -376,6 +379,35 @@ public class Key implements Comparable<Key> { } } + /** + * Copy constructor. + * + * @param key the original key. + */ + protected Key(final Key key) { + // Final attributes. + mCode = key.mCode; + mLabel = key.mLabel; + mHintLabel = key.mHintLabel; + mLabelFlags = key.mLabelFlags; + mIconId = key.mIconId; + mWidth = key.mWidth; + mHeight = key.mHeight; + mX = key.mX; + mY = key.mY; + mHitBox.set(key.mHitBox); + mMoreKeys = key.mMoreKeys; + mMoreKeysColumnAndFlags = key.mMoreKeysColumnAndFlags; + mBackgroundType = key.mBackgroundType; + mActionFlags = key.mActionFlags; + mKeyVisualAttributes = key.mKeyVisualAttributes; + mOptionalAttributes = key.mOptionalAttributes; + mHashCode = key.mHashCode; + // Key state. + mPressed = key.mPressed; + mEnabled = key.mEnabled; + } + private static boolean needsToUpperCase(final int labelFlags, final int keyboardElementId) { if ((labelFlags & LABEL_FLAGS_PRESERVE_CASE) != 0) return false; switch (keyboardElementId) { @@ -465,6 +497,7 @@ public class Key implements Comparable<Key> { private static String backgroundName(final int backgroundType) { switch (backgroundType) { + case BACKGROUND_TYPE_EMPTY: return "empty"; case BACKGROUND_TYPE_NORMAL: return "normal"; case BACKGROUND_TYPE_FUNCTIONAL: return "functional"; case BACKGROUND_TYPE_ACTION: return "action"; @@ -474,6 +507,22 @@ public class Key implements Comparable<Key> { } } + public int getCode() { + return mCode; + } + + public String getLabel() { + return mLabel; + } + + public String getHintLabel() { + return mHintLabel; + } + + public MoreKeySpec[] getMoreKeys() { + return mMoreKeys; + } + public void markAsLeftEdge(final KeyboardParams params) { mHitBox.left = params.mLeftPadding; } @@ -520,6 +569,10 @@ public class Key implements Comparable<Key> { && (mLabelFlags & LABEL_FLAGS_SHIFTED_LETTER_ACTIVATED) == 0; } + public KeyVisualAttributes getVisualAttributes() { + return mKeyVisualAttributes; + } + public final Typeface selectTypeface(final KeyDrawParams params) { // TODO: Handle "bold" here too? if ((mLabelFlags & LABEL_FLAGS_FONT_NORMAL) != 0) { @@ -694,9 +747,26 @@ public class Key implements Comparable<Key> { ? iconSet.getIconDrawable(previewIconId) : iconSet.getIconDrawable(mIconId); } + public int getWidth() { + return mWidth; + } + + public int getHeight() { + return mHeight; + } + + public int getX() { + return mX; + } + + public int getY() { + return mY; + } + public final int getDrawX() { + final int x = getX(); final OptionalAttributes attrs = mOptionalAttributes; - return (attrs == null) ? mX : mX + attrs.mVisualInsetsLeft; + return (attrs == null) ? x : x + attrs.mVisualInsetsLeft; } public final int getDrawWidth() { @@ -731,6 +801,10 @@ public class Key implements Comparable<Key> { mEnabled = enabled; } + public Rect getHitBox() { + return mHitBox; + } + /** * Detects if a point falls on this key. * @param x the x-coordinate of the point @@ -750,9 +824,9 @@ public class Key implements Comparable<Key> { * @return the square of the distance of the point from the nearest edge of the key */ public int squaredDistanceToEdge(final int x, final int y) { - final int left = mX; + final int left = getX(); final int right = left + mWidth; - final int top = mY; + final int top = getY(); final int bottom = top + mHeight; final int edgeX = x < left ? left : (x > right ? right : x); final int edgeY = y < top ? top : (y > bottom ? bottom : y); @@ -788,6 +862,10 @@ public class Key implements Comparable<Key> { android.R.attr.state_pressed }; + private final static int[] KEY_STATE_EMPTY = { + android.R.attr.state_empty + }; + // functional normal state (with properties) private static final int[] KEY_STATE_FUNCTIONAL_NORMAL = { android.R.attr.state_single @@ -825,6 +903,8 @@ public class Key implements Comparable<Key> { return mPressed ? KEY_STATE_PRESSED_HIGHLIGHT_OFF : KEY_STATE_NORMAL_HIGHLIGHT_OFF; case BACKGROUND_TYPE_STICKY_ON: return mPressed ? KEY_STATE_PRESSED_HIGHLIGHT_ON : KEY_STATE_NORMAL_HIGHLIGHT_ON; + case BACKGROUND_TYPE_EMPTY: + return mPressed ? KEY_STATE_PRESSED : KEY_STATE_EMPTY; default: /* BACKGROUND_TYPE_NORMAL */ return mPressed ? KEY_STATE_PRESSED : KEY_STATE_NORMAL; } @@ -842,7 +922,7 @@ public class Key implements Comparable<Key> { protected Spacer(final KeyboardParams params, final int x, final int y, final int width, final int height) { super(params, null, null, ICON_UNDEFINED, CODE_UNSPECIFIED, - null, x, y, width, height, 0); + null, x, y, width, height, 0, BACKGROUND_TYPE_EMPTY); } } } diff --git a/java/src/com/android/inputmethod/keyboard/KeyDetector.java b/java/src/com/android/inputmethod/keyboard/KeyDetector.java index 17e707f95..befb6fa92 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyDetector.java +++ b/java/src/com/android/inputmethod/keyboard/KeyDetector.java @@ -108,8 +108,9 @@ public class KeyDetector { if (distance > minDistance) { continue; } - // To take care of hitbox overlaps, we compare mCode here too. - if (primaryKey == null || distance < minDistance || key.mCode > primaryKey.mCode) { + // To take care of hitbox overlaps, we compare key's code here too. + if (primaryKey == null || distance < minDistance + || key.getCode() > primaryKey.getCode()) { minDistance = distance; primaryKey = key; } @@ -118,7 +119,7 @@ public class KeyDetector { } public static String printableCode(Key key) { - return key != null ? Constants.printableCode(key.mCode) : "none"; + return key != null ? Constants.printableCode(key.getCode()) : "none"; } public static String printableCodes(int[] codes) { diff --git a/java/src/com/android/inputmethod/keyboard/Keyboard.java b/java/src/com/android/inputmethod/keyboard/Keyboard.java index fefac96fd..23f037fbd 100644 --- a/java/src/com/android/inputmethod/keyboard/Keyboard.java +++ b/java/src/com/android/inputmethod/keyboard/Keyboard.java @@ -51,6 +51,11 @@ public class Keyboard { /** Total width of the keyboard, including the padding and keys */ public final int mOccupiedWidth; + /** Base height of the keyboard, used to calculate rows' height */ + public final int mBaseHeight; + /** Base width of the keyboard, used to calculate keys' width */ + public final int mBaseWidth; + /** The padding above the keyboard */ public final int mTopPadding; /** Default gap between rows */ @@ -69,7 +74,7 @@ public class Keyboard { public final int mMaxMoreKeysKeyboardColumn; /** Array of keys and icons in this keyboard */ - public final Key[] mKeys; + private final Key[] mKeys; public final Key[] mShiftKeys; public final Key[] mAltCodeKeysWhileTyping; public final KeyboardIconsSet mIconsSet; @@ -84,6 +89,8 @@ public class Keyboard { mThemeId = params.mThemeId; mOccupiedHeight = params.mOccupiedHeight; mOccupiedWidth = params.mOccupiedWidth; + mBaseHeight = params.mBaseHeight; + mBaseWidth = params.mBaseWidth; mMostCommonKeyHeight = params.mMostCommonKeyHeight; mMostCommonKeyWidth = params.mMostCommonKeyWidth; mMoreKeysTemplate = params.mMoreKeysTemplate; @@ -104,6 +111,30 @@ public class Keyboard { mProximityCharsCorrectionEnabled = params.mProximityCharsCorrectionEnabled; } + protected Keyboard(final Keyboard keyboard) { + mId = keyboard.mId; + mThemeId = keyboard.mThemeId; + mOccupiedHeight = keyboard.mOccupiedHeight; + mOccupiedWidth = keyboard.mOccupiedWidth; + mBaseHeight = keyboard.mBaseHeight; + mBaseWidth = keyboard.mBaseWidth; + mMostCommonKeyHeight = keyboard.mMostCommonKeyHeight; + mMostCommonKeyWidth = keyboard.mMostCommonKeyWidth; + mMoreKeysTemplate = keyboard.mMoreKeysTemplate; + mMaxMoreKeysKeyboardColumn = keyboard.mMaxMoreKeysKeyboardColumn; + mKeyVisualAttributes = keyboard.mKeyVisualAttributes; + mTopPadding = keyboard.mTopPadding; + mVerticalGap = keyboard.mVerticalGap; + + mKeys = keyboard.mKeys; + mShiftKeys = keyboard.mShiftKeys; + mAltCodeKeysWhileTyping = keyboard.mAltCodeKeysWhileTyping; + mIconsSet = keyboard.mIconsSet; + + mProximityInfo = keyboard.mProximityInfo; + mProximityCharsCorrectionEnabled = keyboard.mProximityCharsCorrectionEnabled; + } + public boolean hasProximityCharsCorrection(final int code) { if (!mProximityCharsCorrectionEnabled) { return false; @@ -120,6 +151,10 @@ public class Keyboard { return mProximityInfo; } + public Key[] getKeys() { + return mKeys; + } + public Key getKey(final int code) { if (code == Constants.CODE_UNSPECIFIED) { return null; @@ -130,8 +165,8 @@ public class Keyboard { return mKeyCache.valueAt(index); } - for (final Key key : mKeys) { - if (key.mCode == code) { + for (final Key key : getKeys()) { + if (key.getCode() == code) { mKeyCache.put(code, key); return key; } @@ -146,9 +181,9 @@ public class Keyboard { return true; } - for (final Key key : mKeys) { + for (final Key key : getKeys()) { if (key == aKey) { - mKeyCache.put(key.mCode, key); + mKeyCache.put(key.getCode(), key); return true; } } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardActionListener.java b/java/src/com/android/inputmethod/keyboard/KeyboardActionListener.java index b26698665..dc760e685 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardActionListener.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardActionListener.java @@ -26,10 +26,10 @@ public interface KeyboardActionListener { * * @param primaryCode the unicode of the key being pressed. If the touch is not on a valid key, * the value will be zero. - * @param isRepeatKey true if pressing has occurred while key repeat input. + * @param repeatCount how many times the key was repeated. Zero if it is the first press. * @param isSinglePointer true if pressing has occurred while no other key is being pressed. */ - public void onPressKey(int primaryCode, boolean isRepeatKey, boolean isSinglePointer); + public void onPressKey(int primaryCode, int repeatCount, boolean isSinglePointer); /** * Called when the user releases a key. This is sent after the {@link #onCodeInput} is called. @@ -103,7 +103,7 @@ public interface KeyboardActionListener { public static class Adapter implements KeyboardActionListener { @Override - public void onPressKey(int primaryCode, boolean isRepeatKey, boolean isSinglePointer) {} + public void onPressKey(int primaryCode, int repeatCount, boolean isSinglePointer) {} @Override public void onReleaseKey(int primaryCode, boolean withSliding) {} @Override diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardId.java b/java/src/com/android/inputmethod/keyboard/KeyboardId.java index 1dc3c6a4c..53748bb58 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardId.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardId.java @@ -50,10 +50,16 @@ public final class KeyboardId { public static final int ELEMENT_ALPHABET_SHIFT_LOCKED = 3; public static final int ELEMENT_ALPHABET_SHIFT_LOCK_SHIFTED = 4; public static final int ELEMENT_SYMBOLS = 5; - public static final int ELEMENT_SYMBOLS_SHIFTED = 6; public static final int ELEMENT_PHONE = 7; public static final int ELEMENT_PHONE_SYMBOLS = 8; public static final int ELEMENT_NUMBER = 9; + public static final int ELEMENT_EMOJI_RECENTS = 10; + public static final int ELEMENT_EMOJI_CATEGORY1 = 11; + public static final int ELEMENT_EMOJI_CATEGORY2 = 12; + public static final int ELEMENT_EMOJI_CATEGORY3 = 13; + public static final int ELEMENT_EMOJI_CATEGORY4 = 14; + public static final int ELEMENT_EMOJI_CATEGORY5 = 15; + public static final int ELEMENT_EMOJI_CATEGORY6 = 16; public final InputMethodSubtype mSubtype; public final Locale mLocale; @@ -213,10 +219,16 @@ public final class KeyboardId { case ELEMENT_ALPHABET_SHIFT_LOCKED: return "alphabetShiftLocked"; case ELEMENT_ALPHABET_SHIFT_LOCK_SHIFTED: return "alphabetShiftLockShifted"; case ELEMENT_SYMBOLS: return "symbols"; - case ELEMENT_SYMBOLS_SHIFTED: return "symbolsShifted"; case ELEMENT_PHONE: return "phone"; case ELEMENT_PHONE_SYMBOLS: return "phoneSymbols"; case ELEMENT_NUMBER: return "number"; + case ELEMENT_EMOJI_RECENTS: return "emojiRecents"; + case ELEMENT_EMOJI_CATEGORY1: return "emojiCategory1"; + case ELEMENT_EMOJI_CATEGORY2: return "emojiCategory2"; + case ELEMENT_EMOJI_CATEGORY3: return "emojiCategory3"; + case ELEMENT_EMOJI_CATEGORY4: return "emojiCategory4"; + case ELEMENT_EMOJI_CATEGORY5: return "emojiCategory5"; + case ELEMENT_EMOJI_CATEGORY6: return "emojiCategory6"; default: return null; } } diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java index e97f29452..711de63b3 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardLayoutSet.java @@ -162,7 +162,8 @@ public final class KeyboardLayoutSet { final KeyboardId id = new KeyboardId(keyboardLayoutSetElementId, mParams); try { return getKeyboard(elementParams, id); - } catch (RuntimeException e) { + } catch (final RuntimeException e) { + Log.e(TAG, "Can't create keyboard: " + id, e); throw new KeyboardLayoutSetException(e, id); } } @@ -213,7 +214,6 @@ public final class KeyboardLayoutSet { private final Context mContext; private final String mPackageName; private final Resources mResources; - private final EditorInfo mEditorInfo; private final Params mParams = new Params(); @@ -223,13 +223,12 @@ public final class KeyboardLayoutSet { mContext = context; mPackageName = context.getPackageName(); mResources = context.getResources(); - mEditorInfo = editorInfo; final Params params = mParams; params.mMode = getKeyboardMode(editorInfo); params.mEditorInfo = (editorInfo != null) ? editorInfo : EMPTY_EDITOR_INFO; params.mNoSettingsKey = InputAttributes.inPrivateImeOptions( - mPackageName, NO_SETTINGS_KEY, mEditorInfo); + mPackageName, NO_SETTINGS_KEY, params.mEditorInfo); } public Builder setKeyboardGeometry(final int keyboardWidth, final int keyboardHeight) { @@ -242,7 +241,7 @@ public final class KeyboardLayoutSet { final boolean asciiCapable = subtype.containsExtraValueKey(ASCII_CAPABLE); @SuppressWarnings("deprecation") final boolean deprecatedForceAscii = InputAttributes.inPrivateImeOptions( - mPackageName, FORCE_ASCII, mEditorInfo); + mPackageName, FORCE_ASCII, mParams.mEditorInfo); final boolean forceAscii = EditorInfoCompatUtils.hasFlagForceAscii( mParams.mEditorInfo.imeOptions) || deprecatedForceAscii; @@ -264,9 +263,9 @@ public final class KeyboardLayoutSet { final boolean languageSwitchKeyEnabled) { @SuppressWarnings("deprecation") final boolean deprecatedNoMicrophone = InputAttributes.inPrivateImeOptions( - null, NO_MICROPHONE_COMPAT, mEditorInfo); + null, NO_MICROPHONE_COMPAT, mParams.mEditorInfo); final boolean noMicrophone = InputAttributes.inPrivateImeOptions( - mPackageName, NO_MICROPHONE, mEditorInfo) + mPackageName, NO_MICROPHONE, mParams.mEditorInfo) || deprecatedNoMicrophone; mParams.mVoiceKeyEnabled = voiceKeyEnabled && !noMicrophone; mParams.mVoiceKeyOnMain = voiceKeyOnMain; diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index 1ea0f8b96..c319c57a1 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -58,19 +58,17 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } private static final KeyboardTheme[] KEYBOARD_THEMES = { - new KeyboardTheme(0, R.style.KeyboardTheme), - new KeyboardTheme(1, R.style.KeyboardTheme_HighContrast), - new KeyboardTheme(6, R.style.KeyboardTheme_Stone), - new KeyboardTheme(7, R.style.KeyboardTheme_Stone_Bold), - new KeyboardTheme(8, R.style.KeyboardTheme_Gingerbread), - new KeyboardTheme(5, R.style.KeyboardTheme_IceCreamSandwich), + new KeyboardTheme(0, R.style.KeyboardTheme_ICS), + new KeyboardTheme(1, R.style.KeyboardTheme_GB), }; private SubtypeSwitcher mSubtypeSwitcher; private SharedPreferences mPrefs; private InputView mCurrentInputView; + private View mMainKeyboardFrame; private MainKeyboardView mKeyboardView; + private EmojiKeyboardView mEmojiKeyboardView; private LatinIME mLatinIME; private Resources mResources; @@ -121,8 +119,9 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } catch (NumberFormatException e) { // Format error, keyboard theme is default to 0. } - Log.w(TAG, "Illegal keyboard theme in preference: " + themeIndex + ", default to 0"); - return KEYBOARD_THEMES[0]; + Log.w(TAG, "Illegal keyboard theme in preference: " + themeIndex + ", default to " + + defaultIndex); + return KEYBOARD_THEMES[Integer.valueOf(defaultIndex)]; } private void setContextThemeWrapper(final Context context, final KeyboardTheme keyboardTheme) { @@ -170,6 +169,8 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } private void setKeyboard(final Keyboard keyboard) { + // Make {@link MainKeyboardView} visible and hide {@link EmojiKeyboardView}. + setMainKeyboardFrame(); final MainKeyboardView keyboardView = mKeyboardView; final Keyboard oldKeyboard = keyboardView.getKeyboard(); keyboardView.setKeyboard(keyboard); @@ -256,10 +257,16 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { setKeyboard(mKeyboardLayoutSet.getKeyboard(KeyboardId.ELEMENT_SYMBOLS)); } + private void setMainKeyboardFrame() { + mMainKeyboardFrame.setVisibility(View.VISIBLE); + mEmojiKeyboardView.setVisibility(View.GONE); + } + // Implements {@link KeyboardState.SwitchActions}. @Override - public void setSymbolsShiftedKeyboard() { - setKeyboard(mKeyboardLayoutSet.getKeyboard(KeyboardId.ELEMENT_SYMBOLS_SHIFTED)); + public void setEmojiKeyboard() { + mMainKeyboardFrame.setVisibility(View.GONE); + mEmojiKeyboardView.setVisibility(View.VISIBLE); } // Implements {@link KeyboardState.SwitchActions}. @@ -301,6 +308,20 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { mState.onCodeInput(code, mLatinIME.getCurrentAutoCapsState()); } + public boolean isShowingMoreKeysPanel() { + if (mEmojiKeyboardView.getVisibility() == View.VISIBLE) { + return false; + } + return mKeyboardView.isShowingMoreKeysPanel(); + } + + public View getVisibleKeyboardView() { + if (mEmojiKeyboardView.getVisibility() == View.VISIBLE) { + return mEmojiKeyboardView; + } + return mKeyboardView; + } + public MainKeyboardView getMainKeyboardView() { return mKeyboardView; } @@ -313,10 +334,16 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { setContextThemeWrapper(mLatinIME, mKeyboardTheme); mCurrentInputView = (InputView)LayoutInflater.from(mThemeContext).inflate( R.layout.input_view, null); + mMainKeyboardFrame = mCurrentInputView.findViewById(R.id.main_keyboard_frame); + mEmojiKeyboardView = (EmojiKeyboardView)mCurrentInputView.findViewById( + R.id.emoji_keyboard_view); mKeyboardView = (MainKeyboardView) mCurrentInputView.findViewById(R.id.keyboard_view); mKeyboardView.setHardwareAcceleratedDrawingEnabled(isHardwareAcceleratedDrawingEnabled); mKeyboardView.setKeyboardActionListener(mLatinIME); + mEmojiKeyboardView.setHardwareAcceleratedDrawingEnabled( + isHardwareAcceleratedDrawingEnabled); + mEmojiKeyboardView.setKeyboardActionListener(mLatinIME); // This always needs to be set since the accessibility state can // potentially change without the input view being re-created. diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardView.java b/java/src/com/android/inputmethod/keyboard/KeyboardView.java index 054c503d8..0ef6802ca 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardView.java @@ -196,13 +196,14 @@ public class KeyboardView extends View { @Override protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { - if (mKeyboard != null) { - // The main keyboard expands to the display width. - final int height = mKeyboard.mOccupiedHeight + getPaddingTop() + getPaddingBottom(); - setMeasuredDimension(widthMeasureSpec, height); - } else { + if (mKeyboard == null) { super.onMeasure(widthMeasureSpec, heightMeasureSpec); + return; } + // The main keyboard expands to the entire this {@link KeyboardView}. + final int width = mKeyboard.mOccupiedWidth + getPaddingLeft() + getPaddingRight(); + final int height = mKeyboard.mOccupiedHeight + getPaddingTop() + getPaddingBottom(); + setMeasuredDimension(width, height); } @Override @@ -264,9 +265,9 @@ public class KeyboardView extends View { mClipRegion.setEmpty(); for (final Key key : mInvalidatedKeys) { if (mKeyboard.hasKey(key)) { - final int x = key.mX + getPaddingLeft(); - final int y = key.mY + getPaddingTop(); - mWorkingRect.set(x, y, x + key.mWidth, y + key.mHeight); + final int x = key.getX() + getPaddingLeft(); + final int y = key.getY() + getPaddingTop(); + mWorkingRect.set(x, y, x + key.getWidth(), y + key.getHeight()); mClipRegion.union(mWorkingRect); } } @@ -284,7 +285,7 @@ public class KeyboardView extends View { // TODO: Confirm if it's really required to draw all keys when hardware acceleration is on. if (drawAllKeys || isHardwareAccelerated) { // Draw all keys. - for (final Key key : mKeyboard.mKeys) { + for (final Key key : mKeyboard.getKeys()) { onDrawKey(key, canvas, paint); } } else { @@ -309,11 +310,11 @@ public class KeyboardView extends View { private void onDrawKey(final Key key, final Canvas canvas, final Paint paint) { final int keyDrawX = key.getDrawX() + getPaddingLeft(); - final int keyDrawY = key.mY + getPaddingTop(); + final int keyDrawY = key.getY() + getPaddingTop(); canvas.translate(keyDrawX, keyDrawY); final int keyHeight = mKeyboard.mMostCommonKeyHeight - mKeyboard.mVerticalGap; - final KeyVisualAttributes attr = key.mKeyVisualAttributes; + final KeyVisualAttributes attr = key.getVisualAttributes(); final KeyDrawParams params = mKeyDrawParams.mayCloneAndUpdateParams(keyHeight, attr); params.mAnimAlpha = Constants.Color.ALPHA_OPAQUE; @@ -329,7 +330,7 @@ public class KeyboardView extends View { protected void onDrawKeyBackground(final Key key, final Canvas canvas) { final Rect padding = mKeyBackgroundPadding; final int bgWidth = key.getDrawWidth() + padding.left + padding.right; - final int bgHeight = key.mHeight + padding.top + padding.bottom; + final int bgHeight = key.getHeight() + padding.top + padding.bottom; final int bgX = -padding.left; final int bgY = -padding.top; final int[] drawableState = key.getCurrentDrawableState(); @@ -351,7 +352,7 @@ public class KeyboardView extends View { protected void onDrawKeyTopVisuals(final Key key, final Canvas canvas, final Paint paint, final KeyDrawParams params) { final int keyWidth = key.getDrawWidth(); - final int keyHeight = key.mHeight; + final int keyHeight = key.getHeight(); final float centerX = keyWidth * 0.5f; final float centerY = keyHeight * 0.5f; @@ -362,8 +363,8 @@ public class KeyboardView extends View { // Draw key label. final Drawable icon = key.getIcon(mKeyboard.mIconsSet, params.mAnimAlpha); float positionX = centerX; - if (key.mLabel != null) { - final String label = key.mLabel; + final String label = key.getLabel(); + if (label != null) { paint.setTypeface(key.selectTypeface(params)); paint.setTextSize(key.selectTextSize(params)); final float labelCharHeight = TypefaceUtils.getCharHeight( @@ -440,8 +441,8 @@ public class KeyboardView extends View { } // Draw hint label. - if (key.mHintLabel != null) { - final String hintLabel = key.mHintLabel; + final String hintLabel = key.getHintLabel(); + if (hintLabel != null) { paint.setTextSize(key.selectHintTextSize(params)); paint.setColor(key.selectHintTextColor(params)); blendAlpha(paint, params.mAnimAlpha); @@ -480,7 +481,7 @@ public class KeyboardView extends View { } // Draw key icon. - if (key.mLabel == null && icon != null) { + if (label == null && icon != null) { final int iconWidth = Math.min(icon.getIntrinsicWidth(), keyWidth); final int iconHeight = icon.getIntrinsicHeight(); final int iconX, alignX; @@ -504,7 +505,7 @@ public class KeyboardView extends View { } } - if (key.hasPopupHint() && key.mMoreKeys != null) { + if (key.hasPopupHint() && key.getMoreKeys() != null) { drawKeyPopupHint(key, canvas, paint, params); } } @@ -513,7 +514,7 @@ public class KeyboardView extends View { protected void drawKeyPopupHint(final Key key, final Canvas canvas, final Paint paint, final KeyDrawParams params) { final int keyWidth = key.getDrawWidth(); - final int keyHeight = key.mHeight; + final int keyHeight = key.getHeight(); paint.setTypeface(params.mTypeface); paint.setTextSize(params.mHintLetterSize); @@ -601,9 +602,9 @@ public class KeyboardView extends View { if (mInvalidateAllKeys) return; if (key == null) return; mInvalidatedKeys.add(key); - final int x = key.mX + getPaddingLeft(); - final int y = key.mY + getPaddingTop(); - invalidate(x, y, x + key.mWidth, y + key.mHeight); + final int x = key.getX() + getPaddingLeft(); + final int y = key.getY() + getPaddingTop(); + invalidate(x, y, x + key.getWidth(), y + key.getHeight()); } @Override diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java index 526c2f1ec..e4a8f4cb2 100644 --- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java @@ -217,7 +217,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack startWhileTypingFadeinAnimation(keyboardView); break; case MSG_REPEAT_KEY: - tracker.onKeyRepeat(msg.arg1); + tracker.onKeyRepeat(msg.arg1 /* code */, msg.arg2 /* repeatCount */); break; case MSG_LONGPRESS_KEY: keyboardView.onLongPress(tracker); @@ -230,12 +230,14 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } @Override - public void startKeyRepeatTimer(final PointerTracker tracker, final int delay) { + public void startKeyRepeatTimer(final PointerTracker tracker, final int repeatCount, + final int delay) { final Key key = tracker.getKey(); if (key == null || delay == 0) { return; } - sendMessageDelayed(obtainMessage(MSG_REPEAT_KEY, key.mCode, 0, tracker), delay); + sendMessageDelayed( + obtainMessage(MSG_REPEAT_KEY, key.getCode(), repeatCount, tracker), delay); } public void cancelKeyRepeatTimer() { @@ -296,7 +298,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack final MainKeyboardView keyboardView = getOuterInstance(); // When user hits the space or the enter key, just cancel the while-typing timer. - final int typedCode = typedKey.mCode; + final int typedCode = typedKey.getCode(); if (typedCode == Constants.CODE_SPACE || typedCode == Constants.CODE_ENTER) { if (isTyping) { startWhileTypingFadeinAnimation(keyboardView); @@ -636,7 +638,6 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack mKeyPreviewLingerTimeout = delay; } - private void locatePreviewPlacerView() { if (mPreviewPlacerView.getParent() != null) { return; @@ -805,11 +806,11 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } // The key preview is placed vertically above the top edge of the parent key with an // arbitrary offset. - final int previewY = key.mY - previewHeight + mKeyPreviewOffset + final int previewY = key.getY() - previewHeight + mKeyPreviewOffset + CoordinateUtils.y(mOriginCoords); if (background != null) { - final int hasMoreKeys = (key.mMoreKeys != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL; + final int hasMoreKeys = (key.getMoreKeys() != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL; background.setState(KEY_PREVIEW_BACKGROUND_STATE_TABLE[statePosition][hasMoreKeys]); background.setAlpha(PREVIEW_ALPHA); } @@ -901,7 +902,7 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } private MoreKeysPanel onCreateMoreKeysPanel(final Key key, final Context context) { - if (key.mMoreKeys == null) { + if (key.getMoreKeys() == null) { return null; } Keyboard moreKeysKeyboard = mMoreKeysKeyboardCache.get(key); @@ -936,15 +937,15 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } final KeyboardActionListener listener = mKeyboardActionListener; if (key.hasNoPanelAutoMoreKey()) { - final int moreKeyCode = key.mMoreKeys[0].mCode; + final int moreKeyCode = key.getMoreKeys()[0].mCode; tracker.onLongPressed(); - listener.onPressKey(moreKeyCode, false /* isRepeatKey */, true /* isSinglePointer */); + listener.onPressKey(moreKeyCode, 0 /* repeatCount */, true /* isSinglePointer */); listener.onCodeInput(moreKeyCode, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); listener.onReleaseKey(moreKeyCode, false /* withSliding */); return; } - final int code = key.mCode; + final int code = key.getCode(); if (code == Constants.CODE_SPACE || code == Constants.CODE_LANGUAGE_SWITCH) { // Long pressing the space key invokes IME switcher dialog. if (listener.onCustomRequest(Constants.CUSTOM_CODE_SHOW_INPUT_METHOD_PICKER)) { @@ -970,13 +971,13 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack // keys keyboard is placed at the touch point of the parent key. final int pointX = (mConfigShowMoreKeysKeyboardAtTouchedPoint && !keyPreviewEnabled) ? CoordinateUtils.x(lastCoords) - : key.mX + key.mWidth / 2; + : key.getX() + key.getWidth() / 2; // The more keys keyboard is usually vertically aligned with the top edge of the parent key // (plus vertical gap). If the key preview is enabled, the more keys keyboard is vertically // aligned with the bottom edge of the visible part of the key preview. // {@code mPreviewVisibleOffset} has been set appropriately in // {@link KeyboardView#showKeyPreview(PointerTracker)}. - final int pointY = key.mY + mKeyPreviewDrawParams.mPreviewVisibleOffset; + final int pointY = key.getY() + mKeyPreviewDrawParams.mPreviewVisibleOffset; moreKeysPanel.showMoreKeysPanel(this, this, pointX, pointY, mKeyboardActionListener); tracker.onShowMoreKeysPanel(moreKeysPanel); } @@ -1172,13 +1173,14 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack if (key.altCodeWhileTyping() && key.isEnabled()) { params.mAnimAlpha = mAltCodeKeyWhileTypingAnimAlpha; } - if (key.mCode == Constants.CODE_SPACE) { + final int code = key.getCode(); + if (code == Constants.CODE_SPACE) { drawSpacebar(key, canvas, paint); // Whether space key needs to show the "..." popup hint for special purposes if (key.isLongPressEnabled() && mHasMultipleEnabledIMEsOrSubtypes) { drawKeyPopupHint(key, canvas, paint, params); } - } else if (key.mCode == Constants.CODE_LANGUAGE_SWITCH) { + } else if (code == Constants.CODE_LANGUAGE_SWITCH) { super.onDrawKeyTopVisuals(key, canvas, paint, params); drawKeyPopupHint(key, canvas, paint, params); } else { @@ -1226,8 +1228,8 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack } private void drawSpacebar(final Key key, final Canvas canvas, final Paint paint) { - final int width = key.mWidth; - final int height = key.mHeight; + final int width = key.getWidth(); + final int height = key.getHeight(); // If input language are explicitly selected. if (mNeedsToDisplayLanguage) { diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java b/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java index a2001cb8f..6b76e2461 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysDetector.java @@ -39,7 +39,7 @@ public final class MoreKeysDetector extends KeyDetector { Key nearestKey = null; int nearestDist = (y < 0) ? mSlideAllowanceSquareTop : mSlideAllowanceSquare; - for (final Key key : getKeyboard().mKeys) { + for (final Key key : getKeyboard().getKeys()) { final int dist = key.squaredDistanceToEdge(touchX, touchY); if (dist < nearestDist) { nearestKey = key; diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java index 3fd29dcfd..8256d4623 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboard.java @@ -276,6 +276,7 @@ public final class MoreKeysKeyboard extends Keyboard { mParams.mVerticalGap = parentKeyboard.mVerticalGap / 2; mParentKey = parentKey; + final MoreKeySpec[] moreKeys = parentKey.getMoreKeys(); final int width, height; // {@link KeyPreviewDrawParams#mPreviewVisibleWidth} should have been set at // {@link MainKeyboardView#showKeyPreview(PointerTracker}, though there may be @@ -283,7 +284,7 @@ public final class MoreKeysKeyboard extends Keyboard { // zero-division error at // {@link MoreKeysKeyboardParams#setParameters(int,int,int,int,int,int,boolean,int)}. final boolean singleMoreKeyWithPreview = parentKeyboardView.isKeyPreviewPopupEnabled() - && !parentKey.noKeyPreview() && parentKey.mMoreKeys.length == 1 + && !parentKey.noKeyPreview() && moreKeys.length == 1 && keyPreviewDrawParams.mPreviewVisibleWidth > 0; if (singleMoreKeyWithPreview) { // Use pre-computed width and height if this more keys keyboard has only one key to @@ -312,8 +313,8 @@ public final class MoreKeysKeyboard extends Keyboard { mDivider = null; dividerWidth = 0; } - mParams.setParameters(parentKey.mMoreKeys.length, parentKey.getMoreKeysColumn(), - width, height, parentKey.mX + parentKey.mWidth / 2, + mParams.setParameters(moreKeys.length, parentKey.getMoreKeysColumn(), + width, height, parentKey.getX() + parentKey.getWidth() / 2, parentKeyboard.mId.mWidth, parentKey.isFixedColumnOrderMoreKeys(), dividerWidth); } @@ -321,7 +322,7 @@ public final class MoreKeysKeyboard extends Keyboard { private static int getMaxKeyWidth(final Key parentKey, final int minKeyWidth, final float padding, final Paint paint) { int maxWidth = minKeyWidth; - for (final MoreKeySpec spec : parentKey.mMoreKeys) { + for (final MoreKeySpec spec : parentKey.getMoreKeys()) { final String label = spec.mLabel; // If the label is single letter, minKeyWidth is enough to hold the label. if (label != null && StringUtils.codePointCount(label) > 1) { @@ -336,7 +337,7 @@ public final class MoreKeysKeyboard extends Keyboard { public MoreKeysKeyboard build() { final MoreKeysKeyboardParams params = mParams; final int moreKeyFlags = mParentKey.getMoreKeyLabelFlags(); - final MoreKeySpec[] moreKeys = mParentKey.mMoreKeys; + final MoreKeySpec[] moreKeys = mParentKey.getMoreKeys(); for (int n = 0; n < moreKeys.length; n++) { final MoreKeySpec moreKeySpec = moreKeys[n]; final int row = n / params.mNumColumns; diff --git a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java index f00f5a99e..973128d36 100644 --- a/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MoreKeysKeyboardView.java @@ -127,7 +127,7 @@ public class MoreKeysKeyboardView extends KeyboardView implements MoreKeysPanel public void onUpEvent(final int x, final int y, final int pointerId, final long eventTime) { if (mCurrentKey != null && mActivePointerId == pointerId) { updateReleaseKeyGraphics(mCurrentKey); - onCodeInput(mCurrentKey.mCode, x, y); + onCodeInput(mCurrentKey.getCode(), x, y); mCurrentKey = null; } } diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java index b66ee2a65..d4d0d8718 100644 --- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java +++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java @@ -64,7 +64,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { /** * Get KeyboardActionListener object that is used to register key code and so on. - * @return the KeyboardActionListner for this PointerTracker + * @return the KeyboardActionListner for this PointerTracke */ public KeyboardActionListener getKeyboardActionListener(); @@ -94,7 +94,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { public interface TimerProxy { public void startTypingStateTimer(Key typedKey); public boolean isTypingState(); - public void startKeyRepeatTimer(PointerTracker tracker, int delay); + public void startKeyRepeatTimer(PointerTracker tracker, int repeatCount, int delay); public void startLongPressTimer(PointerTracker tracker, int delay); public void cancelLongPressTimer(); public void startDoubleTapShiftKeyTimer(); @@ -111,7 +111,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { @Override public boolean isTypingState() { return false; } @Override - public void startKeyRepeatTimer(PointerTracker tracker, int delay) {} + public void startKeyRepeatTimer(PointerTracker tracker, int repeatCount, int delay) {} @Override public void startLongPressTimer(PointerTracker tracker, int delay) {} @Override @@ -490,7 +490,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { // Returns true if keyboard has been changed by this callback. private boolean callListenerOnPressAndCheckKeyboardLayoutChange(final Key key, - final boolean isRepeatKey) { + final int repeatCount) { // While gesture input is going on, this method should be a no-operation. But when gesture // input has been canceled, <code>sInGesture</code> and <code>mIsDetectingGesture</code> // are set to false. To keep this method is a no-operation, @@ -504,13 +504,13 @@ public final class PointerTracker implements PointerTrackerQueue.Element { KeyDetector.printableCode(key), ignoreModifierKey ? " ignoreModifier" : "", key.isEnabled() ? "" : " disabled", - isRepeatKey ? " repeat" : "")); + repeatCount > 0 ? " repeatCount=" + repeatCount : "")); } if (ignoreModifierKey) { return false; } if (key.isEnabled()) { - mListener.onPressKey(key.mCode, isRepeatKey, getActivePointerTrackerCount() == 1); + mListener.onPressKey(key.getCode(), repeatCount, getActivePointerTrackerCount() == 1); final boolean keyboardLayoutHasBeenChanged = mKeyboardLayoutHasBeenChanged; mKeyboardLayoutHasBeenChanged = false; mTimerProxy.startTypingStateTimer(key); @@ -776,7 +776,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (sInGesture || !mGestureStrokeWithPreviewPoints.isStartOfAGesture()) { return; } - if (key == null || !Character.isLetter(key.mCode)) { + if (key == null || !Character.isLetter(key.getCode())) { return; } if (DEBUG_LISTENER) { @@ -967,7 +967,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { // This onPress call may have changed keyboard layout. Those cases are detected at // {@link #setKeyboard}. In those cases, we should update key according to the new // keyboard layout. - if (callListenerOnPressAndCheckKeyboardLayoutChange(key, false /* isRepeatKey */)) { + if (callListenerOnPressAndCheckKeyboardLayoutChange(key, 0 /* repeatCount */)) { key = onDownKey(x, y, eventTime); } @@ -1057,7 +1057,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { // at {@link #setKeyboard}. In those cases, we should update key according // to the new keyboard layout. Key key = newKey; - if (callListenerOnPressAndCheckKeyboardLayoutChange(key, false /* isRepeatKey */)) { + if (callListenerOnPressAndCheckKeyboardLayoutChange(key, 0 /* repeatCount */)) { key = onMoveKey(x, y); } onMoveToNewKey(key, x, y); @@ -1075,8 +1075,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { + " phantom sudden move event (distance=%d) is translated to " + "up[%d,%d,%s]/down[%d,%d,%s] events", mPointerId, getDistance(x, y, lastX, lastY), - lastX, lastY, Constants.printableCode(oldKey.mCode), - x, y, Constants.printableCode(key.mCode))); + lastX, lastY, Constants.printableCode(oldKey.getCode()), + x, y, Constants.printableCode(key.getCode()))); } // TODO: This should be moved to outside of this nested if-clause? if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -1098,8 +1098,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { + " bogus down-move-up event (raidus=%.2f key diagonal) is " + " translated to up[%d,%d,%s]/down[%d,%d,%s] events", mPointerId, radiusRatio, - lastX, lastY, Constants.printableCode(oldKey.mCode), - x, y, Constants.printableCode(key.mCode))); + lastX, lastY, Constants.printableCode(oldKey.getCode()), + x, y, Constants.printableCode(key.getCode()))); } onUpEventInternal(x, y, eventTime); onDownEventInternal(x, y, eventTime); @@ -1107,7 +1107,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { private void processSildeOutFromOldKey(final Key oldKey) { setReleasedKeyGraphics(oldKey); - callListenerOnRelease(oldKey, oldKey.mCode, true /* withSliding */); + callListenerOnRelease(oldKey, oldKey.getCode(), true /* withSliding */); startSlidingKeyInput(oldKey); mTimerProxy.cancelKeyTimers(); } @@ -1263,7 +1263,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (sInGesture) { if (currentKey != null) { - callListenerOnRelease(currentKey, currentKey.mCode, true /* withSliding */); + callListenerOnRelease(currentKey, currentKey.getCode(), true /* withSliding */); } mayEndBatchInput(eventTime); return; @@ -1376,9 +1376,9 @@ public final class PointerTracker implements PointerTrackerQueue.Element { // doesn't have its more keys. (e.g. spacebar, globe key) // We always need to start the long press timer if the key has its more keys regardless of // whether or not we are in the sliding input mode. - if (mIsInSlidingKeyInput && key.mMoreKeys == null) return; + if (mIsInSlidingKeyInput && key.getMoreKeys() == null) return; final int delay; - switch (key.mCode) { + switch (key.getCode()) { case Constants.CODE_SHIFT: delay = sParams.mLongPressShiftLockTimeout; break; @@ -1401,7 +1401,7 @@ public final class PointerTracker implements PointerTrackerQueue.Element { return; } - final int code = key.mCode; + final int code = key.getCode(); callListenerOnCodeInput(key, code, x, y, eventTime); callListenerOnRelease(key, code, false /* withSliding */); } @@ -1412,17 +1412,19 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (!key.isRepeatable()) return; // Don't start key repeat when we are in sliding input mode. if (mIsInSlidingKeyInput) return; - detectAndSendKey(key, key.mX, key.mY, SystemClock.uptimeMillis()); - mTimerProxy.startKeyRepeatTimer(this, sParams.mKeyRepeatStartTimeout); + detectAndSendKey(key, key.getX(), key.getY(), SystemClock.uptimeMillis()); + final int startRepeatCount = 1; + mTimerProxy.startKeyRepeatTimer(this, startRepeatCount, sParams.mKeyRepeatStartTimeout); } - public void onKeyRepeat(final int code) { + public void onKeyRepeat(final int code, final int repeatCount) { final Key key = getKey(); - if (key == null || key.mCode != code) { + if (key == null || key.getCode() != code) { return; } - mTimerProxy.startKeyRepeatTimer(this, sParams.mKeyRepeatInterval); - callListenerOnPressAndCheckKeyboardLayoutChange(key, true /* isRepeatKey */); + final int nextRepeatCount = repeatCount + 1; + mTimerProxy.startKeyRepeatTimer(this, nextRepeatCount, sParams.mKeyRepeatInterval); + callListenerOnPressAndCheckKeyboardLayoutChange(key, repeatCount); callListenerOnCodeInput(key, code, mKeyX, mKeyY, SystemClock.uptimeMillis()); } diff --git a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java index 9b0a33cec..cd127c760 100644 --- a/java/src/com/android/inputmethod/keyboard/ProximityInfo.java +++ b/java/src/com/android/inputmethod/keyboard/ProximityInfo.java @@ -96,7 +96,7 @@ public class ProximityInfo { private static boolean needsProximityInfo(final Key key) { // Don't include special keys into ProximityInfo. - return key.mCode >= Constants.CODE_SPACE; + return key.getCode() >= Constants.CODE_SPACE; } private static int getProximityInfoKeysCount(final Key[] keys) { @@ -122,7 +122,7 @@ public class ProximityInfo { if (!needsProximityInfo(neighborKey)) { continue; } - proximityCharsArray[infoIndex] = neighborKey.mCode; + proximityCharsArray[infoIndex] = neighborKey.getCode(); infoIndex++; } } @@ -159,11 +159,11 @@ public class ProximityInfo { if (!needsProximityInfo(key)) { continue; } - keyXCoordinates[infoIndex] = key.mX; - keyYCoordinates[infoIndex] = key.mY; - keyWidths[infoIndex] = key.mWidth; - keyHeights[infoIndex] = key.mHeight; - keyCharCodes[infoIndex] = key.mCode; + keyXCoordinates[infoIndex] = key.getX(); + keyYCoordinates[infoIndex] = key.getY(); + keyWidths[infoIndex] = key.getWidth(); + keyHeights[infoIndex] = key.getHeight(); + keyCharCodes[infoIndex] = key.getCode(); infoIndex++; } @@ -183,7 +183,7 @@ public class ProximityInfo { if (!needsProximityInfo(key)) { continue; } - final Rect hitBox = key.mHitBox; + final Rect hitBox = key.getHitBox(); sweetSpotCenterXs[infoIndex] = hitBox.exactCenterX(); sweetSpotCenterYs[infoIndex] = hitBox.exactCenterY(); sweetSpotRadii[infoIndex] = defaultRadius; @@ -204,7 +204,7 @@ public class ProximityInfo { " [%2d] row=%d x/y/r=%7.2f/%7.2f/%5.2f %s code=%s", infoIndex, row, sweetSpotCenterXs[infoIndex], sweetSpotCenterYs[infoIndex], sweetSpotRadii[infoIndex], (row < rows ? "correct" : "default"), - Constants.printableCode(key.mCode))); + Constants.printableCode(key.getCode()))); } infoIndex++; } @@ -322,25 +322,29 @@ y |---+---+---+---+-v-+-|-+---+---+---+---+---| | thresholdBase and get have to align this on the center of the key. Hence, we don't need a separate value for bottomPixelWithinThreshold and call this yEnd right away. */ - final int topPixelWithinThreshold = key.mY - threshold; + final int keyX = key.getX(); + final int keyY = key.getY(); + final int topPixelWithinThreshold = keyY - threshold; final int yDeltaToGrid = topPixelWithinThreshold % mCellHeight; final int yMiddleOfTopCell = topPixelWithinThreshold - yDeltaToGrid + halfCellHeight; final int yStart = Math.max(halfCellHeight, yMiddleOfTopCell + (yDeltaToGrid <= halfCellHeight ? 0 : mCellHeight)); - final int yEnd = Math.min(fullGridHeight, key.mY + key.mHeight + threshold); + final int yEnd = Math.min(fullGridHeight, keyY + key.getHeight() + threshold); - final int leftPixelWithinThreshold = key.mX - threshold; + final int leftPixelWithinThreshold = keyX - threshold; final int xDeltaToGrid = leftPixelWithinThreshold % mCellWidth; final int xMiddleOfLeftCell = leftPixelWithinThreshold - xDeltaToGrid + halfCellWidth; final int xStart = Math.max(halfCellWidth, xMiddleOfLeftCell + (xDeltaToGrid <= halfCellWidth ? 0 : mCellWidth)); - final int xEnd = Math.min(fullGridWidth, key.mX + key.mWidth + threshold); + final int xEnd = Math.min(fullGridWidth, keyX + key.getWidth() + threshold); int baseIndexOfCurrentRow = (yStart / mCellHeight) * mGridWidth + (xStart / mCellWidth); for (int centerY = yStart; centerY <= yEnd; centerY += mCellHeight) { int index = baseIndexOfCurrentRow; for (int centerX = xStart; centerX <= xEnd; centerX += mCellWidth) { - if (key.squaredDistanceToEdge(centerX, centerY) < thresholdSquared) { + // TODO: Remove "index < neighborCountPerCell.length" below. + if (index < neighborCountPerCell.length + && key.squaredDistanceToEdge(centerX, centerY) < thresholdSquared) { neighborsFlatBuffer[index * keyCount + neighborCountPerCell[index]] = key; ++neighborCountPerCell[index]; } @@ -372,7 +376,7 @@ y |---+---+---+---+-v-+-|-+---+---+---+---+---| | thresholdBase and get if (index >= destLength) { break; } - final int code = key.mCode; + final int code = key.getCode(); if (code <= Constants.CODE_SPACE) { break; } diff --git a/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java new file mode 100644 index 000000000..c10fdbace --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/CodesArrayParser.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import com.android.inputmethod.latin.Constants; + +/** + * The string parser of codesArray specification for <GridRows />. The attribute codesArray is an + * array of string. + * Each element of the array defines a key label by specifying a code point as a hexadecimal string. + * A key label may consist of multiple code points separated by comma. + * Each element of the array optionally can have an output text definition after vertical bar + * marker. An output text may consist of multiple code points separated by comma. + * The format of the codesArray element should be: + * <pre> + * codePointInHex[,codePoint2InHex]*(|outputTextCodePointInHex[,outputTextCodePoint2InHex]*)? + * </pre> + */ +// TODO: Write unit tests for this class. +public final class CodesArrayParser { + // Constants for parsing. + private static final char COMMA = ','; + private static final char VERTICAL_BAR = '|'; + private static final String COMMA_STRING = ","; + private static final int BASE_HEX = 16; + + private CodesArrayParser() { + // This utility class is not publicly instantiable. + } + + private static String getLabelSpec(final String codesArraySpec) { + final int pos = codesArraySpec.indexOf(VERTICAL_BAR); + return (pos < 0) ? codesArraySpec : codesArraySpec.substring(0, pos); + } + + public static String parseLabel(final String codesArraySpec) { + final String labelSpec = getLabelSpec(codesArraySpec); + final StringBuilder sb = new StringBuilder(); + for (final String codeInHex : labelSpec.split(COMMA_STRING)) { + final int codePoint = Integer.parseInt(codeInHex, BASE_HEX); + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } + + private static String getCodeSpec(final String codesArraySpec) { + final int pos = codesArraySpec.indexOf(VERTICAL_BAR); + return (pos < 0) ? codesArraySpec : codesArraySpec.substring(pos + 1); + } + + public static int parseCode(final String codesArraySpec) { + final String codeSpec = getCodeSpec(codesArraySpec); + if (codeSpec.indexOf(COMMA) < 0) { + return Integer.parseInt(codeSpec, BASE_HEX); + } + return Constants.CODE_OUTPUT_TEXT; + } + + public static String parseOutputText(final String codesArraySpec) { + final String codeSpec = getCodeSpec(codesArraySpec); + if (codeSpec.indexOf(COMMA) < 0) { + return null; + } + final StringBuilder sb = new StringBuilder(); + for (final String codeInHex : codeSpec.split(COMMA_STRING)) { + final int codePoint = Integer.parseInt(codeInHex, BASE_HEX); + sb.appendCodePoint(codePoint); + } + return sb.toString(); + } +} diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java b/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java index f65056948..e6a674334 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyStyle.java @@ -24,7 +24,7 @@ public abstract class KeyStyle { public abstract String[] getStringArray(TypedArray a, int index); public abstract String getString(TypedArray a, int index); public abstract int getInt(TypedArray a, int index, int defaultValue); - public abstract int getFlag(TypedArray a, int index); + public abstract int getFlags(TypedArray a, int index); protected KeyStyle(final KeyboardTextsSet textsSet) { mTextsSet = textsSet; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java index 6aab3e7b3..05d855e31 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyStylesSet.java @@ -66,7 +66,7 @@ public final class KeyStylesSet { } @Override - public int getFlag(final TypedArray a, final int index) { + public int getFlags(final TypedArray a, final int index) { return a.getInt(index, 0); } } @@ -123,14 +123,12 @@ public final class KeyStylesSet { } @Override - public int getFlag(final TypedArray a, final int index) { - int flags = a.getInt(index, 0); - final Object value = mStyleAttributes.get(index); - if (value != null) { - flags |= (Integer)value; - } - final KeyStyle parentStyle = mStyles.get(mParentStyleName); - return flags | parentStyle.getFlag(a, index); + public int getFlags(final TypedArray a, final int index) { + final int parentFlags = mStyles.get(mParentStyleName).getFlags(a, index); + final Integer value = (Integer)mStyleAttributes.get(index); + final int styleFlags = (value != null) ? value : 0; + final int flags = a.getInt(index, 0); + return flags | styleFlags | parentFlags; } public void readKeyAttributes(final TypedArray keyAttr) { @@ -142,13 +140,13 @@ public final class KeyStylesSet { readString(keyAttr, R.styleable.Keyboard_Key_keyHintLabel); readStringArray(keyAttr, R.styleable.Keyboard_Key_moreKeys); readStringArray(keyAttr, R.styleable.Keyboard_Key_additionalMoreKeys); - readFlag(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags); + readFlags(keyAttr, R.styleable.Keyboard_Key_keyLabelFlags); readString(keyAttr, R.styleable.Keyboard_Key_keyIcon); readString(keyAttr, R.styleable.Keyboard_Key_keyIconDisabled); readString(keyAttr, R.styleable.Keyboard_Key_keyIconPreview); readInt(keyAttr, R.styleable.Keyboard_Key_maxMoreKeysColumn); readInt(keyAttr, R.styleable.Keyboard_Key_backgroundType); - readFlag(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); + readFlags(keyAttr, R.styleable.Keyboard_Key_keyActionFlags); } private void readString(final TypedArray a, final int index) { @@ -163,10 +161,11 @@ public final class KeyStylesSet { } } - private void readFlag(final TypedArray a, final int index) { + private void readFlags(final TypedArray a, final int index) { if (a.hasValue(index)) { final Integer value = (Integer)mStyleAttributes.get(index); - mStyleAttributes.put(index, a.getInt(index, 0) | (value != null ? value : 0)); + final int styleFlags = value != null ? value : 0; + mStyleAttributes.put(index, a.getInt(index, 0) | styleFlags); } } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java index b34d7c45f..22f7a83fc 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardBuilder.java @@ -29,6 +29,7 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.KeyboardId; +import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.utils.ResourceUtils; import com.android.inputmethod.latin.utils.RunInLocale; @@ -113,6 +114,7 @@ import java.util.Locale; * </pre> */ +// TODO: Write unit tests for this class. public class KeyboardBuilder<KP extends KeyboardParams> { private static final String BUILDER_TAG = "Keyboard.Builder"; private static final boolean DEBUG = false; @@ -120,6 +122,7 @@ public class KeyboardBuilder<KP extends KeyboardParams> { // Keyboard XML Tags private static final String TAG_KEYBOARD = "Keyboard"; private static final String TAG_ROW = "Row"; + private static final String TAG_GRID_ROWS = "GridRows"; private static final String TAG_KEY = "Key"; private static final String TAG_SPACER = "Spacer"; private static final String TAG_INCLUDE = "include"; @@ -218,20 +221,18 @@ public class KeyboardBuilder<KP extends KeyboardParams> { parseKeyboardAttributes(parser); startKeyboard(); parseKeyboardContent(parser, false); - break; - } else { - throw new XmlParseUtils.IllegalStartTag(parser, tag, TAG_KEYBOARD); + return; } + throw new XmlParseUtils.IllegalStartTag(parser, tag, TAG_KEYBOARD); } } } private void parseKeyboardAttributes(final XmlPullParser parser) { + final AttributeSet attr = Xml.asAttributeSet(parser); final TypedArray keyboardAttr = mContext.obtainStyledAttributes( - Xml.asAttributeSet(parser), R.styleable.Keyboard, R.attr.keyboardStyle, - R.style.Keyboard); - final TypedArray keyAttr = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_Key); + attr, R.styleable.Keyboard, R.attr.keyboardStyle, R.style.Keyboard); + final TypedArray keyAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key); try { final KeyboardParams params = mParams; final int height = params.mId.mHeight; @@ -314,6 +315,9 @@ public class KeyboardBuilder<KP extends KeyboardParams> { startRow(row); } parseRowContent(parser, row, skip); + } else if (TAG_GRID_ROWS.equals(tag)) { + if (DEBUG) startTag("<%s>%s", TAG_GRID_ROWS, skip ? " skipped" : ""); + parseGridRows(parser, skip); } else if (TAG_INCLUDE.equals(tag)) { parseIncludeKeyboardContent(parser, skip); } else if (TAG_SWITCH.equals(tag)) { @@ -328,31 +332,30 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (DEBUG) endTag("</%s>", tag); if (TAG_KEYBOARD.equals(tag)) { endKeyboard(); - break; - } else if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) - || TAG_MERGE.equals(tag)) { - break; - } else { - throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); + return; + } + if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) || TAG_MERGE.equals(tag)) { + return; } + throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); } } } private KeyboardRow parseRowAttributes(final XmlPullParser parser) throws XmlPullParserException { - final TypedArray a = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard); + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray keyboardAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard); try { - if (a.hasValue(R.styleable.Keyboard_horizontalGap)) { + if (keyboardAttr.hasValue(R.styleable.Keyboard_horizontalGap)) { throw new XmlParseUtils.IllegalAttribute(parser, TAG_ROW, "horizontalGap"); } - if (a.hasValue(R.styleable.Keyboard_verticalGap)) { + if (keyboardAttr.hasValue(R.styleable.Keyboard_verticalGap)) { throw new XmlParseUtils.IllegalAttribute(parser, TAG_ROW, "verticalGap"); } return new KeyboardRow(mResources, mParams, parser, mCurrentY); } finally { - a.recycle(); + keyboardAttr.recycle(); } } @@ -382,34 +385,97 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (!skip) { endRow(row); } - break; - } else if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) - || TAG_MERGE.equals(tag)) { - break; - } else { - throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); + return; + } + if (TAG_CASE.equals(tag) || TAG_DEFAULT.equals(tag) || TAG_MERGE.equals(tag)) { + return; } + throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_ROW); } } } - private void parseKey(final XmlPullParser parser, final KeyboardRow row, final boolean skip) + private void parseGridRows(final XmlPullParser parser, final boolean skip) throws XmlPullParserException, IOException { if (skip) { - XmlParseUtils.checkEndTag(TAG_KEY, parser); + XmlParseUtils.checkEndTag(TAG_GRID_ROWS, parser); if (DEBUG) { - startEndTag("<%s /> skipped", TAG_KEY); + startEndTag("<%s /> skipped", TAG_GRID_ROWS); } - } else { - final Key key = new Key(mResources, mParams, row, parser); - if (DEBUG) { - startEndTag("<%s%s %s moreKeys=%s />", TAG_KEY, - (key.isEnabled() ? "" : " disabled"), key, - Arrays.toString(key.mMoreKeys)); + return; + } + final KeyboardRow gridRows = new KeyboardRow(mResources, mParams, parser, mCurrentY); + final TypedArray gridRowAttr = mResources.obtainAttributes( + Xml.asAttributeSet(parser), R.styleable.Keyboard_GridRows); + final int codesArrayId = gridRowAttr.getResourceId( + R.styleable.Keyboard_GridRows_codesArray, 0); + final int textsArrayId = gridRowAttr.getResourceId( + R.styleable.Keyboard_GridRows_textsArray, 0); + gridRowAttr.recycle(); + if (codesArrayId == 0 && textsArrayId == 0) { + throw new XmlParseUtils.ParseException( + "Missing codesArray or textsArray attributes", parser); + } + if (codesArrayId != 0 && textsArrayId != 0) { + throw new XmlParseUtils.ParseException( + "Both codesArray and textsArray attributes specifed", parser); + } + final String[] array = mResources.getStringArray( + codesArrayId != 0 ? codesArrayId : textsArrayId); + final int counts = array.length; + final float keyWidth = gridRows.getKeyWidth(null, 0.0f); + final int numColumns = (int)(mParams.mOccupiedWidth / keyWidth); + for (int index = 0; index < counts; index += numColumns) { + final KeyboardRow row = new KeyboardRow(mResources, mParams, parser, mCurrentY); + startRow(row); + for (int c = 0; c < numColumns; c++) { + final int i = index + c; + if (i >= counts) { + break; + } + final String label; + final int code; + final String outputText; + if (codesArrayId != 0) { + final String codeArraySpec = array[i]; + label = CodesArrayParser.parseLabel(codeArraySpec); + code = CodesArrayParser.parseCode(codeArraySpec); + outputText = CodesArrayParser.parseOutputText(codeArraySpec); + } else { + final String textArraySpec = array[i]; + // TODO: Utilize KeySpecParser or write more generic TextsArrayParser. + label = textArraySpec; + code = Constants.CODE_OUTPUT_TEXT; + outputText = textArraySpec + (char)Constants.CODE_SPACE; + } + final int x = (int)row.getKeyX(null); + final int y = row.getKeyY(); + final Key key = new Key(mParams, label, null /* hintLabel */, 0 /* iconId */, + code, outputText, x, y, (int)keyWidth, (int)row.getRowHeight(), + row.getDefaultKeyLabelFlags(), row.getDefaultBackgroundType()); + endKey(key); + row.advanceXPos(keyWidth); } + endRow(row); + } + + XmlParseUtils.checkEndTag(TAG_GRID_ROWS, parser); + } + + private void parseKey(final XmlPullParser parser, final KeyboardRow row, final boolean skip) + throws XmlPullParserException, IOException { + if (skip) { XmlParseUtils.checkEndTag(TAG_KEY, parser); - endKey(key); + if (DEBUG) startEndTag("<%s /> skipped", TAG_KEY); + return; } + final Key key = new Key(mResources, mParams, row, parser); + if (DEBUG) { + startEndTag("<%s%s %s moreKeys=%s />", TAG_KEY, (key.isEnabled() ? "" : " disabled"), + key, Arrays.toString(key.getMoreKeys())); + } + XmlParseUtils.checkEndTag(TAG_KEY, parser); + endKey(key); } private void parseSpacer(final XmlPullParser parser, final KeyboardRow row, final boolean skip) @@ -417,12 +483,12 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (skip) { XmlParseUtils.checkEndTag(TAG_SPACER, parser); if (DEBUG) startEndTag("<%s /> skipped", TAG_SPACER); - } else { - final Key.Spacer spacer = new Key.Spacer(mResources, mParams, row, parser); - if (DEBUG) startEndTag("<%s />", TAG_SPACER); - XmlParseUtils.checkEndTag(TAG_SPACER, parser); - endKey(spacer); + return; } + final Key.Spacer spacer = new Key.Spacer(mResources, mParams, row, parser); + if (DEBUG) startEndTag("<%s />", TAG_SPACER); + XmlParseUtils.checkEndTag(TAG_SPACER, parser); + endKey(spacer); } private void parseIncludeKeyboardContent(final XmlPullParser parser, final boolean skip) @@ -440,66 +506,44 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (skip) { XmlParseUtils.checkEndTag(TAG_INCLUDE, parser); if (DEBUG) startEndTag("</%s> skipped", TAG_INCLUDE); - } else { - final AttributeSet attr = Xml.asAttributeSet(parser); - final TypedArray keyboardAttr = mResources.obtainAttributes(attr, - R.styleable.Keyboard_Include); - final TypedArray keyAttr = mResources.obtainAttributes(attr, - R.styleable.Keyboard_Key); - int keyboardLayout = 0; - float savedDefaultKeyWidth = 0; - int savedDefaultKeyLabelFlags = 0; - int savedDefaultBackgroundType = Key.BACKGROUND_TYPE_NORMAL; - try { - XmlParseUtils.checkAttributeExists(keyboardAttr, - R.styleable.Keyboard_Include_keyboardLayout, "keyboardLayout", - TAG_INCLUDE, parser); - keyboardLayout = keyboardAttr.getResourceId( - R.styleable.Keyboard_Include_keyboardLayout, 0); - if (row != null) { - if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { - // Override current x coordinate. - row.setXPos(row.getKeyX(keyAttr)); - } - // TODO: Remove this if-clause and do the same as backgroundType below. - savedDefaultKeyWidth = row.getDefaultKeyWidth(); - if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyWidth)) { - // Override default key width. - row.setDefaultKeyWidth(row.getKeyWidth(keyAttr)); - } - savedDefaultKeyLabelFlags = row.getDefaultKeyLabelFlags(); - // Bitwise-or default keyLabelFlag if exists. - row.setDefaultKeyLabelFlags(keyAttr.getInt( - R.styleable.Keyboard_Key_keyLabelFlags, 0) - | savedDefaultKeyLabelFlags); - savedDefaultBackgroundType = row.getDefaultBackgroundType(); - // Override default backgroundType if exists. - row.setDefaultBackgroundType(keyAttr.getInt( - R.styleable.Keyboard_Key_backgroundType, - savedDefaultBackgroundType)); - } - } finally { - keyboardAttr.recycle(); - keyAttr.recycle(); + return; + } + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray keyboardAttr = mResources.obtainAttributes( + attr, R.styleable.Keyboard_Include); + final TypedArray keyAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key); + int keyboardLayout = 0; + try { + XmlParseUtils.checkAttributeExists( + keyboardAttr, R.styleable.Keyboard_Include_keyboardLayout, "keyboardLayout", + TAG_INCLUDE, parser); + keyboardLayout = keyboardAttr.getResourceId( + R.styleable.Keyboard_Include_keyboardLayout, 0); + if (row != null) { + // Override current x coordinate. + row.setXPos(row.getKeyX(keyAttr)); + // Push current Row attributes and update with new attributes. + row.pushRowAttributes(keyAttr); } + } finally { + keyboardAttr.recycle(); + keyAttr.recycle(); + } - XmlParseUtils.checkEndTag(TAG_INCLUDE, parser); - if (DEBUG) { - startEndTag("<%s keyboardLayout=%s />",TAG_INCLUDE, - mResources.getResourceEntryName(keyboardLayout)); - } - final XmlResourceParser parserForInclude = mResources.getXml(keyboardLayout); - try { - parseMerge(parserForInclude, row, skip); - } finally { - if (row != null) { - // Restore default keyWidth, keyLabelFlags, and backgroundType. - row.setDefaultKeyWidth(savedDefaultKeyWidth); - row.setDefaultKeyLabelFlags(savedDefaultKeyLabelFlags); - row.setDefaultBackgroundType(savedDefaultBackgroundType); - } - parserForInclude.close(); + XmlParseUtils.checkEndTag(TAG_INCLUDE, parser); + if (DEBUG) { + startEndTag("<%s keyboardLayout=%s />",TAG_INCLUDE, + mResources.getResourceEntryName(keyboardLayout)); + } + final XmlResourceParser parserForInclude = mResources.getXml(keyboardLayout); + try { + parseMerge(parserForInclude, row, skip); + } finally { + if (row != null) { + // Restore Row attributes. + row.popRowAttributes(); } + parserForInclude.close(); } } @@ -516,11 +560,10 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } else { parseRowContent(parser, row, skip); } - break; - } else { - throw new XmlParseUtils.ParseException( - "Included keyboard layout must have <merge> root element", parser); + return; } + throw new XmlParseUtils.ParseException( + "Included keyboard layout must have <merge> root element", parser); } } } @@ -554,10 +597,9 @@ public class KeyboardBuilder<KP extends KeyboardParams> { final String tag = parser.getName(); if (TAG_SWITCH.equals(tag)) { if (DEBUG) endTag("</%s>", TAG_SWITCH); - break; - } else { - throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_SWITCH); + return; } + throw new XmlParseUtils.IllegalEndTag(parser, tag, TAG_SWITCH); } } } @@ -580,86 +622,92 @@ public class KeyboardBuilder<KP extends KeyboardParams> { if (id == null) { return true; } - final TypedArray a = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_Case); + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray caseAttr = mResources.obtainAttributes(attr, R.styleable.Keyboard_Case); try { - final boolean keyboardLayoutSetElementMatched = matchTypedValue(a, + final boolean keyboardLayoutSetMatched = matchString(caseAttr, + R.styleable.Keyboard_Case_keyboardLayoutSet, + SubtypeLocaleUtils.getKeyboardLayoutSetName(id.mSubtype)); + final boolean keyboardLayoutSetElementMatched = matchTypedValue(caseAttr, R.styleable.Keyboard_Case_keyboardLayoutSetElement, id.mElementId, KeyboardId.elementIdToName(id.mElementId)); - final boolean modeMatched = matchTypedValue(a, + final boolean modeMatched = matchTypedValue(caseAttr, R.styleable.Keyboard_Case_mode, id.mMode, KeyboardId.modeName(id.mMode)); - final boolean navigateNextMatched = matchBoolean(a, + final boolean navigateNextMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_navigateNext, id.navigateNext()); - final boolean navigatePreviousMatched = matchBoolean(a, + final boolean navigatePreviousMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_navigatePrevious, id.navigatePrevious()); - final boolean passwordInputMatched = matchBoolean(a, + final boolean passwordInputMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_passwordInput, id.passwordInput()); - final boolean clobberSettingsKeyMatched = matchBoolean(a, + final boolean clobberSettingsKeyMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_clobberSettingsKey, id.mClobberSettingsKey); - final boolean shortcutKeyEnabledMatched = matchBoolean(a, + final boolean shortcutKeyEnabledMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_shortcutKeyEnabled, id.mShortcutKeyEnabled); - final boolean shortcutKeyOnSymbolsMatched = matchBoolean(a, + final boolean shortcutKeyOnSymbolsMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_shortcutKeyOnSymbols, id.mShortcutKeyOnSymbols); - final boolean hasShortcutKeyMatched = matchBoolean(a, + final boolean hasShortcutKeyMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_hasShortcutKey, id.mHasShortcutKey); - final boolean languageSwitchKeyEnabledMatched = matchBoolean(a, + final boolean languageSwitchKeyEnabledMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_languageSwitchKeyEnabled, id.mLanguageSwitchKeyEnabled); - final boolean isMultiLineMatched = matchBoolean(a, + final boolean isMultiLineMatched = matchBoolean(caseAttr, R.styleable.Keyboard_Case_isMultiLine, id.isMultiLine()); - final boolean imeActionMatched = matchInteger(a, + final boolean imeActionMatched = matchInteger(caseAttr, R.styleable.Keyboard_Case_imeAction, id.imeAction()); - final boolean localeCodeMatched = matchString(a, + final boolean localeCodeMatched = matchString(caseAttr, R.styleable.Keyboard_Case_localeCode, id.mLocale.toString()); - final boolean languageCodeMatched = matchString(a, + final boolean languageCodeMatched = matchString(caseAttr, R.styleable.Keyboard_Case_languageCode, id.mLocale.getLanguage()); - final boolean countryCodeMatched = matchString(a, + final boolean countryCodeMatched = matchString(caseAttr, R.styleable.Keyboard_Case_countryCode, id.mLocale.getCountry()); - final boolean selected = keyboardLayoutSetElementMatched && modeMatched - && navigateNextMatched && navigatePreviousMatched && passwordInputMatched - && clobberSettingsKeyMatched && shortcutKeyEnabledMatched - && shortcutKeyOnSymbolsMatched && hasShortcutKeyMatched - && languageSwitchKeyEnabledMatched && isMultiLineMatched && imeActionMatched - && localeCodeMatched && languageCodeMatched && countryCodeMatched; + final boolean selected = keyboardLayoutSetMatched && keyboardLayoutSetElementMatched + && modeMatched && navigateNextMatched && navigatePreviousMatched + && passwordInputMatched && clobberSettingsKeyMatched + && shortcutKeyEnabledMatched && shortcutKeyOnSymbolsMatched + && hasShortcutKeyMatched && languageSwitchKeyEnabledMatched + && isMultiLineMatched && imeActionMatched && localeCodeMatched + && languageCodeMatched && countryCodeMatched; if (DEBUG) { - startTag("<%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s>%s", TAG_CASE, - textAttr(a.getString( + startTag("<%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s>%s", TAG_CASE, + textAttr(caseAttr.getString( + R.styleable.Keyboard_Case_keyboardLayoutSet), "keyboardLayoutSet"), + textAttr(caseAttr.getString( R.styleable.Keyboard_Case_keyboardLayoutSetElement), "keyboardLayoutSetElement"), - textAttr(a.getString(R.styleable.Keyboard_Case_mode), "mode"), - textAttr(a.getString(R.styleable.Keyboard_Case_imeAction), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_mode), "mode"), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_imeAction), "imeAction"), - booleanAttr(a, R.styleable.Keyboard_Case_navigateNext, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_navigateNext, "navigateNext"), - booleanAttr(a, R.styleable.Keyboard_Case_navigatePrevious, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_navigatePrevious, "navigatePrevious"), - booleanAttr(a, R.styleable.Keyboard_Case_clobberSettingsKey, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_clobberSettingsKey, "clobberSettingsKey"), - booleanAttr(a, R.styleable.Keyboard_Case_passwordInput, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_passwordInput, "passwordInput"), - booleanAttr(a, R.styleable.Keyboard_Case_shortcutKeyEnabled, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_shortcutKeyEnabled, "shortcutKeyEnabled"), - booleanAttr(a, R.styleable.Keyboard_Case_shortcutKeyOnSymbols, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_shortcutKeyOnSymbols, "shortcutKeyOnSymbols"), - booleanAttr(a, R.styleable.Keyboard_Case_hasShortcutKey, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_hasShortcutKey, "hasShortcutKey"), - booleanAttr(a, R.styleable.Keyboard_Case_languageSwitchKeyEnabled, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_languageSwitchKeyEnabled, "languageSwitchKeyEnabled"), - booleanAttr(a, R.styleable.Keyboard_Case_isMultiLine, + booleanAttr(caseAttr, R.styleable.Keyboard_Case_isMultiLine, "isMultiLine"), - textAttr(a.getString(R.styleable.Keyboard_Case_localeCode), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_localeCode), "localeCode"), - textAttr(a.getString(R.styleable.Keyboard_Case_languageCode), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_languageCode), "languageCode"), - textAttr(a.getString(R.styleable.Keyboard_Case_countryCode), + textAttr(caseAttr.getString(R.styleable.Keyboard_Case_countryCode), "countryCode"), selected ? "" : " skipped"); } return selected; } finally { - a.recycle(); + caseAttr.recycle(); } } @@ -692,7 +740,8 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } if (ResourceUtils.isIntegerValue(v)) { return intValue == a.getInt(index, 0); - } else if (ResourceUtils.isStringValue(v)) { + } + if (ResourceUtils.isStringValue(v)) { return StringUtils.containsInArray(strValue, a.getString(index).split("\\|")); } return false; @@ -711,10 +760,10 @@ public class KeyboardBuilder<KP extends KeyboardParams> { private void parseKeyStyle(final XmlPullParser parser, final boolean skip) throws XmlPullParserException, IOException { - TypedArray keyStyleAttr = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_KeyStyle); - TypedArray keyAttrs = mResources.obtainAttributes(Xml.asAttributeSet(parser), - R.styleable.Keyboard_Key); + final AttributeSet attr = Xml.asAttributeSet(parser); + final TypedArray keyStyleAttr = mResources.obtainAttributes( + attr, R.styleable.Keyboard_KeyStyle); + final TypedArray keyAttrs = mResources.obtainAttributes(attr, R.styleable.Keyboard_Key); try { if (!keyStyleAttr.hasValue(R.styleable.Keyboard_KeyStyle_styleName)) { throw new XmlParseUtils.ParseException("<" + TAG_KEY_STYLE @@ -756,7 +805,7 @@ public class KeyboardBuilder<KP extends KeyboardParams> { mRightEdgeKey = null; } addEdgeSpace(mParams.mRightPadding, row); - mCurrentY += row.mRowHeight; + mCurrentY += row.getRowHeight(); mCurrentRow = null; mTopEdge = false; } @@ -774,7 +823,10 @@ public class KeyboardBuilder<KP extends KeyboardParams> { } private void endKeyboard() { - // nothing to do here. + // {@link #parseGridRows(XmlPullParser,boolean)} may populate keyboard rows higher than + // previously expected. + final int actualHeight = mCurrentY - mParams.mVerticalGap + mParams.mBottomPadding; + mParams.mOccupiedHeight = Math.max(mParams.mOccupiedHeight, actualHeight); } private void addEdgeSpace(final float width, final KeyboardRow row) { diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java index a57b83ac0..d32bb7581 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardParams.java @@ -85,7 +85,7 @@ public class KeyboardParams { public void onAddKey(final Key newKey) { final Key key = (mKeysCache != null) ? mKeysCache.get(newKey) : newKey; final boolean isSpacer = key.isSpacer(); - if (isSpacer && key.mWidth == 0) { + if (isSpacer && key.getWidth() == 0) { // Ignore zero width {@link Spacer}. return; } @@ -94,7 +94,7 @@ public class KeyboardParams { return; } updateHistogram(key); - if (key.mCode == Constants.CODE_SHIFT) { + if (key.getCode() == Constants.CODE_SHIFT) { mShiftKeys.add(key); } if (key.altCodeWhileTyping()) { @@ -125,14 +125,14 @@ public class KeyboardParams { } private void updateHistogram(final Key key) { - final int height = key.mHeight + mVerticalGap; + final int height = key.getHeight() + mVerticalGap; final int heightCount = updateHistogramCounter(mHeightHistogram, height); if (heightCount > mMaxHeightCount) { mMaxHeightCount = heightCount; mMostCommonKeyHeight = height; } - final int width = key.mWidth + mHorizontalGap; + final int width = key.getWidth() + mHorizontalGap; final int widthCount = updateHistogramCounter(mWidthHistogram, width); if (widthCount > mMaxWidthCount) { mMaxWidthCount = widthCount; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java index 5fe84a704..0f9497c27 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardRow.java @@ -23,10 +23,13 @@ import android.util.Xml; import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.ResourceUtils; import org.xmlpull.v1.XmlPullParser; +import java.util.ArrayDeque; + /** * Container for keys in the keyboard. All keys in a row are at the same Y-coordinate. * Some of the key size defaults can be overridden per row from what the {@link Keyboard} @@ -38,64 +41,100 @@ public final class KeyboardRow { private static final int KEYWIDTH_FILL_RIGHT = -1; private final KeyboardParams mParams; - /** Default width of a key in this row. */ - private float mDefaultKeyWidth; - /** Default height of a key in this row. */ - public final int mRowHeight; - /** Default keyLabelFlags in this row. */ - private int mDefaultKeyLabelFlags; - /** Default backgroundType for this row */ - private int mDefaultBackgroundType; + /** The height of this row. */ + private final int mRowHeight; + + private final ArrayDeque<RowAttributes> mRowAttributesStack = CollectionUtils.newArrayDeque(); + + private static class RowAttributes { + /** Default width of a key in this row. */ + public final float mDefaultKeyWidth; + /** Default keyLabelFlags in this row. */ + public final int mDefaultKeyLabelFlags; + /** Default backgroundType for this row */ + public final int mDefaultBackgroundType; + + /** + * Parse and create key attributes. This constructor is used to parse Row tag. + * + * @param keyAttr an attributes array of Row tag. + * @param defaultKeyWidth a default key width. + * @param keyboardWidth the keyboard width that is required to calculate keyWidth attribute. + */ + public RowAttributes(final TypedArray keyAttr, final float defaultKeyWidth, + final int keyboardWidth) { + mDefaultKeyWidth = keyAttr.getFraction(R.styleable.Keyboard_Key_keyWidth, + keyboardWidth, keyboardWidth, defaultKeyWidth); + mDefaultKeyLabelFlags = keyAttr.getInt(R.styleable.Keyboard_Key_keyLabelFlags, 0); + mDefaultBackgroundType = keyAttr.getInt(R.styleable.Keyboard_Key_backgroundType, + Key.BACKGROUND_TYPE_NORMAL); + } + + /** + * Parse and update key attributes using default attributes. This constructor is used + * to parse include tag. + * + * @param keyAttr an attributes array of include tag. + * @param defaultRowAttr default Row attributes. + * @param keyboardWidth the keyboard width that is required to calculate keyWidth attribute. + */ + public RowAttributes(final TypedArray keyAttr, final RowAttributes defaultRowAttr, + final int keyboardWidth) { + mDefaultKeyWidth = keyAttr.getFraction(R.styleable.Keyboard_Key_keyWidth, + keyboardWidth, keyboardWidth, defaultRowAttr.mDefaultKeyWidth); + mDefaultKeyLabelFlags = keyAttr.getInt(R.styleable.Keyboard_Key_keyLabelFlags, 0) + | defaultRowAttr.mDefaultKeyLabelFlags; + mDefaultBackgroundType = keyAttr.getInt(R.styleable.Keyboard_Key_backgroundType, + defaultRowAttr.mDefaultBackgroundType); + } + } private final int mCurrentY; // Will be updated by {@link Key}'s constructor. private float mCurrentX; - public KeyboardRow(final Resources res, final KeyboardParams params, final XmlPullParser parser, - final int y) { + public KeyboardRow(final Resources res, final KeyboardParams params, + final XmlPullParser parser, final int y) { mParams = params; final TypedArray keyboardAttr = res.obtainAttributes(Xml.asAttributeSet(parser), R.styleable.Keyboard); mRowHeight = (int)ResourceUtils.getDimensionOrFraction(keyboardAttr, - R.styleable.Keyboard_rowHeight, - params.mBaseHeight, params.mDefaultRowHeight); + R.styleable.Keyboard_rowHeight, params.mBaseHeight, params.mDefaultRowHeight); keyboardAttr.recycle(); final TypedArray keyAttr = res.obtainAttributes(Xml.asAttributeSet(parser), R.styleable.Keyboard_Key); - mDefaultKeyWidth = keyAttr.getFraction(R.styleable.Keyboard_Key_keyWidth, - params.mBaseWidth, params.mBaseWidth, params.mDefaultKeyWidth); - mDefaultBackgroundType = keyAttr.getInt(R.styleable.Keyboard_Key_backgroundType, - Key.BACKGROUND_TYPE_NORMAL); + mRowAttributesStack.push(new RowAttributes( + keyAttr, params.mDefaultKeyWidth, params.mBaseWidth)); keyAttr.recycle(); - // TODO: Initialize this with <Row> attribute as backgroundType is done. - mDefaultKeyLabelFlags = 0; mCurrentY = y; mCurrentX = 0.0f; } - public float getDefaultKeyWidth() { - return mDefaultKeyWidth; + public int getRowHeight() { + return mRowHeight; } - public void setDefaultKeyWidth(final float defaultKeyWidth) { - mDefaultKeyWidth = defaultKeyWidth; + public void pushRowAttributes(final TypedArray keyAttr) { + final RowAttributes newAttributes = new RowAttributes( + keyAttr, mRowAttributesStack.peek(), mParams.mBaseWidth); + mRowAttributesStack.push(newAttributes); } - public int getDefaultKeyLabelFlags() { - return mDefaultKeyLabelFlags; + public void popRowAttributes() { + mRowAttributesStack.pop(); } - public void setDefaultKeyLabelFlags(final int keyLabelFlags) { - mDefaultKeyLabelFlags = keyLabelFlags; + public float getDefaultKeyWidth() { + return mRowAttributesStack.peek().mDefaultKeyWidth; } - public int getDefaultBackgroundType() { - return mDefaultBackgroundType; + public int getDefaultKeyLabelFlags() { + return mRowAttributesStack.peek().mDefaultKeyLabelFlags; } - public void setDefaultBackgroundType(final int backgroundType) { - mDefaultBackgroundType = backgroundType; + public int getDefaultBackgroundType() { + return mRowAttributesStack.peek().mDefaultBackgroundType; } public void setXPos(final float keyXPos) { @@ -111,29 +150,27 @@ public final class KeyboardRow { } public float getKeyX(final TypedArray keyAttr) { - if (keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { - final float keyXPos = keyAttr.getFraction(R.styleable.Keyboard_Key_keyXPos, - mParams.mBaseWidth, mParams.mBaseWidth, 0); - if (keyXPos < 0) { - // If keyXPos is negative, the actual x-coordinate will be - // keyboardWidth + keyXPos. - // keyXPos shouldn't be less than mCurrentX because drawable area for this - // key starts at mCurrentX. Or, this key will overlaps the adjacent key on - // its left hand side. - final int keyboardRightEdge = mParams.mOccupiedWidth - mParams.mRightPadding; - return Math.max(keyXPos + keyboardRightEdge, mCurrentX); - } else { - return keyXPos + mParams.mLeftPadding; - } + if (keyAttr == null || !keyAttr.hasValue(R.styleable.Keyboard_Key_keyXPos)) { + return mCurrentX; } - return mCurrentX; - } - - public float getKeyWidth(final TypedArray keyAttr) { - return getKeyWidth(keyAttr, mCurrentX); + final float keyXPos = keyAttr.getFraction(R.styleable.Keyboard_Key_keyXPos, + mParams.mBaseWidth, mParams.mBaseWidth, 0); + if (keyXPos >= 0) { + return keyXPos + mParams.mLeftPadding; + } + // If keyXPos is negative, the actual x-coordinate will be + // keyboardWidth + keyXPos. + // keyXPos shouldn't be less than mCurrentX because drawable area for this + // key starts at mCurrentX. Or, this key will overlaps the adjacent key on + // its left hand side. + final int keyboardRightEdge = mParams.mOccupiedWidth - mParams.mRightPadding; + return Math.max(keyXPos + keyboardRightEdge, mCurrentX); } public float getKeyWidth(final TypedArray keyAttr, final float keyXPos) { + if (keyAttr == null) { + return getDefaultKeyWidth(); + } final int widthType = ResourceUtils.getEnumValue(keyAttr, R.styleable.Keyboard_Key_keyWidth, KEYWIDTH_NOT_ENUM); switch (widthType) { @@ -144,7 +181,7 @@ public final class KeyboardRow { return keyboardRightEdge - keyXPos; default: // KEYWIDTH_NOT_ENUM return keyAttr.getFraction(R.styleable.Keyboard_Key_keyWidth, - mParams.mBaseWidth, mParams.mBaseWidth, mDefaultKeyWidth); + mParams.mBaseWidth, mParams.mBaseWidth, getDefaultKeyWidth()); } } } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java index 164910dd4..089db12a2 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java @@ -46,7 +46,7 @@ public final class KeyboardState { public void setAlphabetShiftLockedKeyboard(); public void setAlphabetShiftLockShiftedKeyboard(); public void setSymbolsKeyboard(); - public void setSymbolsShiftedKeyboard(); + public void setEmojiKeyboard(); /** * Request to call back {@link KeyboardState#onUpdateShiftState(int, int)}. @@ -64,21 +64,20 @@ public final class KeyboardState { private ModifierKeyState mSymbolKeyState = new ModifierKeyState("Symbol"); // TODO: Merge {@link #mSwitchState}, {@link #mIsAlphabetMode}, {@link #mAlphabetShiftState}, - // {@link #mIsSymbolShifted}, {@link #mPrevMainKeyboardWasShiftLocked}, and - // {@link #mPrevSymbolsKeyboardWasShifted} into single state variable. + // {@link #mPrevMainKeyboardWasShiftLocked} into single state variable. private static final int SWITCH_STATE_ALPHA = 0; private static final int SWITCH_STATE_SYMBOL_BEGIN = 1; private static final int SWITCH_STATE_SYMBOL = 2; private static final int SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL = 3; - private static final int SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE = 4; private static final int SWITCH_STATE_MOMENTARY_ALPHA_SHIFT = 5; private int mSwitchState = SWITCH_STATE_ALPHA; + // TODO: Consolidate these two mode booleans into one integer to distinguish between alphabet, + // symbols, and emoji mode. private boolean mIsAlphabetMode; + private boolean mIsEmojiMode; private AlphabetShiftState mAlphabetShiftState = new AlphabetShiftState(); - private boolean mIsSymbolShifted; private boolean mPrevMainKeyboardWasShiftLocked; - private boolean mPrevSymbolsKeyboardWasShifted; private int mRecapitalizeMode; // For handling double tap. @@ -91,6 +90,7 @@ public final class KeyboardState { public boolean mIsValid; public boolean mIsAlphabetMode; public boolean mIsAlphabetShiftLocked; + public boolean mIsEmojiMode; public int mShiftMode; @Override @@ -99,8 +99,10 @@ public final class KeyboardState { if (mIsAlphabetMode) { if (mIsAlphabetShiftLocked) return "ALPHABET_SHIFT_LOCKED"; return "ALPHABET_" + shiftModeToString(mShiftMode); + } else if (mIsEmojiMode) { + return "EMOJI"; } else { - return "SYMBOLS_" + shiftModeToString(mShiftMode); + return "SYMBOLS"; } } } @@ -117,7 +119,6 @@ public final class KeyboardState { // Reset alphabet shift state. mAlphabetShiftState.setShiftLocked(false); mPrevMainKeyboardWasShiftLocked = false; - mPrevSymbolsKeyboardWasShifted = false; mShiftKeyState.onRelease(); mSymbolKeyState.onRelease(); onRestoreKeyboardState(); @@ -131,13 +132,13 @@ public final class KeyboardState { public void onSaveKeyboardState() { final SavedKeyboardState state = mSavedKeyboardState; state.mIsAlphabetMode = mIsAlphabetMode; + state.mIsEmojiMode = mIsEmojiMode; if (mIsAlphabetMode) { state.mIsAlphabetShiftLocked = mAlphabetShiftState.isShiftLocked(); state.mShiftMode = mAlphabetShiftState.isAutomaticShifted() ? AUTOMATIC_SHIFT : (mAlphabetShiftState.isShiftedOrShiftLocked() ? MANUAL_SHIFT : UNSHIFT); } else { state.mIsAlphabetShiftLocked = mPrevMainKeyboardWasShiftLocked; - state.mShiftMode = mIsSymbolShifted ? MANUAL_SHIFT : UNSHIFT; } state.mIsValid = true; if (DEBUG_EVENT) { @@ -152,12 +153,10 @@ public final class KeyboardState { } if (!state.mIsValid || state.mIsAlphabetMode) { setAlphabetKeyboard(); + } else if (state.mIsEmojiMode) { + setEmojiKeyboard(); } else { - if (state.mShiftMode == MANUAL_SHIFT) { - setSymbolsShiftedKeyboard(); - } else { - setSymbolsKeyboard(); - } + setSymbolsKeyboard(); } if (!state.mIsValid) return; @@ -233,14 +232,8 @@ public final class KeyboardState { } if (mIsAlphabetMode) { mPrevMainKeyboardWasShiftLocked = mAlphabetShiftState.isShiftLocked(); - if (mPrevSymbolsKeyboardWasShifted) { - setSymbolsShiftedKeyboard(); - } else { - setSymbolsKeyboard(); - } - mPrevSymbolsKeyboardWasShifted = false; + setSymbolsKeyboard(); } else { - mPrevSymbolsKeyboardWasShifted = mIsSymbolShifted; setAlphabetKeyboard(); if (mPrevMainKeyboardWasShiftLocked) { setShiftLocked(true); @@ -257,7 +250,6 @@ public final class KeyboardState { } if (mIsAlphabetMode) return; - mPrevSymbolsKeyboardWasShifted = mIsSymbolShifted; setAlphabetKeyboard(); if (mPrevMainKeyboardWasShiftLocked) { setShiftLocked(true); @@ -265,14 +257,6 @@ public final class KeyboardState { mPrevMainKeyboardWasShiftLocked = false; } - private void toggleShiftInSymbols() { - if (mIsSymbolShifted) { - setSymbolsKeyboard(); - } else { - setSymbolsShiftedKeyboard(); - } - } - private void setAlphabetKeyboard() { if (DEBUG_ACTION) { Log.d(TAG, "setAlphabetKeyboard"); @@ -280,7 +264,7 @@ public final class KeyboardState { mSwitchActions.setAlphabetKeyboard(); mIsAlphabetMode = true; - mIsSymbolShifted = false; + mIsEmojiMode = false; mRecapitalizeMode = RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE; mSwitchState = SWITCH_STATE_ALPHA; mSwitchActions.requestUpdatingShiftState(); @@ -292,22 +276,18 @@ public final class KeyboardState { } mSwitchActions.setSymbolsKeyboard(); mIsAlphabetMode = false; - mIsSymbolShifted = false; // Reset alphabet shift state. mAlphabetShiftState.setShiftLocked(false); mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; } - private void setSymbolsShiftedKeyboard() { + private void setEmojiKeyboard() { if (DEBUG_ACTION) { - Log.d(TAG, "setSymbolsShiftedKeyboard"); + Log.d(TAG, "setEmojiKeyboard"); } - mSwitchActions.setSymbolsShiftedKeyboard(); mIsAlphabetMode = false; - mIsSymbolShifted = true; - // Reset alphabet shift state. - mAlphabetShiftState.setShiftLocked(false); - mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; + mIsEmojiMode = true; + mSwitchActions.setEmojiKeyboard(); } public void onPressKey(final int code, final boolean isSinglePointer, final int autoCaps) { @@ -357,7 +337,7 @@ public final class KeyboardState { } else if (code == Constants.CODE_CAPSLOCK) { setShiftLocked(!mAlphabetShiftState.isShiftLocked()); } else if (code == Constants.CODE_SWITCH_ALPHA_SYMBOL) { - onReleaseSymbol(withSliding); + onReleaseSymbol(); } } @@ -367,16 +347,11 @@ public final class KeyboardState { mSwitchState = SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL; } - private void onReleaseSymbol(final boolean withSliding) { + private void onReleaseSymbol() { if (mSymbolKeyState.isChording()) { // Switch back to the previous keyboard mode if the user chords the mode change key and // another key, then releases the mode change key. toggleAlphabetAndSymbols(); - } else if (!withSliding) { - // If the mode change key is being released without sliding, we should forget the - // previous symbols keyboard shift state and simply switch back to symbols layout - // (never symbols shifted) next time the mode gets changed to symbols layout. - mPrevSymbolsKeyboardWasShifted = false; } mSymbolKeyState.onRelease(); } @@ -442,47 +417,43 @@ public final class KeyboardState { if (RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE != mRecapitalizeMode) { return; } - if (mIsAlphabetMode) { - mIsInDoubleTapShiftKey = mSwitchActions.isInDoubleTapShiftKeyTimeout(); - if (!mIsInDoubleTapShiftKey) { - // This is first tap. - mSwitchActions.startDoubleTapShiftKeyTimer(); - } - if (mIsInDoubleTapShiftKey) { - if (mAlphabetShiftState.isManualShifted() || mIsInAlphabetUnshiftedFromShifted) { - // Shift key has been double tapped while in manual shifted or automatic - // shifted state. - setShiftLocked(true); - } else { - // Shift key has been double tapped while in normal state. This is the second - // tap to disable shift locked state, so just ignore this. - } + if (!mIsAlphabetMode) { + // There is no shift key on symbols keyboard. + return; + } + mIsInDoubleTapShiftKey = mSwitchActions.isInDoubleTapShiftKeyTimeout(); + if (!mIsInDoubleTapShiftKey) { + // This is first tap. + mSwitchActions.startDoubleTapShiftKeyTimer(); + } + if (mIsInDoubleTapShiftKey) { + if (mAlphabetShiftState.isManualShifted() || mIsInAlphabetUnshiftedFromShifted) { + // Shift key has been double tapped while in manual shifted or automatic + // shifted state. + setShiftLocked(true); } else { - if (mAlphabetShiftState.isShiftLocked()) { - // Shift key is pressed while shift locked state, we will treat this state as - // shift lock shifted state and mark as if shift key pressed while normal state. - setShifted(SHIFT_LOCK_SHIFTED); - mShiftKeyState.onPress(); - } else if (mAlphabetShiftState.isAutomaticShifted()) { - // Shift key is pressed while automatic shifted, we have to move to manual - // shifted. - setShifted(MANUAL_SHIFT); - mShiftKeyState.onPress(); - } else if (mAlphabetShiftState.isShiftedOrShiftLocked()) { - // In manual shifted state, we just record shift key has been pressing while - // shifted state. - mShiftKeyState.onPressOnShifted(); - } else { - // In base layout, chording or manual shifted mode is started. - setShifted(MANUAL_SHIFT); - mShiftKeyState.onPress(); - } + // Shift key has been double tapped while in normal state. This is the second + // tap to disable shift locked state, so just ignore this. } } else { - // In symbol mode, just toggle symbol and symbol more keyboard. - toggleShiftInSymbols(); - mSwitchState = SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE; - mShiftKeyState.onPress(); + if (mAlphabetShiftState.isShiftLocked()) { + // Shift key is pressed while shift locked state, we will treat this state as + // shift lock shifted state and mark as if shift key pressed while normal state. + setShifted(SHIFT_LOCK_SHIFTED); + mShiftKeyState.onPress(); + } else if (mAlphabetShiftState.isAutomaticShifted()) { + // Shift key is pressed while automatic shifted, we have to move to manual shifted. + setShifted(MANUAL_SHIFT); + mShiftKeyState.onPress(); + } else if (mAlphabetShiftState.isShiftedOrShiftLocked()) { + // In manual shifted state, we just record shift key has been pressing while + // shifted state. + mShiftKeyState.onPressOnShifted(); + } else { + // In base layout, chording or manual shifted mode is started. + setShifted(MANUAL_SHIFT); + mShiftKeyState.onPress(); + } } } @@ -537,11 +508,7 @@ public final class KeyboardState { mIsInAlphabetUnshiftedFromShifted = true; } } else { - // In symbol mode, switch back to the previous keyboard mode if the user chords the - // shift key and another key, then releases the shift key. - if (mShiftKeyState.isChording()) { - toggleShiftInSymbols(); - } + // There is no shift key on symbols keyboard. } mShiftKeyState.onRelease(); } @@ -555,9 +522,6 @@ public final class KeyboardState { case SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL: toggleAlphabetAndSymbols(); break; - case SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE: - toggleShiftInSymbols(); - break; case SWITCH_STATE_MOMENTARY_ALPHA_SHIFT: setAlphabetKeyboard(); break; @@ -585,12 +549,6 @@ public final class KeyboardState { } } break; - case SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE: - if (code == Constants.CODE_SHIFT) { - // Detected only the shift key has been pressed on symbol layout, and then released. - mSwitchState = SWITCH_STATE_SYMBOL_BEGIN; - } - break; case SWITCH_STATE_SYMBOL_BEGIN: if (!isSpaceCharacter(code) && (Constants.isLetterCode(code) || code == Constants.CODE_OUTPUT_TEXT)) { @@ -602,7 +560,6 @@ public final class KeyboardState { // characters followed by a space/enter. if (isSpaceCharacter(code)) { toggleAlphabetAndSymbols(); - mPrevSymbolsKeyboardWasShifted = false; } break; } @@ -610,6 +567,8 @@ public final class KeyboardState { // If the code is a letter, update keyboard shift state. if (Constants.isLetterCode(code)) { updateAlphabetShiftState(autoCaps, RecapitalizeStatus.NOT_A_RECAPITALIZE_MODE); + } else if (code == Constants.CODE_EMOJI) { + setEmojiKeyboard(); } } @@ -628,7 +587,6 @@ public final class KeyboardState { case SWITCH_STATE_SYMBOL_BEGIN: return "SYMBOL-BEGIN"; case SWITCH_STATE_SYMBOL: return "SYMBOL"; case SWITCH_STATE_MOMENTARY_ALPHA_AND_SYMBOL: return "MOMENTARY-ALPHA-SYMBOL"; - case SWITCH_STATE_MOMENTARY_SYMBOL_AND_MORE: return "MOMENTARY-SYMBOL-MORE"; case SWITCH_STATE_MOMENTARY_ALPHA_SHIFT: return "MOMENTARY-ALPHA_SHIFT"; default: return null; } @@ -636,8 +594,7 @@ public final class KeyboardState { @Override public String toString() { - return "[keyboard=" + (mIsAlphabetMode ? mAlphabetShiftState.toString() - : (mIsSymbolShifted ? "SYMBOLS_SHIFTED" : "SYMBOLS")) + return "[keyboard=" + (mIsAlphabetMode ? mAlphabetShiftState.toString() : "SYMBOLS") + " shift=" + mShiftKeyState + " symbol=" + mSymbolKeyState + " switch=" + switchStateToString(mSwitchState) + "]"; diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index b55e19d52..6d053f5c0 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -27,18 +27,18 @@ import java.util.HashMap; /** * !!!!! DO NOT EDIT THIS FILE !!!!! * - * This file is generated by tools/maketext. The base template file is - * tools/maketext/res/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.tmpl + * This file is generated by tools/make-keyboard-text. The base template file is + * tools/make-keyboard-text/res/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.tmpl * * This file must be updated when any text resources in keyboard layout files have been changed. * These text resources are referred as "!text/<resource_name>" in keyboard XML definitions, * and should be defined in - * tools/maketext/res/values-<locale>/donottranslate-more-keys.xml + * tools/make-keyboard-text/res/values-<locale>/donottranslate-more-keys.xml * * To update this file, please run the following commands. * $ cd $ANDROID_BUILD_TOP - * $ mmm packages/inputmethods/LatinIME/tools/maketext - * $ maketext -java packages/inputmethods/LatinIME/java/src + * $ mmm packages/inputmethods/LatinIME/tools/make-keyboard-text + * $ make-keyboard-text -java packages/inputmethods/LatinIME/java/src * * The updated source file will be generated to the following path (this file). * packages/inputmethods/LatinIME/java/src/com/android/inputmethod/keyboard/internal/ @@ -153,8 +153,8 @@ public final class KeyboardTextsSet { /* 48 */ "single_angle_quotes", /* 49 */ "double_angle_quotes", /* 50 */ "more_keys_for_currency_dollar", - /* 51 */ "keylabel_for_currency_generic", - /* 52 */ "more_keys_for_currency_generic", + /* 51 */ "keylabel_for_currency", + /* 52 */ "more_keys_for_currency", /* 53 */ "more_keys_for_punctuation", /* 54 */ "more_keys_for_star", /* 55 */ "more_keys_for_bullet", @@ -233,25 +233,24 @@ public final class KeyboardTextsSet { /* 128 */ "label_to_phone_symbols_key", /* 129 */ "label_time_am", /* 130 */ "label_time_pm", - /* 131 */ "label_to_symbol_key_pcqwerty", - /* 132 */ "keylabel_for_popular_domain", - /* 133 */ "more_keys_for_popular_domain", - /* 134 */ "more_keys_for_smiley", - /* 135 */ "single_laqm_raqm", - /* 136 */ "single_laqm_raqm_rtl", - /* 137 */ "single_raqm_laqm", - /* 138 */ "double_laqm_raqm", - /* 139 */ "double_laqm_raqm_rtl", - /* 140 */ "double_raqm_laqm", - /* 141 */ "single_lqm_rqm", - /* 142 */ "single_9qm_lqm", - /* 143 */ "single_9qm_rqm", - /* 144 */ "double_lqm_rqm", - /* 145 */ "double_9qm_lqm", - /* 146 */ "double_9qm_rqm", - /* 147 */ "more_keys_for_single_quote", - /* 148 */ "more_keys_for_double_quote", - /* 149 */ "more_keys_for_tablet_double_quote", + /* 131 */ "keylabel_for_popular_domain", + /* 132 */ "more_keys_for_popular_domain", + /* 133 */ "more_keys_for_smiley", + /* 134 */ "single_laqm_raqm", + /* 135 */ "single_laqm_raqm_rtl", + /* 136 */ "single_raqm_laqm", + /* 137 */ "double_laqm_raqm", + /* 138 */ "double_laqm_raqm_rtl", + /* 139 */ "double_raqm_laqm", + /* 140 */ "single_lqm_rqm", + /* 141 */ "single_9qm_lqm", + /* 142 */ "single_9qm_rqm", + /* 143 */ "double_lqm_rqm", + /* 144 */ "double_9qm_lqm", + /* 145 */ "double_9qm_rqm", + /* 146 */ "more_keys_for_single_quote", + /* 147 */ "more_keys_for_double_quote", + /* 148 */ "more_keys_for_tablet_double_quote", }; private static final String EMPTY = ""; @@ -278,7 +277,7 @@ public final class KeyboardTextsSet { /* 50 */ "\u00A2,\u00A3,\u20AC,\u00A5,\u20B1", /* 51 */ "$", /* 52 */ "$,\u00A2,\u20AC,\u00A3,\u00A5,\u20B1", - /* 53 */ "!fixedColumnOrder!8,\",\',#,-,:,!,\\,,?,@,&,\\%,+,;,/,(,)", + /* 53 */ "!fixedColumnOrder!3,!,\\,,?,:,;,@", // U+2020: "†" DAGGER // U+2021: "‡" DOUBLE DAGGER // U+2605: "★" BLACK STAR @@ -289,8 +288,7 @@ public final class KeyboardTextsSet { // U+2666: "♦" BLACK DIAMOND SUIT // U+2663: "♣" BLACK CLUB SUIT /* 55 */ "\u266A,\u2665,\u2660,\u2666,\u2663", - // U+00B1: "±" PLUS-MINUS SIGN - /* 56 */ "\u00B1", + /* 56 */ EMPTY, // The all letters need to be mirrored are found at // http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt /* 57 */ "!fixedColumnOrder!3,<,{,[", @@ -361,10 +359,9 @@ public final class KeyboardTextsSet { // U+2030: "‰" PER MILLE SIGN /* 103 */ "\u2030", /* 104 */ ",", - /* 105 */ "!", - /* 106 */ "!", - /* 107 */ "?", - /* 108 */ "?", + /* 105~ */ + EMPTY, EMPTY, EMPTY, EMPTY, + /* ~108 */ /* 109 */ "\'", /* 110 */ "\"", /* 111 */ "\"", @@ -396,12 +393,10 @@ public final class KeyboardTextsSet { /* 129 */ "AM", // Key label for "post meridiem" /* 130 */ "PM", - // Label for "switch to symbols" key on PC QWERTY layout - /* 131 */ "Sym", - /* 132 */ ".com", + /* 131 */ ".com", // popular web domains for the locale - most popular, displayed on the keyboard - /* 133 */ "!hasLabels!,.net,.org,.gov,.edu", - /* 134 */ "!fixedColumnOrder!5,!hasLabels!,=-O|=-O ,:-P|:-P ,;-)|;-) ,:-(|:-( ,:-)|:-) ,:-!|:-! ,:-$|:-$ ,B-)|B-) ,:O|:O ,:-*|:-* ,:-D|:-D ,:\'(|:\'( ,:-\\\\|:-\\\\ ,O:-)|O:-) ,:-[|:-[ ", + /* 132 */ "!hasLabels!,.net,.org,.gov,.edu", + /* 133 */ "!fixedColumnOrder!5,!hasLabels!,=-O|=-O ,:-P|:-P ,;-)|;-) ,:-(|:-( ,:-)|:-) ,:-!|:-! ,:-$|:-$ ,B-)|B-) ,:O|:O ,:-*|:-* ,:-D|:-D ,:\'(|:\'( ,:-\\\\|:-\\\\ ,O:-)|O:-) ,:-[|:-[ ", // U+2039: "‹" SINGLE LEFT-POINTING ANGLE QUOTATION MARK // U+203A: "›" SINGLE RIGHT-POINTING ANGLE QUOTATION MARK // U+00AB: "«" LEFT-POINTING DOUBLE ANGLE QUOTATION MARK @@ -423,24 +418,24 @@ public final class KeyboardTextsSet { // The following each quotation mark pair consist of // <opening quotation mark>, <closing quotation mark> // and is named after (single|double)_<opening quotation mark>_<closing quotation mark>. - /* 135 */ "\u2039,\u203A", - /* 136 */ "\u2039|\u203A,\u203A|\u2039", - /* 137 */ "\u203A,\u2039", - /* 138 */ "\u00AB,\u00BB", - /* 139 */ "\u00AB|\u00BB,\u00BB|\u00AB", - /* 140 */ "\u00BB,\u00AB", + /* 134 */ "\u2039,\u203A", + /* 135 */ "\u2039|\u203A,\u203A|\u2039", + /* 136 */ "\u203A,\u2039", + /* 137 */ "\u00AB,\u00BB", + /* 138 */ "\u00AB|\u00BB,\u00BB|\u00AB", + /* 139 */ "\u00BB,\u00AB", // The following each quotation mark triplet consists of // <another quotation mark>, <opening quotation mark>, <closing quotation mark> // and is named after (single|double)_<opening quotation mark>_<closing quotation mark>. - /* 141 */ "\u201A,\u2018,\u2019", - /* 142 */ "\u2019,\u201A,\u2018", - /* 143 */ "\u2018,\u201A,\u2019", - /* 144 */ "\u201E,\u201C,\u201D", - /* 145 */ "\u201D,\u201E,\u201C", - /* 146 */ "\u201C,\u201E,\u201D", - /* 147 */ "!fixedColumnOrder!5,!text/single_quotes,!text/single_angle_quotes", - /* 148 */ "!fixedColumnOrder!5,!text/double_quotes,!text/double_angle_quotes", - /* 149 */ "!fixedColumnOrder!6,!text/double_quotes,!text/single_quotes,!text/double_angle_quotes,!text/single_angle_quotes", + /* 140 */ "\u201A,\u2018,\u2019", + /* 141 */ "\u2019,\u201A,\u2018", + /* 142 */ "\u2018,\u201A,\u2019", + /* 143 */ "\u201E,\u201C,\u201D", + /* 144 */ "\u201D,\u201E,\u201C", + /* 145 */ "\u201C,\u201E,\u201D", + /* 146 */ "!fixedColumnOrder!5,!text/single_quotes,!text/single_angle_quotes", + /* 147 */ "!fixedColumnOrder!5,!text/double_quotes,!text/double_angle_quotes", + /* 148 */ "!fixedColumnOrder!6,!text/double_quotes,!text/single_quotes,!text/double_angle_quotes,!text/single_angle_quotes", }; /* Language af: Afrikaans */ @@ -519,7 +514,7 @@ public final class KeyboardTextsSet { // U+061F: "ØŸ" ARABIC QUESTION MARK // U+060C: "ØŒ" ARABIC COMMA // U+061B: "Ø›" ARABIC SEMICOLON - /* 53 */ "!fixedColumnOrder!8,\",\',#,-,:,!,\u060C,\u061F,@,&,\\%,+,\u061B,/,(,)", + /* 53 */ "!fixedColumnOrder!8,\",\',#,-,:,!,\u060C,\u061F,@,&,\\%,+,\u061B,/,(|),)|(", // U+2605: "★" BLACK STAR // U+066D: "Ù" ARABIC FIVE POINTED STAR /* 54 */ "\u2605,\u066D", @@ -785,7 +780,7 @@ public final class KeyboardTextsSet { null, null, null, null, null, null, null, null, /* ~52 */ // U+00B7: "·" MIDDLE DOT - /* 53 */ "!fixedColumnOrder!9,\u00B7,\",\',#,-,:,!,\\,,?,@,&,\\%,+,;,/,(,)", + /* 53 */ "!fixedColumnOrder!4,\u00B7,!,\\,,?,:,;,@", /* 54~ */ null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, @@ -1257,7 +1252,7 @@ public final class KeyboardTextsSet { /* ~52 */ // U+00A1: "¡" INVERTED EXCLAMATION MARK // U+00BF: "¿" INVERTED QUESTION MARK - /* 53 */ "!fixedColumnOrder!9,\u00A1,\",\',#,-,:,!,\\,,?,\u00BF,@,&,\\%,+,;,/,(,)", + /* 53 */ "!fixedColumnOrder!4,;,!,\\,,?,:,\u00A1,@,\u00BF", /* 54~ */ null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, @@ -1402,13 +1397,14 @@ public final class KeyboardTextsSet { /* 47 */ null, /* 48 */ "!text/single_laqm_raqm_rtl", /* 49 */ "!text/double_laqm_raqm_rtl", - /* 50~ */ - null, null, null, - /* ~52 */ + /* 50 */ null, + // U+FDFC: "ï·¼" RIAL SIGN + /* 51 */ "\uFDFC", + /* 52 */ null, // U+061F: "ØŸ" ARABIC QUESTION MARK // U+060C: "ØŒ" ARABIC COMMA // U+061B: "Ø›" ARABIC SEMICOLON - /* 53 */ "!fixedColumnOrder!8,\",\',#,-,:,!,\u060C,\u061F,@,&,\\%,+,\u061B,/,(,)", + /* 53 */ "!fixedColumnOrder!8,\",\',#,-,:,!,\u060C,\u061F,@,&,\\%,+,\u061B,/,(|),)|(", // U+2605: "★" BLACK STAR // U+066D: "Ù" ARABIC FIVE POINTED STAR /* 54 */ "\u2605,\u066D", @@ -1762,6 +1758,25 @@ public final class KeyboardTextsSet { /* 49 */ "!text/double_raqm_laqm", }; + /* Language hy: Armenian */ + private static final String[] LANGUAGE_hy = { + /* 0~ */ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, + /* ~52 */ + // U+055E: "Õž" ARMENIAN QUESTION MARK + /* 53 */ "!fixedColumnOrder!4,\u055E,!,\\,,?,:,;,@", + /* 54~ */ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, + /* ~107 */ + /* 108 */ "\u055E,?", + }; + /* Language is: Icelandic */ private static final String[] LANGUAGE_is = { // U+00E1: "á" LATIN SMALL LETTER A WITH ACUTE @@ -1899,15 +1914,16 @@ public final class KeyboardTextsSet { /* 47 */ "\u201C,\u201D,\u201E", /* 48 */ "!text/single_laqm_raqm_rtl", /* 49 */ "!text/double_laqm_raqm_rtl", - /* 50~ */ - null, null, null, null, - /* ~53 */ + /* 50 */ null, + // U+20AA: "₪" NEW SHEQEL SIGN + /* 51 */ "\u20AA", + /* 52 */ null, + /* 53 */ null, // U+2605: "★" BLACK STAR /* 54 */ "\u2605", /* 55 */ null, - // U+00B1: "±" PLUS-MINUS SIGN // U+FB29: "﬩" HEBREW LETTER ALTERNATIVE PLUS SIGN - /* 56 */ "\u00B1,\uFB29", + /* 56 */ "\uFB29", // The all letters need to be mirrored are found at // http://www.unicode.org/Public/6.1.0/ucd/BidiMirroring.txt /* 57 */ "!fixedColumnOrder!3,<|>,{|},[|]", @@ -1920,6 +1936,15 @@ public final class KeyboardTextsSet { // U+203A: "›" SINGLE RIGHT-POINTING ANGLE QUOTATION MARK /* 59 */ "!fixedColumnOrder!3,\u2039|\u203A,\u2264|\u2265,\u00AB|\u00BB", /* 60 */ "!fixedColumnOrder!3,\u203A|\u2039,\u2265|\u2264,\u00BB|\u00AB", + /* 61~ */ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, + /* ~104 */ + /* 105 */ "!", + /* 106 */ "!", + /* 107 */ "?", + /* 108 */ "?", }; /* Language ka: Georgian */ @@ -3296,6 +3321,7 @@ public final class KeyboardTextsSet { "hi", LANGUAGE_hi, /* Hindi */ "hr", LANGUAGE_hr, /* Croatian */ "hu", LANGUAGE_hu, /* Hungarian */ + "hy", LANGUAGE_hy, /* Armenian */ "is", LANGUAGE_is, /* Icelandic */ "it", LANGUAGE_it, /* Italian */ "iw", LANGUAGE_iw, /* Hebrew */ diff --git a/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java b/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java new file mode 100644 index 000000000..629c60460 --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import android.text.TextUtils; + +import com.android.inputmethod.keyboard.Key; +import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.latin.utils.CollectionUtils; + +import java.util.ArrayDeque; +import java.util.Random; + +/** + * This is a Keyboard class to host recently used keys. + */ +// TODO: Save/restore recent keys from/to preferences. +public class RecentsKeyboard extends Keyboard { + private static final int TEMPLATE_KEY_CODE_0 = 0x30; + private static final int TEMPLATE_KEY_CODE_1 = 0x31; + + private final int mLeftPadding; + private final int mHorizontalStep; + private final int mVerticalStep; + private final int mColumnsNum; + private final int mMaxRecentKeyCount; + private final ArrayDeque<RecentKey> mRecentKeys = CollectionUtils.newArrayDeque(); + + private Key[] mCachedRecentKeys; + + public RecentsKeyboard(final Keyboard templateKeyboard) { + super(templateKeyboard); + final Key key0 = getTemplateKey(TEMPLATE_KEY_CODE_0); + final Key key1 = getTemplateKey(TEMPLATE_KEY_CODE_1); + mLeftPadding = key0.getX(); + mHorizontalStep = Math.abs(key1.getX() - key0.getX()); + mVerticalStep = key0.getHeight() + mVerticalGap; + mColumnsNum = mBaseWidth / mHorizontalStep; + final int rowsNum = mBaseHeight / mVerticalStep; + mMaxRecentKeyCount = mColumnsNum * rowsNum; + } + + private Key getTemplateKey(final int code) { + for (final Key key : super.getKeys()) { + if (key.getCode() == code) { + return key; + } + } + throw new RuntimeException("Can't find template key: code=" + code); + } + + private final Random random = new Random(); + + public void addRecentKey(final Key usedKey) { + synchronized (mRecentKeys) { + mCachedRecentKeys = null; + final RecentKey key = (usedKey instanceof RecentKey) + ? (RecentKey)usedKey : new RecentKey(usedKey); + while (mRecentKeys.remove(key)) { + // Remove duplicate keys. + } + mRecentKeys.addFirst(key); + while (mRecentKeys.size() > mMaxRecentKeyCount) { + mRecentKeys.removeLast(); + } + int index = 0; + for (final RecentKey recentKey : mRecentKeys) { + final int keyX = getKeyX(index); + final int keyY = getKeyY(index); + final int x = keyX+random.nextInt(recentKey.getWidth()); + final int y = keyY+random.nextInt(recentKey.getHeight()); + recentKey.updateCorrdinates(keyX, keyY); + index++; + } + } + } + + private int getKeyX(final int index) { + final int column = index % mColumnsNum; + return column * mHorizontalStep + mLeftPadding; + } + + private int getKeyY(final int index) { + final int row = index / mColumnsNum; + return row * mVerticalStep + mTopPadding; + } + + @Override + public Key[] getKeys() { + synchronized (mRecentKeys) { + if (mCachedRecentKeys != null) { + return mCachedRecentKeys; + } + mCachedRecentKeys = mRecentKeys.toArray(new Key[mRecentKeys.size()]); + return mCachedRecentKeys; + } + } + + @Override + public Key[] getNearestKeys(final int x, final int y) { + // TODO: Calculate the nearest key index in mRecentKeys from x and y. + return getKeys(); + } + + static final class RecentKey extends Key { + private int mCurrentX; + private int mCurrentY; + + public RecentKey(final Key originalKey) { + super(originalKey); + } + + public void updateCorrdinates(final int x, final int y) { + mCurrentX = x; + mCurrentY = y; + getHitBox().offsetTo(x, y); + } + + @Override + public int getX() { + return mCurrentX; + } + + @Override + public int getY() { + return mCurrentY; + } + + @Override + public boolean equals(final Object o) { + if (!(o instanceof Key)) return false; + final Key key = (Key)o; + if (getCode() != key.getCode()) return false; + if (!TextUtils.equals(getLabel(), key.getLabel())) return false; + return TextUtils.equals(getOutputText(), key.getOutputText()); + } + + @Override + public String toString() { + return "RecentKey: " + super.toString(); + } + } +} diff --git a/java/src/com/android/inputmethod/keyboard/internal/ScrollKeyboardView.java b/java/src/com/android/inputmethod/keyboard/internal/ScrollKeyboardView.java new file mode 100644 index 000000000..b8ee976e8 --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/ScrollKeyboardView.java @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import android.content.Context; +import android.util.AttributeSet; +import android.view.GestureDetector; +import android.view.MotionEvent; +import android.widget.ScrollView; +import android.widget.Scroller; + +import com.android.inputmethod.keyboard.Key; +import com.android.inputmethod.keyboard.KeyDetector; +import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.keyboard.KeyboardView; +import com.android.inputmethod.latin.R; + +/** + * This is an extended {@link KeyboardView} class that hosts a scroll keyboard. + * Multi-touch unsupported. No {@link PointerTracker}s. No gesture support. + */ +// TODO: Implement key popup preview. +public final class ScrollKeyboardView extends KeyboardView implements + ScrollViewWithNotifier.ScrollListener, GestureDetector.OnGestureListener { + private static final boolean PAGINATION = false; + + public interface OnKeyClickListener { + public void onKeyClick(Key key); + } + + private static final OnKeyClickListener EMPTY_LISTENER = new OnKeyClickListener() { + @Override + public void onKeyClick(final Key key) {} + }; + + private OnKeyClickListener mListener = EMPTY_LISTENER; + private final KeyDetector mKeyDetector = new KeyDetector(0.0f /*keyHysteresisDistance */); + private final GestureDetector mGestureDetector; + + private final Scroller mScroller; + private ScrollViewWithNotifier mScrollView; + + public ScrollKeyboardView(final Context context, final AttributeSet attrs) { + this(context, attrs, R.attr.keyboardViewStyle); + } + + public ScrollKeyboardView(final Context context, final AttributeSet attrs, final int defStyle) { + super(context, attrs, defStyle); + mGestureDetector = new GestureDetector(context, this); + mGestureDetector.setIsLongpressEnabled(false /* isLongpressEnabled */); + mScroller = new Scroller(context); + } + + public void setScrollView(final ScrollViewWithNotifier scrollView) { + mScrollView = scrollView; + scrollView.setScrollListener(this); + } + + private final Runnable mScrollTask = new Runnable() { + @Override + public void run() { + final Scroller scroller = mScroller; + final ScrollView scrollView = mScrollView; + scroller.computeScrollOffset(); + scrollView.scrollTo(0, scroller.getCurrY()); + if (!scroller.isFinished()) { + scrollView.post(this); + } + } + }; + + // {@link ScrollViewWithNotified#ScrollListener} methods. + @Override + public void notifyScrollChanged(final int scrollX, final int scrollY, final int oldX, + final int oldY) { + if (PAGINATION) { + mScroller.forceFinished(true /* finished */); + mScrollView.removeCallbacks(mScrollTask); + final int currentTop = mScrollView.getScrollY(); + final int pageHeight = getKeyboard().mBaseHeight; + final int lastPageNo = currentTop / pageHeight; + final int lastPageTop = lastPageNo * pageHeight; + final int nextPageNo = lastPageNo + 1; + final int nextPageTop = Math.min(nextPageNo * pageHeight, getHeight() - pageHeight); + final int scrollTo = (currentTop - lastPageTop) < (nextPageTop - currentTop) + ? lastPageTop : nextPageTop; + final int deltaY = scrollTo - currentTop; + mScroller.startScroll(0, currentTop, 0, deltaY, 300); + mScrollView.post(mScrollTask); + } + } + + @Override + public void notifyOverScrolled(final int scrollX, final int scrollY, final boolean clampedX, + final boolean clampedY) { + releaseCurrentKey(); + } + + public void setOnKeyClickListener(final OnKeyClickListener listener) { + mListener = listener; + } + + /** + * {@inheritDoc} + */ + @Override + public void setKeyboard(final Keyboard keyboard) { + super.setKeyboard(keyboard); + mKeyDetector.setKeyboard(keyboard, 0 /* correctionX */, 0 /* correctionY */); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean onTouchEvent(final MotionEvent e) { + if (mGestureDetector.onTouchEvent(e)) { + return true; + } + final Key key = getKey(e); + if (key != null && key != mCurrentKey) { + releaseCurrentKey(); + } + return true; + } + + // {@link GestureDetector#OnGestureListener} methods. + private Key mCurrentKey; + + private Key getKey(final MotionEvent e) { + final int index = e.getActionIndex(); + final int x = (int)e.getX(index); + final int y = (int)e.getY(index); + return mKeyDetector.detectHitKey(x, y); + } + + public void releaseCurrentKey() { + final Key currentKey = mCurrentKey; + if (currentKey == null) { + return; + } + currentKey.onReleased(); + invalidateKey(currentKey); + mCurrentKey = null; + } + + @Override + public boolean onDown(final MotionEvent e) { + final Key key = getKey(e); + releaseCurrentKey(); + mCurrentKey = key; + if (key == null) { + return false; + } + // TODO: May call {@link KeyboardActionListener#onPressKey(int,int,boolean)}. + key.onPressed(); + invalidateKey(key); + return false; + } + + @Override + public void onShowPress(final MotionEvent e) { + // User feedback is done at {@link #onDown(MotionEvent)}. + } + + @Override + public boolean onSingleTapUp(final MotionEvent e) { + final Key key = getKey(e); + releaseCurrentKey(); + if (key == null) { + return false; + } + // TODO: May call {@link KeyboardActionListener#onReleaseKey(int,boolean)}. + key.onReleased(); + invalidateKey(key); + mListener.onKeyClick(key); + return true; + } + + @Override + public boolean onScroll(final MotionEvent e1, final MotionEvent e2, final float distanceX, + final float distanceY) { + releaseCurrentKey(); + return false; + } + + @Override + public boolean onFling(final MotionEvent e1, final MotionEvent e2, final float velocityX, + final float velocityY) { + releaseCurrentKey(); + return false; + } + + @Override + public void onLongPress(final MotionEvent e) { + // Long press detection of {@link #mGestureDetector} is disabled and not used. + } +} diff --git a/java/src/com/android/inputmethod/keyboard/internal/ScrollViewWithNotifier.java b/java/src/com/android/inputmethod/keyboard/internal/ScrollViewWithNotifier.java new file mode 100644 index 000000000..d1ccdc7b5 --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/internal/ScrollViewWithNotifier.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard.internal; + +import android.content.Context; +import android.util.AttributeSet; +import android.widget.ScrollView; + +/** + * This is an extended {@link ScrollView} that can notify + * {@link ScrollView#onScrollChanged(int,int,int,int} and + * {@link ScrollView#onOverScrolled(int,int,int,int)} to a content view. + */ +public class ScrollViewWithNotifier extends ScrollView { + private ScrollListener mScrollListener = EMPTY_LISTER; + + public interface ScrollListener { + public void notifyScrollChanged(int scrollX, int scrollY, int oldX, int oldY); + public void notifyOverScrolled(int scrollX, int scrollY, boolean clampedX, + boolean clampedY); + } + + private static final ScrollListener EMPTY_LISTER = new ScrollListener() { + @Override + public void notifyScrollChanged(int scrollX, int scrollY, int oldX, int oldY) {} + @Override + public void notifyOverScrolled(int scrollX, int scrollY, boolean clampedX, + boolean clampedY) {} + }; + + public ScrollViewWithNotifier(final Context context, final AttributeSet attrs) { + super(context, attrs); + } + + @Override + protected void onScrollChanged(final int scrollX, final int scrollY, final int oldX, + final int oldY) { + super.onScrollChanged(scrollX, scrollY, oldX, oldY); + mScrollListener.notifyScrollChanged(scrollX, scrollY, oldX, oldY); + } + + @Override + protected void onOverScrolled(final int scrollX, final int scrollY, final boolean clampedX, + final boolean clampedY) { + super.onOverScrolled(scrollX, scrollY, clampedX, clampedY); + mScrollListener.notifyOverScrolled(scrollX, scrollY, clampedX, clampedY); + } + + public void setScrollListener(final ScrollListener listener) { + mScrollListener = listener; + } +} diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java index ebbcedc96..55df263fe 100644 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -19,10 +19,11 @@ package com.android.inputmethod.latin; import android.content.Context; import android.util.Log; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; // TODO: Quit extending Dictionary after implementing dynamic binary dictionary. @@ -42,37 +43,28 @@ abstract public class AbstractDictionaryWriter extends Dictionary { abstract public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, final boolean isNotAWord); + // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve. abstract public void addBigramWords(final String word0, final String word1, - final int frequency, final boolean isValid); + final int frequency, final boolean isValid, + final long lastModifiedTime); abstract public void removeBigramWords(final String word0, final String word1); - abstract protected void writeBinaryDictionary(final FileOutputStream out) + abstract protected void writeDictionary(final DictEncoder dictEncoder) throws IOException, UnsupportedFormatException; public void write(final String fileName) { final String tempFileName = fileName + ".temp"; final File file = new File(mContext.getFilesDir(), fileName); final File tempFile = new File(mContext.getFilesDir(), tempFileName); - FileOutputStream out = null; try { - out = new FileOutputStream(tempFile); - writeBinaryDictionary(out); - out.flush(); - out.close(); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + writeDictionary(dictEncoder); tempFile.renameTo(file); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "Unsupported format", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // ignore - } - } } } } diff --git a/java/src/com/android/inputmethod/latin/AudioAndHapticFeedbackManager.java b/java/src/com/android/inputmethod/latin/AudioAndHapticFeedbackManager.java index 42c57946d..54bc29559 100644 --- a/java/src/com/android/inputmethod/latin/AudioAndHapticFeedbackManager.java +++ b/java/src/com/android/inputmethod/latin/AudioAndHapticFeedbackManager.java @@ -57,10 +57,10 @@ public final class AudioAndHapticFeedbackManager { mVibrator = (Vibrator) context.getSystemService(Context.VIBRATOR_SERVICE); } - public void hapticAndAudioFeedback(final int primaryCode, + public void performHapticAndAudioFeedback(final int code, final View viewToPerformHapticFeedbackOn) { - vibrateInternal(viewToPerformHapticFeedbackOn); - playKeyClick(primaryCode); + performHapticFeedback(viewToPerformHapticFeedbackOn); + performAudioFeedback(code); } public boolean hasVibrator() { @@ -81,14 +81,14 @@ public final class AudioAndHapticFeedbackManager { return mAudioManager.getRingerMode() == AudioManager.RINGER_MODE_NORMAL; } - private void playKeyClick(final int primaryCode) { + public void performAudioFeedback(final int code) { // if mAudioManager is null, we can't play a sound anyway, so return if (mAudioManager == null) { return; } if (mSoundOn) { final int sound; - switch (primaryCode) { + switch (code) { case Constants.CODE_DELETE: sound = AudioManager.FX_KEYPRESS_DELETE; break; @@ -106,7 +106,7 @@ public final class AudioAndHapticFeedbackManager { } } - private void vibrateInternal(final View viewToPerformHapticFeedbackOn) { + public void performHapticFeedback(final View viewToPerformHapticFeedbackOn) { if (!mSettingsValues.mVibrateOn) { return; } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index d181bf697..e8b06570f 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -21,7 +21,6 @@ import android.util.SparseArray; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.settings.AdditionalFeaturesSettingUtils; import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.JniUtils; @@ -120,15 +119,16 @@ public final class BinaryDictionary extends Dictionary { @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, - 0 /* sessionId */); + additionalFeaturesOptions, 0 /* sessionId */); } @Override public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int sessionId) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, + final int sessionId) { if (!isValidDictionary()) return null; Arrays.fill(mInputCodePoints, Constants.NOT_A_CODE); @@ -148,8 +148,7 @@ public final class BinaryDictionary extends Dictionary { final InputPointers ips = composer.getInputPointers(); final int inputSize = isGesture ? ips.getPointerSize() : composerSize; mNativeSuggestOptions.setIsGesture(isGesture); - mNativeSuggestOptions.setAdditionalFeaturesOptions( - AdditionalFeaturesSettingUtils.getAdditionalNativeSuggestOptions()); + mNativeSuggestOptions.setAdditionalFeaturesOptions(additionalFeaturesOptions); // proximityInfo and/or prevWordForBigrams may not be null. final int count = getSuggestionsNative(mNativeDict, proximityInfo.getNativeProximityInfo(), getTraverseSession(sessionId).getSession(), ips.getXCoordinates(), @@ -179,7 +178,8 @@ public final class BinaryDictionary extends Dictionary { // TODO: check that all users of the `kind' parameter are ready to accept // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), - score, kind, mDictType)); + score, kind, this /* sourceDict */, + mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */)); } } return suggestions; diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index fa301b5a6..2b6d983c0 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,17 +21,16 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; -import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.LocaleUtils; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.BufferUnderflowException; -import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; @@ -227,23 +226,12 @@ final public class BinaryDictionaryGetter { // those do not include whitelist entries, the new code with an old version of the dictionary // would lose whitelist functionality. private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { - FileInputStream inStream = null; try { // Read the version of the file - inStream = new FileInputStream(f); - final BinaryDictInputOutput.ByteBufferWrapper buffer = - new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, f.length())); - final int magic = buffer.readInt(); - if (magic != FormatSpec.VERSION_2_MAGIC_NUMBER) { - return false; - } - final int formatVersion = buffer.readInt(); - final int headerSize = buffer.readInt(); - final HashMap<String, String> options = CollectionUtils.newHashMap(); - BinaryDictInputOutput.populateOptions(buffer, headerSize, options); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); + final FileHeader header = dictDecoder.readHeader(); - final String version = options.get(VERSION_KEY); + final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); if (null == version) { // No version in the options : the format is unexpected return false; @@ -259,14 +247,8 @@ final public class BinaryDictionaryGetter { return false; } catch (BufferUnderflowException e) { return false; - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } + } catch (UnsupportedFormatException e) { + return false; } } diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index c99d0e2ea..67eb7f3dd 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -70,7 +70,8 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { private final boolean mUseFirstLastBigrams; public ContactsBinaryDictionary(final Context context, final Locale locale) { - super(context, getFilenameWithLocale(NAME, locale.toString()), Dictionary.TYPE_CONTACTS); + super(context, getFilenameWithLocale(NAME, locale.toString()), Dictionary.TYPE_CONTACTS, + false /* isUpdatable */); mLocale = locale; mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale); registerObserver(context); @@ -208,7 +209,8 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { false /* isNotAWord */); if (!TextUtils.isEmpty(prevWord)) { if (mUseFirstLastBigrams) { - super.setBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM); + super.addBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM, + 0 /* lastModifiedTime */); } } prevWord = word; diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 7c3e4a740..8a3a88438 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -28,9 +28,26 @@ import java.util.ArrayList; public abstract class Dictionary { public static final int NOT_A_PROBABILITY = -1; + // The following types do not actually come from real dictionary instances, so we create + // corresponding instances. public static final String TYPE_USER_TYPED = "user_typed"; + public static final Dictionary DICTIONARY_USER_TYPED = new PhonyDictionary(TYPE_USER_TYPED); + public static final String TYPE_APPLICATION_DEFINED = "application_defined"; + public static final Dictionary DICTIONARY_APPLICATION_DEFINED = + new PhonyDictionary(TYPE_APPLICATION_DEFINED); + public static final String TYPE_HARDCODED = "hardcoded"; // punctuation signs and such + public static final Dictionary DICTIONARY_HARDCODED = + new PhonyDictionary(TYPE_HARDCODED); + + // Spawned by resuming suggestions. Comes from a span that was in the TextView. + public static final String TYPE_RESUMED = "resumed"; + public static final Dictionary DICTIONARY_RESUMED = + new PhonyDictionary(TYPE_RESUMED); + + // The following types of dictionary have actual functional instances. We don't need final + // phony dictionary instances for them. public static final String TYPE_MAIN = "main"; public static final String TYPE_CONTACTS = "contacts"; // User dictionary, the system-managed one. @@ -42,9 +59,7 @@ public abstract class Dictionary { // Personalization prediction dictionary internal to LatinIME's Java code. public static final String TYPE_PERSONALIZATION_PREDICTION_IN_JAVA = "personalization_prediction_in_java"; - // Spawned by resuming suggestions. Comes from a span that was in the TextView. - public static final String TYPE_RESUMED = "resumed"; - protected final String mDictType; + public final String mDictType; public Dictionary(final String dictType) { mDictType = dictType; @@ -57,20 +72,23 @@ public abstract class Dictionary { * @param prevWord the previous word, or null if none * @param proximityInfo the object for key proximity. May be ignored by some implementations. * @param blockOffensiveWords whether to block potentially offensive words + * @param additionalFeaturesOptions options about additional features used for the suggestion. * @return the list of suggestions (possibly null if none) */ // TODO: pass more context than just the previous word, to enable better suggestions (n-gram // and more) abstract public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords); + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions); // The default implementation of this method ignores sessionId. // Subclasses that want to use sessionId need to override this method. public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int sessionId) { - return getSuggestions(composer, prevWord, proximityInfo, blockOffensiveWords); + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, + final int sessionId) { + return getSuggestions(composer, prevWord, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions); } /** @@ -114,8 +132,40 @@ public abstract class Dictionary { /** * Subclasses may override to indicate that this Dictionary is not yet properly initialized. */ - public boolean isInitialized() { return true; } + + /** + * Whether we think this suggestion should trigger an auto-commit. + */ + public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { + // If we don't have support for auto-commit, or if we don't know, we return false to + // avoid auto-committing stuff. Implementations of the Dictionary class that know to + // determine whether we should auto-commit will override this. + return false; + } + + /** + * Not a true dictionary. A placeholder used to indicate suggestions that don't come from any + * real dictionary. + */ + private static class PhonyDictionary extends Dictionary { + // This class is not publicly instantiable. + private PhonyDictionary(final String type) { + super(type); + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + return null; + } + + @Override + public boolean isValidWord(String word) { + return false; + } + } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryCollection.java b/java/src/com/android/inputmethod/latin/DictionaryCollection.java index d05bb1e55..bf075140e 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryCollection.java +++ b/java/src/com/android/inputmethod/latin/DictionaryCollection.java @@ -58,18 +58,18 @@ public final class DictionaryCollection extends Dictionary { @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { final CopyOnWriteArrayList<Dictionary> dictionaries = mDictionaries; if (dictionaries.isEmpty()) return null; // To avoid creating unnecessary objects, we get the list out of the first // dictionary and add the rest to it if not null, hence the get(0) ArrayList<SuggestedWordInfo> suggestions = dictionaries.get(0).getSuggestions(composer, - prevWord, proximityInfo, blockOffensiveWords); + prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions); if (null == suggestions) suggestions = CollectionUtils.newArrayList(); final int length = dictionaries.size(); for (int i = 1; i < length; ++ i) { final ArrayList<SuggestedWordInfo> sugg = dictionaries.get(i).getSuggestions(composer, - prevWord, proximityInfo, blockOffensiveWords); + prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions); if (null != sugg) suggestions.addAll(sugg); } return suggestions; diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 47151bf61..5a453dde5 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -20,15 +20,14 @@ import android.content.Context; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.utils.CollectionUtils; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -51,7 +50,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override public void clear() { final HashMap<String, String> attributes = CollectionUtils.newHashMap(); - mFusionDictionary = new FusionDictionary(new Node(), + mFusionDictionary = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(attributes, false, false)); } @@ -75,7 +74,7 @@ public class DictionaryWriter extends AbstractDictionaryWriter { @Override public void addBigramWords(final String word0, final String word1, final int frequency, - final boolean isValid) { + final boolean isValid, final long lastModifiedTime) { mFusionDictionary.setBigram(word0, word1, frequency); } @@ -85,15 +84,15 @@ public class DictionaryWriter extends AbstractDictionaryWriter { } @Override - protected void writeBinaryDictionary(final FileOutputStream out) + protected void writeDictionary(final DictEncoder dictEncoder) throws IOException, UnsupportedFormatException { - BinaryDictInputOutput.writeDictionaryBinary(out, mFusionDictionary, FORMAT_OPTIONS); + dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS); } @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - boolean blockOffensiveWords) { + boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { // This class doesn't support suggestion. return null; } diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 3f11391ba..b92283c5b 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -20,13 +20,16 @@ import android.content.Context; import android.os.SystemClock; import android.util.Log; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.util.ArrayList; import java.util.HashMap; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -78,12 +81,21 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private final String mFilename; + /** Whether to support dynamically updating the dictionary */ + private final boolean mIsUpdatable; + /** Controls access to the shared binary dictionary file across multiple instances. */ private final DictionaryController mSharedDictionaryController; /** Controls access to the local binary dictionary for this instance. */ private final DictionaryController mLocalDictionaryController = new DictionaryController(); + /* A extension for a binary dictionary file. */ + public static final String DICT_FILE_EXTENSION = ".dict"; + + private final AtomicReference<AsyncWriteBinaryDictionaryTask> mWaitingTask = + new AtomicReference<AsyncWriteBinaryDictionaryTask>(); + /** * Abstract method for loading the unigrams and bigrams of a given dictionary in a background * thread. @@ -110,6 +122,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return controller; } + private static AbstractDictionaryWriter getDictionaryWriter(final Context context, + final String dictType, final boolean isDynamicPersonalizationDictionary) { + if (isDynamicPersonalizationDictionary) { + return new DynamicPersonalizationDictionaryWriter(context, dictType); + } else { + return new DictionaryWriter(context, dictType); + } + } + /** * Creates a new expandable binary dictionary. * @@ -117,19 +138,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * @param filename The filename for this binary dictionary. Multiple dictionaries with the same * filename is supported. * @param dictType the dictionary type, as a human-readable string + * @param isUpdatable whether to support dynamically updating the dictionary. Please note that + * dynamic dictionary has negative effects on memory space and computation time. */ - public ExpandableBinaryDictionary( - final Context context, final String filename, final String dictType) { + public ExpandableBinaryDictionary(final Context context, final String filename, + final String dictType, final boolean isUpdatable) { super(dictType); mFilename = filename; mContext = context; + mIsUpdatable = isUpdatable; mBinaryDictionary = null; mSharedDictionaryController = getSharedDictionaryController(filename); - mDictionaryWriter = new DictionaryWriter(context, dictType); + // Currently, only dynamic personalization dictionary is updatable. + mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); } protected static String getFilenameWithLocale(final String name, final String localeStr) { - return name + "." + localeStr + ".dict"; + return name + "." + localeStr + DICT_FILE_EXTENSION; } /** @@ -137,6 +162,16 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ @Override public void close() { + closeBinaryDictionary(); + mLocalDictionaryController.writeLock().lock(); + try { + mDictionaryWriter.close(); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + + protected void closeBinaryDictionary() { // Ensure that no other threads are accessing the local binary dictionary. mLocalDictionaryController.writeLock().lock(); try { @@ -144,7 +179,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary.close(); mBinaryDictionary = null; } - mDictionaryWriter.close(); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + + protected void clear() { + mLocalDictionaryController.writeLock().lock(); + try { + mDictionaryWriter.clear(); } finally { mLocalDictionaryController.writeLock().unlock(); } @@ -159,59 +202,95 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Sets a word bigram in the dictionary. Used for loading a dictionary. + * Adds a word bigram in the dictionary. Used for loading a dictionary. */ - protected void setBigram(final String prevWord, final String word, final int frequency) { - mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */); + protected void addBigram(final String prevWord, final String word, final int frequency, + final long lastModifiedTime) { + mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */, + lastModifiedTime); } /** - * Dynamically adds a word unigram to the dictionary. + * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. */ protected void addWordDynamically(final String word, final String shortcutTarget, final int frequency, final boolean isNotAWord) { - mLocalDictionaryController.writeLock().lock(); - try { - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); - } finally { - mLocalDictionaryController.writeLock().unlock(); + if (!mIsUpdatable) { + Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename); + return; + } + // TODO: Use a queue to reflect what needs to be reflected. + if (mLocalDictionaryController.writeLock().tryLock()) { + try { + mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } } } /** - * Dynamically sets a word bigram in the dictionary. + * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. */ - protected void setBigramDynamically(final String prevWord, final String word, - final int frequency) { - mLocalDictionaryController.writeLock().lock(); - try { - mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */); - } finally { - mLocalDictionaryController.writeLock().unlock(); + protected void addBigramDynamically(final String word0, final String word1, + final int frequency, final boolean isValid) { + if (!mIsUpdatable) { + Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: " + + mFilename); + return; + } + // TODO: Use a queue to reflect what needs to be reflected. + if (mLocalDictionaryController.writeLock().tryLock()) { + try { + mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, + 0 /* lastTouchedTime */); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + } + + /** + * Dynamically remove a word bigram in the dictionary. + */ + protected void removeBigramDynamically(final String word0, final String word1) { + if (!mIsUpdatable) { + Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: " + + mFilename); + return; + } + // TODO: Use a queue to reflect what needs to be reflected. + if (mLocalDictionaryController.writeLock().tryLock()) { + try { + mDictionaryWriter.removeBigramWords(word0, word1); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } } } @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { asyncReloadDictionaryIfRequired(); // Write lock because getSuggestions in native updates session status. if (mLocalDictionaryController.writeLock().tryLock()) { try { final ArrayList<SuggestedWordInfo> inMemDictSuggestion = mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords); - if (mBinaryDictionary != null) { + blockOffensiveWords, additionalFeaturesOptions); + // TODO: Remove checking mIsUpdatable and use native suggestion. + if (mBinaryDictionary != null && !mIsUpdatable) { final ArrayList<SuggestedWordInfo> binarySuggestion = mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, - blockOffensiveWords); + blockOffensiveWords, additionalFeaturesOptions); if (inMemDictSuggestion == null) { return binarySuggestion; } else if (binarySuggestion == null) { return inMemDictSuggestion; } else { - binarySuggestion.addAll(binarySuggestion); + binarySuggestion.addAll(inMemDictSuggestion); return binarySuggestion; } } else { @@ -277,7 +356,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // Build the new binary dictionary final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length, - true /* useFullEditDistance */, null, mDictType, false /* isUpdatable */); + true /* useFullEditDistance */, null, mDictType, mIsUpdatable); if (mBinaryDictionary != null) { // Ensure all threads accessing the current dictionary have finished before swapping in @@ -302,9 +381,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { abstract protected boolean needsToReloadBeforeWriting(); /** - * Generates and writes a new binary dictionary based on the contents of the fusion dictionary. + * Writes a new binary dictionary based on the contents of the fusion dictionary. */ - private void generateBinaryDictionary() { + private void writeBinaryDictionary() { if (DEBUG) { Log.d(TAG, "Generating binary dictionary: " + mFilename + " request=" + mSharedDictionaryController.mLastUpdateRequestTime + " update=" @@ -337,7 +416,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Reloads the dictionary if required. Reload will occur asynchronously in a separate thread. */ - void asyncReloadDictionaryIfRequired() { + public void asyncReloadDictionaryIfRequired() { if (!isReloadRequired()) return; if (DEBUG) { Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename); @@ -348,7 +427,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Reloads the dictionary if required. */ - protected final void syncReloadDictionaryIfRequired() { + public final void syncReloadDictionaryIfRequired() { if (!isReloadRequired()) return; syncReloadDictionaryInternal(); } @@ -367,41 +446,47 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private final void syncReloadDictionaryInternal() { // Ensure that only one thread attempts to read or write to the shared binary dictionary // file at the same time. - mSharedDictionaryController.writeLock().lock(); + mLocalDictionaryController.writeLock().lock(); try { - final long time = SystemClock.uptimeMillis(); - final boolean dictionaryFileExists = dictionaryFileExists(); - if (mSharedDictionaryController.isOutOfDate() || !dictionaryFileExists) { - // If the shared dictionary file does not exist or is out of date, the first - // instance that acquires the lock will generate a new one. - if (hasContentChanged() || !dictionaryFileExists) { - // If the source content has changed or the dictionary does not exist, rebuild - // the binary dictionary. Empty dictionaries are supported (in the case where - // loadDictionaryAsync() adds nothing) in order to provide a uniform framework. + mSharedDictionaryController.writeLock().lock(); + try { + final long time = SystemClock.uptimeMillis(); + final boolean dictionaryFileExists = dictionaryFileExists(); + if (mSharedDictionaryController.isOutOfDate() || !dictionaryFileExists) { + // If the shared dictionary file does not exist or is out of date, the first + // instance that acquires the lock will generate a new one. + if (hasContentChanged() || !dictionaryFileExists) { + // If the source content has changed or the dictionary does not exist, + // rebuild the binary dictionary. Empty dictionaries are supported (in the + // case where loadDictionaryAsync() adds nothing) in order to provide a + // uniform framework. + mSharedDictionaryController.mLastUpdateTime = time; + writeBinaryDictionary(); + loadBinaryDictionary(); + } else { + // If not, the reload request was unnecessary so revert + // LastUpdateRequestTime to LastUpdateTime. + mSharedDictionaryController.mLastUpdateRequestTime = + mSharedDictionaryController.mLastUpdateTime; + } + } else if (mBinaryDictionary == null || mLocalDictionaryController.mLastUpdateTime + < mSharedDictionaryController.mLastUpdateTime) { + // Otherwise, if the local dictionary is older than the shared dictionary, load + // the shared dictionary. + loadBinaryDictionary(); + } + if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { + // Binary dictionary is not valid. Regenerate the dictionary file. mSharedDictionaryController.mLastUpdateTime = time; - generateBinaryDictionary(); + writeBinaryDictionary(); loadBinaryDictionary(); - } else { - // If not, the reload request was unnecessary so revert LastUpdateRequestTime - // to LastUpdateTime. - mSharedDictionaryController.mLastUpdateRequestTime = - mSharedDictionaryController.mLastUpdateTime; } - } else if (mBinaryDictionary == null || mLocalDictionaryController.mLastUpdateTime - < mSharedDictionaryController.mLastUpdateTime) { - // Otherwise, if the local dictionary is older than the shared dictionary, load the - // shared dictionary. - loadBinaryDictionary(); - } - if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { - // Binary dictionary is not valid. Regenerate the dictionary file. - mSharedDictionaryController.mLastUpdateTime = time; - generateBinaryDictionary(); - loadBinaryDictionary(); + mLocalDictionaryController.mLastUpdateTime = time; + } finally { + mSharedDictionaryController.writeLock().unlock(); } - mLocalDictionaryController.mLastUpdateTime = time; } finally { - mSharedDictionaryController.writeLock().unlock(); + mLocalDictionaryController.writeLock().unlock(); } } @@ -422,6 +507,68 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** + * Load the dictionary to memory. + */ + protected void asyncLoadDictionaryToMemory() { + new AsyncLoadDictionaryToMemoryTask().start(); + } + + /** + * Thread class for asynchronously loading dictionary to memory. + */ + private class AsyncLoadDictionaryToMemoryTask extends Thread { + @Override + public void run() { + mLocalDictionaryController.writeLock().lock(); + try { + mSharedDictionaryController.readLock().lock(); + try { + loadDictionaryAsync(); + } finally { + mSharedDictionaryController.readLock().unlock(); + } + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + } + + /** + * Generate binary dictionary using DictionaryWriter. + */ + protected void asyncWriteBinaryDictionary() { + final AsyncWriteBinaryDictionaryTask newTask = new AsyncWriteBinaryDictionaryTask(); + newTask.start(); + final AsyncWriteBinaryDictionaryTask oldTask = mWaitingTask.getAndSet(newTask); + if (oldTask != null) { + oldTask.interrupt(); + } + } + + /** + * Thread class for asynchronously writing the binary dictionary. + */ + private class AsyncWriteBinaryDictionaryTask extends Thread { + @Override + public void run() { + mSharedDictionaryController.writeLock().lock(); + try { + mLocalDictionaryController.writeLock().lock(); + try { + if (isInterrupted()) { + return; + } + writeBinaryDictionary(); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } finally { + mSharedDictionaryController.writeLock().unlock(); + } + } + } + + /** * Lock for controlling access to a given binary dictionary and for tracking whether the * dictionary is out of date. Can be shared across multiple dictionary instances that access the * same filename. @@ -434,4 +581,45 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return (mLastUpdateRequestTime > mLastUpdateTime); } } + + /** + * Dynamically adds a word unigram to the dictionary for testing with blocking-lock. + */ + @UsedForTesting + protected void addWordDynamicallyForTests(final String word, final String shortcutTarget, + final int frequency, final boolean isNotAWord) { + mLocalDictionaryController.writeLock().lock(); + try { + addWordDynamically(word, shortcutTarget, frequency, isNotAWord); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + + /** + * Dynamically adds a word bigram in the dictionary for testing with blocking-lock. + */ + @UsedForTesting + protected void addBigramDynamicallyForTests(final String word0, final String word1, + final int frequency, final boolean isValid) { + mLocalDictionaryController.writeLock().lock(); + try { + addBigramDynamically(word0, word1, frequency, isValid); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } + + /** + * Dynamically remove a word bigram in the dictionary for testing with blocking-lock. + */ + @UsedForTesting + protected void removeBigramDynamicallyForTests(final String word0, final String word1) { + mLocalDictionaryController.writeLock().lock(); + try { + removeBigramDynamically(word0, word1); + } finally { + mLocalDictionaryController.writeLock().unlock(); + } + } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index bd2d70365..342dcfc5e 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -16,10 +16,10 @@ package com.android.inputmethod.latin; -import android.content.Context; import android.text.TextUtils; import android.util.Log; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -29,9 +29,10 @@ import java.util.ArrayList; import java.util.LinkedList; /** - * Base class for an in-memory dictionary that can grow dynamically and can + * Class for an in-memory dictionary that can grow dynamically and can * be searched for suggestions and valid words. */ +// TODO: Remove after binary dictionary supports dynamic update. public class ExpandableDictionary extends Dictionary { private static final String TAG = ExpandableDictionary.class.getSimpleName(); /** @@ -39,23 +40,11 @@ public class ExpandableDictionary extends Dictionary { */ private static final int FULL_WORD_SCORE_MULTIPLIER = 2; - // Bigram frequency is a fixed point number with 1 meaning 1.2 and 255 meaning 1.8. - protected static final int BIGRAM_MAX_FREQUENCY = 255; - - private Context mContext; private char[] mWordBuilder = new char[Constants.DICTIONARY_MAX_WORD_LENGTH]; private int mMaxDepth; private int mInputLength; - private boolean mRequiresReload; - - private boolean mUpdatingDictionary; - - // Use this lock before touching mUpdatingDictionary & mRequiresDownload - private Object mUpdatingLock = new Object(); - private static final class Node { - Node() {} char mCode; int mFrequency; boolean mTerminal; @@ -157,46 +146,12 @@ public class ExpandableDictionary extends Dictionary { private int[][] mCodes; - public ExpandableDictionary(final Context context, final String dictType) { + public ExpandableDictionary(final String dictType) { super(dictType); - mContext = context; clearDictionary(); mCodes = new int[Constants.DICTIONARY_MAX_WORD_LENGTH][]; } - public void loadDictionary() { - synchronized (mUpdatingLock) { - startDictionaryLoadingTaskLocked(); - } - } - - public void startDictionaryLoadingTaskLocked() { - if (!mUpdatingDictionary) { - mUpdatingDictionary = true; - mRequiresReload = false; - new LoadDictionaryTask().start(); - } - } - - public void setRequiresReload(final boolean reload) { - synchronized (mUpdatingLock) { - mRequiresReload = reload; - } - } - - public boolean getRequiresReload() { - return mRequiresReload; - } - - /** Override to load your dictionary here, on a background thread. */ - public void loadDictionaryAsync() { - // empty base implementation - } - - public Context getContext() { - return mContext; - } - public int getMaxWordLength() { return Constants.DICTIONARY_MAX_WORD_LENGTH; } @@ -231,7 +186,7 @@ public class ExpandableDictionary extends Dictionary { childNode.mShortcutOnly = isShortcutOnly; children.add(childNode); } - if (wordLength == depth + 1 && shortcutTarget != null) { + if (wordLength == depth + 1) { // Terminate this word childNode.mTerminal = true; if (isShortcutOnly) { @@ -255,8 +210,7 @@ public class ExpandableDictionary extends Dictionary { @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { - if (reloadDictionaryIfRequired()) return null; + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { if (composer.size() > 1) { if (composer.size() >= Constants.DICTIONARY_MAX_WORD_LENGTH) { return null; @@ -272,17 +226,7 @@ public class ExpandableDictionary extends Dictionary { } } - // This reloads the dictionary if required, and returns whether it's currently updating its - // contents or not. - private boolean reloadDictionaryIfRequired() { - synchronized (mUpdatingLock) { - // If we need to update, start off a background task - if (mRequiresReload) startDictionaryLoadingTaskLocked(); - return mUpdatingDictionary; - } - } - - protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, + private ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer codes, final String prevWordForBigrams, final ProximityInfo proximityInfo) { final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); mInputLength = codes.size(); @@ -312,11 +256,6 @@ public class ExpandableDictionary extends Dictionary { @Override public synchronized boolean isValidWord(final String word) { - synchronized (mUpdatingLock) { - // If we need to update, start off a background task - if (mRequiresReload) startDictionaryLoadingTaskLocked(); - if (mUpdatingDictionary) return false; - } final Node node = searchNode(mRoots, word, 0, word.length()); // If node is null, we didn't find the word, so it's not valid. // If node.mShortcutOnly is true, then it exists as a shortcut but not as a word, @@ -326,7 +265,7 @@ public class ExpandableDictionary extends Dictionary { return (node == null) ? false : !node.mShortcutOnly; } - protected boolean removeBigram(final String word1, final String word2) { + public boolean removeBigram(final String word1, final String word2) { // Refer to addOrSetBigram() about word1.toLowerCase() final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); final Node secondWord = searchWord(mRoots, word2, 0, null); @@ -351,13 +290,14 @@ public class ExpandableDictionary extends Dictionary { /** * Returns the word's frequency or -1 if not found */ - protected int getWordFrequency(final String word) { + @UsedForTesting + public int getWordFrequency(final String word) { // Case-sensitive search final Node node = searchNode(mRoots, word, 0, word.length()); return (node == null) ? -1 : node.mFrequency; } - protected NextWord getBigramWord(final String word1, final String word2) { + public NextWord getBigramWord(final String word1, final String word2) { // Refer to addOrSetBigram() about word1.toLowerCase() final Node firstWord = searchWord(mRoots, word1.toLowerCase(), 0, null); final Node secondWord = searchWord(mRoots, word2, 0, null); @@ -403,7 +343,8 @@ public class ExpandableDictionary extends Dictionary { // the respective size of the typed word and the suggestion if it matters sometime // in the future. suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, - SuggestedWordInfo.KIND_CORRECTION, mDictType)); + SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; } if (null != node.mShortcutTargets) { @@ -411,7 +352,8 @@ public class ExpandableDictionary extends Dictionary { for (int shortcutIndex = 0; shortcutIndex < length; ++shortcutIndex) { final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), - finalFreq, SuggestedWordInfo.KIND_SHORTCUT, mDictType)); + finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; } } @@ -438,7 +380,7 @@ public class ExpandableDictionary extends Dictionary { * @param suggestions the list in which to add suggestions */ // TODO: Share this routine with the native code for BinaryDictionary - protected void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word, + private void getWordsRec(final NodeArray roots, final WordComposer codes, final char[] word, final int depth, final boolean completion, final int snr, final int inputIndex, final int skipPos, final ArrayList<SuggestedWordInfo> suggestions) { final int count = roots.mLength; @@ -657,7 +599,8 @@ public class ExpandableDictionary extends Dictionary { if (freq >= 0 && node == null) { suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, Constants.DICTIONARY_MAX_WORD_LENGTH - index), - freq, SuggestedWordInfo.KIND_CORRECTION, mDictType)); + freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } @@ -695,21 +638,10 @@ public class ExpandableDictionary extends Dictionary { return null; } - protected void clearDictionary() { + public void clearDictionary() { mRoots = new NodeArray(); } - private final class LoadDictionaryTask extends Thread { - LoadDictionaryTask() {} - @Override - public void run() { - loadDictionaryAsync(); - synchronized (mUpdatingLock) { - mUpdatingDictionary = false; - } - } - } - private static char toLowerCase(final char c) { char baseChar = c; if (c < BASE_CHARS.length) { diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 5c5b7b7c0..6be9ded5c 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -73,9 +73,12 @@ import com.android.inputmethod.keyboard.KeyboardActionListener; import com.android.inputmethod.keyboard.KeyboardId; import com.android.inputmethod.keyboard.KeyboardSwitcher; import com.android.inputmethod.keyboard.MainKeyboardView; +import com.android.inputmethod.latin.Suggest.OnGetSuggestedWordsCallback; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.define.ProductionFlag; -import com.android.inputmethod.latin.personalization.PersonalizationDictionaryHelper; +import com.android.inputmethod.latin.personalization.PersonalizationDictionary; +import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister; +import com.android.inputmethod.latin.personalization.PersonalizationHelper; import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary; import com.android.inputmethod.latin.settings.Settings; @@ -96,6 +99,7 @@ import com.android.inputmethod.latin.utils.RecapitalizeStatus; import com.android.inputmethod.latin.utils.StaticInnerHandlerWrapper; import com.android.inputmethod.latin.utils.TargetPackageInfoGetterTask; import com.android.inputmethod.latin.utils.TextRange; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; import com.android.inputmethod.research.ResearchLogger; import java.io.FileDescriptor; @@ -103,6 +107,8 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Locale; import java.util.TreeSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * Input method implementation for Qwerty'ish keyboard. @@ -123,6 +129,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private static final int PENDING_IMS_CALLBACK_DURATION = 800; + private static final int PERIOD_FOR_AUDIO_AND_HAPTIC_FEEDBACK_IN_KEY_REPEAT = 2; + + // TODO: Set this value appropriately. + private static final int GET_SUGGESTED_WORDS_TIMEOUT = 200; + /** * The name of the scheme used by the Package Manager to warn of a new package installation, * replacement or removal. @@ -171,6 +182,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private UserBinaryDictionary mUserDictionary; private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary; private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary; + private PersonalizationDictionary mPersonalizationDictionary; private boolean mIsUserDictionaryAvailable; private LastComposedWord mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; @@ -190,7 +202,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean mExpectingUpdateSelection; private int mDeleteCount; private long mLastKeyTime; - private TreeSet<Long> mCurrentlyPressedHardwareKeys = CollectionUtils.newTreeSet(); + private final TreeSet<Long> mCurrentlyPressedHardwareKeys = CollectionUtils.newTreeSet(); + // Personalization debugging params + private boolean mUseOnlyPersonalizationDictionaryForDebug = false; + private boolean mBoostPersonalizationDictionaryForDebug = false; // Member variables for remembering the current device orientation. private int mDisplayOrientation; @@ -211,6 +226,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private final boolean mIsHardwareAcceleratedDrawingEnabled; public final UIHandler mHandler = new UIHandler(this); + private InputUpdater mInputUpdater; public static final class UIHandler extends StaticInnerHandlerWrapper<LatinIME> { private static final int MSG_UPDATE_SHIFT_STATE = 0; @@ -219,8 +235,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private static final int MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP = 3; private static final int MSG_RESUME_SUGGESTIONS = 4; private static final int MSG_REOPEN_DICTIONARIES = 5; + private static final int MSG_ON_END_BATCH_INPUT = 6; + private static final int ARG1_NOT_GESTURE_INPUT = 0; private static final int ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT = 1; + private static final int ARG1_SHOW_GESTURE_FLOATING_PREVIEW_TEXT = 2; private int mDelayUpdateSuggestions; private int mDelayUpdateShiftState; @@ -253,8 +272,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen switcher.updateShiftState(); break; case MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP: - latinIme.showGesturePreviewAndSuggestionStrip((SuggestedWords)msg.obj, - msg.arg1 == ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT); + if (msg.arg1 == ARG1_NOT_GESTURE_INPUT) { + latinIme.showSuggestionStrip((SuggestedWords) msg.obj); + } else { + latinIme.showGesturePreviewAndSuggestionStrip((SuggestedWords) msg.obj, + msg.arg1 == ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT); + } break; case MSG_RESUME_SUGGESTIONS: latinIme.restartSuggestionsOnWordTouchedByCursor(); @@ -266,6 +289,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // get any suggestions. Wait one frame. postUpdateSuggestionStrip(); break; + case MSG_ON_END_BATCH_INPUT: + latinIme.onEndBatchInputAsyncInternal((SuggestedWords) msg.obj); + break; } } @@ -307,11 +333,22 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final boolean dismissGestureFloatingPreviewText) { removeMessages(MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP); final int arg1 = dismissGestureFloatingPreviewText - ? ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT : 0; + ? ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT + : ARG1_SHOW_GESTURE_FLOATING_PREVIEW_TEXT; obtainMessage(MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP, arg1, 0, suggestedWords) .sendToTarget(); } + public void showSuggestionStrip(final SuggestedWords suggestedWords) { + removeMessages(MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP); + obtainMessage(MSG_SHOW_GESTURE_PREVIEW_AND_SUGGESTION_STRIP, + ARG1_NOT_GESTURE_INPUT, 0, suggestedWords).sendToTarget(); + } + + public void onEndBatchInput(final SuggestedWords suggestedWords) { + obtainMessage(MSG_ON_END_BATCH_INPUT, suggestedWords).sendToTarget(); + } + public void startDoubleSpacePeriodTimer() { mDoubleSpacePeriodTimerStart = SystemClock.uptimeMillis(); } @@ -472,6 +509,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen KeyboardSwitcher.init(this); AudioAndHapticFeedbackManager.init(this); AccessibilityUtils.init(this); + PersonalizationDictionarySessionRegister.init(this); super.onCreate(); @@ -503,6 +541,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final IntentFilter newDictFilter = new IntentFilter(); newDictFilter.addAction(DictionaryPackConstants.NEW_DICTIONARY_INTENT_ACTION); registerReceiver(mDictionaryPackInstallReceiver, newDictFilter); + + mInputUpdater = new InputUpdater(this); } // Has to be package-visible for unit tests @@ -558,10 +598,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); - mUserHistoryPredictionDictionary = PersonalizationDictionaryHelper + mUserHistoryPredictionDictionary = PersonalizationHelper .getUserHistoryPredictionDictionary(this, localeStr, prefs); newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary); - mPersonalizationPredictionDictionary = PersonalizationDictionaryHelper + mPersonalizationDictionary = PersonalizationHelper + .getPersonalizationDictionary(this, localeStr, prefs); + newSuggest.setPersonalizationDictionary(mPersonalizationDictionary); + mPersonalizationPredictionDictionary = PersonalizationHelper .getPersonalizationPredictionDictionary(this, localeStr, prefs); newSuggest.setPersonalizationPredictionDictionary(mPersonalizationPredictionDictionary); @@ -633,8 +676,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.getInstance().onDestroy(); } unregisterReceiver(mDictionaryPackInstallReceiver); + PersonalizationDictionarySessionRegister.onDestroy(this); LatinImeLogger.commit(); LatinImeLogger.onDestroy(); + if (mInputUpdater != null) { + mInputUpdater.onDestroy(); + mInputUpdater = null; + } super.onDestroy(); } @@ -652,6 +700,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mOptionsDialog.dismiss(); } } + PersonalizationDictionarySessionRegister.onConfigurationChanged(this, conf); super.onConfigurationChanged(conf); } @@ -863,9 +912,35 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // be replaced when the user dictionary reports back with the actual word, which ends // up calling #onWordAddedToUserDictionary() in this class. + initPersonalizationDebugSettings(currentSettingsValues); + if (TRACE) Debug.startMethodTracing("/data/trace/latinime"); } + // Initialization of personalization debug settings. This must be called inside + // onStartInputView. + private void initPersonalizationDebugSettings(SettingsValues currentSettingsValues) { + if (mUseOnlyPersonalizationDictionaryForDebug + != currentSettingsValues.mUseOnlyPersonalizationDictionaryForDebug) { + // Only for debug + initSuggest(); + mUseOnlyPersonalizationDictionaryForDebug = + currentSettingsValues.mUseOnlyPersonalizationDictionaryForDebug; + } + + if (mBoostPersonalizationDictionaryForDebug != + currentSettingsValues.mBoostPersonalizationDictionaryForDebug) { + // Only for debug + mBoostPersonalizationDictionaryForDebug = + currentSettingsValues.mBoostPersonalizationDictionaryForDebug; + if (mBoostPersonalizationDictionaryForDebug) { + UserHistoryForgettingCurveUtils.boostMaxFreqForDebug(); + } else { + UserHistoryForgettingCurveUtils.resetMaxFreqForDebug(); + } + } + } + // Callback for the TargetPackageInfoGetterTask @Override public void onTargetPackageInfoKnown(final PackageInfo info) { @@ -1135,11 +1210,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return currentHeight; } - final MainKeyboardView mainKeyboardView = mKeyboardSwitcher.getMainKeyboardView(); - if (mainKeyboardView == null) { + final View visibleKeyboardView = mKeyboardSwitcher.getVisibleKeyboardView(); + if (visibleKeyboardView == null) { return 0; } - final int keyboardHeight = mainKeyboardView.getHeight(); + // TODO: !!!!!!!!!!!!!!!!!!!! Handle different backing view heights between the main !!! + // keyboard and the emoji keyboard. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + final int keyboardHeight = visibleKeyboardView.getHeight(); final int suggestionsHeight = mSuggestionStripView.getHeight(); final int displayHeight = getResources().getDisplayMetrics().heightPixels; final Rect rect = new Rect(); @@ -1157,8 +1234,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onComputeInsets(final InputMethodService.Insets outInsets) { super.onComputeInsets(outInsets); - final MainKeyboardView mainKeyboardView = mKeyboardSwitcher.getMainKeyboardView(); - if (mainKeyboardView == null || mSuggestionStripView == null) { + final View visibleKeyboardView = mKeyboardSwitcher.getVisibleKeyboardView(); + if (visibleKeyboardView == null || mSuggestionStripView == null) { return; } final int adjustedBackingHeight = getAdjustedBackingViewHeight(); @@ -1173,13 +1250,13 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final int extraHeight = extractHeight + backingHeight + suggestionsHeight; int visibleTopY = extraHeight; // Need to set touchable region only if input view is being shown - if (mainKeyboardView.isShown()) { + if (visibleKeyboardView.isShown()) { if (mSuggestionStripView.getVisibility() == View.VISIBLE) { visibleTopY -= suggestionsHeight; } - final int touchY = mainKeyboardView.isShowingMoreKeysPanel() ? 0 : visibleTopY; - final int touchWidth = mainKeyboardView.getWidth(); - final int touchHeight = mainKeyboardView.getHeight() + extraHeight + final int touchY = mKeyboardSwitcher.isShowingMoreKeysPanel() ? 0 : visibleTopY; + final int touchWidth = visibleKeyboardView.getWidth(); + final int touchHeight = visibleKeyboardView.getHeight() + extraHeight // Extend touchable region below the keyboard. + EXTENDED_TOUCHABLE_REGION_HEIGHT; outInsets.touchableInsets = InputMethodService.Insets.TOUCHABLE_INSETS_REGION; @@ -1368,10 +1445,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } - private static boolean isAlphabet(final int code) { - return Character.isLetter(code); - } - private void onSettingsKeyPressed() { if (isShowingOptionDialog()) return; showSubtypeSelectorAndSettings(); @@ -1479,7 +1552,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen break; case Constants.CODE_SHIFT: // Note: Calling back to the keyboard on Shift key is handled in - // {@link #onPressKey(int,boolean)} and {@link #onReleaseKey(int,boolean)}. + // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}. final Keyboard currentKeyboard = switcher.getKeyboard(); if (null != currentKeyboard && currentKeyboard.mId.isAlphabetKeyboard()) { // TODO: Instead of checking for alphabetic keyboard here, separate keycodes for @@ -1493,7 +1566,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen break; case Constants.CODE_SWITCH_ALPHA_SYMBOL: // Note: Calling back to the keyboard on symbol key is handled in - // {@link #onPressKey(int,boolean)} and {@link #onReleaseKey(int,boolean)}. + // {@link #onPressKey(int,int,boolean)} and {@link #onReleaseKey(int,boolean)}. break; case Constants.CODE_SETTINGS: onSettingsKeyPressed(); @@ -1511,7 +1584,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen handleLanguageSwitchKey(); break; case Constants.CODE_EMOJI: - // TODO: Implement emoji keyboard switch. + // Note: Switching emoji keyboard is being handled in + // {@link KeyboardState#onCodeInput(int,int)}. break; case Constants.CODE_ENTER: final EditorInfo editorInfo = getCurrentInputEditorInfo(); @@ -1627,7 +1701,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onStartBatchInput() { - BatchInputUpdater.getInstance().onStartBatchInput(this); + mInputUpdater.onStartBatchInput(); mHandler.cancelUpdateSuggestionStrip(); mConnection.beginBatchEdit(); final SettingsValues settingsValues = mSettings.getCurrent(); @@ -1667,46 +1741,41 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); } - private static final class BatchInputUpdater implements Handler.Callback { + private static final class InputUpdater implements Handler.Callback { private final Handler mHandler; - private LatinIME mLatinIme; + private final LatinIME mLatinIme; private final Object mLock = new Object(); private boolean mInBatchInput; // synchronized using {@link #mLock}. - private BatchInputUpdater() { + private InputUpdater(final LatinIME latinIme) { final HandlerThread handlerThread = new HandlerThread( - BatchInputUpdater.class.getSimpleName()); + InputUpdater.class.getSimpleName()); handlerThread.start(); mHandler = new Handler(handlerThread.getLooper(), this); - } - - // Initialization-on-demand holder - private static final class OnDemandInitializationHolder { - public static final BatchInputUpdater sInstance = new BatchInputUpdater(); - } - - public static BatchInputUpdater getInstance() { - return OnDemandInitializationHolder.sInstance; + mLatinIme = latinIme; } private static final int MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP = 1; + private static final int MSG_GET_SUGGESTED_WORDS = 2; @Override public boolean handleMessage(final Message msg) { switch (msg.what) { - case MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP: - updateBatchInput((InputPointers)msg.obj); - break; + case MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP: + updateBatchInput((InputPointers)msg.obj); + break; + case MSG_GET_SUGGESTED_WORDS: + mLatinIme.getSuggestedWords(msg.arg1, (OnGetSuggestedWordsCallback) msg.obj); + break; } return true; } // Run in the UI thread. - public void onStartBatchInput(final LatinIME latinIme) { + public void onStartBatchInput() { synchronized (mLock) { mHandler.removeMessages(MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP); mInBatchInput = true; - mLatinIme = latinIme; mLatinIme.mHandler.showGesturePreviewAndSuggestionStrip( SuggestedWords.EMPTY, false /* dismissGestureFloatingPreviewText */); } @@ -1719,9 +1788,14 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Batch input has ended or canceled while the message was being delivered. return; } - final SuggestedWords suggestedWords = getSuggestedWordsGestureLocked(batchPointers); - mLatinIme.mHandler.showGesturePreviewAndSuggestionStrip( - suggestedWords, false /* dismissGestureFloatingPreviewText */); + + getSuggestedWordsGestureLocked(batchPointers, new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + mLatinIme.mHandler.showGesturePreviewAndSuggestionStrip( + suggestedWords, false /* dismissGestureFloatingPreviewText */); + } + }); } } @@ -1744,35 +1818,57 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } // Run in the UI thread. - public SuggestedWords onEndBatchInput(final InputPointers batchPointers) { - synchronized (mLock) { - mInBatchInput = false; - final SuggestedWords suggestedWords = getSuggestedWordsGestureLocked(batchPointers); - mLatinIme.mHandler.showGesturePreviewAndSuggestionStrip( - suggestedWords, true /* dismissGestureFloatingPreviewText */); - return suggestedWords; + public void onEndBatchInput(final InputPointers batchPointers) { + synchronized(mLock) { + getSuggestedWordsGestureLocked(batchPointers, new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + mInBatchInput = false; + mLatinIme.mHandler.showGesturePreviewAndSuggestionStrip(suggestedWords, + true /* dismissGestureFloatingPreviewText */); + mLatinIme.mHandler.onEndBatchInput(suggestedWords); + } + }); } } // {@link LatinIME#getSuggestedWords(int)} method calls with same session id have to // be synchronized. - private SuggestedWords getSuggestedWordsGestureLocked(final InputPointers batchPointers) { + private void getSuggestedWordsGestureLocked(final InputPointers batchPointers, + final OnGetSuggestedWordsCallback callback) { mLatinIme.mWordComposer.setBatchInputPointers(batchPointers); - final SuggestedWords suggestedWords = - mLatinIme.getSuggestedWordsOrOlderSuggestions(Suggest.SESSION_GESTURE); - final int suggestionCount = suggestedWords.size(); - if (suggestionCount <= 1) { - final String mostProbableSuggestion = (suggestionCount == 0) ? null - : suggestedWords.getWord(0); - return mLatinIme.getOlderSuggestions(mostProbableSuggestion); - } - return suggestedWords; + mLatinIme.getSuggestedWordsOrOlderSuggestionsAsync(Suggest.SESSION_GESTURE, + new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(SuggestedWords suggestedWords) { + final int suggestionCount = suggestedWords.size(); + if (suggestionCount <= 1) { + final String mostProbableSuggestion = (suggestionCount == 0) ? null + : suggestedWords.getWord(0); + callback.onGetSuggestedWords( + mLatinIme.getOlderSuggestions(mostProbableSuggestion)); + } + callback.onGetSuggestedWords(suggestedWords); + } + }); + } + + public void getSuggestedWords(final int sessionId, + final OnGetSuggestedWordsCallback callback) { + mHandler.obtainMessage(MSG_GET_SUGGESTED_WORDS, sessionId, 0, callback).sendToTarget(); + } + + private void onDestroy() { + mHandler.removeMessages(MSG_GET_SUGGESTED_WORDS); + mHandler.removeMessages(MSG_UPDATE_GESTURE_PREVIEW_AND_SUGGESTION_STRIP); + mHandler.getLooper().quit(); } } + // This method must run in UI Thread. private void showGesturePreviewAndSuggestionStrip(final SuggestedWords suggestedWords, final boolean dismissGestureFloatingPreviewText) { - showSuggestionStrip(suggestedWords, null); + showSuggestionStrip(suggestedWords); final MainKeyboardView mainKeyboardView = mKeyboardSwitcher.getMainKeyboardView(); mainKeyboardView.showGestureFloatingPreviewText(suggestedWords); if (dismissGestureFloatingPreviewText) { @@ -1782,13 +1878,17 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onUpdateBatchInput(final InputPointers batchPointers) { - BatchInputUpdater.getInstance().onUpdateBatchInput(batchPointers); + final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate(); + if (null != candidate) { + if (candidate.mSourceDict.shouldAutoCommit(candidate)) { + // TODO: implement auto-commit + } + } + mInputUpdater.onUpdateBatchInput(batchPointers); } - @Override - public void onEndBatchInput(final InputPointers batchPointers) { - final SuggestedWords suggestedWords = BatchInputUpdater.getInstance().onEndBatchInput( - batchPointers); + // This method must run in UI Thread. + public void onEndBatchInputAsyncInternal(final SuggestedWords suggestedWords) { final String batchInputText = suggestedWords.isEmpty() ? null : suggestedWords.getWord(0); if (TextUtils.isEmpty(batchInputText)) { @@ -1810,6 +1910,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mKeyboardSwitcher.updateShiftState(); } + @Override + public void onEndBatchInput(final InputPointers batchPointers) { + mInputUpdater.onEndBatchInput(batchPointers); + } + private String specificTldProcessingOnTextInput(final String text) { if (text.length() <= 1 || text.charAt(0) != Constants.CODE_PERIOD || !Character.isLetter(text.charAt(1))) { @@ -1845,7 +1950,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onCancelBatchInput() { - BatchInputUpdater.getInstance().onCancelBatchInput(); + mInputUpdater.onCancelBatchInput(); } private void handleBackspace(final int spaceState) { @@ -1861,23 +1966,23 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // When we exit this if-clause, mWordComposer.isComposingWord() will return false. } if (mWordComposer.isComposingWord()) { - final int length = mWordComposer.size(); - if (length > 0) { - if (mWordComposer.isBatchMode()) { - if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - final String word = mWordComposer.getTypedWord(); - ResearchLogger.latinIME_handleBackspace_batch(word, 1); - } - final String rejectedSuggestion = mWordComposer.getTypedWord(); - mWordComposer.reset(); - mWordComposer.setRejectedBatchModeSuggestion(rejectedSuggestion); - } else { - mWordComposer.deleteLast(); + if (mWordComposer.isBatchMode()) { + if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { + final String word = mWordComposer.getTypedWord(); + ResearchLogger.latinIME_handleBackspace_batch(word, 1); } - mConnection.setComposingText(getTextWithUnderline(mWordComposer.getTypedWord()), 1); - mHandler.postUpdateSuggestionStrip(); + final String rejectedSuggestion = mWordComposer.getTypedWord(); + mWordComposer.reset(); + mWordComposer.setRejectedBatchModeSuggestion(rejectedSuggestion); } else { - mConnection.deleteSurroundingText(1, 0); + mWordComposer.deleteLast(); + } + mConnection.setComposingText(getTextWithUnderline(mWordComposer.getTypedWord()), 1); + mHandler.postUpdateSuggestionStrip(); + if (!mWordComposer.isComposingWord()) { + // If we just removed the last character, auto-caps mode may have changed so we + // need to re-evaluate. + mKeyboardSwitcher.updateShiftState(); } } else { final SettingsValues currentSettings = mSettings.getCurrent(); @@ -1892,8 +1997,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // Cancel multi-character input: remove the text we just entered. // This is triggered on backspace after a key that inputs multiple characters, // like the smiley key or the .com key. - final int length = mEnteredText.length(); - mConnection.deleteSurroundingText(length, 0); + mConnection.deleteSurroundingText(mEnteredText.length(), 0); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_handleBackspace_cancelTextInput(mEnteredText); } @@ -1939,6 +2043,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This should never happen. Log.e(TAG, "Backspace when we don't know the selection position"); } + final int lengthToDelete = Character.isSupplementaryCodePoint( + mConnection.getCodePointBeforeCursor()) ? 2 : 1; if (mAppWorkAroundsUtils.isBeforeJellyBean()) { // Backward compatibility mode. Before Jelly bean, the keyboard would simulate // a hardware keyboard event on pressing enter or delete. This is bad for many @@ -1946,22 +2052,28 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // relying on this behavior so we continue to support it for older apps. sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_DEL); } else { - mConnection.deleteSurroundingText(1, 0); + mConnection.deleteSurroundingText(lengthToDelete, 0); } if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(1, true /* shouldUncommitLogUnit */); + ResearchLogger.latinIME_handleBackspace(lengthToDelete, + true /* shouldUncommitLogUnit */); } if (mDeleteCount > DELETE_ACCELERATE_AT) { - mConnection.deleteSurroundingText(1, 0); + final int lengthToDeleteAgain = Character.isSupplementaryCodePoint( + mConnection.getCodePointBeforeCursor()) ? 2 : 1; + mConnection.deleteSurroundingText(lengthToDeleteAgain, 0); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { - ResearchLogger.latinIME_handleBackspace(1, + ResearchLogger.latinIME_handleBackspace(lengthToDeleteAgain, true /* shouldUncommitLogUnit */); } } } - if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) { + if (currentSettings.isSuggestionsRequested(mDisplayOrientation) + && currentSettings.mCurrentLanguageHasSpaces) { restartSuggestionsOnWordBeforeCursorIfAtEndOfWord(); } + // We just removed a character. We need to update the auto-caps state. + mKeyboardSwitcher.updateShiftState(); } } @@ -1986,6 +2098,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private void handleCharacter(final int primaryCode, final int x, final int y, final int spaceState) { + // TODO: refactor this method to stop flipping isComposingWord around all the time, and + // make it shorter (possibly cut into several pieces). Also factor handleNonSpecialCharacter + // which has the same name as other handle* methods but is not the same. boolean isComposingWord = mWordComposer.isComposingWord(); // TODO: remove isWordConnector() and use isUsuallyFollowedBySpace() instead. @@ -2005,18 +2120,26 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen resetEntireInputState(mLastSelectionStart); isComposingWord = false; } - // NOTE: isCursorTouchingWord() is a blocking IPC call, so it often takes several - // dozen milliseconds. Avoid calling it as much as possible, since we are on the UI - // thread here. - if (!isComposingWord && (isAlphabet(primaryCode) - || currentSettings.isWordConnector(primaryCode)) + // We want to find out whether to start composing a new word with this character. If so, + // we need to reset the composing state and switch isComposingWord. The order of the + // tests is important for good performance. + // We only start composing if we're not already composing. + if (!isComposingWord + // We only start composing if this is a word code point. Essentially that means it's a + // a letter or a word connector. + && currentSettings.isWordCodePoint(primaryCode) + // We never go into composing state if suggestions are not requested. && currentSettings.isSuggestionsRequested(mDisplayOrientation) && - !mConnection.isCursorTouchingWord(currentSettings)) { + // In languages with spaces, we only start composing a word when we are not already + // touching a word. In languages without spaces, the above conditions are sufficient. + (!mConnection.isCursorTouchingWord(currentSettings) + || !currentSettings.mCurrentLanguageHasSpaces)) { // Reset entirely the composing state anyway, then start composing a new word unless - // the character is a single quote. The idea here is, single quote is not a - // separator and it should be treated as a normal character, except in the first - // position where it should not start composing a word. - isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode); + // the character is a single quote or a dash. The idea here is, single quote and dash + // are not separators and they should be treated as normal characters, except in the + // first position where they should not start composing a word. + isComposingWord = (Constants.CODE_SINGLE_QUOTE != primaryCode + && Constants.CODE_DASH != primaryCode); // Here we don't need to reset the last composed word. It will be reset // when we commit this one, if we ever do; if on the other hand we backspace // it entirely and resume suggestions on the previous word, we'd like to still @@ -2099,16 +2222,20 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean handleSeparator(final int primaryCode, final int x, final int y, final int spaceState) { boolean didAutoCorrect = false; + final SettingsValues currentSettings = mSettings.getCurrent(); + // We avoid sending spaces in languages without spaces if we were composing. + final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == primaryCode + && !currentSettings.mCurrentLanguageHasSpaces && mWordComposer.isComposingWord(); if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { // If we are in the middle of a recorrection, we need to commit the recorrection // first so that we can insert the separator at the current cursor position. resetEntireInputState(mLastSelectionStart); } - final SettingsValues currentSettings = mSettings.getCurrent(); - if (mWordComposer.isComposingWord()) { + if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing if (currentSettings.mCorrectionEnabled) { - // TODO: maybe cache Strings in an <String> sparse array or something - commitCurrentAutoCorrection(new String(new int[]{primaryCode}, 0, 1)); + final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR + : new String(new int[] { primaryCode }, 0, 1); + commitCurrentAutoCorrection(separator); didAutoCorrect = true; } else { commitTyped(new String(new int[]{primaryCode}, 0, 1)); @@ -2125,7 +2252,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_handleSeparator(primaryCode, mWordComposer.isComposingWord()); } - sendKeyCodePoint(primaryCode); + + if (!shouldAvoidSendingCode) { + sendKeyCodePoint(primaryCode); + } if (Constants.CODE_SPACE == primaryCode) { if (currentSettings.isSuggestionsRequested(mDisplayOrientation)) { @@ -2255,34 +2385,71 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return; } - final SuggestedWords suggestedWords = - getSuggestedWordsOrOlderSuggestions(Suggest.SESSION_TYPING); - final String typedWord = mWordComposer.getTypedWord(); - showSuggestionStrip(suggestedWords, typedWord); + final CountDownLatch latch = new CountDownLatch(1); + final SuggestedWords[] suggestedWordsArray = new SuggestedWords[1]; + getSuggestedWordsOrOlderSuggestionsAsync(Suggest.SESSION_TYPING, + new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(final SuggestedWords suggestedWords) { + suggestedWordsArray[0] = suggestedWords; + latch.countDown(); + } + } + ); + + // TODO: Quit blocking the main thread. + try { + // Wait for the result of getSuggestedWords + // We set the time out to avoid ANR. + latch.await(GET_SUGGESTED_WORDS_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // TODO: Cancel all pending "getSuggestedWords" tasks when it failed. We may want to add + // "onGetSuggestionFailed" to "OnGetSuggestedWordsCallback". + Log.e(TAG, "InterruptedException while waiting for getSuggestedWords.", e); + return; + } + if (suggestedWordsArray[0] != null) { + showSuggestionStrip(suggestedWordsArray[0]); + } } - private SuggestedWords getSuggestedWords(final int sessionId) { + private void getSuggestedWords(final int sessionId, + final OnGetSuggestedWordsCallback callback) { final Keyboard keyboard = mKeyboardSwitcher.getKeyboard(); final Suggest suggest = mSuggest; if (keyboard == null || suggest == null) { - return SuggestedWords.EMPTY; + callback.onGetSuggestedWords(SuggestedWords.EMPTY); + return; } // Get the word on which we should search the bigrams. If we are composing a word, it's // whatever is *before* the half-committed word in the buffer, hence 2; if we aren't, we // should just skip whitespace if any, so 1. - // TODO: this is slow (2-way IPC) - we should probably cache this instead. final SettingsValues currentSettings = mSettings.getCurrent(); - final String prevWord = - mConnection.getNthPreviousWord(currentSettings.mWordSeparators, - mWordComposer.isComposingWord() ? 2 : 1); - return suggest.getSuggestedWords(mWordComposer, prevWord, keyboard.getProximityInfo(), - currentSettings.mBlockPotentiallyOffensive, - currentSettings.mCorrectionEnabled, sessionId); + final int[] additionalFeaturesOptions = currentSettings.mAdditionalFeaturesSettingValues; + final String prevWord; + if (currentSettings.mCurrentLanguageHasSpaces) { + // If we are typing in a language with spaces we can just look up the previous + // word from textview. + prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, + mWordComposer.isComposingWord() ? 2 : 1); + } else { + prevWord = LastComposedWord.NOT_A_COMPOSED_WORD == mLastComposedWord ? null + : mLastComposedWord.mCommittedWord; + } + suggest.getSuggestedWords(mWordComposer, prevWord, keyboard.getProximityInfo(), + currentSettings.mBlockPotentiallyOffensive, currentSettings.mCorrectionEnabled, + additionalFeaturesOptions, sessionId, callback); } - private SuggestedWords getSuggestedWordsOrOlderSuggestions(final int sessionId) { - return maybeRetrieveOlderSuggestions(mWordComposer.getTypedWord(), - getSuggestedWords(sessionId)); + private void getSuggestedWordsOrOlderSuggestionsAsync(final int sessionId, + final OnGetSuggestedWordsCallback callback) { + mInputUpdater.getSuggestedWords(sessionId, new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords(SuggestedWords suggestedWords) { + callback.onGetSuggestedWords(maybeRetrieveOlderSuggestions( + mWordComposer.getTypedWord(), suggestedWords)); + } + }); } private SuggestedWords maybeRetrieveOlderSuggestions(final String typedWord, @@ -2322,16 +2489,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen false /* isPrediction */); } - private void showSuggestionStrip(final SuggestedWords suggestedWords, final String typedWord) { + private void showSuggestionStrip(final SuggestedWords suggestedWords) { if (suggestedWords.isEmpty()) { clearSuggestionStrip(); return; } final String autoCorrection; if (suggestedWords.mWillAutoCorrect) { - autoCorrection = suggestedWords.getWord(1); + autoCorrection = suggestedWords.getWord(SuggestedWords.INDEX_OF_AUTO_CORRECTION); } else { - autoCorrection = typedWord; + autoCorrection = suggestedWords.getWord(SuggestedWords.INDEX_OF_TYPED_WORD); } mWordComposer.setAutoCorrection(autoCorrection); final boolean isAutoCorrection = suggestedWords.willAutoCorrect(); @@ -2440,7 +2607,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_pickSuggestionManually(replacedWord, index, suggestion, mWordComposer.isBatchMode(), suggestionInfo.mScore, suggestionInfo.mKind, - suggestionInfo.mSourceDict); + suggestionInfo.mSourceDict.mDictType); } mConnection.endBatchEdit(); // Don't allow cancellation of manual pick @@ -2544,6 +2711,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // recorrection. This is a temporary, stopgap measure that will be removed later. // TODO: remove this. if (mAppWorkAroundsUtils.isBrokenByRecorrection()) return; + // Recorrection is not supported in languages without spaces because we don't know + // how to segment them yet. + if (!mSettings.getCurrent().mCurrentLanguageHasSpaces) return; // If the cursor is not touching a word, or if there is a selection, return right away. if (mLastSelectionStart != mLastSelectionEnd) return; // If we don't know the cursor location, return. @@ -2566,51 +2736,63 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!TextUtils.equals(s, typedWord)) { suggestions.add(new SuggestedWordInfo(s, SuggestionStripView.MAX_SUGGESTIONS - i, - SuggestedWordInfo.KIND_RESUMED, Dictionary.TYPE_RESUMED)); + SuggestedWordInfo.KIND_RESUMED, Dictionary.DICTIONARY_RESUMED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } } mWordComposer.setComposingWord(typedWord, mKeyboardSwitcher.getKeyboard()); - // TODO: this is in chars but the callee expects code points! - mWordComposer.setCursorPositionWithinWord(numberOfCharsInWordBeforeCursor); + mWordComposer.setCursorPositionWithinWord( + typedWord.codePointCount(0, numberOfCharsInWordBeforeCursor)); mConnection.setComposingRegion( mLastSelectionStart - numberOfCharsInWordBeforeCursor, mLastSelectionEnd + range.getNumberOfCharsInWordAfterCursor()); - final SuggestedWords suggestedWords; if (suggestions.isEmpty()) { // We come here if there weren't any suggestion spans on this word. We will try to // compute suggestions for it instead. - final SuggestedWords suggestedWordsIncludingTypedWord = - getSuggestedWords(Suggest.SESSION_TYPING); - if (suggestedWordsIncludingTypedWord.size() > 1) { - // We were able to compute new suggestions for this word. - // Remove the typed word, since we don't want to display it in this case. - // The #getSuggestedWordsExcludingTypedWord() method sets willAutoCorrect to false. - suggestedWords = - suggestedWordsIncludingTypedWord.getSuggestedWordsExcludingTypedWord(); - } else { - // No saved suggestions, and we were unable to compute any good one either. - // Rather than displaying an empty suggestion strip, we'll display the original - // word alone in the middle. - // Since there is only one word, willAutoCorrect is false. - suggestedWords = suggestedWordsIncludingTypedWord; - } + mInputUpdater.getSuggestedWords(Suggest.SESSION_TYPING, + new OnGetSuggestedWordsCallback() { + @Override + public void onGetSuggestedWords( + final SuggestedWords suggestedWordsIncludingTypedWord) { + final SuggestedWords suggestedWords; + if (suggestedWordsIncludingTypedWord.size() > 1) { + // We were able to compute new suggestions for this word. + // Remove the typed word, since we don't want to display it in this case. + // The #getSuggestedWordsExcludingTypedWord() method sets willAutoCorrect to + // false. + suggestedWords = suggestedWordsIncludingTypedWord + .getSuggestedWordsExcludingTypedWord(); + } else { + // No saved suggestions, and we were unable to compute any good one either. + // Rather than displaying an empty suggestion strip, we'll display the + // original word alone in the middle. + // Since there is only one word, willAutoCorrect is false. + suggestedWords = suggestedWordsIncludingTypedWord; + } + unsetIsAutoCorrectionIndicatorOnAndCallShowSuggestionStrip(suggestedWords); + }}); } else { // We found suggestion spans in the word. We'll create the SuggestedWords out of // them, and make willAutoCorrect false. - suggestedWords = new SuggestedWords(suggestions, + final SuggestedWords suggestedWords = new SuggestedWords(suggestions, true /* typedWordValid */, false /* willAutoCorrect */, false /* isPunctuationSuggestions */, false /* isObsoleteSuggestions */, false /* isPrediction */); + unsetIsAutoCorrectionIndicatorOnAndCallShowSuggestionStrip(suggestedWords); } + } + public void unsetIsAutoCorrectionIndicatorOnAndCallShowSuggestionStrip( + final SuggestedWords suggestedWords) { // Note that it's very important here that suggestedWords.mWillAutoCorrect is false. - // We never want to auto-correct on a resumed suggestion. Please refer to the three - // places above where suggestedWords is affected. We also need to reset - // mIsAutoCorrectionIndicatorOn to avoid showSuggestionStrip touching the text to adapt it. - // TODO: remove mIsAutoCorrectionIndicator on (see comment on definition) + // We never want to auto-correct on a resumed suggestion. Please refer to the three places + // above in restartSuggestionsOnWordTouchedByCursor() where suggestedWords is affected. + // We also need to unset mIsAutoCorrectionIndicatorOn to avoid showSuggestionStrip touching + // the text to adapt it. + // TODO: remove mIsAutoCorrectionIndicatorOn (see comment on definition) mIsAutoCorrectionIndicatorOn = false; - showSuggestionStrip(suggestedWords, typedWord); + mHandler.showSuggestionStrip(suggestedWords); } /** @@ -2666,7 +2848,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) { mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord); } - mConnection.commitText(originallyTypedWord + mLastComposedWord.mSeparatorString, 1); + final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString; + if (mSettings.getCurrent().mCurrentLanguageHasSpaces) { + // For languages with spaces, we revert to the typed string, but the cursor is still + // after the separator so we don't resume suggestions. If the user wants to correct + // the word, they have to press backspace again. + mConnection.commitText(stringToCommit, 1); + } else { + // For languages without spaces, we revert the typed string but the cursor is flush + // with the typed word, so we need to resume suggestions right away. + mWordComposer.setComposingWord(stringToCommit, mKeyboardSwitcher.getKeyboard()); + mConnection.setComposingText(stringToCommit, 1); + } if (mSettings.isInternal()) { LatinImeLoggerUtils.onSeparator(mLastComposedWord.mSeparatorString, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE); @@ -2684,7 +2877,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // This essentially inserts a space, and that's it. public void promotePhantomSpace() { - if (mSettings.getCurrent().shouldInsertSpacesAutomatically() + final SettingsValues currentSettings = mSettings.getCurrent(); + if (currentSettings.shouldInsertSpacesAutomatically() + && currentSettings.mCurrentLanguageHasSpaces && !mConnection.textBeforeCursorLooksLikeURL()) { if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.latinIME_promotePhantomSpace(); @@ -2710,30 +2905,43 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } - private void hapticAndAudioFeedback(final int code, final boolean isRepeatKey) { + private void hapticAndAudioFeedback(final int code, final int repeatCount) { final MainKeyboardView keyboardView = mKeyboardSwitcher.getMainKeyboardView(); if (keyboardView != null && keyboardView.isInSlidingKeyInput()) { // No need to feedback while sliding input. return; } - if (isRepeatKey && code == Constants.CODE_DELETE && !mConnection.canDeleteCharacters()) { - // No need to feedback when repeating delete key will have no effect. - return; + if (repeatCount > 0) { + if (code == Constants.CODE_DELETE && !mConnection.canDeleteCharacters()) { + // No need to feedback when repeat delete key will have no effect. + return; + } + // TODO: Use event time that the last feedback has been generated instead of relying on + // a repeat count to thin out feedback. + if (repeatCount % PERIOD_FOR_AUDIO_AND_HAPTIC_FEEDBACK_IN_KEY_REPEAT == 0) { + return; + } + } + final AudioAndHapticFeedbackManager feedbackManager = + AudioAndHapticFeedbackManager.getInstance(); + if (repeatCount == 0) { + // TODO: Reconsider how to perform haptic feedback when repeating key. + feedbackManager.performHapticFeedback(keyboardView); } - AudioAndHapticFeedbackManager.getInstance().hapticAndAudioFeedback(code, keyboardView); + feedbackManager.performAudioFeedback(code); } // Callback of the {@link KeyboardActionListener}. This is called when a key is depressed; // release matching call is {@link #onReleaseKey(int,boolean)} below. @Override - public void onPressKey(final int primaryCode, final boolean isRepeatKey, + public void onPressKey(final int primaryCode, final int repeatCount, final boolean isSinglePointer) { mKeyboardSwitcher.onPressKey(primaryCode, isSinglePointer); - hapticAndAudioFeedback(primaryCode, isRepeatKey); + hapticAndAudioFeedback(primaryCode, repeatCount); } // Callback of the {@link KeyboardActionListener}. This is called when a key is released; - // press matching call is {@link #onPressKey(int,boolean,boolean)} above. + // press matching call is {@link #onPressKey(int,int,boolean)} above. @Override public void onReleaseKey(final int primaryCode, final boolean withSliding) { mKeyboardSwitcher.onReleaseKey(primaryCode, withSliding); @@ -2749,17 +2957,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen break; } } - - if (Constants.CODE_DELETE == primaryCode) { - // This is a stopgap solution to avoid leaving a high surrogate alone in a text view. - // In the future, we need to deprecate deteleSurroundingText() and have a surrogate - // pair-friendly way of deleting characters in InputConnection. - // TODO: use getCodePointBeforeCursor instead to improve performance - final CharSequence lastChar = mConnection.getTextBeforeCursor(1, 0); - if (!TextUtils.isEmpty(lastChar) && Character.isHighSurrogate(lastChar.charAt(0))) { - mConnection.deleteSurroundingText(1, 0); - } - } } // Hooks for hardware keyboard @@ -2895,6 +3092,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return mSuggest.hasMainDictionary(); } + // DO NOT USE THIS for any other purpose than testing. This can break the keyboard badly. + @UsedForTesting + /* package for test */ void replaceMainDictionaryForTest(final Locale locale) { + mSuggest.resetMainDict(this, locale, null); + } + public void debugDumpStateAndCrashWithException(final String context) { final StringBuilder s = new StringBuilder(mAppWorkAroundsUtils.toString()); s.append("\nAttributes : ").append(mSettings.getCurrent().mInputAttributes) diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 35920f8cb..a031bb3be 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -169,7 +169,6 @@ public final class RichInputConnection { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); mCommittedTextBeforeComposingText.append(mComposingText); - mExpectedCursorPosition += mComposingText.length(); mComposingText.setLength(0); if (null != mIC) { mIC.finishComposingText(); diff --git a/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java b/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java index be03d4ae5..0889f22ca 100644 --- a/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java +++ b/java/src/com/android/inputmethod/latin/SubtypeSwitcher.java @@ -53,12 +53,22 @@ public final class SubtypeSwitcher { private InputMethodInfo mShortcutInputMethodInfo; private InputMethodSubtype mShortcutSubtype; private InputMethodSubtype mNoLanguageSubtype; + private InputMethodSubtype mEmojiSubtype; private boolean mIsNetworkConnected; // Dummy no language QWERTY subtype. See {@link R.xml.method}. private static final InputMethodSubtype DUMMY_NO_LANGUAGE_SUBTYPE = new InputMethodSubtype( - R.string.subtype_no_language_qwerty, R.drawable.ic_subtype_keyboard, "zz", "keyboard", - "KeyboardLayoutSet=qwerty,AsciiCapable,EnabledWhenDefaultIsNotAsciiCapable", + R.string.subtype_no_language_qwerty, R.drawable.ic_subtype_keyboard, + SubtypeLocaleUtils.NO_LANGUAGE, "keyboard", + "KeyboardLayoutSet=" + SubtypeLocaleUtils.QWERTY + + ",AsciiCapable,EnabledWhenDefaultIsNotAsciiCapable", + false /* isAuxiliary */, false /* overridesImplicitlyEnabledSubtype */); + // Caveat: We probably should remove this when we add an Emoji subtype in {@link R.xml.method}. + // Dummy Emoji subtype. See {@link R.xml.method}. + private static final InputMethodSubtype DUMMY_EMOJI_SUBTYPE = new InputMethodSubtype( + R.string.subtype_emoji, R.drawable.ic_subtype_keyboard, + SubtypeLocaleUtils.NO_LANGUAGE, "keyboard", + "KeyboardLayoutSet=" + SubtypeLocaleUtils.EMOJI, false /* isAuxiliary */, false /* overridesImplicitlyEnabledSubtype */); static final class NeedsToDisplayLanguage { @@ -271,4 +281,17 @@ public final class SubtypeSwitcher { + DUMMY_NO_LANGUAGE_SUBTYPE); return DUMMY_NO_LANGUAGE_SUBTYPE; } + + public InputMethodSubtype getEmojiSubtype() { + if (mEmojiSubtype == null) { + mEmojiSubtype = mRichImm.findSubtypeByLocaleAndKeyboardLayoutSet( + SubtypeLocaleUtils.NO_LANGUAGE, SubtypeLocaleUtils.EMOJI); + } + if (mEmojiSubtype != null) { + return mEmojiSubtype; + } + Log.w(TAG, "Can't find Emoji subtype"); + Log.w(TAG, "No input method subtype found; return dummy subtype: " + DUMMY_EMOJI_SUBTYPE); + return DUMMY_EMOJI_SUBTYPE; + } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index c2fdcb552..18ba15872 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -24,6 +24,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary; import com.android.inputmethod.latin.settings.Settings; @@ -107,7 +108,7 @@ public final class Suggest { } private void addOrReplaceDictionaryInternal(final String key, final Dictionary dict) { - if (mOnlyDictionarySetForDebug != null && mOnlyDictionarySetForDebug.contains(key)) { + if (mOnlyDictionarySetForDebug != null && !mOnlyDictionarySetForDebug.contains(key)) { Log.w(TAG, "Ignore add " + key + " dictionary for debug."); return; } @@ -200,28 +201,41 @@ public final class Suggest { personalizationPredictionDictionary); } + public void setPersonalizationDictionary( + final PersonalizationDictionary personalizationDictionary) { + addOrReplaceDictionaryInternal(Dictionary.TYPE_PERSONALIZATION, + personalizationDictionary); + } + public void setAutoCorrectionThreshold(float threshold) { mAutoCorrectionThreshold = threshold; } - public SuggestedWords getSuggestedWords(final WordComposer wordComposer, + public interface OnGetSuggestedWordsCallback { + public void onGetSuggestedWords(final SuggestedWords suggestedWords); + } + + public void getSuggestedWords(final WordComposer wordComposer, final String prevWordForBigram, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final boolean isCorrectionEnabled, - final int sessionId) { + final int[] additionalFeaturesOptions, final int sessionId, + final OnGetSuggestedWordsCallback callback) { LatinImeLogger.onStartSuggestion(prevWordForBigram); if (wordComposer.isBatchMode()) { - return getSuggestedWordsForBatchInput( - wordComposer, prevWordForBigram, proximityInfo, blockOffensiveWords, sessionId); + getSuggestedWordsForBatchInput(wordComposer, prevWordForBigram, proximityInfo, + blockOffensiveWords, additionalFeaturesOptions, sessionId, callback); } else { - return getSuggestedWordsForTypingInput(wordComposer, prevWordForBigram, proximityInfo, - blockOffensiveWords, isCorrectionEnabled); + getSuggestedWordsForTypingInput(wordComposer, prevWordForBigram, proximityInfo, + blockOffensiveWords, isCorrectionEnabled, additionalFeaturesOptions, callback); } } - // Retrieves suggestions for the typing input. - private SuggestedWords getSuggestedWordsForTypingInput(final WordComposer wordComposer, + // Retrieves suggestions for the typing input + // and calls the callback function with the suggestions. + private void getSuggestedWordsForTypingInput(final WordComposer wordComposer, final String prevWordForBigram, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final boolean isCorrectionEnabled) { + final boolean blockOffensiveWords, final boolean isCorrectionEnabled, + final int[] additionalFeaturesOptions, final OnGetSuggestedWordsCallback callback) { final int trailingSingleQuotesCount = wordComposer.trailingSingleQuotesCount(); final BoundedTreeSet suggestionsSet = new BoundedTreeSet(sSuggestedWordInfoComparator, MAX_SUGGESTIONS); @@ -244,8 +258,9 @@ public final class Suggest { for (final String key : mDictionaries.keySet()) { final Dictionary dictionary = mDictionaries.get(key); - suggestionsSet.addAll(dictionary.getSuggestions( - wordComposerForLookup, prevWordForBigram, proximityInfo, blockOffensiveWords)); + suggestionsSet.addAll(dictionary.getSuggestions(wordComposerForLookup, + prevWordForBigram, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions)); } final String whitelistedWord; @@ -303,13 +318,15 @@ public final class Suggest { for (int i = 0; i < suggestionsCount; ++i) { final SuggestedWordInfo wordInfo = suggestionsContainer.get(i); - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord.toString(), + wordInfo.mSourceDict.mDictType); } if (!TextUtils.isEmpty(typedWord)) { suggestionsContainer.add(0, new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, - Dictionary.TYPE_USER_TYPED)); + Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } SuggestedWordInfo.removeDups(suggestionsContainer); @@ -320,7 +337,7 @@ public final class Suggest { suggestionsList = suggestionsContainer; } - return new SuggestedWords(suggestionsList, + callback.onGetSuggestedWords(new SuggestedWords(suggestionsList, // TODO: this first argument is lying. If this is a whitelisted word which is an // actual word, it says typedWordValid = false, which looks wrong. We should either // rename the attribute or change the value. @@ -328,13 +345,15 @@ public final class Suggest { hasAutoCorrection, /* willAutoCorrect */ false /* isPunctuationSuggestions */, false /* isObsoleteSuggestions */, - !wordComposer.isComposingWord() /* isPrediction */); + !wordComposer.isComposingWord() /* isPrediction */)); } - // Retrieves suggestions for the batch input. - private SuggestedWords getSuggestedWordsForBatchInput(final WordComposer wordComposer, + // Retrieves suggestions for the batch input + // and calls the callback function with the suggestions. + private void getSuggestedWordsForBatchInput(final WordComposer wordComposer, final String prevWordForBigram, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int sessionId) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, + final int sessionId, final OnGetSuggestedWordsCallback callback) { final BoundedTreeSet suggestionsSet = new BoundedTreeSet(sSuggestedWordInfoComparator, MAX_SUGGESTIONS); @@ -348,11 +367,12 @@ public final class Suggest { } final Dictionary dictionary = mDictionaries.get(key); suggestionsSet.addAll(dictionary.getSuggestionsWithSessionId(wordComposer, - prevWordForBigram, proximityInfo, blockOffensiveWords, sessionId)); + prevWordForBigram, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId)); } for (SuggestedWordInfo wordInfo : suggestionsSet) { - LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict); + LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType); } final ArrayList<SuggestedWordInfo> suggestionsContainer = @@ -387,12 +407,12 @@ public final class Suggest { // In the batch input mode, the most relevant suggested word should act as a "typed word" // (typedWordValid=true), not as an "auto correct word" (willAutoCorrect=false). - return new SuggestedWords(suggestionsContainer, + callback.onGetSuggestedWords(new SuggestedWords(suggestionsContainer, true /* typedWordValid */, false /* willAutoCorrect */, false /* isPunctuationSuggestions */, false /* isObsoleteSuggestions */, - false /* isPrediction */); + false /* isPrediction */)); } private static ArrayList<SuggestedWordInfo> getSuggestionsInfoListWithDebugInfo( @@ -453,7 +473,7 @@ public final class Suggest { sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE); } return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind, - wordInfo.mSourceDict); + wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord); } public void close() { diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index 22beaefee..b27fd81e9 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -113,7 +113,8 @@ public final class SuggestedWords { if (null == text) continue; final SuggestedWordInfo suggestedWordInfo = new SuggestedWordInfo(text.toString(), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_APP_DEFINED, - Dictionary.TYPE_APPLICATION_DEFINED); + Dictionary.DICTIONARY_APPLICATION_DEFINED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */); result.add(suggestedWordInfo); } return result; @@ -126,7 +127,8 @@ public final class SuggestedWords { final ArrayList<SuggestedWordInfo> suggestionsList = CollectionUtils.newArrayList(); final HashSet<String> alreadySeen = CollectionUtils.newHashSet(); suggestionsList.add(new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, - SuggestedWordInfo.KIND_TYPED, Dictionary.TYPE_USER_TYPED)); + SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); alreadySeen.add(typedWord.toString()); final int previousSize = previousSuggestions.size(); for (int index = 1; index < previousSize; index++) { @@ -141,7 +143,14 @@ public final class SuggestedWords { return suggestionsList; } + public SuggestedWordInfo getAutoCommitCandidate() { + if (mSuggestedWordInfoList.size() <= 0) return null; + final SuggestedWordInfo candidate = mSuggestedWordInfoList.get(0); + return candidate.isEligibleForAutoCommit() ? candidate : null; + } + public static final class SuggestedWordInfo { + public static final int NOT_AN_INDEX = -1; public static final int MAX_SCORE = Integer.MAX_VALUE; public static final int KIND_MASK_KIND = 0xFF; // Mask to get only the kind public static final int KIND_TYPED = 0; // What user typed @@ -166,18 +175,26 @@ public final class SuggestedWords { public final int mScore; public final int mKind; // one of the KIND_* constants above public final int mCodePointCount; - public final String mSourceDict; + public final Dictionary mSourceDict; + // For auto-commit. This keeps track of the index inside the touch coordinates array + // passed to native code to get suggestions for a gesture that corresponds to the first + // letter of the second word. + public final int mIndexOfTouchPointOfSecondWord; private String mDebugString = ""; public SuggestedWordInfo(final String word, final int score, final int kind, - final String sourceDict) { + final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord) { mWord = word; mScore = score; mKind = kind; mSourceDict = sourceDict; mCodePointCount = StringUtils.codePointCount(mWord); + mIndexOfTouchPointOfSecondWord = indexOfTouchPointOfSecondWord; } + public boolean isEligibleForAutoCommit() { + return (KIND_CORRECTION == mKind && NOT_AN_INDEX != mIndexOfTouchPointOfSecondWord); + } public void setDebugString(final String str) { if (null == str) throw new NullPointerException("Debug info is null"); diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java index 92f96c027..67ef538ac 100644 --- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java @@ -34,9 +34,10 @@ public final class SynchronouslyLoadedContactsBinaryDictionary extends ContactsB @Override public synchronized ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer codes, final String prevWordForBigrams, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { syncReloadDictionaryIfRequired(); - return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, blockOffensiveWords); + return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions); } @Override diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java index 33fe89611..bea522320 100644 --- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java @@ -37,9 +37,10 @@ public final class SynchronouslyLoadedUserBinaryDictionary extends UserBinaryDic @Override public synchronized ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer codes, final String prevWordForBigrams, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { syncReloadDictionaryIfRequired(); - return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, blockOffensiveWords); + return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions); } @Override diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index ed6fefae4..b2bb61596 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -75,7 +75,8 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { public UserBinaryDictionary(final Context context, final String locale, final boolean alsoUseMoreRestrictiveLocales) { - super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_USER); + super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_USER, + false /* isUpdatable */); if (null == locale) throw new NullPointerException(); // Catch the error earlier if (SubtypeLocaleUtils.NO_LANGUAGE.equals(locale)) { // If we don't have a locale, insert into the "all locales" user dictionary. diff --git a/java/src/com/android/inputmethod/latin/WordComposer.java b/java/src/com/android/inputmethod/latin/WordComposer.java index a09ca605c..039dadc66 100644 --- a/java/src/com/android/inputmethod/latin/WordComposer.java +++ b/java/src/com/android/inputmethod/latin/WordComposer.java @@ -272,8 +272,8 @@ public final class WordComposer { final int x, y; final Key key; if (keyboard != null && (key = keyboard.getKey(codePoint)) != null) { - x = key.mX + key.mWidth / 2; - y = key.mY + key.mHeight / 2; + x = key.getX() + key.getWidth() / 2; + y = key.getY() + key.getHeight() / 2; } else { x = Constants.NOT_A_COORDINATE; y = Constants.NOT_A_COORDINATE; diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java new file mode 100644 index 000000000..ceb8fa81f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -0,0 +1,634 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; + +/** + * Decodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + * + * TODO: Remove calls from classes except Ver3DictDecoder + * TODO: Move this file to makedict/internal. + * TODO: Rename this class to DictDecoderUtils. + */ +public final class BinaryDictDecoderUtils { + + private static final boolean DBG = MakedictLog.DBG; + + private BinaryDictDecoderUtils() { + // This utility class is not publicly instantiable. + } + + private static final int MAX_JUMPS = 12; + + @UsedForTesting + public interface DictBuffer { + public int readUnsignedByte(); + public int readUnsignedShort(); + public int readUnsignedInt24(); + public int readInt(); + public int position(); + public void position(int newPosition); + public void put(final byte b); + public int limit(); + @UsedForTesting + public int capacity(); + } + + public static final class ByteBufferDictBuffer implements DictBuffer { + private ByteBuffer mBuffer; + + public ByteBufferDictBuffer(final ByteBuffer buffer) { + mBuffer = buffer; + } + + @Override + public int readUnsignedByte() { + return mBuffer.get() & 0xFF; + } + + @Override + public int readUnsignedShort() { + return mBuffer.getShort() & 0xFFFF; + } + + @Override + public int readUnsignedInt24() { + final int retval = readUnsignedByte(); + return (retval << 16) + readUnsignedShort(); + } + + @Override + public int readInt() { + return mBuffer.getInt(); + } + + @Override + public int position() { + return mBuffer.position(); + } + + @Override + public void position(int newPos) { + mBuffer.position(newPos); + } + + @Override + public void put(final byte b) { + mBuffer.put(b); + } + + @Override + public int limit() { + return mBuffer.limit(); + } + + @Override + public int capacity() { + return mBuffer.capacity(); + } + } + + /** + * A class grouping utility function for our specific character encoding. + */ + static final class CharEncoding { + private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; + private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; + + /** + * Helper method to find out whether this code fits on one byte + */ + private static boolean fitsOnOneByte(final int character) { + return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE + && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; + } + + /** + * Compute the size of a character given its character code. + * + * Char format is: + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + * + * @param character the character code. + * @return the size in binary encoded-form, either 1 or 3 bytes. + */ + static int getCharSize(final int character) { + // See char encoding in FusionDictionary.java + if (fitsOnOneByte(character)) return 1; + if (FormatSpec.INVALID_CHARACTER == character) return 1; + return 3; + } + + /** + * Compute the byte size of a character array. + */ + static int getCharArraySize(final int[] chars) { + int size = 0; + for (int character : chars) size += getCharSize(character); + return size; + } + + /** + * Writes a char array to a byte buffer. + * + * @param codePoints the code point array to write. + * @param buffer the byte buffer to write to. + * @param index the index in buffer to write the character array to. + * @return the index after the last character. + */ + static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { + for (int codePoint : codePoints) { + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + return index; + } + + /** + * Writes a string with our character format to a byte buffer. + * + * This will also write the terminator byte. + * + * @param buffer the byte buffer to write to. + * @param origin the offset to write from. + * @param word the string to write. + * @return the size written, in bytes. + */ + static int writeString(final byte[] buffer, final int origin, + final String word) { + final int length = word.length(); + int index = origin; + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + return index - origin; + } + + /** + * Writes a string with our character format to a ByteArrayOutputStream. + * + * This will also write the terminator byte. + * + * @param buffer the ByteArrayOutputStream to write to. + * @param word the string to write. + */ + static void writeString(final ByteArrayOutputStream buffer, final String word) { + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer.write((byte) codePoint); + } else { + buffer.write((byte) (0xFF & (codePoint >> 16))); + buffer.write((byte) (0xFF & (codePoint >> 8))); + buffer.write((byte) (0xFF & codePoint)); + } + } + buffer.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + } + + /** + * Reads a string from a DictBuffer. This is the converse of the above method. + */ + static String readString(final DictBuffer dictBuffer) { + final StringBuilder s = new StringBuilder(); + int character = readChar(dictBuffer); + while (character != FormatSpec.INVALID_CHARACTER) { + s.appendCodePoint(character); + character = readChar(dictBuffer); + } + return s.toString(); + } + + /** + * Reads a character from the buffer. + * + * This follows the character format documented earlier in this source file. + * + * @param dictBuffer the buffer, positioned over an encoded character. + * @return the character code. + */ + static int readChar(final DictBuffer dictBuffer) { + int character = dictBuffer.readUnsignedByte(); + if (!fitsOnOneByte(character)) { + if (FormatSpec.PTNODE_CHARACTERS_TERMINATOR == character) { + return FormatSpec.INVALID_CHARACTER; + } + character <<= 16; + character += dictBuffer.readUnsignedShort(); + } + return character; + } + } + + // Input methods: Read a binary dictionary to memory. + // readDictionaryBinary is the public entry point for them. + + static int readSInt24(final DictBuffer dictBuffer) { + final int retval = dictBuffer.readUnsignedInt24(); + final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (retval & FormatSpec.SINT24_MAX); + } + + static int readChildrenAddress(final DictBuffer dictBuffer, + final int optionFlags, final FormatOptions options) { + if (options.mSupportsDynamicUpdate) { + final int address = dictBuffer.readUnsignedInt24(); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + if ((address & FormatSpec.MSB24) != 0) { + return -(address & FormatSpec.SINT24_MAX); + } else { + return address; + } + } + int address; + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + + static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int parentAddress = dictBuffer.readUnsignedInt24(); + final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (parentAddress & FormatSpec.SINT24_MAX); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + /** + * Reads and returns the PtNode count out of a buffer and forwards the pointer. + */ + /* package */ static int readPtNodeCount(final DictBuffer dictBuffer) { + final int msb = dictBuffer.readUnsignedByte(); + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= msb) { + return msb; + } else { + return ((FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT & msb) << 8) + + dictBuffer.readUnsignedByte(); + } + } + + /** + * Finds, as a string, the word at the position passed as an argument. + * + * @param dictDecoder the dict decoder. + * @param headerSize the size of the header. + * @param pos the position to seek. + * @param formatOptions file format options. + * @return the word with its frequency, as a weighted string. + */ + /* package for tests */ static WeightedString getWordAtPosition( + final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, + final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + final WeightedString result; + final int originalPos = dictBuffer.position(); + dictBuffer.position(pos); + + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions); + } else { + result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos, + formatOptions); + } + + dictBuffer.position(originalPos); + return result; + } + + @SuppressWarnings("unused") + private static WeightedString getWordAtPositionWithParentAddress( + final Ver3DictDecoder dictDecoder, final int pos, final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + int currentPos = pos; + int frequency = Integer.MIN_VALUE; + final StringBuilder builder = new StringBuilder(); + // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH + for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { + PtNodeInfo currentInfo; + int loopCounter = 0; + do { + dictBuffer.position(currentPos); + currentInfo = dictDecoder.readPtNode(currentPos, options); + if (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)) { + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + } + if (DBG && loopCounter++ > MAX_JUMPS) { + MakedictLog.d("Too many jumps - probably a bug"); + } + } while (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)); + if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; + builder.insert(0, + new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); + if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + } + return new WeightedString(builder.toString(), frequency); + } + + private static WeightedString getWordAtPositionWithoutParentAddress( + final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, + final FormatOptions options) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + dictBuffer.position(headerSize); + final int count = readPtNodeCount(dictBuffer); + int groupPos = headerSize + BinaryDictIOUtils.getPtNodeCountSize(count); + final StringBuilder builder = new StringBuilder(); + WeightedString result = null; + + PtNodeInfo last = null; + for (int i = count - 1; i >= 0; --i) { + PtNodeInfo info = dictDecoder.readPtNode(groupPos, options); + groupPos = info.mEndAddress; + if (info.mOriginalAddress == pos) { + builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); + result = new WeightedString(builder.toString(), info.mFrequency); + break; // and return + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + if (info.mChildrenAddress > pos) { + if (null == last) continue; + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + dictBuffer.position(last.mChildrenAddress); + i = readPtNodeCount(dictBuffer); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); + last = null; + continue; + } + last = info; + } + if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + dictBuffer.position(last.mChildrenAddress); + i = readPtNodeCount(dictBuffer); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); + last = null; + continue; + } + } + return result; + } + + /** + * Reads a single node array from a buffer. + * + * This methods reads the file at the current position. A node array is fully expected to start + * at the current position. + * This will recursively read other node arrays into the structure, populating the reverse + * maps on the fly and using them to keep track of already read nodes. + * + * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. + * @param headerSize the size, in bytes, of the file header. + * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. + * @param reversePtNodeMap a mapping from addresses to already read PtNodes. + * @param options file format options. + * @return the read node array with all his children already read. + */ + private static PtNodeArray readNodeArray(final Ver3DictDecoder dictDecoder, + final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, + final Map<Integer, PtNode> reversePtNodeMap, final FormatOptions options) + throws IOException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + final ArrayList<PtNode> nodeArrayContents = new ArrayList<PtNode>(); + final int nodeArrayOriginPos = dictBuffer.position(); + + do { // Scan the linked-list node. + final int nodeArrayHeadPos = dictBuffer.position(); + final int count = readPtNodeCount(dictBuffer); + int groupOffsetPos = nodeArrayHeadPos + BinaryDictIOUtils.getPtNodeCountSize(count); + for (int i = count; i > 0; --i) { // Scan the array of PtNode. + PtNodeInfo info = dictDecoder.readPtNode(groupOffsetPos, options); + if (BinaryDictIOUtils.isMovedPtNode(info.mFlags, options)) continue; + ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; + ArrayList<WeightedString> bigrams = null; + if (null != info.mBigrams) { + bigrams = new ArrayList<WeightedString>(); + for (PendingAttribute bigram : info.mBigrams) { + final WeightedString word = getWordAtPosition(dictDecoder, headerSize, + bigram.mAddress, options); + final int reconstructedFrequency = + BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, + bigram.mFrequency); + bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); + } + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); + if (null == children) { + final int currentPosition = dictBuffer.position(); + dictBuffer.position(info.mChildrenAddress); + children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, + reversePtNodeMap, options); + dictBuffer.position(currentPosition); + } + nodeArrayContents.add( + new PtNode(info.mCharacters, shortcutTargets, bigrams, + info.mFrequency, + 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), + 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); + } else { + nodeArrayContents.add( + new PtNode(info.mCharacters, shortcutTargets, bigrams, + info.mFrequency, + 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), + 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); + } + groupOffsetPos = info.mEndAddress; + } + + // reach the end of the array. + if (options.mSupportsDynamicUpdate) { + final int nextAddress = dictBuffer.readUnsignedInt24(); + if (nextAddress >= 0 && nextAddress < dictBuffer.limit()) { + dictBuffer.position(nextAddress); + } else { + break; + } + } + } while (options.mSupportsDynamicUpdate && + dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); + + final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; + nodeArray.mCachedAddressAfterUpdate = nodeArrayOriginPos; + reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); + return nodeArray; + } + + /** + * Helper function to get the binary format version from the header. + * @throws IOException + */ + private static int getFormatVersion(final DictBuffer dictBuffer) + throws IOException { + final int magic = dictBuffer.readInt(); + if (FormatSpec.MAGIC_NUMBER == magic) return dictBuffer.readUnsignedShort(); + return FormatSpec.NOT_A_VERSION_NUMBER; + } + + /** + * Helper function to get and validate the binary format version. + * @throws UnsupportedFormatException + * @throws IOException + */ + static int checkFormatVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + final int version = getFormatVersion(dictBuffer); + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("This file has version " + version + + ", but this implementation does not support versions above " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + return version; + } + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. The optional dict argument is an existing dictionary to + * which words from the buffer should be added. If it is null, a new dictionary is created. + * + * @param dictDecoder the dict decoder. + * @param dict an optional dictionary to add words to, or null. + * @return the created (or merged) dictionary. + */ + @UsedForTesting + /* package */ static FusionDictionary readDictionaryBinary(final Ver3DictDecoder dictDecoder, + final FusionDictionary dict) throws IOException, UnsupportedFormatException { + // Read header + final FileHeader fileHeader = dictDecoder.readHeader(); + + Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); + Map<Integer, PtNode> reversePtNodeMapping = new TreeMap<Integer, PtNode>(); + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, + reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions); + + FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); + if (null != dict) { + for (final Word w : dict) { + if (w.mIsBlacklistEntry) { + newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); + } else { + newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); + } + } + for (final Word w : dict) { + // By construction a binary dictionary may not have bigrams pointing to + // words that are not also registered as unigrams so we don't have to avoid + // them explicitly here. + for (final WeightedString bigram : w.mBigrams) { + newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); + } + } + } + + return newDict; + } + + /** + * Helper method to pass a file name instead of a File object to isBinaryDictionary. + */ + public static boolean isBinaryDictionary(final String filename) { + final File file = new File(filename); + return isBinaryDictionary(file); + } + + /** + * Basic test to find out whether the file is a binary dictionary or not. + * + * Concretely this only tests the magic number. + * + * @param file The file to test. + * @return true if it's a binary dictionary, false otherwise + */ + public static boolean isBinaryDictionary(final File file) { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final ByteBuffer buffer = inStream.getChannel().map( + FileChannel.MapMode.READ_ONLY, 0, file.length()); + final int version = getFormatVersion(new ByteBufferDictBuffer(buffer)); + return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION + && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } catch (FileNotFoundException e) { + return false; + } catch (IOException e) { + return false; + } finally { + if (inStream != null) { + try { + inStream.close(); + } catch (IOException e) { + // do nothing + } + } + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java new file mode 100644 index 000000000..79f5ad8bd --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -0,0 +1,1007 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * Encodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + * + * TODO: Rename this class to DictEncoderUtils. + */ +public class BinaryDictEncoderUtils { + + private static final boolean DBG = MakedictLog.DBG; + + private BinaryDictEncoderUtils() { + // This utility class is not publicly instantiable. + } + + // Arbitrary limit to how much passes we consider address size compression should + // terminate in. At the time of this writing, our largest dictionary completes + // compression in five passes. + // If the number of passes exceeds this number, makedict bails with an exception on + // suspicion that a bug might be causing an infinite loop. + private static final int MAX_PASSES = 24; + + /** + * Compute the binary size of the character array. + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param characters the character array + * @return the size of the char array, including the terminator if any + */ + static int getPtNodeCharactersSize(final int[] characters) { + int size = CharEncoding.getCharArraySize(characters); + if (characters.length > 1) size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the binary size of the character array in a PtNode + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param ptNode the PtNode + * @return the size of the char array, including the terminator if any + */ + private static int getPtNodeCharactersSize(final PtNode ptNode) { + return getPtNodeCharactersSize(ptNode.mChars); + } + + /** + * Compute the binary size of the PtNode count for a node array. + * @param nodeArray the nodeArray + * @return the size of the PtNode count, either 1 or 2 bytes. + */ + private static int getPtNodeCountSize(final PtNodeArray nodeArray) { + return BinaryDictIOUtils.getPtNodeCountSize(nodeArray.mData.size()); + } + + /** + * Compute the size of a shortcut in bytes. + */ + private static int getShortcutSize(final WeightedString shortcut) { + int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + final String word = shortcut.mWord; + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + size += CharEncoding.getCharSize(codePoint); + } + size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the size of a shortcut list in bytes. + * + * This is known in advance and does not change according to position in the file + * like address lists do. + */ + static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { + if (null == shortcutList || shortcutList.isEmpty()) return 0; + int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + for (final WeightedString shortcut : shortcutList) { + size += getShortcutSize(shortcut); + } + return size; + } + + /** + * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. + * + * @param ptNode the PtNode to compute the size of. + * @param options file format options. + * @return the maximum size of the PtNode. + */ + private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { + int size = getNodeHeaderSize(ptNode, options); + // If terminal, one byte for the frequency + if (ptNode.isTerminal()) size += FormatSpec.PTNODE_FREQUENCY_SIZE; + size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address + size += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE + + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE) + * ptNode.mBigrams.size(); + } + return size; + } + + /** + * Compute the maximum size of each PtNode of a PtNode array, assuming 3-byte addresses for + * everything, and caches it in the `mCachedSize' member of the nodes; deduce the size of + * the containing node array, and cache it it its 'mCachedSize' member. + * + * @param ptNodeArray the node array to compute the maximum size of. + * @param options file format options. + */ + private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray, + final FormatOptions options) { + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode node : ptNodeArray.mData) { + final int nodeSize = getPtNodeMaximumSize(node, options); + node.mCachedSize = nodeSize; + size += nodeSize; + } + if (options.mSupportsDynamicUpdate) { + size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + ptNodeArray.mCachedSize = size; + } + + /** + * Compute the size of the header (flag + [parent address] + characters size) of a PtNode. + * + * @param ptNode the PtNode of which to compute the size of the header + * @param options file format options. + */ + private static int getNodeHeaderSize(final PtNode ptNode, final FormatOptions options) { + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + getPtNodeCharactersSize(ptNode); + } else { + return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode); + } + } + + /** + * Compute the size, in bytes, that an address will occupy. + * + * This can be used either for children addresses (which are always positive) or for + * attribute, which may be positive or negative but + * store their sign bit separately. + * + * @param address the address + * @return the byte size. + */ + static int getByteSize(final int address) { + assert(address <= FormatSpec.UINT24_MAX); + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + return 0; + } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) { + return 1; + } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) { + return 2; + } else { + return 3; + } + } + + // End utility methods + + // This method is responsible for finding a nice ordering of the nodes that favors run-time + // cache performance and dictionary size. + /* package for tests */ static ArrayList<PtNodeArray> flattenTree( + final PtNodeArray rootNodeArray) { + final int treeSize = FusionDictionary.countPtNodes(rootNodeArray); + MakedictLog.i("Counted nodes : " + treeSize); + final ArrayList<PtNodeArray> flatTree = new ArrayList<PtNodeArray>(treeSize); + return flattenTreeInner(flatTree, rootNodeArray); + } + + private static ArrayList<PtNodeArray> flattenTreeInner(final ArrayList<PtNodeArray> list, + final PtNodeArray ptNodeArray) { + // Removing the node is necessary if the tails are merged, because we would then + // add the same node several times when we only want it once. A number of places in + // the code also depends on any node being only once in the list. + // Merging tails can only be done if there are no attributes. Searching for attributes + // in LatinIME code depends on a total breadth-first ordering, which merging tails + // breaks. If there are no attributes, it should be fine (and reduce the file size) + // to merge tails, and removing the node from the list would be necessary. However, + // we don't merge tails because breaking the breadth-first ordering would result in + // extreme overhead at bigram lookup time (it would make the search function O(n) instead + // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty + // high). + // If no nodes are ever merged, we can't have the same node twice in the list, hence + // searching for duplicates in unnecessary. It is also very performance consuming, + // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making + // this simple list.remove operation O(n*n) overall. On Android this overhead is very + // high. + // For future reference, the code to remove duplicate is a simple : list.remove(node); + list.add(ptNodeArray); + final ArrayList<PtNode> branches = ptNodeArray.mData; + final int nodeSize = branches.size(); + for (PtNode ptNode : branches) { + if (null != ptNode.mChildren) flattenTreeInner(list, ptNode.mChildren); + } + return list; + } + + /** + * Get the offset from a position inside a current node array to a target node array, during + * update. + * + * If the current node array is before the target node array, the target node array has not + * been updated yet, so we should return the offset from the old position of the current node + * array to the old position of the target node array. If on the other hand the target is + * before the current node array, it already has been updated, so we should return the offset + * from the new position in the current node array to the new position in the target node + * array. + * + * @param currentNodeArray node array containing the PtNode where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetNodeArray the target node array to get the offset to + * @return the offset to the target node array + */ + private static int getOffsetToTargetNodeArrayDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNodeArray targetNodeArray) { + final boolean isTargetBeforeCurrent = (targetNodeArray.mCachedAddressBeforeUpdate + < currentNodeArray.mCachedAddressBeforeUpdate); + if (isTargetBeforeCurrent) { + return targetNodeArray.mCachedAddressAfterUpdate + - (currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray); + } else { + return targetNodeArray.mCachedAddressBeforeUpdate + - (currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray); + } + } + + /** + * Get the offset from a position inside a current node array to a target PtNode, during + * update. + * + * @param currentNodeArray node array containing the PtNode where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetPtNode the target PtNode to get the offset to + * @return the offset to the target PtNode + */ + // TODO: is there any way to factorize this method with the one above? + private static int getOffsetToTargetPtNodeDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNode targetPtNode) { + final int oldOffsetBasePoint = currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray; + final boolean isTargetBeforeCurrent = (targetPtNode.mCachedAddressBeforeUpdate + < oldOffsetBasePoint); + // If the target is before the current node array, then its address has already been + // updated. We can use the AfterUpdate member, and compare it to our own member after + // update. Otherwise, the AfterUpdate member is not updated yet, so we need to use the + // BeforeUpdate member, and of course we have to compare this to our own address before + // update. + if (isTargetBeforeCurrent) { + final int newOffsetBasePoint = currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray; + return targetPtNode.mCachedAddressAfterUpdate - newOffsetBasePoint; + } else { + return targetPtNode.mCachedAddressBeforeUpdate - oldOffsetBasePoint; + } + } + + /** + * Computes the actual node array size, based on the cached addresses of the children nodes. + * + * Each node array stores its tentative address. During dictionary address computing, these + * are not final, but they can be used to compute the node array size (the node array size + * depends on the address of the children because the number of bytes necessary to store an + * address depends on its numeric value. The return value indicates whether the node array + * contents (as in, any of the addresses stored in the cache fields) have changed with + * respect to their previous value. + * + * @param ptNodeArray the node array to compute the size of. + * @param dict the dictionary in which the word/attributes are to be found. + * @param formatOptions file format options. + * @return false if none of the cached addresses inside the node array changed, true otherwise. + */ + private static boolean computeActualPtNodeArraySize(final PtNodeArray ptNodeArray, + final FusionDictionary dict, final FormatOptions formatOptions) { + boolean changed = false; + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode ptNode : ptNodeArray.mData) { + ptNode.mCachedAddressAfterUpdate = ptNodeArray.mCachedAddressAfterUpdate + size; + if (ptNode.mCachedAddressAfterUpdate != ptNode.mCachedAddressBeforeUpdate) { + changed = true; + } + int nodeSize = getNodeHeaderSize(ptNode, formatOptions); + if (ptNode.isTerminal()) nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (formatOptions.mSupportsDynamicUpdate) { + nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + } else if (null != ptNode.mChildren) { + nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, + nodeSize + size, ptNode.mChildren)); + } + nodeSize += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + for (WeightedString bigram : ptNode.mBigrams) { + final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, + nodeSize + size + FormatSpec.PTNODE_FLAGS_SIZE, + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); + nodeSize += getByteSize(offset) + FormatSpec.PTNODE_FLAGS_SIZE; + } + } + ptNode.mCachedSize = nodeSize; + size += nodeSize; + } + if (formatOptions.mSupportsDynamicUpdate) { + size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + if (ptNodeArray.mCachedSize != size) { + ptNodeArray.mCachedSize = size; + changed = true; + } + return changed; + } + + /** + * Initializes the cached addresses of node arrays and their containing nodes from their size. + * + * @param flatNodes the list of node arrays. + * @param formatOptions file format options. + * @return the byte size of the entire stack. + */ + private static int initializePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes, + final FormatOptions formatOptions) { + int nodeArrayOffset = 0; + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset; + int nodeCountSize = getPtNodeCountSize(nodeArray); + int nodeffset = 0; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate = + nodeCountSize + nodeArrayOffset + nodeffset; + nodeffset += ptNode.mCachedSize; + } + final int nodeSize = nodeCountSize + nodeffset + + (formatOptions.mSupportsDynamicUpdate + ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); + nodeArrayOffset += nodeArray.mCachedSize; + } + return nodeArrayOffset; + } + + /** + * Updates the cached addresses of node arrays after recomputing their new positions. + * + * @param flatNodes the list of node arrays. + */ + private static void updatePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArray.mCachedAddressAfterUpdate; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate; + } + } + } + + /** + * Compute the cached parent addresses after all has been updated. + * + * The parent addresses are used by some binary formats at write-to-disk time. Not all formats + * need them. In particular, version 2 does not need them, and version 3 does. + * + * @param flatNodes the flat array of node arrays to fill in + */ + private static void computeParentAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + for (final PtNode ptNode : nodeArray.mData) { + if (null != ptNode.mChildren) { + // Assign my address to children's parent address + // Here BeforeUpdate and AfterUpdate addresses have the same value, so it + // does not matter which we use. + ptNode.mChildren.mCachedParentAddress = ptNode.mCachedAddressAfterUpdate + - ptNode.mChildren.mCachedAddressAfterUpdate; + } + } + } + } + + /** + * Compute the addresses and sizes of an ordered list of PtNode arrays. + * + * This method takes a list of PtNode arrays and will update their cached address and size + * values so that they can be written into a file. It determines the smallest size each of the + * PtNode arrays can be given the addresses of its children and attributes, and store that into + * each PtNode. + * The order of the PtNode is given by the order of the array. This method makes no effort + * to find a good order; it only mechanically computes the size this order results in. + * + * @param dict the dictionary + * @param flatNodes the ordered list of PtNode arrays + * @param formatOptions file format options. + * @return the same array it was passed. The nodes have been updated for address and size. + */ + private static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, + final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) { + // First get the worst possible sizes and offsets + for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions); + final int offset = initializePtNodeArraysCachedAddresses(flatNodes, formatOptions); + + MakedictLog.i("Compressing the array addresses. Original size : " + offset); + MakedictLog.i("(Recursively seen size : " + offset + ")"); + + int passes = 0; + boolean changesDone = false; + do { + changesDone = false; + int ptNodeArrayStartOffset = 0; + for (final PtNodeArray ptNodeArray : flatNodes) { + ptNodeArray.mCachedAddressAfterUpdate = ptNodeArrayStartOffset; + final int oldNodeArraySize = ptNodeArray.mCachedSize; + final boolean changed = + computeActualPtNodeArraySize(ptNodeArray, dict, formatOptions); + final int newNodeArraySize = ptNodeArray.mCachedSize; + if (oldNodeArraySize < newNodeArraySize) { + throw new RuntimeException("Increased size ?!"); + } + ptNodeArrayStartOffset += newNodeArraySize; + changesDone |= changed; + } + updatePtNodeArraysCachedAddresses(flatNodes); + ++passes; + if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); + } while (changesDone); + + if (formatOptions.mSupportsDynamicUpdate) { + computeParentAddresses(flatNodes); + } + final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); + MakedictLog.i("Compression complete in " + passes + " passes."); + MakedictLog.i("After address compression : " + + (lastPtNodeArray.mCachedAddressAfterUpdate + lastPtNodeArray.mCachedSize)); + + return flatNodes; + } + + /** + * Sanity-checking method. + * + * This method checks a list of PtNode arrays for juxtaposition, that is, it will do + * nothing if each node array's cached address is actually the previous node array's address + * plus the previous node's size. + * If this is not the case, it will throw an exception. + * + * @param arrays the list of node arrays to check + */ + private static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) { + int offset = 0; + int index = 0; + for (final PtNodeArray ptNodeArray : arrays) { + // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter + // which we use. + if (ptNodeArray.mCachedAddressAfterUpdate != offset) { + throw new RuntimeException("Wrong address for node " + index + + " : expected " + offset + ", got " + + ptNodeArray.mCachedAddressAfterUpdate); + } + ++index; + offset += ptNodeArray.mCachedSize; + } + } + + /** + * Helper method to write a variable-size address to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param address the address to write. + * @return the size in bytes the address actually took. + */ + private static int writeVariableAddress(final byte[] buffer, int index, final int address) { + switch (getByteSize(address)) { + case 1: + buffer[index++] = (byte)address; + return 1; + case 2: + buffer[index++] = (byte)(0xFF & (address >> 8)); + buffer[index++] = (byte)(0xFF & address); + return 2; + case 3: + buffer[index++] = (byte)(0xFF & (address >> 16)); + buffer[index++] = (byte)(0xFF & (address >> 8)); + buffer[index++] = (byte)(0xFF & address); + return 3; + case 0: + return 0; + default: + throw new RuntimeException("Address " + address + " has a strange size"); + } + } + + /** + * Helper method to write a variable-size signed address to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param address the address to write. + * @return the size in bytes the address actually took. + */ + private static int writeVariableSignedAddress(final byte[] buffer, int index, + final int address) { + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absAddress = Math.abs(address); + buffer[index++] = + (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16))); + buffer[index++] = (byte)(0xFF & (absAddress >> 8)); + buffer[index++] = (byte)(0xFF & absAddress); + } + return 3; + } + + /** + * Makes the flag value for a PtNode. + * + * @param hasMultipleChars whether the PtNode has multiple chars. + * @param isTerminal whether the PtNode is terminal. + * @param childrenAddressSize the size of a children address. + * @param hasShortcuts whether the PtNode has shortcuts. + * @param hasBigrams whether the PtNode has bigrams. + * @param isNotAWord whether the PtNode is not a word. + * @param isBlackListEntry whether the PtNode is a blacklist entry. + * @param formatOptions file format options. + * @return the flags + */ + static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, + final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, + final boolean isNotAWord, final boolean isBlackListEntry, + final FormatOptions formatOptions) { + byte flags = 0; + if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; + if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; + if (formatOptions.mSupportsDynamicUpdate) { + flags |= FormatSpec.FLAG_IS_NOT_MOVED; + } else if (true) { + switch (childrenAddressSize) { + case 1: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES; + break; + case 0: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS; + break; + default: + throw new RuntimeException("Node with a strange address"); + } + } + if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; + if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; + if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; + if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; + return flags; + } + + private static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress, + final int childrenOffset, final FormatOptions formatOptions) { + return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0, + getByteSize(childrenOffset), + node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), + node.mBigrams != null, node.mIsNotAWord, node.mIsBlacklistEntry, formatOptions); + } + + /** + * Makes the flag value for a bigram. + * + * @param more whether there are more bigrams after this one. + * @param offset the offset of the bigram. + * @param bigramFrequency the frequency of the bigram, 0..255. + * @param unigramFrequency the unigram frequency of the same word, 0..255. + * @param word the second bigram, for debugging purposes + * @return the flags + */ + private static final int makeBigramFlags(final boolean more, final int offset, + int bigramFrequency, final int unigramFrequency, final String word) { + int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); + switch (getByteSize(offset)) { + case 1: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; + break; + default: + throw new RuntimeException("Strange offset size"); + } + if (unigramFrequency > bigramFrequency) { + MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word + + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " + + word + " is " + unigramFrequency); + bigramFrequency = unigramFrequency; + } + // We compute the difference between 255 (which means probability = 1) and the + // unigram score. We split this into a number of discrete steps. + // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 + // represents an increase of 16 steps: a value of 15 will be interpreted as the median + // value of the 16th step. In all justice, if the bigram frequency is low enough to be + // rounded below the first step (which means it is less than half a step higher than the + // unigram frequency) then the unigram frequency itself is the best approximation of the + // bigram freq that we could possibly supply, hence we should *not* include this bigram + // in the file at all. + // until this is done, we'll write 0 and slightly overestimate this case. + // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step + // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to + // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the + // step size. Then we compute the start of the first step (the one where value 0 starts) + // by adding half-a-step to the unigramFrequency. From there, we compute the integer + // number of steps to the bigramFrequency. One last thing: we want our steps to include + // their lower bound and exclude their higher bound so we need to have the first step + // start at exactly 1 unit higher than floor(unigramFreq + half a step). + // Note : to reconstruct the score, the dictionary reader will need to divide + // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, + // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best + // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the + // step pointed by the discretized frequency. + final float stepSize = + (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); + final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); + // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 + // here. The best approximation would be the unigram freq itself, so we should not + // include this bigram in the dictionary. For now, register as 0, and live with the + // small over-estimation that we get in this case. TODO: actually remove this bigram + // if discretizedFrequency < 0. + final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; + bigramFlags += finalBigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; + return bigramFlags; + } + + /** + * Makes the 2-byte value for options flags. + */ + private static final int makeOptionsValue(final FusionDictionary dictionary, + final FormatOptions formatOptions) { + final DictionaryOptions options = dictionary.mOptions; + final boolean hasBigrams = dictionary.hasBigrams(); + return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) + + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) + + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) + + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); + } + + /** + * Makes the flag value for a shortcut. + * + * @param more whether there are more attributes after this one. + * @param frequency the frequency of the attribute, 0..15 + * @return the flags + */ + static final int makeShortcutFlags(final boolean more, final int frequency) { + return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); + } + + private static final int writeParentAddress(final byte[] buffer, final int index, + final int address, final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + if (address == FormatSpec.NO_PARENT_ADDRESS) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absAddress = Math.abs(address); + assert(absAddress <= FormatSpec.SINT24_MAX); + buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0) + | ((absAddress >> 16) & 0xFF)); + buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); + buffer[index + 2] = (byte)(absAddress & 0xFF); + } + return index + 3; + } else { + return index; + } + } + + /** + * Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached. + * + * @param dict the dictionary the node array is a part of (for relative offsets). + * @param buffer the memory buffer to write to. + * @param ptNodeArray the node array to write. + * @param formatOptions file format options. + * @return the address of the END of the node. + */ + @SuppressWarnings("unused") + private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, + final PtNodeArray ptNodeArray, final FormatOptions formatOptions) { + // TODO: Make the code in common with BinaryDictIOUtils#writePtNode + int index = ptNodeArray.mCachedAddressAfterUpdate; + + final int ptNodeCount = ptNodeArray.mData.size(); + final int countSize = getPtNodeCountSize(ptNodeArray); + final int parentAddress = ptNodeArray.mCachedParentAddress; + if (1 == countSize) { + buffer[index++] = (byte)ptNodeCount; + } else if (2 == countSize) { + // We need to signal 2-byte size by setting the top bit of the MSB to 1, so + // we | 0x80 to do this. + buffer[index++] = (byte)((ptNodeCount >> 8) | 0x80); + buffer[index++] = (byte)(ptNodeCount & 0xFF); + } else { + throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); + } + int ptNodeAddress = index; + for (int i = 0; i < ptNodeCount; ++i) { + final PtNode ptNode = ptNodeArray.mData.get(i); + if (index != ptNode.mCachedAddressAfterUpdate) { + throw new RuntimeException("Bug: write index is not the same as the cached address " + + "of the node : " + index + " <> " + ptNode.mCachedAddressAfterUpdate); + } + ptNodeAddress += getNodeHeaderSize(ptNode, formatOptions); + // Sanity checks. + if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { + throw new RuntimeException("A node has a frequency > " + + FormatSpec.MAX_TERMINAL_FREQUENCY + + " : " + ptNode.mFrequency); + } + if (ptNode.mFrequency >= 0) ptNodeAddress += FormatSpec.PTNODE_FREQUENCY_SIZE; + final int childrenOffset = null == ptNode.mChildren + ? FormatSpec.NO_CHILDREN_ADDRESS + : ptNode.mChildren.mCachedAddressAfterUpdate - ptNodeAddress; + buffer[index++] = + makePtNodeFlags(ptNode, ptNodeAddress, childrenOffset, formatOptions); + + if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { + index = writeParentAddress(buffer, index, parentAddress, formatOptions); + } else { + index = writeParentAddress(buffer, index, parentAddress + + (ptNodeArray.mCachedAddressAfterUpdate + - ptNode.mCachedAddressAfterUpdate), + formatOptions); + } + + index = CharEncoding.writeCharArray(ptNode.mChars, buffer, index); + if (ptNode.hasSeveralChars()) { + buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + if (ptNode.mFrequency >= 0) { + buffer[index++] = (byte) ptNode.mFrequency; + } + + final int shift; + if (formatOptions.mSupportsDynamicUpdate) { + shift = writeVariableSignedAddress(buffer, index, childrenOffset); + } else { + shift = writeVariableAddress(buffer, index, childrenOffset); + } + index += shift; + ptNodeAddress += shift; + + // Write shortcuts + if (null != ptNode.mShortcutTargets && !ptNode.mShortcutTargets.isEmpty()) { + final int indexOfShortcutByteSize = index; + index += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + ptNodeAddress += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = + ptNode.mShortcutTargets.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + ++ptNodeAddress; + int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), + target.mFrequency); + buffer[index++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); + index += shortcutShift; + ptNodeAddress += shortcutShift; + } + final int shortcutByteSize = index - indexOfShortcutByteSize; + if (shortcutByteSize > 0xFFFF) { + throw new RuntimeException("Shortcut list too large"); + } + buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); + buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); + } + // Write bigrams + if (null != ptNode.mBigrams) { + final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + ++ptNodeAddress; + final int offset = addressOfBigram - ptNodeAddress; + int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, + bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + buffer[index++] = (byte)bigramFlags; + final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); + index += bigramShift; + ptNodeAddress += bigramShift; + } + } + + } + if (formatOptions.mSupportsDynamicUpdate) { + buffer[index] = buffer[index + 1] = buffer[index + 2] + = FormatSpec.NO_FORWARD_LINK_ADDRESS; + index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + if (index != ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize) { + throw new RuntimeException( + "Not the same size : written " + (index - ptNodeArray.mCachedAddressAfterUpdate) + + " bytes from a node that should have " + ptNodeArray.mCachedSize + " bytes"); + } + return index; + } + + /** + * Dumps a collection of useful statistics about a list of PtNode arrays. + * + * This prints purely informative stuff, like the total estimated file size, the + * number of PtNode arrays, of PtNodes, the repartition of each address size, etc + * + * @param ptNodeArrays the list of PtNode arrays. + */ + private static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) { + int firstTerminalAddress = Integer.MAX_VALUE; + int lastTerminalAddress = Integer.MIN_VALUE; + int size = 0; + int ptNodes = 0; + int maxNodes = 0; + int maxRuns = 0; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + if (maxNodes < ptNodeArray.mData.size()) maxNodes = ptNodeArray.mData.size(); + for (final PtNode ptNode : ptNodeArray.mData) { + ++ptNodes; + if (ptNode.mChars.length > maxRuns) maxRuns = ptNode.mChars.length; + if (ptNode.mFrequency >= 0) { + if (ptNodeArray.mCachedAddressAfterUpdate < firstTerminalAddress) + firstTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + if (ptNodeArray.mCachedAddressAfterUpdate > lastTerminalAddress) + lastTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + } + } + if (ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize > size) { + size = ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize; + } + } + final int[] ptNodeCounts = new int[maxNodes + 1]; + final int[] runCounts = new int[maxRuns + 1]; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + ++ptNodeCounts[ptNodeArray.mData.size()]; + for (final PtNode ptNode : ptNodeArray.mData) { + ++runCounts[ptNode.mChars.length]; + } + } + + MakedictLog.i("Statistics:\n" + + " total file size " + size + "\n" + + " " + ptNodeArrays.size() + " node arrays\n" + + " " + ptNodes + " PtNodes (" + ((float)ptNodes / ptNodeArrays.size()) + + " PtNodes per node)\n" + + " first terminal at " + firstTerminalAddress + "\n" + + " last terminal at " + lastTerminalAddress + "\n" + + " PtNode stats : max = " + maxNodes); + for (int i = 0; i < ptNodeCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + ptNodeCounts[i]); + } + MakedictLog.i(" Character run stats : max = " + maxRuns); + for (int i = 0; i < runCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + runCounts[i]); + } + } + + /** + * Dumps a FusionDictionary to a file. + * + * @param destination the stream to write the binary data to. + * @param dict the dictionary to write. + * @param formatOptions file format options. + */ + /* package */ static void writeDictionaryBinary(final OutputStream destination, + final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + + // Addresses are limited to 3 bytes, but since addresses can be relative to each node + // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding + // the order of the PtNode arrays becomes a quite complicated problem, because though the + // dictionary itself does not have a size limit, each node array must still be within 16MB + // of all its children and parents. As long as this is ensured, the dictionary file may + // grow to any size. + + final int version = formatOptions.mVersion; + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("Requested file format version " + version + + ", but this implementation only supports versions " + + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + + ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); + + // The magic number in big-endian order. + // Magic number for all versions. + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8))); + headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER)); + // Dictionary version. + headerBuffer.write((byte) (0xFF & (version >> 8))); + headerBuffer.write((byte) (0xFF & version)); + + // Options flags + final int options = makeOptionsValue(dict, formatOptions); + headerBuffer.write((byte) (0xFF & (options >> 8))); + headerBuffer.write((byte) (0xFF & options)); + final int headerSizeOffset = headerBuffer.size(); + // Placeholder to be written later with header size. + for (int i = 0; i < 4; ++i) { + headerBuffer.write(0); + } + // Write out the options. + for (final String key : dict.mOptions.mAttributes.keySet()) { + final String value = dict.mOptions.mAttributes.get(key); + CharEncoding.writeString(headerBuffer, key); + CharEncoding.writeString(headerBuffer, value); + } + final int size = headerBuffer.size(); + final byte[] bytes = headerBuffer.toByteArray(); + // Write out the header size. + bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); + bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); + bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); + bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); + destination.write(bytes); + + headerBuffer.close(); + + // Leave the choice of the optimal node order to the flattenTree function. + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = flattenTree(dict.mRootNodeArray); + + MakedictLog.i("Computing addresses..."); + computeAddresses(dict, flatNodes, formatOptions); + MakedictLog.i("Checking PtNode array..."); + if (DBG) checkFlatPtNodeArrayList(flatNodes); + + // Create a buffer that matches the final dictionary size. + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + final byte[] buffer = new byte[bufferSize]; + int index = 0; + + MakedictLog.i("Writing file..."); + int dataEndOffset = 0; + for (PtNodeArray nodeArray : flatNodes) { + dataEndOffset = writePlacedNode(dict, buffer, nodeArray, formatOptions); + } + + if (DBG) showStatistics(flatNodes); + + destination.write(buffer, 0, dataEndOffset); + + destination.close(); + MakedictLog.i("Done"); + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 167c6915c..106f02519 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -18,58 +18,55 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; -import java.nio.channels.FileChannel; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.Map; import java.util.Stack; public final class BinaryDictIOUtils { private static final boolean DBG = false; - private static final int MSB24 = 0x800000; - private static final int SINT24_MAX = 0x7FFFFF; - private static final int MAX_JUMPS = 10000; private BinaryDictIOUtils() { // This utility class is not publicly instantiable. } private static final class Position { - public static final int NOT_READ_GROUPCOUNT = -1; + public static final int NOT_READ_PTNODE_COUNT = -1; public int mAddress; - public int mNumOfCharGroup; + public int mNumOfPtNode; public int mPosition; public int mLength; public Position(int address, int length) { mAddress = address; mLength = length; - mNumOfCharGroup = NOT_READ_GROUPCOUNT; + mNumOfPtNode = NOT_READ_PTNODE_COUNT; } } /** - * Tours all node without recursive call. + * Retrieves all node arrays without recursive call. */ private static void readUnigramsAndBigramsBinaryInner( - final FusionDictionaryBufferInterface buffer, final int headerSize, + final Ver3DictDecoder dictDecoder, final int headerSize, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack<Position> stack = new Stack<Position>(); @@ -82,46 +79,45 @@ public final class BinaryDictIOUtils { Position p = stack.peek(); if (DBG) { - MakedictLog.d("read: address=" + p.mAddress + ", numOfCharGroup=" + - p.mNumOfCharGroup + ", position=" + p.mPosition + ", length=" + p.mLength); + MakedictLog.d("read: address=" + p.mAddress + ", numOfPtNode=" + + p.mNumOfPtNode + ", position=" + p.mPosition + ", length=" + p.mLength); } - if (buffer.position() != p.mAddress) buffer.position(p.mAddress); + if (dictBuffer.position() != p.mAddress) dictBuffer.position(p.mAddress); if (index != p.mLength) index = p.mLength; - if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) { - p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer); - p.mAddress += BinaryDictInputOutput.getGroupCountSize(p.mNumOfCharGroup); + if (p.mNumOfPtNode == Position.NOT_READ_PTNODE_COUNT) { + p.mNumOfPtNode = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + p.mAddress += getPtNodeCountSize(p.mNumOfPtNode); p.mPosition = 0; } - if (p.mNumOfCharGroup == 0) { + if (p.mNumOfPtNode == 0) { stack.pop(); continue; } - CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer, - p.mAddress - headerSize, formatOptions); + PtNodeInfo info = dictDecoder.readPtNode(p.mAddress, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; } p.mPosition++; - final boolean isMovedGroup = BinaryDictInputOutput.isMovedGroup(info.mFlags, + final boolean isMovedPtNode = isMovedPtNode(info.mFlags, formatOptions); - final boolean isDeletedGroup = BinaryDictInputOutput.isDeletedGroup(info.mFlags, + final boolean isDeletedPtNode = isDeletedPtNode(info.mFlags, formatOptions); - if (!isMovedGroup && !isDeletedGroup - && info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) {// found word + if (!isMovedPtNode && !isDeletedPtNode + && info.mFrequency != FusionDictionary.PtNode.NOT_A_TERMINAL) {// found word words.put(info.mOriginalAddress, new String(pushedChars, 0, index)); frequencies.put(info.mOriginalAddress, info.mFrequency); if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams); } - if (p.mPosition == p.mNumOfCharGroup) { + if (p.mPosition == p.mNumOfPtNode) { if (formatOptions.mSupportsDynamicUpdate) { - final int forwardLinkAddress = buffer.readUnsignedInt24(); + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); if (forwardLinkAddress != FormatSpec.NO_FORWARD_LINK_ADDRESS) { - // the node has a forward link. - p.mNumOfCharGroup = Position.NOT_READ_GROUPCOUNT; + // The node array has a forward link. + p.mNumOfPtNode = Position.NOT_READ_PTNODE_COUNT; p.mAddress = forwardLinkAddress; } else { stack.pop(); @@ -130,12 +126,12 @@ public final class BinaryDictIOUtils { stack.pop(); } } else { - // the node has more groups. - p.mAddress = buffer.position(); + // The Ptnode array has more PtNodes. + p.mAddress = dictBuffer.position(); } - if (!isMovedGroup && BinaryDictInputOutput.hasChildrenAddress(info.mChildrenAddress)) { - Position childrenPos = new Position(info.mChildrenAddress + headerSize, index); + if (!isMovedPtNode && hasChildrenAddress(info.mChildrenAddress)) { + final Position childrenPos = new Position(info.mChildrenAddress, index); stack.push(childrenPos); } } @@ -143,61 +139,60 @@ public final class BinaryDictIOUtils { /** * Reads unigrams and bigrams from the binary file. - * Doesn't make the memory representation of the dictionary. + * Doesn't store a full memory representation of the dictionary. * - * @param buffer the buffer to read. + * @param dictDecoder the dict decoder. * @param words the map to store the address as a key and the word as a value. * @param frequencies the map to store the address as a key and the frequency as a value. * @param bigrams the map to store the address as a key and the list of address as a value. - * @throws IOException - * @throws UnsupportedFormatException + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer, + /* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams, - header.mFormatOptions); + final FileHeader header = dictDecoder.readHeader(); + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, + frequencies, bigrams, header.mFormatOptions); } /** - * Gets the address of the last CharGroup of the exact matching word in the dictionary. + * Gets the address of the last PtNode of the exact matching word in the dictionary. * If no match is found, returns NOT_VALID_WORD. * - * @param buffer the buffer to read. + * @param dictDecoder the dict decoder. * @param word the word we search for. * @return the address of the terminal node. - * @throws IOException - * @throws UnsupportedFormatException + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final FusionDictionaryBufferInterface buffer, + /* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; - if (buffer.position() != 0) buffer.position(0); + if (dictBuffer.position() != 0) dictBuffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); + final FileHeader header = dictDecoder.readHeader(); int wordPos = 0; final int wordLen = word.codePointCount(0, word.length()); for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; do { - final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); - boolean foundNextCharGroup = false; - for (int i = 0; i < charGroupCount; ++i) { - final int charGroupPos = buffer.position(); - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, - buffer.position(), header.mFormatOptions); - final boolean isMovedGroup = - BinaryDictInputOutput.isMovedGroup(currentInfo.mFlags, - header.mFormatOptions); - final boolean isDeletedGroup = - BinaryDictInputOutput.isDeletedGroup(currentInfo.mFlags, - header.mFormatOptions); - if (isMovedGroup) continue; + final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + boolean foundNextPtNode = false; + for (int i = 0; i < ptNodeCount; ++i) { + final int ptNodePos = dictBuffer.position(); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos, + header.mFormatOptions); + final boolean isMovedNode = isMovedPtNode(currentInfo.mFlags, + header.mFormatOptions); + final boolean isDeletedNode = isDeletedPtNode(currentInfo.mFlags, + header.mFormatOptions); + if (isMovedNode) continue; boolean same = true; for (int p = 0, j = word.offsetByCodePoints(0, wordPos); p < currentInfo.mCharacters.length; @@ -210,86 +205,60 @@ public final class BinaryDictIOUtils { } if (same) { - // found the group matches the word. + // found the PtNode matches the word. if (wordPos + currentInfo.mCharacters.length == wordLen) { - if (currentInfo.mFrequency == CharGroup.NOT_A_TERMINAL - || isDeletedGroup) { + if (currentInfo.mFrequency == PtNode.NOT_A_TERMINAL + || isDeletedNode) { return FormatSpec.NOT_VALID_WORD; } else { - return charGroupPos; + return ptNodePos; } } wordPos += currentInfo.mCharacters.length; if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { return FormatSpec.NOT_VALID_WORD; } - foundNextCharGroup = true; - buffer.position(currentInfo.mChildrenAddress); + foundNextPtNode = true; + dictBuffer.position(currentInfo.mChildrenAddress); break; } } - // If we found the next char group, it is under the file pointer. - // But if not, we are at the end of this node so we expect to have + // If we found the next PtNode, it is under the file pointer. + // But if not, we are at the end of this node array so we expect to have // a forward link address that we need to consult and possibly resume - // search on the next node in the linked list. - if (foundNextCharGroup) break; + // search on the next node array in the linked list. + if (foundNextPtNode) break; if (!header.mFormatOptions.mSupportsDynamicUpdate) { return FormatSpec.NOT_VALID_WORD; } - final int forwardLinkAddress = buffer.readUnsignedInt24(); + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { return FormatSpec.NOT_VALID_WORD; } - buffer.position(forwardLinkAddress); + dictBuffer.position(forwardLinkAddress); } while(true); } return FormatSpec.NOT_VALID_WORD; } - private static int markAsDeleted(final int flags) { - return (flags & (~FormatSpec.MASK_GROUP_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; - } - - /** - * Delete the word from the binary file. - * - * @param buffer the buffer to write. - * @param word the word we delete - * @throws IOException - * @throws UnsupportedFormatException - */ - @UsedForTesting - public static void deleteWord(final FusionDictionaryBufferInterface buffer, - final String word) throws IOException, UnsupportedFormatException { - buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - final int wordPosition = getTerminalPosition(buffer, word); - if (wordPosition == FormatSpec.NOT_VALID_WORD) return; - - buffer.position(wordPosition); - final int flags = buffer.readUnsignedByte(); - buffer.position(wordPosition); - buffer.put((byte)markAsDeleted(flags)); - } - /** * @return the size written, in bytes. Always 3 bytes. */ - private static int writeSInt24ToBuffer(final FusionDictionaryBufferInterface buffer, + static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) { final int absValue = Math.abs(value); - buffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); - buffer.put((byte)((absValue >> 8) & 0xFF)); - buffer.put((byte)(absValue & 0xFF)); + dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); + dictBuffer.put((byte)((absValue >> 8) & 0xFF)); + dictBuffer.put((byte)(absValue & 0xFF)); return 3; } /** * @return the size written, in bytes. Always 3 bytes. */ - private static int writeSInt24ToStream(final OutputStream destination, final int value) + static int writeSInt24ToStream(final OutputStream destination, final int value) throws IOException { final int absValue = Math.abs(value); destination.write((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); @@ -303,7 +272,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictInputOutput.getByteSize(value)) { + switch (BinaryDictEncoderUtils.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -317,111 +286,52 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictInputOutput.getByteSize(value); - } - - /** - * Update a parent address in a CharGroup that is referred to by groupOriginAddress. - * - * @param buffer the buffer to write. - * @param groupOriginAddress the address of the group. - * @param newParentAddress the absolute address of the parent. - * @param formatOptions file format options. - */ - public static void updateParentAddress(final FusionDictionaryBufferInterface buffer, - final int groupOriginAddress, final int newParentAddress, - final FormatOptions formatOptions) { - final int originalPosition = buffer.position(); - buffer.position(groupOriginAddress); - if (!formatOptions.mSupportsDynamicUpdate) { - throw new RuntimeException("this file format does not support parent addresses"); - } - final int flags = buffer.readUnsignedByte(); - if (BinaryDictInputOutput.isMovedGroup(flags, formatOptions)) { - // if the group is moved, the parent address is stored in the destination group. - // We are guaranteed to process the destination group later, so there is no need to - // update anything here. - buffer.position(originalPosition); - return; - } - if (DBG) { - MakedictLog.d("update parent address flags=" + flags + ", " + groupOriginAddress); - } - final int parentOffset = newParentAddress - groupOriginAddress; - writeSInt24ToBuffer(buffer, parentOffset); - buffer.position(originalPosition); + return BinaryDictEncoderUtils.getByteSize(value); } - private static void skipCharGroup(final FusionDictionaryBufferInterface buffer, - final FormatOptions formatOptions) { - final int flags = buffer.readUnsignedByte(); - BinaryDictInputOutput.readParentAddress(buffer, formatOptions); - skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); + static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) { + final int flags = dictBuffer.readUnsignedByte(); + BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); + skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + BinaryDictDecoderUtils.readChildrenAddress(dictBuffer, flags, formatOptions); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { - final int shortcutsSize = buffer.readUnsignedShort(); - buffer.position(buffer.position() + shortcutsSize - - FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE); + final int shortcutsSize = dictBuffer.readUnsignedShort(); + dictBuffer.position(dictBuffer.position() + shortcutsSize + - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); } if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = buffer.readUnsignedByte(); - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - buffer.readUnsignedByte(); + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + dictBuffer.readUnsignedByte(); break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - buffer.readUnsignedShort(); + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + dictBuffer.readUnsignedShort(); break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - buffer.readUnsignedInt24(); + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + dictBuffer.readUnsignedInt24(); break; } - if ((bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT) == 0) break; + if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - throw new RuntimeException("Too many bigrams in a group."); + if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode."); } } } - /** - * Update parent addresses in a Node that is referred to by nodeOriginAddress. - * - * @param buffer the buffer to be modified. - * @param nodeOriginAddress the address of a modified Node. - * @param newParentAddress the address to be written. - * @param formatOptions file format options. - */ - public static void updateParentAddresses(final FusionDictionaryBufferInterface buffer, - final int nodeOriginAddress, final int newParentAddress, - final FormatOptions formatOptions) { - final int originalPosition = buffer.position(); - buffer.position(nodeOriginAddress); - do { - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); - for (int i = 0; i < count; ++i) { - updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions); - skipCharGroup(buffer, formatOptions); - } - final int forwardLinkAddress = buffer.readUnsignedInt24(); - buffer.position(forwardLinkAddress); - } while (formatOptions.mSupportsDynamicUpdate - && buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); - buffer.position(originalPosition); - } - - private static void skipString(final FusionDictionaryBufferInterface buffer, + static void skipString(final DictBuffer dictBuffer, final boolean hasMultipleChars) { if (hasMultipleChars) { - int character = CharEncoding.readChar(buffer); + int character = CharEncoding.readChar(dictBuffer); while (character != FormatSpec.INVALID_CHARACTER) { - character = CharEncoding.readChar(buffer); + character = CharEncoding.readChar(dictBuffer); } } else { - CharEncoding.readChar(buffer); + CharEncoding.readChar(dictBuffer); } } @@ -449,46 +359,24 @@ public final class BinaryDictIOUtils { size += 3; } } - destination.write((byte)FormatSpec.GROUP_CHARACTERS_TERMINATOR); - size += FormatSpec.GROUP_TERMINATOR_SIZE; + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + size += FormatSpec.PTNODE_TERMINATOR_SIZE; return size; } /** - * Update a children address in a CharGroup that is addressed by groupOriginAddress. - * - * @param buffer the buffer to write. - * @param groupOriginAddress the address of the group. - * @param newChildrenAddress the absolute address of the child. - * @param formatOptions file format options. - */ - public static void updateChildrenAddress(final FusionDictionaryBufferInterface buffer, - final int groupOriginAddress, final int newChildrenAddress, - final FormatOptions formatOptions) { - final int originalPosition = buffer.position(); - buffer.position(groupOriginAddress); - final int flags = buffer.readUnsignedByte(); - final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions); - skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); - final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS - ? FormatSpec.NO_CHILDREN_ADDRESS : newChildrenAddress - buffer.position(); - writeSInt24ToBuffer(buffer, childrenOffset); - buffer.position(originalPosition); - } - - /** - * Write a char group to an output stream. - * A char group is an in-memory representation of a node in trie. - * A char group info is an on-disk representation of a node. + * Write a PtNode to an output stream from a PtNodeInfo. + * A PtNode is an in-memory representation of a node in the patricia trie. + * A PtNode info is a container for low-level information about how the + * PtNode is stored in the binary format. * * @param destination the stream to write. - * @param info the char group info to be written. + * @param info the PtNode info to be written. * @return the size written, in bytes. */ - public static int writeCharGroup(final OutputStream destination, final CharGroupInfo info) + private static int writePtNode(final OutputStream destination, final PtNodeInfo info) throws IOException { - int size = FormatSpec.GROUP_FLAGS_SIZE; + int size = FormatSpec.PTNODE_FLAGS_SIZE; destination.write((byte)info.mFlags); final int parentOffset = info.mParentAddress == FormatSpec.NO_PARENT_ADDRESS ? FormatSpec.NO_PARENT_ADDRESS : info.mParentAddress - info.mOriginalAddress; @@ -503,7 +391,7 @@ public final class BinaryDictIOUtils { } } if (info.mCharacters.length > 1) { - destination.write((byte)FormatSpec.GROUP_CHARACTERS_TERMINATOR); + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); size++; } @@ -513,7 +401,7 @@ public final class BinaryDictIOUtils { } if (DBG) { - MakedictLog.d("writeCharGroup origin=" + info.mOriginalAddress + ", size=" + size + MakedictLog.d("writePtNode origin=" + info.mOriginalAddress + ", size=" + size + ", child=" + info.mChildrenAddress + ", characters =" + new String(info.mCharacters, 0, info.mCharacters.length)); } @@ -524,14 +412,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictInputOutput.makeShortcutFlags( + destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -540,28 +428,28 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictionaryInputOutput#computeActualNodeSize + // in BinaryDictEncoderUtils#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; int bigramFlags = (i < info.mBigrams.size() - 1) - ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0; + ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0; size++; final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); - bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictInputOutput.getByteSize(bigramOffset)) { + bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0; + switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; break; case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; break; case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; break; } - bigramFlags |= bigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; + bigramFlags |= bigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; destination.write((byte)bigramFlags); size += writeVariableAddress(destination, Math.abs(bigramOffset)); } @@ -569,82 +457,40 @@ public final class BinaryDictIOUtils { return size; } - @SuppressWarnings("unused") - private static void updateForwardLink(final FusionDictionaryBufferInterface buffer, - final int nodeOriginAddress, final int newNodeAddress, - final FormatOptions formatOptions) { - buffer.position(nodeOriginAddress); - int jumpCount = 0; - while (jumpCount++ < MAX_JUMPS) { - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); - for (int i = 0; i < count; ++i) skipCharGroup(buffer, formatOptions); - final int forwardLinkAddress = buffer.readUnsignedInt24(); - if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { - buffer.position(buffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE); - writeSInt24ToBuffer(buffer, newNodeAddress); - return; - } - buffer.position(forwardLinkAddress); - } - if (DBG && jumpCount >= MAX_JUMPS) { - throw new RuntimeException("too many jumps, probably a bug."); - } - } - /** - * Helper method to move a char group to the tail of the file. + * Compute the size of the PtNode. */ - private static int moveCharGroup(final OutputStream destination, - final FusionDictionaryBufferInterface buffer, final CharGroupInfo info, - final int nodeOriginAddress, final int oldGroupAddress, - final FormatOptions formatOptions) throws IOException { - updateParentAddress(buffer, oldGroupAddress, buffer.limit() + 1, formatOptions); - buffer.position(oldGroupAddress); - final int currentFlags = buffer.readUnsignedByte(); - buffer.position(oldGroupAddress); - buffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags - & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); - int size = FormatSpec.GROUP_FLAGS_SIZE; - updateForwardLink(buffer, nodeOriginAddress, buffer.limit(), formatOptions); - size += writeNode(destination, new CharGroupInfo[] { info }); - return size; - } - - /** - * Compute the size of the char group. - */ - private static int computeGroupSize(final CharGroupInfo info, - final FormatOptions formatOptions) { - int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters) - + BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions); + static int computePtNodeSize(final PtNodeInfo info, final FormatOptions formatOptions) { + int size = FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + BinaryDictEncoderUtils.getPtNodeCharactersSize(info.mCharacters) + + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - size += FormatSpec.GROUP_FREQUENCY_SIZE; + size += FormatSpec.PTNODE_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { - size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictInputOutput.getByteSize(attr.mAddress); + size += FormatSpec.PTNODE_FLAGS_SIZE; + size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); } } return size; } /** - * Write a node to the stream. + * Write a node array to the stream. * * @param destination the stream to write. - * @param infos groups to be written. + * @param infos an array of PtNodeInfo to be written. * @return the size written, in bytes. * @throws IOException */ - private static int writeNode(final OutputStream destination, final CharGroupInfo[] infos) + static int writeNodes(final OutputStream destination, final PtNodeInfo[] infos) throws IOException { - int size = BinaryDictInputOutput.getGroupCountSize(infos.length); - switch (BinaryDictInputOutput.getGroupCountSize(infos.length)) { + int size = getPtNodeCountSize(infos.length); + switch (getPtNodeCountSize(infos.length)) { case 1: destination.write((byte)infos.length); break; @@ -653,335 +499,13 @@ public final class BinaryDictIOUtils { destination.write((byte)(infos.length & 0xFF)); break; default: - throw new RuntimeException("Invalid group count size."); + throw new RuntimeException("Invalid node count size."); } - for (final CharGroupInfo info : infos) size += writeCharGroup(destination, info); + for (final PtNodeInfo info : infos) size += writePtNode(destination, info); writeSInt24ToStream(destination, FormatSpec.NO_FORWARD_LINK_ADDRESS); return size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } - /** - * Move a group that is referred to by oldGroupOrigin to the tail of the file. - * And set the children address to the byte after the group. - * - * @param nodeOrigin the address of the tail of the file. - * @param characters - * @param length - * @param flags - * @param frequency - * @param parentAddress - * @param shortcutTargets - * @param bigrams - * @param destination the stream representing the tail of the file. - * @param buffer the buffer representing the (constant-size) body of the file. - * @param oldNodeOrigin - * @param oldGroupOrigin - * @param formatOptions - * @return the size written, in bytes. - * @throws IOException - */ - private static int moveGroup(final int nodeOrigin, final int[] characters, final int length, - final int flags, final int frequency, final int parentAddress, - final ArrayList<WeightedString> shortcutTargets, - final ArrayList<PendingAttribute> bigrams, final OutputStream destination, - final FusionDictionaryBufferInterface buffer, final int oldNodeOrigin, - final int oldGroupOrigin, final FormatOptions formatOptions) throws IOException { - int size = 0; - final int newGroupOrigin = nodeOrigin + 1; - final int[] writtenCharacters = Arrays.copyOfRange(characters, 0, length); - final CharGroupInfo tmpInfo = new CharGroupInfo(newGroupOrigin, -1 /* endAddress */, - flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, - shortcutTargets, bigrams); - size = computeGroupSize(tmpInfo, formatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newGroupOrigin, newGroupOrigin + size, - flags, writtenCharacters, frequency, parentAddress, - nodeOrigin + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, - bigrams); - moveCharGroup(destination, buffer, newInfo, oldNodeOrigin, oldGroupOrigin, formatOptions); - return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - - /** - * Insert a word into a binary dictionary. - * - * @param buffer - * @param destination - * @param word - * @param frequency - * @param bigramStrings - * @param shortcuts - * @throws IOException - * @throws UnsupportedFormatException - */ - // TODO: Support batch insertion. - // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. - @UsedForTesting - public static void insertWord(final FusionDictionaryBufferInterface buffer, - final OutputStream destination, final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, - final boolean isBlackListEntry) - throws IOException, UnsupportedFormatException { - final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); - if (bigramStrings != null) { - for (final WeightedString bigram : bigramStrings) { - int position = getTerminalPosition(buffer, bigram.mWord); - if (position == FormatSpec.NOT_VALID_WORD) { - // TODO: figure out what is the correct thing to do here. - } else { - bigrams.add(new PendingAttribute(bigram.mFrequency, position)); - } - } - } - - final boolean isTerminal = true; - final boolean hasBigrams = !bigrams.isEmpty(); - final boolean hasShortcuts = shortcuts != null && !shortcuts.isEmpty(); - - // find the insert position of the word. - if (buffer.position() != 0) buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - - int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position(); - final int[] codePoints = FusionDictionary.getCodePoints(word); - final int wordLen = codePoints.length; - - for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { - if (wordPos >= wordLen) break; - nodeOriginAddress = buffer.position(); - int nodeParentAddress = -1; - final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); - boolean foundNextGroup = false; - - for (int i = 0; i < charGroupCount; ++i) { - address = buffer.position(); - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, - buffer.position(), header.mFormatOptions); - final boolean isMovedGroup = BinaryDictInputOutput.isMovedGroup(currentInfo.mFlags, - header.mFormatOptions); - if (isMovedGroup) continue; - nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) - ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; - boolean matched = true; - for (int p = 0; p < currentInfo.mCharacters.length; ++p) { - if (wordPos + p >= wordLen) { - /* - * splitting - * before - * abcd - ef - * - * insert "abc" - * - * after - * abc - d - ef - */ - final int newNodeAddress = buffer.limit(); - final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, - isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, - false /* isBlackListEntry */, header.mFormatOptions); - int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, - frequency, nodeParentAddress, shortcuts, bigrams, destination, - buffer, nodeOriginAddress, address, header.mFormatOptions); - - final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, - currentInfo.mCharacters.length); - if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(buffer, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, header.mFormatOptions); - } - final CharGroupInfo newInfo2 = new CharGroupInfo( - newNodeAddress + written + 1, -1 /* endAddress */, - currentInfo.mFlags, characters2, currentInfo.mFrequency, - newNodeAddress + 1, currentInfo.mChildrenAddress, - currentInfo.mShortcutTargets, currentInfo.mBigrams); - writeNode(destination, new CharGroupInfo[] { newInfo2 }); - return; - } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { - if (p > 0) { - /* - * splitting - * before - * ab - cd - * - * insert "ac" - * - * after - * a - b - cd - * | - * - c - */ - - final int newNodeAddress = buffer.limit(); - final int childrenAddress = currentInfo.mChildrenAddress; - - // move prefix - final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, - false /* isTerminal */, 0 /* childrenAddressSize*/, - false /* hasShortcut */, false /* hasBigrams */, - false /* isNotAWord */, false /* isBlackListEntry */, - header.mFormatOptions); - int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, - prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, - destination, buffer, nodeOriginAddress, address, - header.mFormatOptions); - - final int[] suffixCharacters = Arrays.copyOfRange( - currentInfo.mCharacters, p, currentInfo.mCharacters.length); - if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(buffer, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, header.mFormatOptions); - } - final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags( - suffixCharacters.length > 1, - (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, - 0 /* childrenAddressSize */, - (currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) - != 0, - (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo suffixInfo = new CharGroupInfo( - newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, - suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, - currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, - currentInfo.mBigrams); - written += computeGroupSize(suffixInfo, header.mFormatOptions) + 1; - - final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, - codePoints.length); - final int flags = BinaryDictInputOutput.makeCharGroupFlags( - newCharacters.length > 1, isTerminal, - 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo( - newNodeAddress + written, -1 /* endAddress */, flags, - newCharacters, frequency, newNodeAddress + 1, - FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - writeNode(destination, new CharGroupInfo[] { suffixInfo, newInfo }); - return; - } - matched = false; - break; - } - } - - if (matched) { - if (wordPos + currentInfo.mCharacters.length == wordLen) { - // the word exists in the dictionary. - // only update group. - final int newNodeAddress = buffer.limit(); - final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, - -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, - nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, - bigrams); - moveCharGroup(destination, buffer, newInfo, nodeOriginAddress, address, - header.mFormatOptions); - return; - } - wordPos += currentInfo.mCharacters.length; - if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { - /* - * found the prefix of the word. - * make new node and link to the node from this group. - * - * before - * ab - cd - * - * insert "abcde" - * - * after - * ab - cd - e - */ - final int newNodeAddress = buffer.limit(); - updateChildrenAddress(buffer, address, newNodeAddress, - header.mFormatOptions); - final int newGroupAddress = newNodeAddress + 1; - final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final CharGroupInfo newInfo = new CharGroupInfo(newGroupAddress, -1, flags, - characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, - shortcuts, bigrams); - writeNode(destination, new CharGroupInfo[] { newInfo }); - return; - } - buffer.position(currentInfo.mChildrenAddress); - foundNextGroup = true; - break; - } - } - - if (foundNextGroup) continue; - - // reached the end of the array. - final int linkAddressPosition = buffer.position(); - int nextLink = buffer.readUnsignedInt24(); - if ((nextLink & MSB24) != 0) { - nextLink = -(nextLink & SINT24_MAX); - } - if (nextLink == FormatSpec.NO_FORWARD_LINK_ADDRESS) { - /* - * expand this node. - * - * before - * ab - cd - * - * insert "abef" - * - * after - * ab - cd - * | - * - ef - */ - - // change the forward link address. - final int newNodeAddress = buffer.limit(); - buffer.position(linkAddressPosition); - writeSInt24ToBuffer(buffer, newNodeAddress); - - final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, - -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, - FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - writeNode(destination, new CharGroupInfo[]{ newInfo }); - return; - } else { - depth--; - buffer.position(nextLink); - } - } - } - - /** - * Find a word from the buffer. - * - * @param buffer the buffer representing the body of the dictionary file. - * @param word the word searched - * @return the found group - * @throws IOException - * @throws UnsupportedFormatException - */ - @UsedForTesting - public static CharGroupInfo findWordFromBuffer(final FusionDictionaryBufferInterface buffer, - final String word) throws IOException, UnsupportedFormatException { - int position = getTerminalPosition(buffer, word); - if (position != FormatSpec.NOT_VALID_WORD) { - buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - buffer.position(position); - return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions); - } - return null; - } - private static final int HEADER_READING_BUFFER_SIZE = 16384; /** * Convenience method to read the header of a binary file. @@ -992,20 +516,27 @@ public final class BinaryDictIOUtils { * @param offset The offset in the file where to start reading the data. * @param length The length of the data file. */ - public static FileHeader getDictionaryFileHeader( + private static FileHeader getDictionaryFileHeader( final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final FileInputStream inStream = new FileInputStream(file); - try { - inStream.read(buffer); - final BinaryDictInputOutput.ByteBufferWrapper wrapper = - new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, offset, length)); - return BinaryDictInputOutput.readHeader(wrapper); - } finally { - inStream.close(); - } + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + new DictDecoder.DictionaryBufferFactory() { + @Override + public DictBuffer getDictionaryBuffer(File file) + throws FileNotFoundException, IOException { + final FileInputStream inStream = new FileInputStream(file); + try { + inStream.skip(offset); + inStream.read(buffer); + return new ByteArrayDictBuffer(buffer); + } finally { + inStream.close(); + } + } + } + ); + return dictDecoder.readHeader(); } public static FileHeader getDictionaryFileHeaderOrNull(final File file, final long offset, @@ -1019,4 +550,83 @@ public final class BinaryDictIOUtils { return null; } } + + /** + * Helper method to hide the actual value of the no children address. + */ + public static boolean hasChildrenAddress(final int address) { + return FormatSpec.NO_CHILDREN_ADDRESS != address; + } + + /** + * Helper method to check whether the node is moved. + */ + public static boolean isMovedPtNode(final int flags, final FormatOptions options) { + return options.mSupportsDynamicUpdate + && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); + } + + /** + * Helper method to check whether the dictionary can be updated dynamically. + */ + public static boolean supportsDynamicUpdate(final FormatOptions options) { + return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE + && options.mSupportsDynamicUpdate; + } + + /** + * Helper method to check whether the node is deleted. + */ + public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { + return formatOptions.mSupportsDynamicUpdate + && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); + } + + /** + * Compute the binary size of the node count + * @param count the node count + * @return the size of the node count, either 1 or 2 bytes. + */ + public static int getPtNodeCountSize(final int count) { + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= count) { + return 1; + } else if (FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY >= count) { + return 2; + } else { + throw new RuntimeException("Can't have more than " + + FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY + " PtNode in a PtNodeArray (found " + + count + ")"); + } + } + + static int getChildrenAddressSize(final int optionFlags, + final FormatOptions formatOptions) { + if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return 1; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return 2; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return 3; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return 0; + } + } + + /** + * Calculate bigram frequency from compressed value + * + * @param unigramFrequency + * @param bigramFrequency compressed frequency + * @return approximate bigram frequency + */ + public static int reconstructBigramFrequency(final int unigramFrequency, + final int bigramFrequency) { + final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); + return (int)resultFreqFloat; + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java deleted file mode 100644 index 1b187d85d..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ /dev/null @@ -1,1838 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - -/** - * Reads and writes XML files for a FusionDictionary. - * - * All the methods in this class are static. - */ -public final class BinaryDictInputOutput { - - private static final boolean DBG = MakedictLog.DBG; - - // Arbitrary limit to how much passes we consider address size compression should - // terminate in. At the time of this writing, our largest dictionary completes - // compression in five passes. - // If the number of passes exceeds this number, makedict bails with an exception on - // suspicion that a bug might be causing an infinite loop. - private static final int MAX_PASSES = 24; - private static final int MAX_JUMPS = 12; - - @UsedForTesting - public interface FusionDictionaryBufferInterface { - public int readUnsignedByte(); - public int readUnsignedShort(); - public int readUnsignedInt24(); - public int readInt(); - public int position(); - public void position(int newPosition); - public void put(final byte b); - public int limit(); - public int capacity(); - } - - public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface { - private ByteBuffer mBuffer; - - public ByteBufferWrapper(final ByteBuffer buffer) { - mBuffer = buffer; - } - - @Override - public int readUnsignedByte() { - return mBuffer.get() & 0xFF; - } - - @Override - public int readUnsignedShort() { - return mBuffer.getShort() & 0xFFFF; - } - - @Override - public int readUnsignedInt24() { - final int retval = readUnsignedByte(); - return (retval << 16) + readUnsignedShort(); - } - - @Override - public int readInt() { - return mBuffer.getInt(); - } - - @Override - public int position() { - return mBuffer.position(); - } - - @Override - public void position(int newPos) { - mBuffer.position(newPos); - } - - @Override - public void put(final byte b) { - mBuffer.put(b); - } - - @Override - public int limit() { - return mBuffer.limit(); - } - - @Override - public int capacity() { - return mBuffer.capacity(); - } - } - - /** - * A class grouping utility function for our specific character encoding. - */ - static final class CharEncoding { - private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; - private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; - - /** - * Helper method to find out whether this code fits on one byte - */ - private static boolean fitsOnOneByte(final int character) { - return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE - && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; - } - - /** - * Compute the size of a character given its character code. - * - * Char format is: - * 1 byte = bbbbbbbb match - * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte - * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because - * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with - * 00011111 would be outside unicode. - * else: iso-latin-1 code - * This allows for the whole unicode range to be encoded, including chars outside of - * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control - * characters which should never happen anyway (and still work, but take 3 bytes). - * - * @param character the character code. - * @return the size in binary encoded-form, either 1 or 3 bytes. - */ - static int getCharSize(final int character) { - // See char encoding in FusionDictionary.java - if (fitsOnOneByte(character)) return 1; - if (FormatSpec.INVALID_CHARACTER == character) return 1; - return 3; - } - - /** - * Compute the byte size of a character array. - */ - private static int getCharArraySize(final int[] chars) { - int size = 0; - for (int character : chars) size += getCharSize(character); - return size; - } - - /** - * Writes a char array to a byte buffer. - * - * @param codePoints the code point array to write. - * @param buffer the byte buffer to write to. - * @param index the index in buffer to write the character array to. - * @return the index after the last character. - */ - private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { - for (int codePoint : codePoints) { - if (1 == getCharSize(codePoint)) { - buffer[index++] = (byte)codePoint; - } else { - buffer[index++] = (byte)(0xFF & (codePoint >> 16)); - buffer[index++] = (byte)(0xFF & (codePoint >> 8)); - buffer[index++] = (byte)(0xFF & codePoint); - } - } - return index; - } - - /** - * Writes a string with our character format to a byte buffer. - * - * This will also write the terminator byte. - * - * @param buffer the byte buffer to write to. - * @param origin the offset to write from. - * @param word the string to write. - * @return the size written, in bytes. - */ - private static int writeString(final byte[] buffer, final int origin, - final String word) { - final int length = word.length(); - int index = origin; - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer[index++] = (byte)codePoint; - } else { - buffer[index++] = (byte)(0xFF & (codePoint >> 16)); - buffer[index++] = (byte)(0xFF & (codePoint >> 8)); - buffer[index++] = (byte)(0xFF & codePoint); - } - } - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; - return index - origin; - } - - /** - * Writes a string with our character format to a ByteArrayOutputStream. - * - * This will also write the terminator byte. - * - * @param buffer the ByteArrayOutputStream to write to. - * @param word the string to write. - */ - private static void writeString(final ByteArrayOutputStream buffer, final String word) { - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer.write((byte) codePoint); - } else { - buffer.write((byte) (0xFF & (codePoint >> 16))); - buffer.write((byte) (0xFF & (codePoint >> 8))); - buffer.write((byte) (0xFF & codePoint)); - } - } - buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); - } - - /** - * Reads a string from a buffer. This is the converse of the above method. - */ - private static String readString(final FusionDictionaryBufferInterface buffer) { - final StringBuilder s = new StringBuilder(); - int character = readChar(buffer); - while (character != FormatSpec.INVALID_CHARACTER) { - s.appendCodePoint(character); - character = readChar(buffer); - } - return s.toString(); - } - - /** - * Reads a character from the buffer. - * - * This follows the character format documented earlier in this source file. - * - * @param buffer the buffer, positioned over an encoded character. - * @return the character code. - */ - static int readChar(final FusionDictionaryBufferInterface buffer) { - int character = buffer.readUnsignedByte(); - if (!fitsOnOneByte(character)) { - if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { - return FormatSpec.INVALID_CHARACTER; - } - character <<= 16; - character += buffer.readUnsignedShort(); - } - return character; - } - } - - /** - * Compute the binary size of the character array. - * - * If only one character, this is the size of this character. If many, it's the sum of their - * sizes + 1 byte for the terminator. - * - * @param characters the character array - * @return the size of the char array, including the terminator if any - */ - static int getGroupCharactersSize(final int[] characters) { - int size = CharEncoding.getCharArraySize(characters); - if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the binary size of the character array in a group - * - * If only one character, this is the size of this character. If many, it's the sum of their - * sizes + 1 byte for the terminator. - * - * @param group the group - * @return the size of the char array, including the terminator if any - */ - private static int getGroupCharactersSize(final CharGroup group) { - return getGroupCharactersSize(group.mChars); - } - - /** - * Compute the binary size of the group count - * @param count the group count - * @return the size of the group count, either 1 or 2 bytes. - */ - public static int getGroupCountSize(final int count) { - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { - return 1; - } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) { - return 2; - } else { - throw new RuntimeException("Can't have more than " - + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count - + ")"); - } - } - - /** - * Compute the binary size of the group count for a node - * @param node the node - * @return the size of the group count, either 1 or 2 bytes. - */ - private static int getGroupCountSize(final Node node) { - return getGroupCountSize(node.mData.size()); - } - - /** - * Compute the size of a shortcut in bytes. - */ - private static int getShortcutSize(final WeightedString shortcut) { - int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE; - final String word = shortcut.mWord; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - size += CharEncoding.getCharSize(codePoint); - } - size += FormatSpec.GROUP_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the size of a shortcut list in bytes. - * - * This is known in advance and does not change according to position in the file - * like address lists do. - */ - static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { - if (null == shortcutList) return 0; - int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - for (final WeightedString shortcut : shortcutList) { - size += getShortcutSize(shortcut); - } - return size; - } - - /** - * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. - * - * @param group the CharGroup to compute the size of. - * @param options file format options. - * @return the maximum size of the group. - */ - private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) { - int size = getGroupHeaderSize(group, options); - // If terminal, one byte for the frequency - if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE; - size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address - size += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE - + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) - * group.mBigrams.size(); - } - return size; - } - - /** - * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches - * it in the 'actualSize' member of the node. - * - * @param node the node to compute the maximum size of. - * @param options file format options. - */ - private static void calculateNodeMaximumSize(final Node node, final FormatOptions options) { - int size = getGroupCountSize(node); - for (CharGroup g : node.mData) { - final int groupSize = getCharGroupMaximumSize(g, options); - g.mCachedSize = groupSize; - size += groupSize; - } - if (options.mSupportsDynamicUpdate) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - node.mCachedSize = size; - } - - /** - * Helper method to hide the actual value of the no children address. - */ - public static boolean hasChildrenAddress(final int address) { - return FormatSpec.NO_CHILDREN_ADDRESS != address; - } - - /** - * Helper method to check whether the group is moved. - */ - public static boolean isMovedGroup(final int flags, final FormatOptions options) { - return options.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); - } - - /** - * Helper method to check whether the group is deleted. - */ - public static boolean isDeletedGroup(final int flags, final FormatOptions formatOptions) { - return formatOptions.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); - } - - /** - * Helper method to check whether the dictionary can be updated dynamically. - */ - public static boolean supportsDynamicUpdate(final FormatOptions options) { - return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.mSupportsDynamicUpdate; - } - - /** - * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. - * - * @param group the group of which to compute the size of the header - * @param options file format options. - */ - private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { - if (supportsDynamicUpdate(options)) { - return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + getGroupCharactersSize(group); - } else { - return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group); - } - } - - private static final int UINT8_MAX = 0xFF; - private static final int UINT16_MAX = 0xFFFF; - private static final int UINT24_MAX = 0xFFFFFF; - - /** - * Compute the size, in bytes, that an address will occupy. - * - * This can be used either for children addresses (which are always positive) or for - * attribute, which may be positive or negative but - * store their sign bit separately. - * - * @param address the address - * @return the byte size. - */ - static int getByteSize(final int address) { - assert(address <= UINT24_MAX); - if (!hasChildrenAddress(address)) { - return 0; - } else if (Math.abs(address) <= UINT8_MAX) { - return 1; - } else if (Math.abs(address) <= UINT16_MAX) { - return 2; - } else { - return 3; - } - } - - private static final int SINT24_MAX = 0x7FFFFF; - private static final int MSB8 = 0x80; - private static final int MSB24 = 0x800000; - - // End utility methods. - - // This method is responsible for finding a nice ordering of the nodes that favors run-time - // cache performance and dictionary size. - /* package for tests */ static ArrayList<Node> flattenTree(final Node root) { - final int treeSize = FusionDictionary.countCharGroups(root); - MakedictLog.i("Counted nodes : " + treeSize); - final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize); - return flattenTreeInner(flatTree, root); - } - - private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) { - // Removing the node is necessary if the tails are merged, because we would then - // add the same node several times when we only want it once. A number of places in - // the code also depends on any node being only once in the list. - // Merging tails can only be done if there are no attributes. Searching for attributes - // in LatinIME code depends on a total breadth-first ordering, which merging tails - // breaks. If there are no attributes, it should be fine (and reduce the file size) - // to merge tails, and removing the node from the list would be necessary. However, - // we don't merge tails because breaking the breadth-first ordering would result in - // extreme overhead at bigram lookup time (it would make the search function O(n) instead - // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty - // high). - // If no nodes are ever merged, we can't have the same node twice in the list, hence - // searching for duplicates in unnecessary. It is also very performance consuming, - // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making - // this simple list.remove operation O(n*n) overall. On Android this overhead is very - // high. - // For future reference, the code to remove duplicate is a simple : list.remove(node); - list.add(node); - final ArrayList<CharGroup> branches = node.mData; - final int nodeSize = branches.size(); - for (CharGroup group : branches) { - if (null != group.mChildren) flattenTreeInner(list, group.mChildren); - } - return list; - } - - /** - * Get the offset from a position inside a current node to a target node, during update. - * - * If the current node is before the target node, the target node has not been updated yet, - * so we should return the offset from the old position of the current node to the old position - * of the target node. If on the other hand the target is before the current node, it already - * has been updated, so we should return the offset from the new position in the current node - * to the new position in the target node. - * @param currentNode the node containing the CharGroup where the offset will be written - * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode - * @param targetNode the target node to get the offset to - * @return the offset to the target node - */ - private static int getOffsetToTargetNodeDuringUpdate(final Node currentNode, - final int offsetFromStartOfCurrentNode, final Node targetNode) { - final boolean isTargetBeforeCurrent = (targetNode.mCachedAddressBeforeUpdate - < currentNode.mCachedAddressBeforeUpdate); - if (isTargetBeforeCurrent) { - return targetNode.mCachedAddressAfterUpdate - - (currentNode.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNode); - } else { - return targetNode.mCachedAddressBeforeUpdate - - (currentNode.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNode); - } - } - - /** - * Get the offset from a position inside a current node to a target CharGroup, during update. - * @param currentNode the node containing the CharGroup where the offset will be written - * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode - * @param targetCharGroup the target CharGroup to get the offset to - * @return the offset to the target CharGroup - */ - // TODO: is there any way to factorize this method with the one above? - private static int getOffsetToTargetCharGroupDuringUpdate(final Node currentNode, - final int offsetFromStartOfCurrentNode, final CharGroup targetCharGroup) { - final int oldOffsetBasePoint = currentNode.mCachedAddressBeforeUpdate - + offsetFromStartOfCurrentNode; - final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate - < oldOffsetBasePoint); - // If the target is before the current node, then its address has already been updated. - // We can use the AfterUpdate member, and compare it to our own member after update. - // Otherwise, the AfterUpdate member is not updated yet, so we need to use the BeforeUpdate - // member, and of course we have to compare this to our own address before update. - if (isTargetBeforeCurrent) { - final int newOffsetBasePoint = currentNode.mCachedAddressAfterUpdate - + offsetFromStartOfCurrentNode; - return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint; - } else { - return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint; - } - } - - /** - * Computes the actual node size, based on the cached addresses of the children nodes. - * - * Each node stores its tentative address. During dictionary address computing, these - * are not final, but they can be used to compute the node size (the node size depends - * on the address of the children because the number of bytes necessary to store an - * address depends on its numeric value. The return value indicates whether the node - * contents (as in, any of the addresses stored in the cache fields) have changed with - * respect to their previous value. - * - * @param node the node to compute the size of. - * @param dict the dictionary in which the word/attributes are to be found. - * @param formatOptions file format options. - * @return false if none of the cached addresses inside the node changed, true otherwise. - */ - private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict, - final FormatOptions formatOptions) { - boolean changed = false; - int size = getGroupCountSize(node); - for (CharGroup group : node.mData) { - group.mCachedAddressAfterUpdate = node.mCachedAddressAfterUpdate + size; - if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) { - changed = true; - } - int groupSize = getGroupHeaderSize(group, formatOptions); - if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE; - if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else if (null != group.mChildren) { - if (formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else { - groupSize += getByteSize(getOffsetToTargetNodeDuringUpdate(node, - groupSize + size, group.mChildren)); - } - } - groupSize += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - for (WeightedString bigram : group.mBigrams) { - final int offset = getOffsetToTargetCharGroupDuringUpdate(node, - groupSize + size + FormatSpec.GROUP_FLAGS_SIZE, - FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord)); - groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE; - } - } - group.mCachedSize = groupSize; - size += groupSize; - } - if (formatOptions.mSupportsDynamicUpdate) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - if (node.mCachedSize != size) { - node.mCachedSize = size; - changed = true; - } - return changed; - } - - /** - * Initializes the cached addresses of nodes from their size. - * - * @param flatNodes the array of nodes. - * @param formatOptions file format options. - * @return the byte size of the entire stack. - */ - private static int initializeNodesCachedAddresses(final ArrayList<Node> flatNodes, - final FormatOptions formatOptions) { - int nodeOffset = 0; - for (final Node n : flatNodes) { - n.mCachedAddressBeforeUpdate = nodeOffset; - int groupCountSize = getGroupCountSize(n); - int groupOffset = 0; - for (final CharGroup g : n.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate = - groupCountSize + nodeOffset + groupOffset; - groupOffset += g.mCachedSize; - } - final int nodeSize = groupCountSize + groupOffset - + (formatOptions.mSupportsDynamicUpdate - ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); - nodeOffset += n.mCachedSize; - } - return nodeOffset; - } - - /** - * Updates the cached addresses of nodes after recomputing their new positions. - * - * @param flatNodes the array of nodes. - */ - private static void updateNodeCachedAddresses(final ArrayList<Node> flatNodes) { - for (final Node n : flatNodes) { - n.mCachedAddressBeforeUpdate = n.mCachedAddressAfterUpdate; - for (final CharGroup g : n.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate; - } - } - } - - /** - * Compute the cached parent addresses after all has been updated. - * - * The parent addresses are used by some binary formats at write-to-disk time. Not all formats - * need them. In particular, version 2 does not need them, and version 3 does. - * - * @param flatNodes the flat array of nodes to fill in - */ - private static void computeParentAddresses(final ArrayList<Node> flatNodes) { - for (final Node node : flatNodes) { - for (final CharGroup group : node.mData) { - if (null != group.mChildren) { - // Assign my address to children's parent address - // Here BeforeUpdate and AfterUpdate addresses have the same value, so it - // does not matter which we use. - group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate - - group.mChildren.mCachedAddressAfterUpdate; - } - } - } - } - - /** - * Compute the addresses and sizes of an ordered node array. - * - * This method takes a node array and will update its cached address and size values - * so that they can be written into a file. It determines the smallest size each of the - * nodes can be given the addresses of its children and attributes, and store that into - * each node. - * The order of the node is given by the order of the array. This method makes no effort - * to find a good order; it only mechanically computes the size this order results in. - * - * @param dict the dictionary - * @param flatNodes the ordered array of nodes - * @param formatOptions file format options. - * @return the same array it was passed. The nodes have been updated for address and size. - */ - private static ArrayList<Node> computeAddresses(final FusionDictionary dict, - final ArrayList<Node> flatNodes, final FormatOptions formatOptions) { - // First get the worst possible sizes and offsets - for (final Node n : flatNodes) calculateNodeMaximumSize(n, formatOptions); - final int offset = initializeNodesCachedAddresses(flatNodes, formatOptions); - - MakedictLog.i("Compressing the array addresses. Original size : " + offset); - MakedictLog.i("(Recursively seen size : " + offset + ")"); - - int passes = 0; - boolean changesDone = false; - do { - changesDone = false; - int nodeStartOffset = 0; - for (final Node n : flatNodes) { - n.mCachedAddressAfterUpdate = nodeStartOffset; - final int oldNodeSize = n.mCachedSize; - final boolean changed = computeActualNodeSize(n, dict, formatOptions); - final int newNodeSize = n.mCachedSize; - if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!"); - nodeStartOffset += newNodeSize; - changesDone |= changed; - } - updateNodeCachedAddresses(flatNodes); - ++passes; - if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); - } while (changesDone); - - if (formatOptions.mSupportsDynamicUpdate) { - computeParentAddresses(flatNodes); - } - final Node lastNode = flatNodes.get(flatNodes.size() - 1); - MakedictLog.i("Compression complete in " + passes + " passes."); - MakedictLog.i("After address compression : " - + (lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize)); - - return flatNodes; - } - - /** - * Sanity-checking method. - * - * This method checks an array of node for juxtaposition, that is, it will do - * nothing if each node's cached address is actually the previous node's address - * plus the previous node's size. - * If this is not the case, it will throw an exception. - * - * @param array the array node to check - */ - private static void checkFlatNodeArray(final ArrayList<Node> array) { - int offset = 0; - int index = 0; - for (final Node n : array) { - // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter - // which we use. - if (n.mCachedAddressAfterUpdate != offset) { - throw new RuntimeException("Wrong address for node " + index - + " : expected " + offset + ", got " + n.mCachedAddressAfterUpdate); - } - ++index; - offset += n.mCachedSize; - } - } - - /** - * Helper method to write a variable-size address to a file. - * - * @param buffer the buffer to write to. - * @param index the index in the buffer to write the address to. - * @param address the address to write. - * @return the size in bytes the address actually took. - */ - private static int writeVariableAddress(final byte[] buffer, int index, final int address) { - switch (getByteSize(address)) { - case 1: - buffer[index++] = (byte)address; - return 1; - case 2: - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); - return 2; - case 3: - buffer[index++] = (byte)(0xFF & (address >> 16)); - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); - return 3; - case 0: - return 0; - default: - throw new RuntimeException("Address " + address + " has a strange size"); - } - } - - /** - * Helper method to write a variable-size signed address to a file. - * - * @param buffer the buffer to write to. - * @param index the index in the buffer to write the address to. - * @param address the address to write. - * @return the size in bytes the address actually took. - */ - private static int writeVariableSignedAddress(final byte[] buffer, int index, - final int address) { - if (!hasChildrenAddress(address)) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - buffer[index++] = (byte)((address < 0 ? MSB8 : 0) | (0xFF & (absAddress >> 16))); - buffer[index++] = (byte)(0xFF & (absAddress >> 8)); - buffer[index++] = (byte)(0xFF & absAddress); - } - return 3; - } - - /** - * Makes the flag value for a char group. - * - * @param hasMultipleChars whether the group has multiple chars. - * @param isTerminal whether the group is terminal. - * @param childrenAddressSize the size of a children address. - * @param hasShortcuts whether the group has shortcuts. - * @param hasBigrams whether the group has bigrams. - * @param isNotAWord whether the group is not a word. - * @param isBlackListEntry whether the group is a blacklist entry. - * @param formatOptions file format options. - * @return the flags - */ - static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal, - final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, - final boolean isNotAWord, final boolean isBlackListEntry, - final FormatOptions formatOptions) { - byte flags = 0; - if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; - if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.mSupportsDynamicUpdate) { - flags |= FormatSpec.FLAG_IS_NOT_MOVED; - } else if (true) { - switch (childrenAddressSize) { - case 1: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; - break; - case 0: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS; - break; - default: - throw new RuntimeException("Node with a strange address"); - } - } - if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; - if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; - if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; - if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; - return flags; - } - - private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress, - final int childrenOffset, final FormatOptions formatOptions) { - return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0, - getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null, - group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions); - } - - /** - * Makes the flag value for a bigram. - * - * @param more whether there are more bigrams after this one. - * @param offset the offset of the bigram. - * @param bigramFrequency the frequency of the bigram, 0..255. - * @param unigramFrequency the unigram frequency of the same word, 0..255. - * @param word the second bigram, for debugging purposes - * @return the flags - */ - private static final int makeBigramFlags(final boolean more, final int offset, - int bigramFrequency, final int unigramFrequency, final String word) { - int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0); - switch (getByteSize(offset)) { - case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; - break; - default: - throw new RuntimeException("Strange offset size"); - } - if (unigramFrequency > bigramFrequency) { - MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word - + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " - + word + " is " + unigramFrequency); - bigramFrequency = unigramFrequency; - } - // We compute the difference between 255 (which means probability = 1) and the - // unigram score. We split this into a number of discrete steps. - // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 - // represents an increase of 16 steps: a value of 15 will be interpreted as the median - // value of the 16th step. In all justice, if the bigram frequency is low enough to be - // rounded below the first step (which means it is less than half a step higher than the - // unigram frequency) then the unigram frequency itself is the best approximation of the - // bigram freq that we could possibly supply, hence we should *not* include this bigram - // in the file at all. - // until this is done, we'll write 0 and slightly overestimate this case. - // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step - // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to - // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the - // step size. Then we compute the start of the first step (the one where value 0 starts) - // by adding half-a-step to the unigramFrequency. From there, we compute the integer - // number of steps to the bigramFrequency. One last thing: we want our steps to include - // their lower bound and exclude their higher bound so we need to have the first step - // start at exactly 1 unit higher than floor(unigramFreq + half a step). - // Note : to reconstruct the score, the dictionary reader will need to divide - // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, - // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best - // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the - // step pointed by the discretized frequency. - final float stepSize = - (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) - / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); - final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); - final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); - // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 - // here. The best approximation would be the unigram freq itself, so we should not - // include this bigram in the dictionary. For now, register as 0, and live with the - // small over-estimation that we get in this case. TODO: actually remove this bigram - // if discretizedFrequency < 0. - final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; - bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; - return bigramFlags; - } - - /** - * Makes the 2-byte value for options flags. - */ - private static final int makeOptionsValue(final FusionDictionary dictionary, - final FormatOptions formatOptions) { - final DictionaryOptions options = dictionary.mOptions; - final boolean hasBigrams = dictionary.hasBigrams(); - return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) - + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) - + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) - + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); - } - - /** - * Makes the flag value for a shortcut. - * - * @param more whether there are more attributes after this one. - * @param frequency the frequency of the attribute, 0..15 - * @return the flags - */ - static final int makeShortcutFlags(final boolean more, final int frequency) { - return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY); - } - - private static final int writeParentAddress(final byte[] buffer, final int index, - final int address, final FormatOptions formatOptions) { - if (supportsDynamicUpdate(formatOptions)) { - if (address == FormatSpec.NO_PARENT_ADDRESS) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - assert(absAddress <= SINT24_MAX); - buffer[index] = (byte)((address < 0 ? MSB8 : 0) - | ((absAddress >> 16) & 0xFF)); - buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); - buffer[index + 2] = (byte)(absAddress & 0xFF); - } - return index + 3; - } else { - return index; - } - } - - /** - * Write a node to memory. The node is expected to have its final position cached. - * - * This can be an empty map, but the more is inside the faster the lookups will be. It can - * be carried on as long as nodes do not move. - * - * @param dict the dictionary the node is a part of (for relative offsets). - * @param buffer the memory buffer to write to. - * @param node the node to write. - * @param formatOptions file format options. - * @return the address of the END of the node. - */ - @SuppressWarnings("unused") - private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, - final Node node, final FormatOptions formatOptions) { - // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup - int index = node.mCachedAddressAfterUpdate; - - final int groupCount = node.mData.size(); - final int countSize = getGroupCountSize(node); - final int parentAddress = node.mCachedParentAddress; - if (1 == countSize) { - buffer[index++] = (byte)groupCount; - } else if (2 == countSize) { - // We need to signal 2-byte size by setting the top bit of the MSB to 1, so - // we | 0x80 to do this. - buffer[index++] = (byte)((groupCount >> 8) | 0x80); - buffer[index++] = (byte)(groupCount & 0xFF); - } else { - throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); - } - int groupAddress = index; - for (int i = 0; i < groupCount; ++i) { - final CharGroup group = node.mData.get(i); - if (index != group.mCachedAddressAfterUpdate) { - throw new RuntimeException("Bug: write index is not the same as the cached address " - + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate); - } - groupAddress += getGroupHeaderSize(group, formatOptions); - // Sanity checks. - if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { - throw new RuntimeException("A node has a frequency > " - + FormatSpec.MAX_TERMINAL_FREQUENCY - + " : " + group.mFrequency); - } - if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE; - final int childrenOffset = null == group.mChildren - ? FormatSpec.NO_CHILDREN_ADDRESS - : group.mChildren.mCachedAddressAfterUpdate - groupAddress; - buffer[index++] = - makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions); - - if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { - index = writeParentAddress(buffer, index, parentAddress, formatOptions); - } else { - index = writeParentAddress(buffer, index, parentAddress - + (node.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate), - formatOptions); - } - - index = CharEncoding.writeCharArray(group.mChars, buffer, index); - if (group.hasSeveralChars()) { - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; - } - if (group.mFrequency >= 0) { - buffer[index++] = (byte) group.mFrequency; - } - - final int shift; - if (formatOptions.mSupportsDynamicUpdate) { - shift = writeVariableSignedAddress(buffer, index, childrenOffset); - } else { - shift = writeVariableAddress(buffer, index, childrenOffset); - } - index += shift; - groupAddress += shift; - - // Write shortcuts - if (null != group.mShortcutTargets) { - final int indexOfShortcutByteSize = index; - index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - ++groupAddress; - int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), - target.mFrequency); - buffer[index++] = (byte)shortcutFlags; - final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); - index += shortcutShift; - groupAddress += shortcutShift; - } - final int shortcutByteSize = index - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { - throw new RuntimeException("Shortcut list too large"); - } - buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); - buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); - } - // Write bigrams - if (null != group.mBigrams) { - final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); - while (bigramIterator.hasNext()) { - final WeightedString bigram = bigramIterator.next(); - final CharGroup target = - FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord); - final int addressOfBigram = target.mCachedAddressAfterUpdate; - final int unigramFrequencyForThisWord = target.mFrequency; - ++groupAddress; - final int offset = addressOfBigram - groupAddress; - int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, - bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); - buffer[index++] = (byte)bigramFlags; - final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); - index += bigramShift; - groupAddress += bigramShift; - } - } - - } - if (formatOptions.mSupportsDynamicUpdate) { - buffer[index] = buffer[index + 1] = buffer[index + 2] - = FormatSpec.NO_FORWARD_LINK_ADDRESS; - index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - if (index != node.mCachedAddressAfterUpdate + node.mCachedSize) throw new RuntimeException( - "Not the same size : written " - + (index - node.mCachedAddressAfterUpdate) + " bytes from a node that should have " - + node.mCachedSize + " bytes"); - return index; - } - - /** - * Dumps a collection of useful statistics about a node array. - * - * This prints purely informative stuff, like the total estimated file size, the - * number of nodes, of character groups, the repartition of each address size, etc - * - * @param nodes the node array. - */ - private static void showStatistics(ArrayList<Node> nodes) { - int firstTerminalAddress = Integer.MAX_VALUE; - int lastTerminalAddress = Integer.MIN_VALUE; - int size = 0; - int charGroups = 0; - int maxGroups = 0; - int maxRuns = 0; - for (final Node n : nodes) { - if (maxGroups < n.mData.size()) maxGroups = n.mData.size(); - for (final CharGroup cg : n.mData) { - ++charGroups; - if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length; - if (cg.mFrequency >= 0) { - if (n.mCachedAddressAfterUpdate < firstTerminalAddress) - firstTerminalAddress = n.mCachedAddressAfterUpdate; - if (n.mCachedAddressAfterUpdate > lastTerminalAddress) - lastTerminalAddress = n.mCachedAddressAfterUpdate; - } - } - if (n.mCachedAddressAfterUpdate + n.mCachedSize > size) { - size = n.mCachedAddressAfterUpdate + n.mCachedSize; - } - } - final int[] groupCounts = new int[maxGroups + 1]; - final int[] runCounts = new int[maxRuns + 1]; - for (final Node n : nodes) { - ++groupCounts[n.mData.size()]; - for (final CharGroup cg : n.mData) { - ++runCounts[cg.mChars.length]; - } - } - - MakedictLog.i("Statistics:\n" - + " total file size " + size + "\n" - + " " + nodes.size() + " nodes\n" - + " " + charGroups + " groups (" + ((float)charGroups / nodes.size()) - + " groups per node)\n" - + " first terminal at " + firstTerminalAddress + "\n" - + " last terminal at " + lastTerminalAddress + "\n" - + " Group stats : max = " + maxGroups); - for (int i = 0; i < groupCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + groupCounts[i]); - } - MakedictLog.i(" Character run stats : max = " + maxRuns); - for (int i = 0; i < runCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + runCounts[i]); - } - } - - /** - * Dumps a FusionDictionary to a file. - * - * This is the public entry point to write a dictionary to a file. - * - * @param destination the stream to write the binary data to. - * @param dict the dictionary to write. - * @param formatOptions file format options. - */ - public static void writeDictionaryBinary(final OutputStream destination, - final FusionDictionary dict, final FormatOptions formatOptions) - throws IOException, UnsupportedFormatException { - - // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the - // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order - // of the nodes becomes a quite complicated problem, because though the dictionary itself - // does not have a size limit, each node must still be within 16MB of all its children and - // parents. As long as this is ensured, the dictionary file may grow to any size. - - final int version = formatOptions.mVersion; - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("Requested file format version " + version - + ", but this implementation only supports versions " - + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - - ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); - - // The magic number in big-endian order. - if (version >= FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { - // Magic number for version 2+. - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 24))); - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 16))); - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 8))); - headerBuffer.write((byte) (0xFF & FormatSpec.VERSION_2_MAGIC_NUMBER)); - // Dictionary version. - headerBuffer.write((byte) (0xFF & (version >> 8))); - headerBuffer.write((byte) (0xFF & version)); - } else { - // Magic number for version 1. - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_1_MAGIC_NUMBER >> 8))); - headerBuffer.write((byte) (0xFF & FormatSpec.VERSION_1_MAGIC_NUMBER)); - // Dictionary version. - headerBuffer.write((byte) (0xFF & version)); - } - // Options flags - final int options = makeOptionsValue(dict, formatOptions); - headerBuffer.write((byte) (0xFF & (options >> 8))); - headerBuffer.write((byte) (0xFF & options)); - if (version >= FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { - final int headerSizeOffset = headerBuffer.size(); - // Placeholder to be written later with header size. - for (int i = 0; i < 4; ++i) { - headerBuffer.write(0); - } - // Write out the options. - for (final String key : dict.mOptions.mAttributes.keySet()) { - final String value = dict.mOptions.mAttributes.get(key); - CharEncoding.writeString(headerBuffer, key); - CharEncoding.writeString(headerBuffer, value); - } - final int size = headerBuffer.size(); - final byte[] bytes = headerBuffer.toByteArray(); - // Write out the header size. - bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); - bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); - bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); - bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); - destination.write(bytes); - } else { - headerBuffer.writeTo(destination); - } - - headerBuffer.close(); - - // Leave the choice of the optimal node order to the flattenTree function. - MakedictLog.i("Flattening the tree..."); - ArrayList<Node> flatNodes = flattenTree(dict.mRoot); - - MakedictLog.i("Computing addresses..."); - computeAddresses(dict, flatNodes, formatOptions); - MakedictLog.i("Checking array..."); - if (DBG) checkFlatNodeArray(flatNodes); - - // Create a buffer that matches the final dictionary size. - final Node lastNode = flatNodes.get(flatNodes.size() - 1); - final int bufferSize = lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize; - final byte[] buffer = new byte[bufferSize]; - int index = 0; - - MakedictLog.i("Writing file..."); - int dataEndOffset = 0; - for (Node n : flatNodes) { - dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions); - } - - if (DBG) showStatistics(flatNodes); - - destination.write(buffer, 0, dataEndOffset); - - destination.close(); - MakedictLog.i("Done"); - } - - - // Input methods: Read a binary dictionary to memory. - // readDictionaryBinary is the public entry point for them. - - static int getChildrenAddressSize(final int optionFlags, - final FormatOptions formatOptions) { - if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - return 1; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - return 2; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - return 3; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: - default: - return 0; - } - } - - static int readChildrenAddress(final FusionDictionaryBufferInterface buffer, - final int optionFlags, final FormatOptions options) { - if (options.mSupportsDynamicUpdate) { - final int address = buffer.readUnsignedInt24(); - if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; - if ((address & MSB24) != 0) { - return -(address & SINT24_MAX); - } else { - return address; - } - } - int address; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - return buffer.readUnsignedByte(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - return buffer.readUnsignedShort(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - return buffer.readUnsignedInt24(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - - static int readParentAddress(final FusionDictionaryBufferInterface buffer, - final FormatOptions formatOptions) { - if (supportsDynamicUpdate(formatOptions)) { - final int parentAddress = buffer.readUnsignedInt24(); - final int sign = ((parentAddress & MSB24) != 0) ? -1 : 1; - return sign * (parentAddress & SINT24_MAX); - } else { - return FormatSpec.NO_PARENT_ADDRESS; - } - } - - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = buffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(buffer, options); - if (supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = buffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(buffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = buffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - buffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = buffer.readUnsignedByte(); - final String word = CharEncoding.readString(buffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += buffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = buffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * buffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * buffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (buffer.readUnsignedByte() << 16) - + buffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - - /** - * Reads and returns the char group count out of a buffer and forwards the pointer. - */ - public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) { - final int msb = buffer.readUnsignedByte(); - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { - return msb; - } else { - return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) - + buffer.readUnsignedByte(); - } - } - - // The word cache here is a stopgap bandaid to help the catastrophic performance - // of this method. Since it performs direct, unbuffered random access to the file and - // may be called hundreds of thousands of times, the resulting performance is not - // reasonable without some kind of cache. Thus: - private static TreeMap<Integer, WeightedString> wordCache = - new TreeMap<Integer, WeightedString>(); - /** - * Finds, as a string, the word at the address passed as an argument. - * - * @param buffer the buffer to read from. - * @param headerSize the size of the header. - * @param address the address to seek. - * @param formatOptions file format options. - * @return the word with its frequency, as a weighted string. - */ - /* package for tests */ static WeightedString getWordAtAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions formatOptions) { - final WeightedString cachedString = wordCache.get(address); - if (null != cachedString) return cachedString; - - final WeightedString result; - final int originalPointer = buffer.position(); - buffer.position(address); - - if (supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions); - } else { - result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address, - formatOptions); - } - - wordCache.put(address, result); - buffer.position(originalPointer); - return result; - } - - // TODO: static!? This will behave erratically when used in multi-threaded code. - // We need to fix this - private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; - @SuppressWarnings("unused") - private static WeightedString getWordAtAddressWithParentAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions options) { - int currentAddress = address; - int index = FormatSpec.MAX_WORD_LENGTH - 1; - int frequency = Integer.MIN_VALUE; - // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH - for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { - CharGroupInfo currentInfo; - int loopCounter = 0; - do { - buffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(buffer, currentAddress, options); - if (isMovedGroup(currentInfo.mFlags, options)) { - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - if (DBG && loopCounter++ > MAX_JUMPS) { - MakedictLog.d("Too many jumps - probably a bug"); - } - } while (isMovedGroup(currentInfo.mFlags, options)); - if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; - for (int i = 0; i < currentInfo.mCharacters.length; ++i) { - sGetWordBuffer[index--] = - currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1]; - } - if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - - return new WeightedString( - new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1), - frequency); - } - - private static WeightedString getWordAtAddressWithoutParentAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions options) { - buffer.position(headerSize); - final int count = readCharGroupCount(buffer); - int groupOffset = getGroupCountSize(count); - final StringBuilder builder = new StringBuilder(); - WeightedString result = null; - - CharGroupInfo last = null; - for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - groupOffset = info.mEndAddress; - if (info.mOriginalAddress == address) { - builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); - result = new WeightedString(builder.toString(), info.mFrequency); - break; // and return - } - if (hasChildrenAddress(info.mChildrenAddress)) { - if (info.mChildrenAddress > address) { - if (null == last) continue; - builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - buffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + getGroupCountSize(i); - last = null; - continue; - } - last = info; - } - if (0 == i && hasChildrenAddress(last.mChildrenAddress)) { - builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - buffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + getGroupCountSize(i); - last = null; - continue; - } - } - return result; - } - - /** - * Reads a single node from a buffer. - * - * This methods reads the file at the current position. A node is fully expected to start at - * the current position. - * This will recursively read other nodes into the structure, populating the reverse - * maps on the fly and using them to keep track of already read nodes. - * - * @param buffer the buffer, correctly positioned at the start of a node. - * @param headerSize the size, in bytes, of the file header. - * @param reverseNodeMap a mapping from addresses to already read nodes. - * @param reverseGroupMap a mapping from addresses to already read character groups. - * @param options file format options. - * @return the read node with all his children already read. - */ - private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize, - final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap, - final FormatOptions options) - throws IOException { - final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>(); - final int nodeOrigin = buffer.position() - headerSize; - - do { // Scan the linked-list node. - final int nodeHeadPosition = buffer.position() - headerSize; - final int count = readCharGroupCount(buffer); - int groupOffset = nodeHeadPosition + getGroupCountSize(count); - for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - if (isMovedGroup(info.mFlags, options)) continue; - ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; - ArrayList<WeightedString> bigrams = null; - if (null != info.mBigrams) { - bigrams = new ArrayList<WeightedString>(); - for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - buffer, headerSize, bigram.mAddress, options); - final int reconstructedFrequency = - reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); - bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); - } - } - if (hasChildrenAddress(info.mChildrenAddress)) { - Node children = reverseNodeMap.get(info.mChildrenAddress); - if (null == children) { - final int currentPosition = buffer.position(); - buffer.position(info.mChildrenAddress + headerSize); - children = readNode( - buffer, headerSize, reverseNodeMap, reverseGroupMap, options); - buffer.position(currentPosition); - } - nodeContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, - info.mFrequency, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); - } else { - nodeContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, - info.mFrequency, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); - } - groupOffset = info.mEndAddress; - } - - // reach the end of the array. - if (options.mSupportsDynamicUpdate) { - final int nextAddress = buffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < buffer.limit()) { - buffer.position(nextAddress); - } else { - break; - } - } - } while (options.mSupportsDynamicUpdate && - buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); - - final Node node = new Node(nodeContents); - node.mCachedAddressBeforeUpdate = nodeOrigin; - node.mCachedAddressAfterUpdate = nodeOrigin; - reverseNodeMap.put(node.mCachedAddressAfterUpdate, node); - return node; - } - - /** - * Helper function to get the binary format version from the header. - * @throws IOException - */ - private static int getFormatVersion(final FusionDictionaryBufferInterface buffer) - throws IOException { - final int magic_v1 = buffer.readUnsignedShort(); - if (FormatSpec.VERSION_1_MAGIC_NUMBER == magic_v1) return buffer.readUnsignedByte(); - final int magic_v2 = (magic_v1 << 16) + buffer.readUnsignedShort(); - if (FormatSpec.VERSION_2_MAGIC_NUMBER == magic_v2) return buffer.readUnsignedShort(); - return FormatSpec.NOT_A_VERSION_NUMBER; - } - - /** - * Helper function to get and validate the binary format version. - * @throws UnsupportedFormatException - * @throws IOException - */ - private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer) - throws IOException, UnsupportedFormatException { - final int version = getFormatVersion(buffer); - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("This file has version " + version - + ", but this implementation does not support versions above " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - return version; - } - - /** - * Reads a header from a buffer. - * @param buffer the buffer to read. - * @throws IOException - * @throws UnsupportedFormatException - */ - public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer) - throws IOException, UnsupportedFormatException { - final int version = checkFormatVersion(buffer); - final int optionsFlags = buffer.readUnsignedShort(); - - final HashMap<String, String> attributes = new HashMap<String, String>(); - final int headerSize; - if (version < FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { - headerSize = buffer.position(); - } else { - headerSize = buffer.readInt(); - populateOptions(buffer, headerSize, attributes); - buffer.position(headerSize); - } - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } - - /** - * Reads options from a buffer and populate a map with their contents. - * - * The buffer is read at the current position, so the caller must take care the pointer - * is in the right place before calling this. - */ - public static void populateOptions(final FusionDictionaryBufferInterface buffer, - final int headerSize, final HashMap<String, String> options) { - while (buffer.position() < headerSize) { - final String key = CharEncoding.readString(buffer); - final String value = CharEncoding.readString(buffer); - options.put(key, value); - } - } - - /** - * Reads a buffer and returns the memory representation of the dictionary. - * - * This high-level method takes a buffer and reads its contents, populating a - * FusionDictionary structure. The optional dict argument is an existing dictionary to - * which words from the buffer should be added. If it is null, a new dictionary is created. - * - * @param buffer the buffer to read. - * @param dict an optional dictionary to add words to, or null. - * @return the created (or merged) dictionary. - */ - @UsedForTesting - public static FusionDictionary readDictionaryBinary( - final FusionDictionaryBufferInterface buffer, final FusionDictionary dict) - throws IOException, UnsupportedFormatException { - // clear cache - wordCache.clear(); - - // Read header - final FileHeader header = readHeader(buffer); - - Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>(); - Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping, - reverseGroupMapping, header.mFormatOptions); - - FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions); - if (null != dict) { - for (final Word w : dict) { - if (w.mIsBlacklistEntry) { - newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); - } else { - newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); - } - } - for (final Word w : dict) { - // By construction a binary dictionary may not have bigrams pointing to - // words that are not also registered as unigrams so we don't have to avoid - // them explicitly here. - for (final WeightedString bigram : w.mBigrams) { - newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); - } - } - } - - return newDict; - } - - /** - * Helper method to pass a file name instead of a File object to isBinaryDictionary. - */ - public static boolean isBinaryDictionary(final String filename) { - final File file = new File(filename); - return isBinaryDictionary(file); - } - - /** - * Basic test to find out whether the file is a binary dictionary or not. - * - * Concretely this only tests the magic number. - * - * @param file The file to test. - * @return true if it's a binary dictionary, false otherwise - */ - public static boolean isBinaryDictionary(final File file) { - FileInputStream inStream = null; - try { - inStream = new FileInputStream(file); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - final int version = getFormatVersion(new ByteBufferWrapper(buffer)); - return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION - && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } catch (FileNotFoundException e) { - return false; - } catch (IOException e) { - return false; - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } - } - } - - /** - * Calculate bigram frequency from compressed value - * - * @see #makeBigramFlags - * - * @param unigramFrequency - * @param bigramFrequency compressed frequency - * @return approximate bigram frequency - */ - public static int reconstructBigramFrequency(final int unigramFrequency, - final int bigramFrequency) { - final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) - / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); - final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); - return (int)resultFreqFloat; - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java new file mode 100644 index 000000000..11a3f0b3a --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.TreeMap; + +/** + * An interface of binary dictionary decoder. + */ +public interface DictDecoder { + public FileHeader readHeader() throws IOException, UnsupportedFormatException; + + /** + * Reads PtNode from nodeAddress. + * @param ptNodePos the position of PtNode. + * @param formatOptions the format options. + * @return PtNodeInfo. + */ + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. The optional dict argument is an existing dictionary to + * which words from the buffer should be added. If it is null, a new dictionary is created. + * + * @param dict an optional dictionary to add words to, or null. + * @return the created (or merged) dictionary. + */ + @UsedForTesting + public FusionDictionary readDictionaryBinary(final FusionDictionary dict) + throws FileNotFoundException, IOException, UnsupportedFormatException; + + /** + * Gets the address of the last PtNode of the exact matching word in the dictionary. + * If no match is found, returns NOT_VALID_WORD. + * + * @param word the word we search for. + * @return the address of the terminal node. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + public int getTerminalPosition(final String word) + throws IOException, UnsupportedFormatException; + + /** + * Reads unigrams and bigrams from the binary file. + * Doesn't store a full memory representation of the dictionary. + * + * @param words the map to store the address as a key and the word as a value. + * @param frequencies the map to store the address as a key and the frequency as a value. + * @param bigrams the map to store the address as a key and the list of address as a value. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException; + + // Flags for DictionaryBufferFactory. + public static final int USE_READONLY_BYTEBUFFER = 0x01000000; + public static final int USE_BYTEARRAY = 0x02000000; + public static final int USE_WRITABLE_BYTEBUFFER = 0x04000000; + public static final int MASK_DICTBUFFER = 0x0F000000; + + public interface DictionaryBufferFactory { + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException; + } + + /** + * Creates DictionaryBuffer using a ByteBuffer + * + * This class uses less memory than DictionaryBufferFromByteArrayFactory, + * but doesn't perform as fast. + * When operating on a big dictionary, this class is preferred. + */ + public static final class DictionaryBufferFromReadOnlyByteBufferFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + ByteBuffer buffer = null; + try { + inStream = new FileInputStream(file); + buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, + 0, file.length()); + } finally { + if (inStream != null) { + inStream.close(); + } + } + if (buffer != null) { + return new BinaryDictDecoderUtils.ByteBufferDictBuffer(buffer); + } + return null; + } + } + + /** + * Creates DictionaryBuffer using a byte array + * + * This class performs faster than other classes, but consumes more memory. + * When operating on a small dictionary, this class is preferred. + */ + public static final class DictionaryBufferFromByteArrayFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final byte[] array = new byte[(int) file.length()]; + inStream.read(array); + return new ByteArrayDictBuffer(array); + } finally { + if (inStream != null) { + inStream.close(); + } + } + } + } + + /** + * Creates DictionaryBuffer using a writable ByteBuffer and a RandomAccessFile. + * + * This class doesn't perform as fast as other classes, + * but this class is the only option available for destructive operations (insert or delete) + * on a dictionary. + */ + @UsedForTesting + public static final class DictionaryBufferFromWritableByteBufferFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + RandomAccessFile raFile = null; + ByteBuffer buffer = null; + try { + raFile = new RandomAccessFile(file, "rw"); + buffer = raFile.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, file.length()); + } finally { + if (raFile != null) { + raFile.close(); + } + } + if (buffer != null) { + return new BinaryDictDecoderUtils.ByteBufferDictBuffer(buffer); + } + return null; + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java new file mode 100644 index 000000000..89c982e7b --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; + +import java.io.IOException; + +/** + * An interface of binary dictionary encoder. + */ +public interface DictEncoder { + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException; +} diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java new file mode 100644 index 000000000..bf3d19101 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -0,0 +1,502 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * The utility class to help dynamic updates on the binary dictionary. + * + * All the methods in this class are static. + */ +@UsedForTesting +public final class DynamicBinaryDictIOUtils { + private static final boolean DBG = false; + private static final int MAX_JUMPS = 10000; + + private DynamicBinaryDictIOUtils() { + // This utility class is not publicly instantiable. + } + + private static int markAsDeleted(final int flags) { + return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; + } + + /** + * Delete the word from the binary file. + * + * @param dictDecoder the dict decoder. + * @param word the word we delete + * @throws IOException + * @throws UnsupportedFormatException + */ + @UsedForTesting + public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word) + throws IOException, UnsupportedFormatException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + dictBuffer.position(0); + final FileHeader header = dictDecoder.readHeader(); + final int wordPosition = dictDecoder.getTerminalPosition(word); + if (wordPosition == FormatSpec.NOT_VALID_WORD) return; + + dictBuffer.position(wordPosition); + final int flags = dictBuffer.readUnsignedByte(); + dictBuffer.position(wordPosition); + dictBuffer.put((byte)markAsDeleted(flags)); + } + + /** + * Update a parent address in a PtNode that is referred to by ptNodeOriginAddress. + * + * @param dictBuffer the DictBuffer to write. + * @param ptNodeOriginAddress the address of the PtNode. + * @param newParentAddress the absolute address of the parent. + * @param formatOptions file format options. + */ + public static void updateParentAddress(final DictBuffer dictBuffer, + final int ptNodeOriginAddress, final int newParentAddress, + final FormatOptions formatOptions) { + final int originalPosition = dictBuffer.position(); + dictBuffer.position(ptNodeOriginAddress); + if (!formatOptions.mSupportsDynamicUpdate) { + throw new RuntimeException("this file format does not support parent addresses"); + } + final int flags = dictBuffer.readUnsignedByte(); + if (BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { + // If the node is moved, the parent address is stored in the destination node. + // We are guaranteed to process the destination node later, so there is no need to + // update anything here. + dictBuffer.position(originalPosition); + return; + } + if (DBG) { + MakedictLog.d("update parent address flags=" + flags + ", " + ptNodeOriginAddress); + } + final int parentOffset = newParentAddress - ptNodeOriginAddress; + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, parentOffset); + dictBuffer.position(originalPosition); + } + + /** + * Update parent addresses in a node array stored at ptNodeOriginAddress. + * + * @param dictBuffer the DictBuffer to be modified. + * @param ptNodeOriginAddress the address of the node array to update. + * @param newParentAddress the address to be written. + * @param formatOptions file format options. + */ + public static void updateParentAddresses(final DictBuffer dictBuffer, + final int ptNodeOriginAddress, final int newParentAddress, + final FormatOptions formatOptions) { + final int originalPosition = dictBuffer.position(); + dictBuffer.position(ptNodeOriginAddress); + do { + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + for (int i = 0; i < count; ++i) { + updateParentAddress(dictBuffer, dictBuffer.position(), newParentAddress, + formatOptions); + BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); + } + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); + dictBuffer.position(forwardLinkAddress); + } while (formatOptions.mSupportsDynamicUpdate + && dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); + dictBuffer.position(originalPosition); + } + + /** + * Update a children address in a PtNode that is addressed by ptNodeOriginAddress. + * + * @param dictBuffer the DictBuffer to write. + * @param ptNodeOriginAddress the address of the PtNode. + * @param newChildrenAddress the absolute address of the child. + * @param formatOptions file format options. + */ + public static void updateChildrenAddress(final DictBuffer dictBuffer, + final int ptNodeOriginAddress, final int newChildrenAddress, + final FormatOptions formatOptions) { + final int originalPosition = dictBuffer.position(); + dictBuffer.position(ptNodeOriginAddress); + final int flags = dictBuffer.readUnsignedByte(); + final int parentAddress = BinaryDictDecoderUtils.readParentAddress(dictBuffer, + formatOptions); + BinaryDictIOUtils.skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); + final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS + ? FormatSpec.NO_CHILDREN_ADDRESS : newChildrenAddress - dictBuffer.position(); + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, childrenOffset); + dictBuffer.position(originalPosition); + } + + /** + * Helper method to move a PtNode to the tail of the file. + */ + private static int movePtNode(final OutputStream destination, + final DictBuffer dictBuffer, final PtNodeInfo info, + final int nodeArrayOriginAddress, final int oldNodeAddress, + final FormatOptions formatOptions) throws IOException { + updateParentAddress(dictBuffer, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); + dictBuffer.position(oldNodeAddress); + final int currentFlags = dictBuffer.readUnsignedByte(); + dictBuffer.position(oldNodeAddress); + dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags + & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); + int size = FormatSpec.PTNODE_FLAGS_SIZE; + updateForwardLink(dictBuffer, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); + size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info }); + return size; + } + + @SuppressWarnings("unused") + private static void updateForwardLink(final DictBuffer dictBuffer, + final int nodeArrayOriginAddress, final int newNodeArrayAddress, + final FormatOptions formatOptions) { + dictBuffer.position(nodeArrayOriginAddress); + int jumpCount = 0; + while (jumpCount++ < MAX_JUMPS) { + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + for (int i = 0; i < count; ++i) { + BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); + } + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); + if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { + dictBuffer.position(dictBuffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeArrayAddress); + return; + } + dictBuffer.position(forwardLinkAddress); + } + if (DBG && jumpCount >= MAX_JUMPS) { + throw new RuntimeException("too many jumps, probably a bug."); + } + } + + /** + * Move a PtNode that is referred to by oldPtNodeOrigin to the tail of the file, and set the + * children address to the byte after the PtNode. + * + * @param fileEndAddress the address of the tail of the file. + * @param codePoints the characters to put inside the PtNode. + * @param length how many code points to read from codePoints. + * @param flags the flags for this PtNode. + * @param frequency the frequency of this terminal. + * @param parentAddress the address of the parent PtNode of this PtNode. + * @param shortcutTargets the shortcut targets for this PtNode. + * @param bigrams the bigrams for this PtNode. + * @param destination the stream representing the tail of the file. + * @param dictBuffer the DictBuffer representing the (constant-size) body of the file. + * @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of. + * @param oldPtNodeOrigin the old origin where this PtNode used to be stored. + * @param formatOptions format options for this dictionary. + * @return the size written, in bytes. + * @throws IOException if the file can't be accessed + */ + private static int movePtNode(final int fileEndAddress, final int[] codePoints, + final int length, final int flags, final int frequency, final int parentAddress, + final ArrayList<WeightedString> shortcutTargets, + final ArrayList<PendingAttribute> bigrams, final OutputStream destination, + final DictBuffer dictBuffer, final int oldPtNodeArrayOrigin, + final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException { + int size = 0; + final int newPtNodeOrigin = fileEndAddress + 1; + final int[] writtenCharacters = Arrays.copyOfRange(codePoints, 0, length); + final PtNodeInfo tmpInfo = new PtNodeInfo(newPtNodeOrigin, -1 /* endAddress */, + flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, + shortcutTargets, bigrams); + size = BinaryDictIOUtils.computePtNodeSize(tmpInfo, formatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newPtNodeOrigin, newPtNodeOrigin + size, + flags, writtenCharacters, frequency, parentAddress, + fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, + bigrams); + movePtNode(destination, dictBuffer, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, + formatOptions); + return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + + /** + * Insert a word into a binary dictionary. + * + * @param dictDecoder the dict decoder. + * @param destination a stream to the underlying file, with the pointer at the end of the file. + * @param word the word to insert. + * @param frequency the frequency of the new word. + * @param bigramStrings bigram list, or null if none. + * @param shortcuts shortcut list, or null if none. + * @param isBlackListEntry whether this should be a blacklist entry. + * @throws IOException if the file can't be accessed. + * @throws UnsupportedFormatException if the existing dictionary is in an unexpected format. + */ + // TODO: Support batch insertion. + // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. + @UsedForTesting + public static void insertWord(final Ver3DictDecoder dictDecoder, + final OutputStream destination, final String word, final int frequency, + final ArrayList<WeightedString> bigramStrings, + final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, + final boolean isBlackListEntry) + throws IOException, UnsupportedFormatException { + final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + if (bigramStrings != null) { + for (final WeightedString bigram : bigramStrings) { + int position = dictDecoder.getTerminalPosition(bigram.mWord); + if (position == FormatSpec.NOT_VALID_WORD) { + // TODO: figure out what is the correct thing to do here. + } else { + bigrams.add(new PendingAttribute(bigram.mFrequency, position)); + } + } + } + + final boolean isTerminal = true; + final boolean hasBigrams = !bigrams.isEmpty(); + final boolean hasShortcuts = shortcuts != null && !shortcuts.isEmpty(); + + // find the insert position of the word. + if (dictBuffer.position() != 0) dictBuffer.position(0); + final FileHeader fileHeader = dictDecoder.readHeader(); + + int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position(); + final int[] codePoints = FusionDictionary.getCodePoints(word); + final int wordLen = codePoints.length; + + for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { + if (wordPos >= wordLen) break; + nodeOriginAddress = dictBuffer.position(); + int nodeParentAddress = -1; + final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + boolean foundNextNode = false; + + for (int i = 0; i < ptNodeCount; ++i) { + address = dictBuffer.position(); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(address, + fileHeader.mFormatOptions); + final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, + fileHeader.mFormatOptions); + if (isMovedNode) continue; + nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) + ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; + boolean matched = true; + for (int p = 0; p < currentInfo.mCharacters.length; ++p) { + if (wordPos + p >= wordLen) { + /* + * splitting + * before + * abcd - ef + * + * insert "abc" + * + * after + * abc - d - ef + */ + final int newNodeAddress = dictBuffer.limit(); + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, + isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, + false /* isBlackListEntry */, fileHeader.mFormatOptions); + int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags, + frequency, nodeParentAddress, shortcuts, bigrams, destination, + dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions); + + final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, + currentInfo.mCharacters.length); + if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, + newNodeAddress + written + 1, fileHeader.mFormatOptions); + } + final PtNodeInfo newInfo2 = new PtNodeInfo( + newNodeAddress + written + 1, -1 /* endAddress */, + currentInfo.mFlags, characters2, currentInfo.mFrequency, + newNodeAddress + 1, currentInfo.mChildrenAddress, + currentInfo.mShortcutTargets, currentInfo.mBigrams); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo2 }); + return; + } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { + if (p > 0) { + /* + * splitting + * before + * ab - cd + * + * insert "ac" + * + * after + * a - b - cd + * | + * - c + */ + + final int newNodeAddress = dictBuffer.limit(); + final int childrenAddress = currentInfo.mChildrenAddress; + + // move prefix + final int prefixFlags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, + false /* isTerminal */, 0 /* childrenAddressSize*/, + false /* hasShortcut */, false /* hasBigrams */, + false /* isNotAWord */, false /* isBlackListEntry */, + fileHeader.mFormatOptions); + int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, + prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, + destination, dictBuffer, nodeOriginAddress, address, + fileHeader.mFormatOptions); + + final int[] suffixCharacters = Arrays.copyOfRange( + currentInfo.mCharacters, p, currentInfo.mCharacters.length); + if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, + newNodeAddress + written + 1, fileHeader.mFormatOptions); + } + final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags( + suffixCharacters.length > 1, + (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, + 0 /* childrenAddressSize */, + (currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) + != 0, + (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo suffixInfo = new PtNodeInfo( + newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, + suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, + currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, + currentInfo.mBigrams); + written += BinaryDictIOUtils.computePtNodeSize(suffixInfo, + fileHeader.mFormatOptions) + 1; + + final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, + codePoints.length); + final int flags = BinaryDictEncoderUtils.makePtNodeFlags( + newCharacters.length > 1, isTerminal, + 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo newInfo = new PtNodeInfo( + newNodeAddress + written, -1 /* endAddress */, flags, + newCharacters, frequency, newNodeAddress + 1, + FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); + BinaryDictIOUtils.writeNodes(destination, + new PtNodeInfo[] { suffixInfo, newInfo }); + return; + } + matched = false; + break; + } + } + + if (matched) { + if (wordPos + currentInfo.mCharacters.length == wordLen) { + // the word exists in the dictionary. + // only update the PtNode. + final int newNodeAddress = dictBuffer.limit(); + final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, + isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, + -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, + nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, + bigrams); + movePtNode(destination, dictBuffer, newInfo, nodeOriginAddress, address, + fileHeader.mFormatOptions); + return; + } + wordPos += currentInfo.mCharacters.length; + if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { + /* + * found the prefix of the word. + * make new PtNode and link to the PtNode from this PtNode. + * + * before + * ab - cd + * + * insert "abcde" + * + * after + * ab - cd - e + */ + final int newNodeArrayAddress = dictBuffer.limit(); + updateChildrenAddress(dictBuffer, address, newNodeArrayAddress, + fileHeader.mFormatOptions); + final int newNodeAddress = newNodeArrayAddress + 1; + final boolean hasMultipleChars = (wordLen - wordPos) > 1; + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, + isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress, -1, flags, + characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, + shortcuts, bigrams); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo }); + return; + } + dictBuffer.position(currentInfo.mChildrenAddress); + foundNextNode = true; + break; + } + } + + if (foundNextNode) continue; + + // reached the end of the array. + final int linkAddressPosition = dictBuffer.position(); + int nextLink = dictBuffer.readUnsignedInt24(); + if ((nextLink & FormatSpec.MSB24) != 0) { + nextLink = -(nextLink & FormatSpec.SINT24_MAX); + } + if (nextLink == FormatSpec.NO_FORWARD_LINK_ADDRESS) { + /* + * expand this node. + * + * before + * ab - cd + * + * insert "abef" + * + * after + * ab - cd + * | + * - ef + */ + + // change the forward link address. + final int newNodeAddress = dictBuffer.limit(); + dictBuffer.position(linkAddressPosition); + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); + + final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(characters.length > 1, + isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, + -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, + FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[]{ newInfo }); + return; + } else { + depth--; + dictBuffer.position(nextLink); + } + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 46266aa50..b8ef57696 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -26,21 +26,55 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions public final class FormatSpec { /* - * Array of Node(FusionDictionary.Node) layout is as follows: + * File header layout is as follows: * - * g | - * r | the number of groups, 1 or 2 bytes. - * o | 1 byte = bbbbbbbb match - * u | case 1xxxxxxx => xxxxxxx << 8 + next byte - * p | otherwise => bbbbbbbb - * c | - * ount + * v | + * e | MAGIC_NUMBER + version of the file format, 2 bytes. + * r | + * sion * - * g | - * r | sequence of groups, - * o | the layout of each group is described below. - * u | - * ps + * o | + * p | not used 4 bits + * t | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG + * i | FRENCH_LIGATURE_PROCESSING_FLAG + * o | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE + * n | GERMAN_UMLAUT_PROCESSING_FLAG + * f | + * lags + * + * h | + * e | size of the file header, 4bytes + * a | including the size of the magic number, the option flags and the header size + * d | + * ersize + * + * | attributes list + * + * attributes list is: + * <key> = | string of characters at the char format described below, with the terminator used + * | to signal the end of the string. + * <value> = | string of characters at the char format described below, with the terminator used + * | to signal the end of the string. + * if the size of already read < headersize, goto key. + * + */ + + /* + * Node array (FusionDictionary.PtNodeArray) layout is as follows: + * + * n | + * o | the number of PtNodes, 1 or 2 bytes. + * d | 1 byte = bbbbbbbb match + * e | case 1xxxxxxx => xxxxxxx << 8 + next byte + * c | otherwise => bbbbbbbb + * o | + * unt + * + * n | + * o | sequence of PtNodes, + * d | the layout of each PtNode is described below. + * e | + * s * * f | * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) @@ -52,19 +86,19 @@ public final class FormatSpec { * linkaddress */ - /* Node(CharGroup) layout is as follows: + /* Node (FusionDictionary.PtNode) layout is as follows: * | IF !SUPPORTS_DYNAMIC_UPDATE - * | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE - * | 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS - * f | 01 = 1 byte : FLAG_GROUP_ADDRESS_TYPE_ONEBYTE - * l | 10 = 2 bytes : FLAG_GROUP_ADDRESS_TYPE_TWOBYTES - * a | 11 = 3 bytes : FLAG_GROUP_ADDRESS_TYPE_THREEBYTES + * | addressType xx : mask with MASK_CHILDREN_ADDRESS_TYPE + * | 2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS + * f | 01 = 1 byte : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE + * l | 10 = 2 bytes : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES + * a | 11 = 3 bytes : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES * g | ELSE - * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED - * | This must be the same as FLAG_GROUP_ADDRESS_TYPE_THREEBYTES - * | 01 = yes : FLAG_IS_MOVED + * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED + * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 01 = yes : FLAG_IS_MOVED * | the new address is stored in the same place as the parent address - * | is deleted? 10 = yes : FLAG_IS_DELETED + * | is deleted? 10 = yes : FLAG_IS_DELETED * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS @@ -82,7 +116,7 @@ public final class FormatSpec { * ddress * * c | IF FLAG_HAS_MULTIPLE_CHARS - * h | char, char, char, char n * (1 or 3 bytes) : use CharGroupInfo for i/o helpers + * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers * a | end 1 byte, = 0 * r | ELSE * s | char 1 or 3 bytes @@ -93,17 +127,22 @@ public final class FormatSpec { * e | frequency 1 byte * q | * - * c | IF 00 = FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = addressType - * h | // nothing - * i | ELSIF 01 = FLAG_GROUP_ADDRESS_TYPE_ONEBYTE == addressType - * l | children address, 1 byte - * d | ELSIF 10 = FLAG_GROUP_ADDRESS_TYPE_TWOBYTES == addressType - * r | children address, 2 bytes - * e | ELSE // 11 = FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = addressType - * n | children address, 3 bytes - * A | END - * d - * dress + * c | IF SUPPORTS_DYNAMIC_UPDATE + * h | children address, 3 bytes + * i | 1 byte = bbbbbbbb match + * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte + * r | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType + * e | // nothing + * n | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType + * A | children address, 1 byte + * d | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType + * d | children address, 2 bytes + * r | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType + * e | children address, 3 bytes + * s | END + * s + * ress * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list @@ -122,41 +161,39 @@ public final class FormatSpec { * characters which should never happen anyway (and still work, but take 3 bytes). * * bigram address list is: - * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT - * | addressSign = 1 bit, : FLAG_ATTRIBUTE_OFFSET_NEGATIVE + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT + * | addressSign = 1 bit, : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE * | 1 = must take -address, 0 = must take +address - * | xx : mask with MASK_ATTRIBUTE_ADDRESS_TYPE - * | addressFormat = 2 bits, 00 = unused : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE - * | 01 = 1 byte : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE - * | 10 = 2 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES - * | 11 = 3 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES - * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY - * <address> | IF (01 == FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE == addressFormat) + * | xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE + * | addressFormat = 2 bits, 00 = unused : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 01 = 1 byte : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES + * | 11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY + * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat) * | read 1 byte, add top 4 bits - * | ELSIF (10 == FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES == addressFormat) + * | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat) * | read 2 bytes, add top 4 bits - * | ELSE // 11 == FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES == addressFormat + * | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat * | read 3 bytes, add top 4 bits * | END - * | if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE) then address = -address - * if (FLAG_ATTRIBUTE_HAS_NEXT) goto bigram_and_shortcut_address_list_is + * | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is * * shortcut string list is: - * <byte size> = GROUP_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. - * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT + * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT * | reserved = 3 bits, must be 0 - * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY * <shortcut> = | string of characters at the char format described above, with the terminator * | used to signal the end of the string. - * if (FLAG_ATTRIBUTE_HAS_NEXT goto flags + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags */ - static final int VERSION_1_MAGIC_NUMBER = 0x78B1; - public static final int VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; - static final int MINIMUM_SUPPORTED_VERSION = 1; + public static final int MAGIC_NUMBER = 0x9BC13AFE; + static final int MINIMUM_SUPPORTED_VERSION = 2; static final int MAXIMUM_SUPPORTED_VERSION = 3; static final int NOT_A_VERSION_NUMBER = -1; - static final int FIRST_VERSION_WITH_HEADER_SIZE = 2; static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; // These options need to be the same numeric values as the one in the native reading code. @@ -174,11 +211,11 @@ public final class FormatSpec { static final int FORWARD_LINK_ADDRESS_SIZE = 3; // These flags are used only in the static dictionary. - static final int MASK_GROUP_ADDRESS_TYPE = 0xC0; - static final int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; - static final int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; - static final int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; - static final int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; + static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0; + static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00; + static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40; + static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80; + static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0; static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; @@ -195,32 +232,32 @@ public final class FormatSpec { static final int FLAG_IS_NOT_MOVED = 0x80 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE; static final int FLAG_IS_DELETED = 0x80; - static final int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; - static final int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; - static final int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; - static final int FLAG_ATTRIBUTE_FREQUENCY = 0x0F; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80; + static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40; + static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F; - static final int GROUP_CHARACTERS_TERMINATOR = 0x1F; + static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F; - static final int GROUP_TERMINATOR_SIZE = 1; - static final int GROUP_FLAGS_SIZE = 1; - static final int GROUP_FREQUENCY_SIZE = 1; - static final int GROUP_MAX_ADDRESS_SIZE = 3; - static final int GROUP_ATTRIBUTE_FLAGS_SIZE = 1; - static final int GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; - static final int GROUP_SHORTCUT_LIST_SIZE_SIZE = 2; + static final int PTNODE_TERMINATOR_SIZE = 1; + static final int PTNODE_FLAGS_SIZE = 1; + static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; + static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; static final int INVALID_CHARACTER = -1; - static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127 - static final int MAX_CHARGROUPS_IN_A_NODE = 0x7FFF; // 32767 - static final int MAX_BIGRAMS_IN_A_GROUP = 10000; + static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 + static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 + static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; @@ -231,6 +268,13 @@ public final class FormatSpec { static final int NOT_VALID_WORD = -99; static final int SIGNED_CHILDREN_ADDRESS_SIZE = 3; + static final int UINT8_MAX = 0xFF; + static final int UINT16_MAX = 0xFFFF; + static final int UINT24_MAX = 0xFFFFFF; + static final int SINT24_MAX = 0x7FFFFF; + static final int MSB8 = 0x80; + static final int MSB24 = 0x800000; + /** * Options about file format. */ diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 118dc22b8..3e685a3d7 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -37,15 +37,15 @@ public final class FusionDictionary implements Iterable<Word> { private static int CHARACTER_NOT_FOUND_INDEX = -1; /** - * A node of the dictionary, containing several CharGroups. + * A node array of the dictionary, containing several PtNodes. * - * A node is but an ordered array of CharGroups, which essentially contain all the + * A PtNodeArray is but an ordered array of PtNodes, which essentially contain all the * real information. * This class also contains fields to cache size and address, to help with binary * generation. */ - public static final class Node { - ArrayList<CharGroup> mData; + public static final class PtNodeArray { + ArrayList<PtNode> mData; // To help with binary generation int mCachedSize = Integer.MIN_VALUE; // mCachedAddressBefore/AfterUpdate are helpers for binary dictionary generation. They @@ -57,10 +57,10 @@ public final class FusionDictionary implements Iterable<Word> { int mCachedAddressAfterUpdate = Integer.MIN_VALUE; int mCachedParentAddress = 0; - public Node() { - mData = new ArrayList<CharGroup>(); + public PtNodeArray() { + mData = new ArrayList<PtNode>(); } - public Node(ArrayList<CharGroup> data) { + public PtNodeArray(ArrayList<PtNode> data) { mData = data; } } @@ -93,24 +93,25 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * A group of characters, with a frequency, shortcut targets, bigrams, and children. + * PtNode is a group of characters, with a frequency, shortcut targets, bigrams, and children + * (Pt means Patricia Trie). * - * This is the central class of the in-memory representation. A CharGroup is what can + * This is the central class of the in-memory representation. A PtNode is what can * be seen as a traditional "trie node", except it can hold several characters at the - * same time. A CharGroup essentially represents one or several characters in the middle - * of the trie trie; as such, it can be a terminal, and it can have children. - * In this in-memory representation, whether the CharGroup is a terminal or not is represented + * same time. A PtNode essentially represents one or several characters in the middle + * of the trie tree; as such, it can be a terminal, and it can have children. + * In this in-memory representation, whether the PtNode is a terminal or not is represented * in the frequency, where NOT_A_TERMINAL (= -1) means this is not a terminal and any other * value is the frequency of this terminal. A terminal may have non-null shortcuts and/or * bigrams, but a non-terminal may not. Moreover, children, if present, are null. */ - public static final class CharGroup { + public static final class PtNode { public static final int NOT_A_TERMINAL = -1; final int mChars[]; ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal. - Node mChildren; + PtNodeArray mChildren; boolean mIsNotAWord; // Only a shortcut boolean mIsBlacklistEntry; // mCachedSize and mCachedAddressBefore/AfterUpdate are helpers for binary dictionary @@ -119,11 +120,11 @@ public final class FusionDictionary implements Iterable<Word> { // same time. Updating will update the AfterUpdate value, and the code will move them // to BeforeUpdate before the next update pass. // The update process does not need two versions of mCachedSize. - int mCachedSize; // The size, in bytes, of this char group. - int mCachedAddressBeforeUpdate; // The address of this char group (before update) - int mCachedAddressAfterUpdate; // The address of this char group (after update) + int mCachedSize; // The size, in bytes, of this PtNode. + int mCachedAddressBeforeUpdate; // The address of this PtNode (before update) + int mCachedAddressAfterUpdate; // The address of this PtNode (after update) - public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, final boolean isNotAWord, final boolean isBlacklistEntry) { mChars = chars; @@ -135,9 +136,10 @@ public final class FusionDictionary implements Iterable<Word> { mIsBlacklistEntry = isBlacklistEntry; } - public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, - final boolean isNotAWord, final boolean isBlacklistEntry, final Node children) { + final boolean isNotAWord, final boolean isBlacklistEntry, + final PtNodeArray children) { mChars = chars; mFrequency = frequency; mShortcutTargets = shortcutTargets; @@ -147,9 +149,9 @@ public final class FusionDictionary implements Iterable<Word> { mIsBlacklistEntry = isBlacklistEntry; } - public void addChild(CharGroup n) { + public void addChild(PtNode n) { if (null == mChildren) { - mChildren = new Node(); + mChildren = new PtNodeArray(); } mChildren.mData.add(n); } @@ -244,7 +246,7 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Updates the CharGroup with the given properties. Adds the shortcut and bigram lists to + * Updates the PtNode with the given properties. Adds the shortcut and bigram lists to * the existing ones if any. Note: unigram, bigram, and shortcut frequencies are only * updated if they are higher than the existing ones. */ @@ -344,10 +346,10 @@ public final class FusionDictionary implements Iterable<Word> { } public final DictionaryOptions mOptions; - public final Node mRoot; + public final PtNodeArray mRootNodeArray; - public FusionDictionary(final Node root, final DictionaryOptions options) { - mRoot = root; + public FusionDictionary(final PtNodeArray rootNodeArray, final DictionaryOptions options) { + mRootNodeArray = rootNodeArray; mOptions = options; } @@ -406,13 +408,13 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Sanity check for a node. + * Sanity check for a PtNode array. * - * This method checks that all CharGroups in a node are ordered as expected. + * This method checks that all PtNodes in a node array are ordered as expected. * If they are, nothing happens. If they aren't, an exception is thrown. */ - private void checkStack(Node node) { - ArrayList<CharGroup> stack = node.mData; + private void checkStack(PtNodeArray ptNodeArray) { + ArrayList<PtNode> stack = ptNodeArray.mData; int lastValue = -1; for (int i = 0; i < stack.size(); ++i) { int currentValue = stack.get(i).mChars[0]; @@ -431,18 +433,18 @@ public final class FusionDictionary implements Iterable<Word> { * @param frequency the bigram frequency */ public void setBigram(final String word1, final String word2, final int frequency) { - CharGroup charGroup = findWordInTree(mRoot, word1); - if (charGroup != null) { - final CharGroup charGroup2 = findWordInTree(mRoot, word2); - if (charGroup2 == null) { + PtNode ptNode = findWordInTree(mRootNodeArray, word1); + if (ptNode != null) { + final PtNode ptNode2 = findWordInTree(mRootNodeArray, word2); + if (ptNode2 == null) { add(getCodePoints(word2), 0, null, false /* isNotAWord */, false /* isBlacklistEntry */); - // The chargroup for the first word may have moved by the above insertion, + // The PtNode for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been - // a cutting point until now. In this case, we need to refresh charGroup. - charGroup = findWordInTree(mRoot, word1); + // a cutting point until now. In this case, we need to refresh ptNode. + ptNode = findWordInTree(mRootNodeArray, word1); } - charGroup.addBigram(word2, frequency); + ptNode.addBigram(word2, frequency); } else { throw new RuntimeException("First word of bigram not found"); } @@ -469,92 +471,91 @@ public final class FusionDictionary implements Iterable<Word> { return; } - Node currentNode = mRoot; + PtNodeArray currentNodeArray = mRootNodeArray; int charIndex = 0; - CharGroup currentGroup = null; + PtNode currentPtNode = null; int differentCharIndex = 0; // Set by the loop to the index of the char that differs - int nodeIndex = findIndexOfChar(mRoot, word[charIndex]); + int nodeIndex = findIndexOfChar(mRootNodeArray, word[charIndex]); while (CHARACTER_NOT_FOUND_INDEX != nodeIndex) { - currentGroup = currentNode.mData.get(nodeIndex); - differentCharIndex = compareArrays(currentGroup.mChars, word, charIndex); + currentPtNode = currentNodeArray.mData.get(nodeIndex); + differentCharIndex = compareCharArrays(currentPtNode.mChars, word, charIndex); if (ARRAYS_ARE_EQUAL != differentCharIndex - && differentCharIndex < currentGroup.mChars.length) break; - if (null == currentGroup.mChildren) break; - charIndex += currentGroup.mChars.length; + && differentCharIndex < currentPtNode.mChars.length) break; + if (null == currentPtNode.mChildren) break; + charIndex += currentPtNode.mChars.length; if (charIndex >= word.length) break; - currentNode = currentGroup.mChildren; - nodeIndex = findIndexOfChar(currentNode, word[charIndex]); + currentNodeArray = currentPtNode.mChildren; + nodeIndex = findIndexOfChar(currentNodeArray, word[charIndex]); } if (CHARACTER_NOT_FOUND_INDEX == nodeIndex) { // No node at this point to accept the word. Create one. - final int insertionIndex = findInsertionIndex(currentNode, word[charIndex]); - final CharGroup newGroup = new CharGroup( - Arrays.copyOfRange(word, charIndex, word.length), + final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); + final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentNode.mData.add(insertionIndex, newGroup); - if (DBG) checkStack(currentNode); + currentNodeArray.mData.add(insertionIndex, newPtNode); + if (DBG) checkStack(currentNodeArray); } else { // There is a word with a common prefix. - if (differentCharIndex == currentGroup.mChars.length) { + if (differentCharIndex == currentPtNode.mChars.length) { if (charIndex + differentCharIndex >= word.length) { // The new word is a prefix of an existing word, but the node on which it - // should end already exists as is. Since the old CharNode was not a terminal, + // should end already exists as is. Since the old PtNode was not a terminal, // make it one by filling in its frequency and other attributes - currentGroup.update(frequency, shortcutTargets, null, isNotAWord, + currentPtNode.update(frequency, shortcutTargets, null, isNotAWord, isBlacklistEntry); } else { // The new word matches the full old word and extends past it. // We only have to create a new node and add it to the end of this. - final CharGroup newNode = new CharGroup( + final PtNode newNode = new PtNode( Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentGroup.mChildren = new Node(); - currentGroup.mChildren.mData.add(newNode); + currentPtNode.mChildren = new PtNodeArray(); + currentPtNode.mChildren.mData.add(newNode); } } else { if (0 == differentCharIndex) { // Exact same word. Update the frequency if higher. This will also add the // new shortcuts to the existing shortcut list if it already exists. - currentGroup.update(frequency, shortcutTargets, null, - currentGroup.mIsNotAWord && isNotAWord, - currentGroup.mIsBlacklistEntry || isBlacklistEntry); + currentPtNode.update(frequency, shortcutTargets, null, + currentPtNode.mIsNotAWord && isNotAWord, + currentPtNode.mIsBlacklistEntry || isBlacklistEntry); } else { // Partial prefix match only. We have to replace the current node with a node // containing the current prefix and create two new ones for the tails. - Node newChildren = new Node(); - final CharGroup newOldWord = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, differentCharIndex, - currentGroup.mChars.length), currentGroup.mShortcutTargets, - currentGroup.mBigrams, currentGroup.mFrequency, - currentGroup.mIsNotAWord, currentGroup.mIsBlacklistEntry, - currentGroup.mChildren); + PtNodeArray newChildren = new PtNodeArray(); + final PtNode newOldWord = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex, + currentPtNode.mChars.length), currentPtNode.mShortcutTargets, + currentPtNode.mBigrams, currentPtNode.mFrequency, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry, + currentPtNode.mChildren); newChildren.mData.add(newOldWord); - final CharGroup newParent; + final PtNode newParent; if (charIndex + differentCharIndex >= word.length) { - newParent = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, 0, differentCharIndex), + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry, newChildren); } else { - newParent = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, 0, differentCharIndex), + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), null /* shortcutTargets */, null /* bigrams */, -1, false /* isNotAWord */, false /* isBlacklistEntry */, newChildren); - final CharGroup newWord = new CharGroup(Arrays.copyOfRange(word, + final PtNode newWord = new PtNode(Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); final int addIndex = word[charIndex + differentCharIndex] - > currentGroup.mChars[differentCharIndex] ? 1 : 0; + > currentPtNode.mChars[differentCharIndex] ? 1 : 0; newChildren.mData.add(addIndex, newWord); } - currentNode.mData.set(nodeIndex, newParent); + currentNodeArray.mData.set(nodeIndex, newParent); } - if (DBG) checkStack(currentNode); + if (DBG) checkStack(currentNodeArray); } } } @@ -576,7 +577,7 @@ public final class FusionDictionary implements Iterable<Word> { * @param dstOffset the offset in the right-hand side string. * @return the index at which the strings differ, or ARRAYS_ARE_EQUAL = 0 if they don't. */ - private static int compareArrays(final int[] src, final int[] dst, int dstOffset) { + private static int compareCharArrays(final int[] src, final int[] dst, int dstOffset) { // We do NOT test the first char, because we come from a method that already // tested it. for (int i = 1; i < src.length; ++i) { @@ -588,43 +589,43 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Helper class that compares and sorts two chargroups according to their + * Helper class that compares and sorts two PtNodes according to their * first element only. I repeat: ONLY the first element is considered, the rest * is ignored. * This comparator imposes orderings that are inconsistent with equals. */ - static private final class CharGroupComparator implements java.util.Comparator<CharGroup> { + static private final class PtNodeComparator implements java.util.Comparator<PtNode> { @Override - public int compare(CharGroup c1, CharGroup c2) { - if (c1.mChars[0] == c2.mChars[0]) return 0; - return c1.mChars[0] < c2.mChars[0] ? -1 : 1; + public int compare(PtNode p1, PtNode p2) { + if (p1.mChars[0] == p2.mChars[0]) return 0; + return p1.mChars[0] < p2.mChars[0] ? -1 : 1; } } - final static private CharGroupComparator CHARGROUP_COMPARATOR = new CharGroupComparator(); + final static private PtNodeComparator PTNODE_COMPARATOR = new PtNodeComparator(); /** - * Finds the insertion index of a character within a node. + * Finds the insertion index of a character within a node array. */ - private static int findInsertionIndex(final Node node, int character) { - final ArrayList<CharGroup> data = node.mData; - final CharGroup reference = new CharGroup(new int[] { character }, + private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { + final ArrayList<PtNode> data = nodeArray.mData; + final PtNode reference = new PtNode(new int[] { character }, null /* shortcutTargets */, null /* bigrams */, 0, false /* isNotAWord */, false /* isBlacklistEntry */); - int result = Collections.binarySearch(data, reference, CHARGROUP_COMPARATOR); + int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR); return result >= 0 ? result : -result - 1; } /** - * Find the index of a char in a node, if it exists. + * Find the index of a char in a node array, if it exists. * - * @param node the node to search in. + * @param nodeArray the node array to search in. * @param character the character to search for. * @return the position of the character if it's there, or CHARACTER_NOT_FOUND_INDEX = -1 else. */ - private static int findIndexOfChar(final Node node, int character) { - final int insertionIndex = findInsertionIndex(node, character); - if (node.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; - return character == node.mData.get(insertionIndex).mChars[0] ? insertionIndex + private static int findIndexOfChar(final PtNodeArray nodeArray, int character) { + final int insertionIndex = findInsertionIndex(nodeArray, character); + if (nodeArray.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; + return character == nodeArray.mData.get(insertionIndex).mChars[0] ? insertionIndex : CHARACTER_NOT_FOUND_INDEX; } @@ -632,35 +633,37 @@ public final class FusionDictionary implements Iterable<Word> { * Helper method to find a word in a given branch. */ @SuppressWarnings("unused") - public static CharGroup findWordInTree(Node node, final String string) { + public static PtNode findWordInTree(PtNodeArray nodeArray, final String string) { int index = 0; final StringBuilder checker = DBG ? new StringBuilder() : null; final int[] codePoints = getCodePoints(string); - CharGroup currentGroup; + PtNode currentPtNode; do { - int indexOfGroup = findIndexOfChar(node, codePoints[index]); + int indexOfGroup = findIndexOfChar(nodeArray, codePoints[index]); if (CHARACTER_NOT_FOUND_INDEX == indexOfGroup) return null; - currentGroup = node.mData.get(indexOfGroup); + currentPtNode = nodeArray.mData.get(indexOfGroup); - if (codePoints.length - index < currentGroup.mChars.length) return null; + if (codePoints.length - index < currentPtNode.mChars.length) return null; int newIndex = index; - while (newIndex < codePoints.length && newIndex - index < currentGroup.mChars.length) { - if (currentGroup.mChars[newIndex - index] != codePoints[newIndex]) return null; + while (newIndex < codePoints.length && newIndex - index < currentPtNode.mChars.length) { + if (currentPtNode.mChars[newIndex - index] != codePoints[newIndex]) return null; newIndex++; } index = newIndex; - if (DBG) checker.append(new String(currentGroup.mChars, 0, currentGroup.mChars.length)); + if (DBG) { + checker.append(new String(currentPtNode.mChars, 0, currentPtNode.mChars.length)); + } if (index < codePoints.length) { - node = currentGroup.mChildren; + nodeArray = currentPtNode.mChildren; } - } while (null != node && index < codePoints.length); + } while (null != nodeArray && index < codePoints.length); if (index < codePoints.length) return null; - if (!currentGroup.isTerminal()) return null; + if (!currentPtNode.isTerminal()) return null; if (DBG && !string.equals(checker.toString())) return null; - return currentGroup; + return currentPtNode; } /** @@ -670,22 +673,22 @@ public final class FusionDictionary implements Iterable<Word> { if (null == s || "".equals(s)) { throw new RuntimeException("Can't search for a null or empty string"); } - return null != findWordInTree(mRoot, s); + return null != findWordInTree(mRootNodeArray, s); } /** - * Recursively count the number of character groups in a given branch of the trie. + * Recursively count the number of PtNodes in a given branch of the trie. * - * @param node the parent node. - * @return the number of char groups in all the branch under this node. + * @param nodeArray the parent node. + * @return the number of PtNodes in all the branch under this node. */ - public static int countCharGroups(final Node node) { - final int nodeSize = node.mData.size(); + public static int countPtNodes(final PtNodeArray nodeArray) { + final int nodeSize = nodeArray.mData.size(); int size = nodeSize; for (int i = nodeSize - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); - if (null != group.mChildren) - size += countCharGroups(group.mChildren); + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countPtNodes(ptNode.mChildren); } return size; } @@ -693,15 +696,15 @@ public final class FusionDictionary implements Iterable<Word> { /** * Recursively count the number of nodes in a given branch of the trie. * - * @param node the node to count. + * @param nodeArray the node array to count. * @return the number of nodes in this branch. */ - public static int countNodes(final Node node) { + public static int countNodeArrays(final PtNodeArray nodeArray) { int size = 1; - for (int i = node.mData.size() - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); - if (null != group.mChildren) - size += countNodes(group.mChildren); + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countNodeArrays(ptNode.mChildren); } return size; } @@ -709,12 +712,12 @@ public final class FusionDictionary implements Iterable<Word> { // Recursively find out whether there are any bigrams. // This can be pretty expensive especially if there aren't any (we return as soon // as we find one, so it's much cheaper if there are bigrams) - private static boolean hasBigramsInternal(final Node node) { - if (null == node) return false; - for (int i = node.mData.size() - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); - if (null != group.mBigrams) return true; - if (hasBigramsInternal(group.mChildren)) return true; + private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { + if (null == nodeArray) return false; + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mBigrams) return true; + if (hasBigramsInternal(ptNode.mChildren)) return true; } return false; } @@ -729,7 +732,7 @@ public final class FusionDictionary implements Iterable<Word> { // find a more efficient way of doing this, without compromising too much on memory // and ease of use. public boolean hasBigrams() { - return hasBigramsInternal(mRoot); + return hasBigramsInternal(mRootNodeArray); } // Historically, the tails of the words were going to be merged to save space. @@ -747,16 +750,16 @@ public final class FusionDictionary implements Iterable<Word> { MakedictLog.i("Do not merge tails"); return; -// MakedictLog.i("Merging nodes. Number of nodes : " + countNodes(root)); -// MakedictLog.i("Number of groups : " + countCharGroups(root)); +// MakedictLog.i("Merging PtNodes. Number of PtNodes : " + countPtNodes(root)); +// MakedictLog.i("Number of PtNodes : " + countPtNodes(root)); // -// final HashMap<String, ArrayList<Node>> repository = -// new HashMap<String, ArrayList<Node>>(); +// final HashMap<String, ArrayList<PtNodeArray>> repository = +// new HashMap<String, ArrayList<PtNodeArray>>(); // mergeTailsInner(repository, root); // // MakedictLog.i("Number of different pseudohashes : " + repository.size()); // int size = 0; -// for (ArrayList<Node> a : repository.values()) { +// for (ArrayList<PtNodeArray> a : repository.values()) { // size += a.size(); // } // MakedictLog.i("Number of nodes after merge : " + (1 + size)); @@ -764,58 +767,58 @@ public final class FusionDictionary implements Iterable<Word> { } // The following methods are used by the deactivated mergeTails() -// private static boolean isEqual(Node a, Node b) { +// private static boolean isEqual(PtNodeArray a, PtNodeArray b) { // if (null == a && null == b) return true; // if (null == a || null == b) return false; // if (a.data.size() != b.data.size()) return false; // final int size = a.data.size(); // for (int i = size - 1; i >= 0; --i) { -// CharGroup aGroup = a.data.get(i); -// CharGroup bGroup = b.data.get(i); -// if (aGroup.frequency != bGroup.frequency) return false; -// if (aGroup.alternates == null && bGroup.alternates != null) return false; -// if (aGroup.alternates != null && !aGroup.equals(bGroup.alternates)) return false; -// if (!Arrays.equals(aGroup.chars, bGroup.chars)) return false; -// if (!isEqual(aGroup.children, bGroup.children)) return false; +// PtNode aPtNode = a.data.get(i); +// PtNode bPtNode = b.data.get(i); +// if (aPtNode.frequency != bPtNode.frequency) return false; +// if (aPtNode.alternates == null && bPtNode.alternates != null) return false; +// if (aPtNode.alternates != null && !aPtNode.equals(bPtNode.alternates)) return false; +// if (!Arrays.equals(aPtNode.chars, bPtNode.chars)) return false; +// if (!isEqual(aPtNode.children, bPtNode.children)) return false; // } // return true; // } -// static private HashMap<String, ArrayList<Node>> mergeTailsInner( -// final HashMap<String, ArrayList<Node>> map, final Node node) { -// final ArrayList<CharGroup> branches = node.data; +// static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner( +// final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) { +// final ArrayList<PtNode> branches = nodeArray.data; // final int nodeSize = branches.size(); // for (int i = 0; i < nodeSize; ++i) { -// CharGroup group = branches.get(i); -// if (null != group.children) { -// String pseudoHash = getPseudoHash(group.children); -// ArrayList<Node> similarList = map.get(pseudoHash); +// PtNode ptNode = branches.get(i); +// if (null != ptNode.children) { +// String pseudoHash = getPseudoHash(ptNode.children); +// ArrayList<PtNodeArray> similarList = map.get(pseudoHash); // if (null == similarList) { -// similarList = new ArrayList<Node>(); +// similarList = new ArrayList<PtNodeArray>(); // map.put(pseudoHash, similarList); // } // boolean merged = false; -// for (Node similar : similarList) { -// if (isEqual(group.children, similar)) { -// group.children = similar; +// for (PtNodeArray similar : similarList) { +// if (isEqual(ptNode.children, similar)) { +// ptNode.children = similar; // merged = true; // break; // } // } // if (!merged) { -// similarList.add(group.children); +// similarList.add(ptNode.children); // } -// mergeTailsInner(map, group.children); +// mergeTailsInner(map, ptNode.children); // } // } // return map; // } -// private static String getPseudoHash(final Node node) { +// private static String getPseudoHash(final PtNodeArray nodeArray) { // StringBuilder s = new StringBuilder(); -// for (CharGroup g : node.data) { -// s.append(g.frequency); -// for (int ch : g.chars) { +// for (PtNode ptNode : nodeArray.data) { +// s.append(ptNode.frequency); +// for (int ch : ptNode.chars) { // s.append(Character.toChars(ch)); // } // } @@ -829,20 +832,20 @@ public final class FusionDictionary implements Iterable<Word> { */ public static final class DictionaryIterator implements Iterator<Word> { private static final class Position { - public Iterator<CharGroup> pos; + public Iterator<PtNode> pos; public int length; - public Position(ArrayList<CharGroup> groups) { - pos = groups.iterator(); + public Position(ArrayList<PtNode> ptNodes) { + pos = ptNodes.iterator(); length = 0; } } final StringBuilder mCurrentString; final LinkedList<Position> mPositions; - public DictionaryIterator(ArrayList<CharGroup> root) { + public DictionaryIterator(ArrayList<PtNode> ptRoot) { mCurrentString = new StringBuilder(); mPositions = new LinkedList<Position>(); - final Position rootPos = new Position(root); + final Position rootPos = new Position(ptRoot); mPositions.add(rootPos); } @@ -863,20 +866,20 @@ public final class FusionDictionary implements Iterable<Word> { do { if (currentPos.pos.hasNext()) { - final CharGroup currentGroup = currentPos.pos.next(); + final PtNode currentPtNode = currentPos.pos.next(); currentPos.length = mCurrentString.length(); - for (int i : currentGroup.mChars) { + for (int i : currentPtNode.mChars) { mCurrentString.append(Character.toChars(i)); } - if (null != currentGroup.mChildren) { - currentPos = new Position(currentGroup.mChildren.mData); + if (null != currentPtNode.mChildren) { + currentPos = new Position(currentPtNode.mChildren.mData); currentPos.length = mCurrentString.length(); mPositions.addLast(currentPos); } - if (currentGroup.mFrequency >= 0) { - return new Word(mCurrentString.toString(), currentGroup.mFrequency, - currentGroup.mShortcutTargets, currentGroup.mBigrams, - currentGroup.mIsNotAWord, currentGroup.mIsBlacklistEntry); + if (currentPtNode.mFrequency >= 0) { + return new Word(mCurrentString.toString(), currentPtNode.mFrequency, + currentPtNode.mShortcutTargets, currentPtNode.mBigrams, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry); } } else { mPositions.removeLast(); @@ -901,6 +904,6 @@ public final class FusionDictionary implements Iterable<Word> { */ @Override public Iterator<Word> iterator() { - return new DictionaryIterator(mRoot.mData); + return new DictionaryIterator(mRootNodeArray.mData); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/CharGroupInfo.java b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java index b3617443e..188de7a0f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/CharGroupInfo.java +++ b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java @@ -21,9 +21,9 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.util.ArrayList; /** - * Raw char group info straight out of a file. This will contain numbers for addresses. + * Raw PtNode info straight out of a file. This will contain numbers for addresses. */ -public final class CharGroupInfo { +public final class PtNodeInfo { public final int mOriginalAddress; public final int mEndAddress; @@ -35,7 +35,7 @@ public final class CharGroupInfo { public final ArrayList<WeightedString> mShortcutTargets; public final ArrayList<PendingAttribute> mBigrams; - public CharGroupInfo(final int originalAddress, final int endAddress, final int flags, + public PtNodeInfo(final int originalAddress, final int endAddress, final int flags, final int[] characters, final int frequency, final int parentAddress, final int childrenAddress, final ArrayList<WeightedString> shortcutTargets, final ArrayList<PendingAttribute> bigrams) { diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java new file mode 100644 index 000000000..1a5023ef6 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -0,0 +1,350 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.JniUtils; + +import android.util.Log; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.TreeMap; + +/** + * An implementation of DictDecoder for version 3 binary dictionary. + */ +@UsedForTesting +public class Ver3DictDecoder implements DictDecoder { + private static final String TAG = Ver3DictDecoder.class.getSimpleName(); + + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + + private final static class HeaderReader { + protected static int readVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); + } + + protected static int readOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedShort(); + } + + protected static int readHeaderSize(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + + protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, + final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (dictBuffer.position() < headerSize) { + // We can avoid an infinite loop here since dictBuffer.position() is always + // increased by calling CharEncoding.readString. + final String key = CharEncoding.readString(dictBuffer); + final String value = CharEncoding.readString(dictBuffer); + attributes.put(key, value); + } + dictBuffer.position(headerSize); + return attributes; + } + } + + private final static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readFrequency(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigrams(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + final int offset = (dictBuffer.readUnsignedByte() << 16) + + dictBuffer.readUnsignedShort(); + bigramAddress += sign * offset; + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return readLength; + } + } + + private final File mDictionaryBinaryFile; + private final DictionaryBufferFactory mBufferFactory; + private DictBuffer mDictBuffer; + + public Ver3DictDecoder(final File file) { + this(file, USE_READONLY_BYTEBUFFER); + } + + public Ver3DictDecoder(final File file, final int factoryFlag) { + mDictionaryBinaryFile = file; + mDictBuffer = null; + + if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { + mBufferFactory = new DictionaryBufferFromByteArrayFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); + } else { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } + } + + public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { + mDictionaryBinaryFile = file; + mBufferFactory = factory; + } + + public void openDictBuffer() throws FileNotFoundException, IOException { + mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); + } + + /* package */ DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @UsedForTesting + /* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException { + openDictBuffer(); + return getDictBuffer(); + } + + @Override + public FileHeader readHeader() throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + + final int version = HeaderReader.readVersion(mDictBuffer); + final int optionsFlags = HeaderReader.readOptionFlags(mDictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(mDictBuffer); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = HeaderReader.readAttributes(mDictBuffer, + headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { + int addressPointer = ptNodePos; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + ++addressPointer; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += 3; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (-1 != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + ++addressPointer; + frequency = PtNodeReader.readFrequency(mDictBuffer); + } else { + frequency = PtNode.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + final ArrayList<WeightedString> shortcutTargets; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + // readShortcut will add shortcuts to shortcutTargets. + shortcutTargets = new ArrayList<WeightedString>(); + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } else { + shortcutTargets = null; + } + + final ArrayList<PendingAttribute> bigrams; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigrams(mDictBuffer, bigrams, addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + MakedictLog.d("too many bigrams in a PtNode."); + } + } else { + bigrams = null; + } + return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } + + @Override + public FusionDictionary readDictionaryBinary(final FusionDictionary dict) + throws FileNotFoundException, IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + try { + return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); + } catch (IOException e) { + Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); + if (!mDictionaryBinaryFile.delete()) { + Log.e(TAG, "Failed to delete the broken dictionary."); + } + throw e; + } catch (UnsupportedFormatException e) { + Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); + if (!mDictionaryBinaryFile.delete()) { + Log.e(TAG, "Failed to delete the broken dictionary."); + } + throw e; + } + } + + @Override + public int getTerminalPosition(String word) throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + return BinaryDictIOUtils.getTerminalPosition(this, word); + } + + @Override + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); + } + +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java new file mode 100644 index 000000000..e81fd4514 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * An implementation of DictEncoder for version 3 binary dictionary. + */ +public class Ver3DictEncoder implements DictEncoder { + + private final File mDictFile; + private OutputStream mOutStream; + + public Ver3DictEncoder(final File dictFile) { + mDictFile = dictFile; + mOutStream = null; + } + + // This constructor is used only by BinaryDictOffdeviceUtilsTests. + // If you want to use this in the production code, you should consider keeping consistency of + // the interface of Ver3DictDecoder by using factory. + public Ver3DictEncoder(final OutputStream outStream) { + mDictFile = null; + mOutStream = outStream; + } + + private void openStream() throws FileNotFoundException { + mOutStream = new FileOutputStream(mDictFile); + } + + private void close() throws IOException { + if (mOutStream != null) { + mOutStream.close(); + mOutStream = null; + } + } + + @Override + public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (mOutStream == null) { + openStream(); + } + BinaryDictEncoderUtils.writeDictionaryBinary(mOutStream, dict, formatOptions); + close(); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java new file mode 100644 index 000000000..d44660623 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; + +import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.AbstractDictionaryWriter; +import com.android.inputmethod.latin.ExpandableDictionary; +import com.android.inputmethod.latin.WordComposer; +import com.android.inputmethod.latin.ExpandableDictionary.NextWord; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; +import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; +import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; + +import java.io.IOException; +import java.util.ArrayList; + +// Currently this class is used to implement dynamic prodiction dictionary. +// TODO: Move to native code. +public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { + private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); + /** Maximum number of pairs. Pruning will start when databases goes above this number. */ + public static final int MAX_HISTORY_BIGRAMS = 10000; + + /** Any pair being typed or picked */ + private static final int FREQUENCY_FOR_TYPED = 2; + + private static final int BINARY_DICT_VERSION = 3; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(BINARY_DICT_VERSION, true /* supportsDynamicUpdate */); + + private final UserHistoryDictionaryBigramList mBigramList = + new UserHistoryDictionaryBigramList(); + private final ExpandableDictionary mExpandableDictionary; + + public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { + super(context, dictType); + mExpandableDictionary = new ExpandableDictionary(dictType); + } + + @Override + public void clear() { + mBigramList.evictAll(); + mExpandableDictionary.clearDictionary(); + } + + /** + * Adds a word unigram to the fusion dictionary. Call updateBinaryDictionary when all changes + * are done to update the binary dictionary. + */ + @Override + public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, + final boolean isNotAWord) { + mExpandableDictionary.addWord(word, shortcutTarget, frequency); + mBigramList.addBigram(null, word, (byte)frequency); + } + + @Override + public void addBigramWords(final String word0, final String word1, final int frequency, + final boolean isValid, final long lastModifiedTime) { + if (lastModifiedTime > 0) { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(frequency, System.currentTimeMillis(), + lastModifiedTime)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } else { + mExpandableDictionary.setBigramAndGetFrequency(word0, word1, + new ForgettingCurveParams(isValid)); + mBigramList.addBigram(word0, word1, (byte)frequency); + } + } + + @Override + public void removeBigramWords(final String word0, final String word1) { + if (mBigramList.removeBigram(word0, word1)) { + mExpandableDictionary.removeBigram(word0, word1); + } + } + + @Override + protected void writeDictionary(final DictEncoder dictEncoder) + throws IOException, UnsupportedFormatException { + UserHistoryDictIOUtils.writeDictionary(dictEncoder, + new FrequencyProvider(mBigramList, mExpandableDictionary), mBigramList, + FORMAT_OPTIONS); + } + + private static class FrequencyProvider implements BigramDictionaryInterface { + final private UserHistoryDictionaryBigramList mBigramList; + final private ExpandableDictionary mExpandableDictionary; + + public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, + final ExpandableDictionary expandableDictionary) { + mBigramList = bigramList; + mExpandableDictionary = expandableDictionary; + } + @Override + public int getFrequency(final String word0, final String word1) { + final int freq; + if (word0 == null) { // unigram + freq = FREQUENCY_FOR_TYPED; + } else { // bigram + final NextWord nw = mExpandableDictionary.getBigramWord(word0, word1); + if (nw != null) { + final ForgettingCurveParams forgettingCurveParams = nw.getFcParams(); + final byte prevFc = mBigramList.getBigrams(word0).get(word1); + final byte fc = forgettingCurveParams.getFc(); + final boolean isValid = forgettingCurveParams.isValid(); + if (prevFc > 0 && prevFc == fc) { + freq = fc & 0xFF; + } else if (UserHistoryForgettingCurveUtils. + needsToSave(fc, isValid, mBigramList.size() <= MAX_HISTORY_BIGRAMS)) { + freq = fc & 0xFF; + } else { + // Delete this entry + freq = -1; + } + } else { + // Delete this entry + freq = -1; + } + } + return freq; + } + } + + @Override + public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, + final String prevWord, final ProximityInfo proximityInfo, + boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + return mExpandableDictionary.getSuggestions(composer, prevWord, proximityInfo, + blockOffensiveWords, additionalFeaturesOptions); + } + + @Override + public boolean isValidWord(final String word) { + return mExpandableDictionary.isValidWord(word); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index bb6ec6b1c..a08145b33 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -18,59 +18,41 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; import android.content.SharedPreferences; -import android.os.AsyncTask; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.ExpandableDictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.LatinImeLogger; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; -import com.android.inputmethod.latin.WordComposer; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.settings.Settings; -import com.android.inputmethod.latin.utils.ByteArrayWrapper; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; -import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils; -import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.concurrent.locks.ReentrantLock; /** * This class is a base class of a dictionary for the personalized prediction language model. */ -public abstract class DynamicPredictionDictionaryBase extends ExpandableDictionary { - +public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary { private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName(); public static final boolean DBG_SAVE_RESTORE = false; private static final boolean DBG_STRESS_TEST = false; private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; - private static final FormatOptions VERSION3 = new FormatOptions(3, - true /* supportsDynamicUpdate */); - /** Any pair being typed or picked */ - private static final int FREQUENCY_FOR_TYPED = 2; - - /** Maximum number of pairs. Pruning will start when databases goes above this number. */ - private static final int MAX_HISTORY_BIGRAMS = 10000; + public static final int FREQUENCY_FOR_TYPED = 2; /** Locale for which this user history dictionary is storing words */ private final String mLocale; - private final UserHistoryDictionaryBigramList mBigramList = - new UserHistoryDictionaryBigramList(); - private final ReentrantLock mBigramListLock = new ReentrantLock(); + private final String mFileName; + private final SharedPreferences mPrefs; private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = @@ -80,41 +62,44 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona @UsedForTesting boolean mIsTest = false; /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale, - final SharedPreferences sp, final String dictionaryType) { - super(context, dictionaryType); + final SharedPreferences sp, final String dictionaryType, final String fileName) { + super(context, fileName, dictionaryType, true); mLocale = locale; + mFileName = fileName; mPrefs = sp; if (mLocale != null && mLocale.length() > 1) { - loadDictionary(); + asyncLoadDictionaryToMemory(); + asyncReloadDictionaryIfRequired(); } } @Override public void close() { - flushPendingWrites(); - // Don't close the database as locale changes will require it to be reopened anyway - // Also, the database is written to somewhat frequently, so it needs to be kept alive - // throughout the life of the process. - // mOpenHelper.close(); - // Ignore close because we cache PersonalizationPredictionDictionary for each language. - // See getInstance() above. + // Close only binary dictionary to reuse this dictionary. // super.close(); + closeBinaryDictionary(); + // Flush pending writes. + // TODO: Remove after this class become to use a dynamic binary dictionary. + asyncWriteBinaryDictionary(); + Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); } @Override - protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo) { - // Inhibit suggestions (not predictions) for user history for now. Removing this method - // is enough to use it through the standard ExpandableDictionary way. - return null; + protected boolean hasContentChanged() { + return false; + } + + @Override + protected boolean needsToReloadBeforeWriting() { + return false; } /** * Return whether the passed charsequence is in the dictionary. */ @Override - public synchronized boolean isValidWord(final String word) { - // TODO: figure out what is the correct thing to do here. + public boolean isValidWord(final String word) { + // Words included only in the user history should be treated as not in dictionary words. return false; } @@ -126,70 +111,29 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public int addToPersonalizationPredictionDictionary( - final String word1, final String word2, final boolean isValid) { - if (word2.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || - (word1 != null && word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { - return -1; + public void addToPersonalizationPredictionDictionary( + final String word0, final String word1, final boolean isValid) { + if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || + (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { + return; } - if (mBigramListLock.tryLock()) { - try { - super.addWord( - word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED); - mBigramList.addBigram(null, word2, (byte)FREQUENCY_FOR_TYPED); - // Do not insert a word as a bigram of itself - if (word2.equals(word1)) { - return 0; - } - final int freq; - if (null == word1) { - freq = FREQUENCY_FOR_TYPED; - } else { - freq = super.setBigramAndGetFrequency( - word1, word2, new ForgettingCurveParams(isValid)); - } - mBigramList.addBigram(word1, word2); - return freq; - } finally { - mBigramListLock.unlock(); - } + addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED, + false /* isNotAWord */); + // Do not insert a word as a bigram of itself + if (word1.equals(word0)) { + return; } - return -1; - } - - public boolean cancelAddingUserHistory(final String word1, final String word2) { - if (mBigramListLock.tryLock()) { - try { - if (mBigramList.removeBigram(word1, word2)) { - return super.removeBigram(word1, word2); - } - } finally { - mBigramListLock.unlock(); - } + if (null != word0) { + addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid); } - return false; } - /** - * Schedules a background thread to write any pending words to the database. - */ - private void flushPendingWrites() { - // Create a background thread to write the pending entries - new UpdateBinaryTask(mBigramList, mLocale, this, mPrefs, getContext()).execute(); + public void cancelAddingUserHistory(final String word0, final String word1) { + removeBigramDynamically(word0, word1); } @Override - public final void loadDictionaryAsync() { - // This must be run on non-main thread - mBigramListLock.lock(); - try { - loadDictionaryAsyncLocked(); - } finally { - mBigramListLock.unlock(); - } - } - - private void loadDictionaryAsyncLocked() { + protected void loadDictionaryAsync() { final int[] profTotalCount = { 0 }; final String locale = getLocale(); if (DBG_STRESS_TEST) { @@ -201,10 +145,8 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona } } final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale); - final boolean initializing = last == 0; final long now = System.currentTimeMillis(); - final String fileName = getDictionaryFileName(); - final ExpandableDictionary dictionary = this; + final ExpandableBinaryDictionary dictionary = this; final OnAddWordListener listener = new OnAddWordListener() { @Override public void setUnigram(final String word, final String shortcutTarget, @@ -212,49 +154,36 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona if (DBG_SAVE_RESTORE) { Log.d(TAG, "load unigram: " + word + "," + frequency); } - dictionary.addWord(word, shortcutTarget, frequency); + addWord(word, shortcutTarget, frequency, false /* isNotAWord */); ++profTotalCount[0]; - addToBigramListLocked(null, word, (byte)frequency); } @Override - public void setBigram(final String word1, final String word2, final int frequency) { - if (word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH - && word2.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { + public void setBigram(final String word0, final String word1, final int frequency) { + if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH + && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) { if (DBG_SAVE_RESTORE) { - Log.d(TAG, "load bigram: " + word1 + "," + word2 + "," + frequency); + Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency); } ++profTotalCount[0]; - dictionary.setBigramAndGetFrequency( - word1, word2, initializing ? new ForgettingCurveParams(true) - : new ForgettingCurveParams(frequency, now, last)); + addBigram(word0, word1, frequency, last); } - addToBigramListLocked(word1, word2, (byte)frequency); } }; // Load the dictionary from binary file - FileInputStream inStream = null; + final File dictFile = new File(mContext.getFilesDir(), mFileName); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile, + DictDecoder.USE_BYTEARRAY); try { - final File file = new File(getContext().getFilesDir(), fileName); - final byte[] buffer = new byte[(int)file.length()]; - inStream = new FileInputStream(file); - inStream.read(buffer); - UserHistoryDictIOUtils.readDictionaryBinary( - new ByteArrayWrapper(buffer), listener); + dictDecoder.openDictBuffer(); + UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); } catch (FileNotFoundException e) { // This is an expected condition: we don't have a user history dictionary for this // language yet. It will be created sometime later. } catch (IOException e) { Log.e(TAG, "IOException on opening a bytebuffer", e); } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } if (PROFILE_SAVE_RESTORE) { final long diff = System.currentTimeMillis() - now; Log.d(TAG, "PROF: Load UserHistoryDictionary: " @@ -263,146 +192,30 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona } } - protected abstract String getDictionaryFileName(); - protected String getLocale() { return mLocale; } - private void addToBigramListLocked(String word0, String word1, byte fcValue) { - mBigramList.addBigram(word0, word1, fcValue); - } - - /** - * Async task to write pending words to the binarydicts. - */ - private static final class UpdateBinaryTask extends AsyncTask<Void, Void, Void> - implements BigramDictionaryInterface { - private final UserHistoryDictionaryBigramList mBigramList; - private final boolean mAddLevel0Bigrams; - private final String mLocale; - private final DynamicPredictionDictionaryBase mDynamicPredictionDictionary; - private final SharedPreferences mPrefs; - private final Context mContext; - - public UpdateBinaryTask(final UserHistoryDictionaryBigramList pendingWrites, - final String locale, final DynamicPredictionDictionaryBase dict, - final SharedPreferences prefs, final Context context) { - mBigramList = pendingWrites; - mLocale = locale; - mDynamicPredictionDictionary = dict; - mPrefs = prefs; - mContext = context; - mAddLevel0Bigrams = mBigramList.size() <= MAX_HISTORY_BIGRAMS; - } - - @Override - protected Void doInBackground(final Void... v) { - if (mDynamicPredictionDictionary.mIsTest) { - // If mIsTest == true, wait until the lock is released. - mDynamicPredictionDictionary.mBigramListLock.lock(); - try { - doWriteTaskLocked(); - } finally { - mDynamicPredictionDictionary.mBigramListLock.unlock(); - } - } else if (mDynamicPredictionDictionary.mBigramListLock.tryLock()) { - try { - doWriteTaskLocked(); - } finally { - mDynamicPredictionDictionary.mBigramListLock.unlock(); - } - } - return null; - } - - private void doWriteTaskLocked() { - if (DBG_STRESS_TEST) { - try { - Log.w(TAG, "Start stress in closing: " + mLocale); - Thread.sleep(15000); - Log.w(TAG, "End stress in closing"); - } catch (InterruptedException e) { - Log.e(TAG, "In stress test", e); - } - } - - final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0; - final String fileName = - mDynamicPredictionDictionary.getDictionaryFileName(); - final File file = new File(mContext.getFilesDir(), fileName); - FileOutputStream out = null; - - try { - out = new FileOutputStream(file); - UserHistoryDictIOUtils.writeDictionaryBinary(out, this, mBigramList, VERSION3); - out.flush(); - out.close(); - } catch (IOException e) { - Log.e(TAG, "IO Exception while writing file", e); - } finally { - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // ignore - } - } - } - - // Save the timestamp after we finish writing the binary dictionary. - Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); - if (PROFILE_SAVE_RESTORE) { - final long diff = System.currentTimeMillis() - now; - Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", " + diff + "ms."); - } - } - - @Override - public int getFrequency(final String word1, final String word2) { - final int freq; - if (word1 == null) { // unigram - freq = FREQUENCY_FOR_TYPED; - final byte prevFc = mBigramList.getBigrams(word1).get(word2); - } else { // bigram - final NextWord nw = - mDynamicPredictionDictionary.getBigramWord(word1, word2); - if (nw != null) { - final ForgettingCurveParams fcp = nw.getFcParams(); - final byte prevFc = mBigramList.getBigrams(word1).get(word2); - final byte fc = fcp.getFc(); - final boolean isValid = fcp.isValid(); - if (prevFc > 0 && prevFc == fc) { - freq = fc & 0xFF; - } else if (UserHistoryForgettingCurveUtils. - needsToSave(fc, isValid, mAddLevel0Bigrams)) { - freq = fc & 0xFF; - } else { - // Delete this entry - freq = -1; - } - } else { - // Delete this entry - freq = -1; - } - } - return freq; - } - } - @UsedForTesting /* package for test */ void forceAddWordForTest( - final String word1, final String word2, final boolean isValid) { - mBigramListLock.lock(); - try { - addToPersonalizationPredictionDictionary(word1, word2, isValid); - } finally { - mBigramListLock.unlock(); - } + final String word0, final String word1, final boolean isValid) { + addToPersonalizationPredictionDictionary(word0, word1, isValid); } public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { - session.setDictionary(this); + session.setPredictionDictionary(this); mSessions.add(session); + session.onDictionaryReady(); + } + + public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { + mSessions.remove(session); + } + + public void clearAndFlushDictionary() { + // Clear the node structure on memory + clear(); + // Then flush the cleared state of the dictionary on disk. + asyncWriteBinaryDictionary(); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java index e38a235e9..f257165cb 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -18,26 +18,32 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; +import com.android.inputmethod.latin.utils.CollectionUtils; import android.content.Context; +import android.content.SharedPreferences; + +import java.util.ArrayList; /** * This class is a dictionary for the personalized language model that uses binary dictionary. */ public class PersonalizationDictionary extends ExpandableBinaryDictionary { private static final String NAME = "personalization"; - - public static void registerUpdateListener(PersonalizationDictionaryUpdateSession listener) { - // TODO: Implement - } + private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions = + CollectionUtils.newArrayList(); /** Locale for which this user history dictionary is storing words */ private final String mLocale; - // Singleton - private PersonalizationDictionary(final Context context, final String locale) { - super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION); + public PersonalizationDictionary(final Context context, final String locale, + final SharedPreferences prefs) { + // TODO: Make isUpdatable true. + super(context, getFilenameWithLocale(NAME, locale), Dictionary.TYPE_PERSONALIZATION, + false /* isUpdatable */); mLocale = locale; + // TODO: Restore last updated time + loadDictionary(); } @Override @@ -47,15 +53,21 @@ public class PersonalizationDictionary extends ExpandableBinaryDictionary { @Override protected boolean hasContentChanged() { - // TODO: Implement return false; } @Override protected boolean needsToReloadBeforeWriting() { - // TODO: Implement return false; } - // TODO: Implement + public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) { + session.setDictionary(this); + mSessions.add(session); + session.onDictionaryReady(); + } + + public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) { + mSessions.remove(session); + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java new file mode 100644 index 000000000..c1833ff14 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionarySessionRegister.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +import android.content.Context; +import android.content.res.Configuration; + +public class PersonalizationDictionarySessionRegister { + public static void init(Context context) { + } + + public static void onConfigurationChanged(final Context context, final Configuration conf) { + } + + public static void onUpdateData(Context context, String type) { + } + + public static void onRemoveData(Context context, String type) { + } + + public static void onDestroy(Context context) { + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index d62aec19d..ab3de801c 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -16,6 +16,8 @@ package com.android.inputmethod.latin.personalization; +import android.content.Context; + import java.lang.ref.WeakReference; import java.util.ArrayList; @@ -43,18 +45,78 @@ public abstract class PersonalizationDictionaryUpdateSession { } // TODO: Use a dynamic binary dictionary instead - public WeakReference<DynamicPredictionDictionaryBase> mDictionary; + public WeakReference<PersonalizationDictionary> mDictionary; + public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary; + public final String mSystemLocale; + public PersonalizationDictionaryUpdateSession(String locale) { + mSystemLocale = locale; + } public abstract void onDictionaryReady(); - public void setDictionary(DynamicPredictionDictionaryBase dictionary) { - mDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); + public abstract void onDictionaryClosed(Context context); + + public void setDictionary(PersonalizationDictionary dictionary) { + mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); + } + + public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) { + mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); + } + + protected PersonalizationDictionary getDictionary() { + return mDictionary == null ? null : mDictionary.get(); + } + + protected DynamicPredictionDictionaryBase getPredictionDictionary() { + return mPredictionDictionary == null ? null : mPredictionDictionary.get(); + } + + private void unsetDictionary() { + final PersonalizationDictionary dictionary = getDictionary(); + if (dictionary == null) { + return; + } + dictionary.unRegisterUpdateSession(this); + } + + private void unsetPredictionDictionary() { + final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.unRegisterUpdateSession(this); + } + + public void clearAndFlushPredictionDictionary(Context context) { + final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.clearAndFlushDictionary(); + } + + public void closeSession(Context context) { + unsetDictionary(); + unsetPredictionDictionary(); + onDictionaryClosed(context); + } + + // TODO: Support multi locale to add bigram + public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, + int frequency) { + final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + if (dictionary == null) { + return; + } + dictionary.addToPersonalizationPredictionDictionary(word0, word1, isValid); } - public void addToPersonalizationDictionary( + // Bulk import + // TODO: Support multi locale to add bigram + public void addBigramsToPersonalizationDictionary( final ArrayList<PersonalizationLanguageModelParam> lmParams) { - final DynamicPredictionDictionaryBase dictionary = mDictionary == null - ? null : mDictionary.get(); + final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index da256f827..c8deaf90d 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -26,16 +26,20 @@ import android.util.Log; import java.lang.ref.SoftReference; import java.util.concurrent.ConcurrentHashMap; -public class PersonalizationDictionaryHelper { - private static final String TAG = PersonalizationDictionaryHelper.class.getSimpleName(); +public class PersonalizationHelper { + private static final String TAG = PersonalizationHelper.class.getSimpleName(); private static final boolean DEBUG = false; private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>> sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> + sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, SoftReference<PersonalizationPredictionDictionary>> - sLangPersonalizationDictCache = CollectionUtils.newConcurrentHashMap(); + sLangPersonalizationPredictionDictCache = + CollectionUtils.newConcurrentHashMap(); public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary( final Context context, final String locale, final SharedPreferences sp) { @@ -48,6 +52,7 @@ public class PersonalizationDictionaryHelper { if (DEBUG) { Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale); } + dict.asyncReloadDictionaryIfRequired(); return dict; } } @@ -59,22 +64,46 @@ public class PersonalizationDictionaryHelper { } } - public static void - registerPersonalizationDictionaryUpdateSession(final Context context, - final PersonalizationDictionaryUpdateSession session) { - final PersonalizationPredictionDictionary dictionary = - getPersonalizationPredictionDictionary(context, - context.getResources().getConfiguration().locale.toString(), + public static void registerPersonalizationDictionaryUpdateSession(final Context context, + final PersonalizationDictionaryUpdateSession session, String locale) { + final PersonalizationPredictionDictionary predictionDictionary = + getPersonalizationPredictionDictionary(context, locale, + PreferenceManager.getDefaultSharedPreferences(context)); + predictionDictionary.registerUpdateSession(session); + final PersonalizationDictionary dictionary = + getPersonalizationDictionary(context, locale, PreferenceManager.getDefaultSharedPreferences(context)); dictionary.registerUpdateSession(session); } - public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( + public static PersonalizationDictionary getPersonalizationDictionary( final Context context, final String locale, final SharedPreferences sp) { synchronized (sLangPersonalizationDictCache) { if (sLangPersonalizationDictCache.containsKey(locale)) { - final SoftReference<PersonalizationPredictionDictionary> ref = + final SoftReference<PersonalizationDictionary> ref = sLangPersonalizationDictCache.get(locale); + final PersonalizationDictionary dict = ref == null ? null : ref.get(); + if (dict != null) { + if (DEBUG) { + Log.w(TAG, "Use cached PersonalizationDictCache for " + locale); + } + return dict; + } + } + final PersonalizationDictionary dict = + new PersonalizationDictionary(context, locale, sp); + sLangPersonalizationDictCache.put( + locale, new SoftReference<PersonalizationDictionary>(dict)); + return dict; + } + } + + public static PersonalizationPredictionDictionary getPersonalizationPredictionDictionary( + final Context context, final String locale, final SharedPreferences sp) { + synchronized (sLangPersonalizationPredictionDictCache) { + if (sLangPersonalizationPredictionDictCache.containsKey(locale)) { + final SoftReference<PersonalizationPredictionDictionary> ref = + sLangPersonalizationPredictionDictCache.get(locale); final PersonalizationPredictionDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { @@ -85,7 +114,7 @@ public class PersonalizationDictionaryHelper { } final PersonalizationPredictionDictionary dict = new PersonalizationPredictionDictionary(context, locale, sp); - sLangPersonalizationDictCache.put( + sLangPersonalizationPredictionDictCache.put( locale, new SoftReference<PersonalizationPredictionDictionary>(dict)); return dict; } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java index 955bd2762..e80953c05 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; import android.content.SharedPreferences; @@ -26,11 +27,11 @@ public class PersonalizationPredictionDictionary extends DynamicPredictionDictio /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, final SharedPreferences sp) { - super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA); + super(context, locale, sp, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA, + getDictionaryFileName(locale)); } - @Override - protected String getDictionaryFileName() { - return NAME + "." + getLocale() + ".dict"; + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java index f21db25a6..6c2c9e26e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java @@ -45,6 +45,7 @@ public final class UserHistoryDictionaryBigramList { /** * Called when the user typed a word. */ + @UsedForTesting public void addBigram(String word1, String word2) { addBigram(word1, word2, FORGETTING_CURVE_INITIAL_VALUE); } @@ -53,7 +54,7 @@ public final class UserHistoryDictionaryBigramList { * Called when loaded from the SQL DB. */ public void addBigram(String word1, String word2, byte fcValue) { - if (UserHistoryPredictionDictionary.DBG_SAVE_RESTORE) { + if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; @@ -73,7 +74,7 @@ public final class UserHistoryDictionaryBigramList { * Called when inserted to the SQL DB. */ public void updateBigram(String word1, String word2, byte fcValue) { - if (UserHistoryPredictionDictionary.DBG_SAVE_RESTORE) { + if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java index d11784454..b140c919b 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.personalization; import com.android.inputmethod.latin.Dictionary; +import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; import android.content.SharedPreferences; @@ -26,14 +27,14 @@ import android.content.SharedPreferences; * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. */ public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase { - private static final String NAME = UserHistoryPredictionDictionary.class.getSimpleName(); + /* package for tests */ static final String NAME = + UserHistoryPredictionDictionary.class.getSimpleName(); /* package */ UserHistoryPredictionDictionary(final Context context, final String locale, final SharedPreferences sp) { - super(context, locale, sp, Dictionary.TYPE_USER_HISTORY); + super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); } - @Override - protected String getDictionaryFileName() { - return NAME + "." + getLocale() + ".dict"; + private static String getDictionaryFileName(final String locale) { + return NAME + "." + locale + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; } } diff --git a/java/src/com/android/inputmethod/latin/settings/AdditionalFeaturesSettingUtils.java b/java/src/com/android/inputmethod/latin/settings/AdditionalFeaturesSettingUtils.java index 139f5e290..6543003e8 100644 --- a/java/src/com/android/inputmethod/latin/settings/AdditionalFeaturesSettingUtils.java +++ b/java/src/com/android/inputmethod/latin/settings/AdditionalFeaturesSettingUtils.java @@ -40,8 +40,4 @@ public class AdditionalFeaturesSettingUtils { final SharedPreferences prefs, final int[] additionalFeaturesPreferences) { // do nothing. } - - public static int[] getAdditionalNativeSuggestOptions() { - return Settings.getInstance().getCurrent().mAdditionalFeaturesSettingValues; - } } diff --git a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java index b1cd88729..1b592b565 100644 --- a/java/src/com/android/inputmethod/latin/settings/DebugSettings.java +++ b/java/src/com/android/inputmethod/latin/settings/DebugSettings.java @@ -39,6 +39,8 @@ public final class DebugSettings extends PreferenceFragment public static final String PREF_STATISTICS_LOGGING = "enable_logging"; public static final String PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG = "use_only_personalization_dictionary_for_debug"; + public static final String PREF_BOOST_PERSONALIZATION_DICTIONARY_FOR_DEBUG = + "boost_personalization_dictionary_for_debug"; private static final String PREF_READ_EXTERNAL_DICTIONARY = "read_external_dictionary"; private static final boolean SHOW_STATISTICS_LOGGING = false; diff --git a/java/src/com/android/inputmethod/latin/settings/NativeSuggestOptions.java b/java/src/com/android/inputmethod/latin/settings/NativeSuggestOptions.java index 878c505bd..cd726c969 100644 --- a/java/src/com/android/inputmethod/latin/settings/NativeSuggestOptions.java +++ b/java/src/com/android/inputmethod/latin/settings/NativeSuggestOptions.java @@ -34,6 +34,9 @@ public class NativeSuggestOptions { } public void setAdditionalFeaturesOptions(final int[] additionalOptions) { + if (additionalOptions == null) { + return; + } for (int i = 0; i < additionalOptions.length; i++) { setIntegerOption(OPTIONS_SIZE + i, additionalOptions[i]); } diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java index d432087d3..fd83865ba 100644 --- a/java/src/com/android/inputmethod/latin/settings/Settings.java +++ b/java/src/com/android/inputmethod/latin/settings/Settings.java @@ -27,10 +27,10 @@ import com.android.inputmethod.latin.AudioAndHapticFeedbackManager; import com.android.inputmethod.latin.InputAttributes; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.utils.AdditionalSubtypeUtils; -import com.android.inputmethod.latin.utils.DebugLogUtils; import com.android.inputmethod.latin.utils.LocaleUtils; import com.android.inputmethod.latin.utils.ResourceUtils; import com.android.inputmethod.latin.utils.RunInLocale; +import com.android.inputmethod.latin.utils.StringUtils; import java.util.HashMap; import java.util.Locale; @@ -90,6 +90,8 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang private static final String PREF_SUPPRESS_LANGUAGE_SWITCH_KEY = "pref_suppress_language_switch_key"; + private static final String PREF_LAST_USED_PERSONALIZATION_TOKEN = + "pref_last_used_personalization_token"; public static final String PREF_SEND_FEEDBACK = "send_feedback"; public static final String PREF_ABOUT_KEYBOARD = "about_keyboard"; @@ -343,4 +345,20 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang return prefs.getBoolean( DebugSettings.PREF_USE_ONLY_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false); } + + public static boolean readBoostPersonalizationDictionaryForDebug( + final SharedPreferences prefs) { + return prefs.getBoolean( + DebugSettings.PREF_BOOST_PERSONALIZATION_DICTIONARY_FOR_DEBUG, false); + } + + public void writeLastUsedPersonalizationToken(byte[] token) { + final String tokenStr = StringUtils.byteArrayToHexString(token); + mPrefs.edit().putString(PREF_LAST_USED_PERSONALIZATION_TOKEN, tokenStr).apply(); + } + + public byte[] readLastUsedPersonalizationToken() { + final String tokenStr = mPrefs.getString(PREF_LAST_USED_PERSONALIZATION_TOKEN, null); + return StringUtils.hexStringToByteArray(tokenStr); + } } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java b/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java index 446777704..1677e1828 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsFragment.java @@ -229,10 +229,10 @@ public final class SettingsFragment extends InputMethodSettingsFragment if (!Settings.readFromBuildConfigIfGestureInputEnabled(res)) { removePreference(Settings.PREF_GESTURE_SETTINGS, getPreferenceScreen()); - } else { - AdditionalFeaturesSettingUtils.addAdditionalFeaturesPreferences(context, this); } + AdditionalFeaturesSettingUtils.addAdditionalFeaturesPreferences(context, this); + setupKeyLongpressTimeoutSettings(prefs, res); setupKeypressVibrationDurationSettings(prefs, res); setupKeypressSoundVolumeSettings(prefs, res); diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index a25cf620c..32730d23f 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -22,6 +22,7 @@ import android.content.res.Resources; import android.util.Log; import android.view.inputmethod.EditorInfo; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.internal.KeySpecParser; import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.InputAttributes; @@ -56,6 +57,7 @@ public final class SettingsValues { public final SuggestedWords mSuggestPuncList; public final String mWordSeparators; public final CharSequence mHintToSaveText; + public final boolean mCurrentLanguageHasSpaces; // From preferences, in the same order as xml/prefs.xml: public final boolean mAutoCap; @@ -90,6 +92,8 @@ public final class SettingsValues { public final int mSuggestionVisibility; private final boolean mVoiceKeyEnabled; private final boolean mVoiceKeyOnMain; + public final boolean mBoostPersonalizationDictionaryForDebug; + public final boolean mUseOnlyPersonalizationDictionaryForDebug; // Setting values for additional features public final int[] mAdditionalFeaturesSettingValues = @@ -117,6 +121,7 @@ public final class SettingsValues { mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec); mWordSeparators = res.getString(R.string.symbols_word_separators); mHintToSaveText = res.getText(R.string.hint_add_to_dictionary); + mCurrentLanguageHasSpaces = res.getBoolean(R.bool.current_language_has_spaces); // Store the input attributes if (null == inputAttributes) { @@ -168,6 +173,62 @@ public final class SettingsValues { AdditionalFeaturesSettingUtils.readAdditionalFeaturesPreferencesIntoArray( prefs, mAdditionalFeaturesSettingValues); mIsInternal = Settings.isInternal(prefs); + mBoostPersonalizationDictionaryForDebug = + Settings.readBoostPersonalizationDictionaryForDebug(prefs); + mUseOnlyPersonalizationDictionaryForDebug = + Settings.readUseOnlyPersonalizationDictionaryForDebug(prefs); + } + + // Only for tests + private SettingsValues(final Locale locale) { + // TODO: locale is saved, but not used yet. May have to change this if tests require. + mLocale = locale; + mDelayUpdateOldSuggestions = 0; + mSymbolsPrecededBySpace = new int[] { '(', '[', '{', '&' }; + Arrays.sort(mSymbolsPrecededBySpace); + mSymbolsFollowedBySpace = new int[] { '.', ',', ';', ':', '!', '?', ')', ']', '}', '&' }; + Arrays.sort(mSymbolsFollowedBySpace); + mWordConnectors = new int[] { '\'', '-' }; + Arrays.sort(mWordConnectors); + final String[] suggestPuncsSpec = new String[] { "!", "?", ",", ":", ";" }; + mSuggestPuncList = createSuggestPuncList(suggestPuncsSpec); + mWordSeparators = "&\t \n()[]{}*&<>+=|.,;:!?/_\""; + mHintToSaveText = "Touch again to save"; + mCurrentLanguageHasSpaces = true; + mInputAttributes = new InputAttributes(null, false /* isFullscreenMode */); + mAutoCap = true; + mVibrateOn = true; + mSoundOn = true; + mKeyPreviewPopupOn = true; + mSlidingKeyInputPreviewEnabled = true; + mVoiceMode = "0"; + mIncludesOtherImesInLanguageSwitchList = false; + mShowsLanguageSwitchKey = true; + mUseContactsDict = true; + mUseDoubleSpacePeriod = true; + mBlockPotentiallyOffensive = true; + mAutoCorrectEnabled = true; + mBigramPredictionEnabled = true; + mKeyLongpressTimeout = 300; + mKeypressVibrationDuration = 5; + mKeypressSoundVolume = 1; + mKeyPreviewPopupDismissDelay = 70; + mAutoCorrectionThreshold = 1; + mVoiceKeyEnabled = true; + mVoiceKeyOnMain = true; + mGestureInputEnabled = true; + mGestureTrailEnabled = true; + mGestureFloatingPreviewTextEnabled = true; + mCorrectionEnabled = mAutoCorrectEnabled && !mInputAttributes.mInputTypeNoAutoCorrect; + mSuggestionVisibility = 0; + mIsInternal = false; + mBoostPersonalizationDictionaryForDebug = false; + mUseOnlyPersonalizationDictionaryForDebug = false; + } + + @UsedForTesting + public static SettingsValues makeDummySettingsValuesForTest(final Locale locale) { + return new SettingsValues(locale); } public boolean isApplicationSpecifiedCompletionsOn() { @@ -194,6 +255,10 @@ public final class SettingsValues { return Arrays.binarySearch(mWordConnectors, code) >= 0; } + public boolean isWordCodePoint(final int code) { + return Character.isLetter(code) || isWordConnector(code); + } + public boolean isUsuallyPrecededBySpace(final int code) { return Arrays.binarySearch(mSymbolsPrecededBySpace, code) >= 0; } @@ -241,7 +306,8 @@ public final class SettingsValues { // TODO: Stop using KeySpceParser.getLabel(). puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, - Dictionary.TYPE_HARDCODED)); + Dictionary.DICTIONARY_HARDCODED, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); } } return new SuggestedWords(puncList, diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java index 6719e98da..69f9a467f 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidWordLevelSpellCheckerSession.java @@ -301,12 +301,14 @@ public abstract class AndroidWordLevelSpellCheckerSession extends Session { // TODO: make a spell checker option to block offensive words or not final ArrayList<SuggestedWordInfo> suggestions = dictInfo.mDictionary.getSuggestions(composer, prevWord, - dictInfo.getProximityInfo(), - true /* blockOffensiveWords */); - for (final SuggestedWordInfo suggestion : suggestions) { - final String suggestionStr = suggestion.mWord; - suggestionsGatherer.addWord(suggestionStr.toCharArray(), null, 0, - suggestionStr.length(), suggestion.mScore); + dictInfo.getProximityInfo(), true /* blockOffensiveWords */, + null /* additionalFeaturesOptions */); + if (suggestions != null) { + for (final SuggestedWordInfo suggestion : suggestions) { + final String suggestionStr = suggestion.mWord; + suggestionsGatherer.addWord(suggestionStr.toCharArray(), null, 0, + suggestionStr.length(), suggestion.mScore); + } } isInDict = isInDictForAnyCapitalization(dictInfo.mDictionary, text, capitalizeType); } finally { diff --git a/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java b/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java index ac8f68781..a0aed2829 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/DictionaryPool.java @@ -52,7 +52,7 @@ public final class DictionaryPool extends LinkedBlockingQueue<DictAndKeyboard> { @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { return noSuggestions; } @Override diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java index e97069dff..acd47450b 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestions.java @@ -210,7 +210,8 @@ public final class MoreSuggestions extends Keyboard { final int indexInMoreSuggestions = index + SUGGESTION_CODE_BASE; final Key key = new Key( params, word, info, KeyboardIconsSet.ICON_UNDEFINED, indexInMoreSuggestions, - null, x, y, width, params.mDefaultRowHeight, 0); + null /* outputText */, x, y, width, params.mDefaultRowHeight, + 0 /* labelFlags */, Key.BACKGROUND_TYPE_NORMAL); params.markAsEdgeKey(key, index); params.onAddKey(key); final int columnNumber = params.getColumnNumber(index); diff --git a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java index d585b5c7f..0ebe37782 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/MoreSuggestionsView.java @@ -34,7 +34,7 @@ public final class MoreSuggestionsView extends MoreKeysKeyboardView { private static final String TAG = MoreSuggestionsView.class.getSimpleName(); public MoreSuggestionsView(final Context context, final AttributeSet attrs) { - this(context, attrs, R.attr.moreSuggestionsViewStyle); + this(context, attrs, R.attr.moreKeysKeyboardViewStyle); } public MoreSuggestionsView(final Context context, final AttributeSet attrs, diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java index 1dd04fc4d..8d2689a7d 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripLayoutHelper.java @@ -122,27 +122,15 @@ final class SuggestionStripLayoutHelper { mSuggestionsStripHeight = res.getDimensionPixelSize(R.dimen.suggestions_strip_height); final TypedArray a = context.obtainStyledAttributes(attrs, - R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripViewStyle); + R.styleable.SuggestionStripView, defStyle, R.style.SuggestionStripView); mSuggestionStripOption = a.getInt( R.styleable.SuggestionStripView_suggestionStripOption, 0); - final float alphaValidTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaValidTypedWord, 1.0f); - final float alphaTypedWord = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaTypedWord, 1.0f); - final float alphaAutoCorrect = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaAutoCorrect, 1.0f); - final float alphaSuggested = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); mAlphaObsoleted = ResourceUtils.getFraction(a, - R.styleable.SuggestionStripView_alphaSuggested, 1.0f); - mColorValidTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorValidTypedWord, 0), alphaValidTypedWord); - mColorTypedWord = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorTypedWord, 0), alphaTypedWord); - mColorAutoCorrect = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorAutoCorrect, 0), alphaAutoCorrect); - mColorSuggested = applyAlpha(a.getColor( - R.styleable.SuggestionStripView_colorSuggested, 0), alphaSuggested); + R.styleable.SuggestionStripView_alphaObsoleted, 1.0f); + mColorValidTypedWord = a.getColor(R.styleable.SuggestionStripView_colorValidTypedWord, 0); + mColorTypedWord = a.getColor(R.styleable.SuggestionStripView_colorTypedWord, 0); + mColorAutoCorrect = a.getColor(R.styleable.SuggestionStripView_colorAutoCorrect, 0); + mColorSuggested = a.getColor(R.styleable.SuggestionStripView_colorSuggested, 0); mSuggestionsCountInStrip = a.getInt( R.styleable.SuggestionStripView_suggestionsCountInStrip, DEFAULT_SUGGESTIONS_COUNT_IN_STRIP); diff --git a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java index a8a14a825..75f17c559 100644 --- a/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java +++ b/java/src/com/android/inputmethod/latin/suggestions/SuggestionStripView.java @@ -198,7 +198,7 @@ public final class SuggestionStripView extends RelativeLayout implements OnClick @Override public boolean onLongClick(final View view) { - AudioAndHapticFeedbackManager.getInstance().hapticAndAudioFeedback( + AudioAndHapticFeedbackManager.getInstance().performHapticAndAudioFeedback( Constants.NOT_A_CODE, this); return showMoreSuggestions(); } diff --git a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java b/java/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java index 1bb27aa2b..2028298f2 100644 --- a/java/src/com/android/inputmethod/latin/utils/ByteArrayWrapper.java +++ b/java/src/com/android/inputmethod/latin/utils/ByteArrayDictBuffer.java @@ -16,17 +16,17 @@ package com.android.inputmethod.latin.utils; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; /** * This class provides an implementation for the FusionDictionary buffer interface that is backed * by a simpled byte array. It allows to create a binary dictionary in memory. */ -public final class ByteArrayWrapper implements FusionDictionaryBufferInterface { +public final class ByteArrayDictBuffer implements DictBuffer { private byte[] mBuffer; private int mPosition; - public ByteArrayWrapper(final byte[] buffer) { + public ByteArrayDictBuffer(final byte[] buffer) { mBuffer = buffer; mPosition = 0; } diff --git a/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java b/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java index 98f0d8b68..cc25102ce 100644 --- a/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/CollectionUtils.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin.utils; import android.util.SparseArray; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -94,6 +95,10 @@ public final class CollectionUtils { return new CopyOnWriteArrayList<E>(array); } + public static <E> ArrayDeque<E> newArrayDeque() { + return new ArrayDeque<E>(); + } + public static <E> SparseArray<E> newSparseArray() { return new SparseArray<E>(); } diff --git a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java index 34eccd65b..021bf0825 100644 --- a/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/DictionaryInfoUtils.java @@ -27,10 +27,8 @@ import com.android.inputmethod.latin.BinaryDictionaryGetter; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.Locale; @@ -281,13 +279,7 @@ public class DictionaryInfoUtils { } public static FileHeader getDictionaryFileHeaderOrNull(final File file) { - try { - return BinaryDictIOUtils.getDictionaryFileHeader(file, 0, file.length()); - } catch (UnsupportedFormatException e) { - return null; - } catch (IOException e) { - return null; - } + return BinaryDictIOUtils.getDictionaryFileHeaderOrNull(file, 0, file.length()); } private static DictionaryInfo createDictionaryInfoFromFileAddress( diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index 7406d855a..be4184093 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -18,7 +18,9 @@ package com.android.inputmethod.latin.utils; import android.text.TextUtils; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.settings.SettingsValues; import java.util.ArrayList; import java.util.Locale; @@ -193,27 +195,56 @@ public final class StringUtils { } public static boolean isIdenticalAfterUpcase(final String text) { - final int len = text.length(); - for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) { + final int length = text.length(); + int i = 0; + while (i < length) { final int codePoint = text.codePointAt(i); if (Character.isLetter(codePoint) && !Character.isUpperCase(codePoint)) { return false; } + i += Character.charCount(codePoint); } return true; } public static boolean isIdenticalAfterDowncase(final String text) { - final int len = text.length(); - for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) { + final int length = text.length(); + int i = 0; + while (i < length) { final int codePoint = text.codePointAt(i); if (Character.isLetter(codePoint) && !Character.isLowerCase(codePoint)) { return false; } + i += Character.charCount(codePoint); } return true; } + @UsedForTesting + public static boolean looksValidForDictionaryInsertion(final CharSequence text, + final SettingsValues settings) { + if (TextUtils.isEmpty(text)) return false; + final int length = text.length(); + int i = 0; + int digitCount = 0; + while (i < length) { + final int codePoint = Character.codePointAt(text, i); + final int charCount = Character.charCount(codePoint); + i += charCount; + if (Character.isDigit(codePoint)) { + // Count digits: see below + digitCount += charCount; + continue; + } + if (!settings.isWordCodePoint(codePoint)) return false; + } + // We reject strings entirely comprised of digits to avoid using PIN codes or credit + // card numbers. It would come in handy for word prediction though; a good example is + // when writing one's address where the street number is usually quite discriminative, + // as well as the postal code. + return digitCount < length; + } + public static boolean isIdenticalAfterCapitalizeEachWord(final String text, final String separators) { boolean needCapsNext = true; @@ -316,4 +347,47 @@ public final class StringUtils { // Otherwise, it doesn't look like an URL. return false; } + + public static boolean isEmptyStringOrWhiteSpaces(String s) { + final int N = codePointCount(s); + for (int i = 0; i < N; ++i) { + if (!Character.isWhitespace(s.codePointAt(i))) { + return false; + } + } + return true; + } + + @UsedForTesting + public static String byteArrayToHexString(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return ""; + } + final StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b & 0xff)); + } + return sb.toString(); + } + + /** + * Convert hex string to byte array. The string length must be an even number. + */ + @UsedForTesting + public static byte[] hexStringToByteArray(String hexString) { + if (TextUtils.isEmpty(hexString)) { + return null; + } + final int N = hexString.length(); + if (N % 2 != 0) { + throw new NumberFormatException("Input hex string length must be an even number." + + " Length = " + N); + } + final byte[] bytes = new byte[N / 2]; + for (int i = 0; i < N; i += 2) { + bytes[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4) + + Character.digit(hexString.charAt(i + 1), 16)); + } + return bytes; + } } diff --git a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java index 16728092d..102a41b4e 100644 --- a/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/SubtypeLocaleUtils.java @@ -40,6 +40,7 @@ public final class SubtypeLocaleUtils { // Special language code to represent "no language". public static final String NO_LANGUAGE = "zz"; public static final String QWERTY = "qwerty"; + public static final String EMOJI = "emoji"; public static final int UNKNOWN_KEYBOARD_LAYOUT = R.string.subtype_generic; private static boolean sInitialized = false; diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index d02f7187e..99788f6f2 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -20,20 +20,20 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; -import java.io.OutputStream; import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; /** * Reads and writes Binary files for a UserHistoryDictionary. @@ -57,12 +57,12 @@ public final class UserHistoryDictIOUtils { /** * Writes dictionary to file. */ - public static void writeDictionaryBinary(final OutputStream destination, + public static void writeDictionary(final DictEncoder dictEncoder, final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams, final FormatOptions formatOptions) { final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams); try { - BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions); + dictEncoder.writeDictionary(fusionDict, formatOptions); Log.d(TAG, "end writing"); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); @@ -77,7 +77,7 @@ public final class UserHistoryDictIOUtils { @UsedForTesting static FusionDictionary constructFusionDictionary( final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams) { - final FusionDictionary fusionDict = new FusionDictionary(new Node(), + final FusionDictionary fusionDict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, false)); int profTotal = 0; @@ -101,7 +101,7 @@ public final class UserHistoryDictIOUtils { if (word1 == null) { // unigram fusionDict.add(word2, freq, null, false /* isNotAWord */); } else { // bigram - if (FusionDictionary.findWordInTree(fusionDict.mRoot, word1) == null) { + if (FusionDictionary.findWordInTree(fusionDict.mRootNodeArray, word1) == null) { fusionDict.add(word1, 2, null, false /* isNotAWord */); } fusionDict.setBigram(word1, word2, freq); @@ -118,14 +118,13 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer, + public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, final OnAddWordListener dict) { - final Map<Integer, String> unigrams = CollectionUtils.newTreeMap(); - final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); - final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap(); + final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap(); + final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap(); try { - BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies, - bigrams); + dictDecoder.readUnigramsAndBigramsBinary(unigrams, frequencies, bigrams); } catch (IOException e) { Log.e(TAG, "IO exception while reading file", e); } catch (UnsupportedFormatException e) { @@ -140,10 +139,11 @@ public final class UserHistoryDictIOUtils { * Adds all unigrams and bigrams in maps to OnAddWordListener. */ @UsedForTesting - static void addWordsFromWordMap(final Map<Integer, String> unigrams, - final Map<Integer, Integer> frequencies, - final Map<Integer, ArrayList<PendingAttribute>> bigrams, final OnAddWordListener to) { - for (Map.Entry<Integer, String> entry : unigrams.entrySet()) { + static void addWordsFromWordMap(final TreeMap<Integer, String> unigrams, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams, + final OnAddWordListener to) { + for (Entry<Integer, String> entry : unigrams.entrySet()) { final String word1 = entry.getValue(); final int unigramFrequency = frequencies.get(entry.getKey()); to.setUnigram(word1, null, unigramFrequency); @@ -156,7 +156,7 @@ public final class UserHistoryDictIOUtils { continue; } to.setBigram(word1, word2, - BinaryDictInputOutput.reconstructBigramFrequency(unigramFrequency, + BinaryDictIOUtils.reconstructBigramFrequency(unigramFrequency, attr.mFrequency)); } } diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java index 713a45bda..1992b2f5d 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryForgettingCurveUtils.java @@ -23,7 +23,9 @@ import java.util.concurrent.TimeUnit; public final class UserHistoryForgettingCurveUtils { private static final String TAG = UserHistoryForgettingCurveUtils.class.getSimpleName(); private static final boolean DEBUG = false; - private static final int FC_FREQ_MAX = 127; + private static final int DEFAULT_FC_FREQ = 127; + private static final int BOOSTED_FC_FREQ = 200; + private static int FC_FREQ_MAX = DEFAULT_FC_FREQ; /* package */ static final int COUNT_MAX = 3; private static final int FC_LEVEL_MAX = 3; /* package */ static final int ELAPSED_TIME_MAX = 15; @@ -33,6 +35,14 @@ public final class UserHistoryForgettingCurveUtils { private static final int HALF_LIFE_HOURS = 48; private static final int MAX_PUSH_ELAPSED = (FC_LEVEL_MAX + 1) * (ELAPSED_TIME_MAX + 1); + public static void boostMaxFreqForDebug() { + FC_FREQ_MAX = BOOSTED_FC_FREQ; + } + + public static void resetMaxFreqForDebug() { + FC_FREQ_MAX = DEFAULT_FC_FREQ; + } + private UserHistoryForgettingCurveUtils() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/utils/UserLogRingCharBuffer.java b/java/src/com/android/inputmethod/latin/utils/UserLogRingCharBuffer.java index 161386e2e..a75d353c9 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserLogRingCharBuffer.java +++ b/java/src/com/android/inputmethod/latin/utils/UserLogRingCharBuffer.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin.utils; import android.inputmethodservice.InputMethodService; import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.settings.Settings; public final class UserLogRingCharBuffer { @@ -64,6 +65,9 @@ public final class UserLogRingCharBuffer { if (!mEnabled) { return; } + if (LatinImeLogger.sUsabilityStudy) { + UsabilityStudyLogUtils.getInstance().writeChar(c, x, y); + } mCharBuf[mEnd] = c; mXBuf[mEnd] = x; mYBuf[mEnd] = y; diff --git a/java/src/com/android/inputmethod/research/JsonUtils.java b/java/src/com/android/inputmethod/research/JsonUtils.java index 63d524df7..2beebdfae 100644 --- a/java/src/com/android/inputmethod/research/JsonUtils.java +++ b/java/src/com/android/inputmethod/research/JsonUtils.java @@ -75,12 +75,12 @@ import java.util.Map; private static void writeJson(final Key key, final JsonWriter jsonWriter) throws IOException { jsonWriter.beginObject(); - jsonWriter.name("code").value(key.mCode); + jsonWriter.name("code").value(key.getCode()); jsonWriter.name("altCode").value(key.getAltCode()); - jsonWriter.name("x").value(key.mX); - jsonWriter.name("y").value(key.mY); - jsonWriter.name("w").value(key.mWidth); - jsonWriter.name("h").value(key.mHeight); + jsonWriter.name("x").value(key.getX()); + jsonWriter.name("y").value(key.getY()); + jsonWriter.name("w").value(key.getWidth()); + jsonWriter.name("h").value(key.getHeight()); jsonWriter.endObject(); } @@ -103,7 +103,7 @@ import java.util.Map; jsonWriter.name("word").value(wordInfo.toString()); jsonWriter.name("score").value(wordInfo.mScore); jsonWriter.name("kind").value(wordInfo.mKind); - jsonWriter.name("sourceDict").value(wordInfo.mSourceDict); + jsonWriter.name("sourceDict").value(wordInfo.mSourceDict.mDictType); jsonWriter.endObject(); } jsonWriter.endArray(); diff --git a/java/src/com/android/inputmethod/research/ResearchLogger.java b/java/src/com/android/inputmethod/research/ResearchLogger.java index 3a3408266..da9c61103 100644 --- a/java/src/com/android/inputmethod/research/ResearchLogger.java +++ b/java/src/com/android/inputmethod/research/ResearchLogger.java @@ -1427,7 +1427,7 @@ public class ResearchLogger implements SharedPreferences.OnSharedPreferenceChang kid.navigateNext(), kid.navigatePrevious(), kid.mClobberSettingsKey, isPasswordView, kid.mShortcutKeyEnabled, kid.mHasShortcutKey, kid.mLanguageSwitchKeyEnabled, kid.isMultiLine(), keyboard.mOccupiedWidth, - keyboard.mOccupiedHeight, keyboard.mKeys); + keyboard.mOccupiedHeight, keyboard.getKeys()); } /** |