diff options
54 files changed, 1020 insertions, 331 deletions
diff --git a/java/res/layout/emoji_keyboard_view.xml b/java/res/layout/emoji_keyboard_view.xml index ccbcfdc6f..5fee419d0 100644 --- a/java/res/layout/emoji_keyboard_view.xml +++ b/java/res/layout/emoji_keyboard_view.xml @@ -76,18 +76,24 @@ android:id="@+id/emoji_action_bar" android:orientation="horizontal" android:layout_width="match_parent" - android:layout_height="@dimen/suggestions_strip_height" + android:layout_height="0dip" + android:layout_weight="1" > <ImageButton android:id="@+id/emoji_keyboard_alphabet" android:layout_width="0dip" - android:layout_weight="0.825" + android:layout_weight="0.15" android:layout_height="match_parent" android:src="@drawable/ic_ime_light" /> <ImageButton + android:id="@+id/emoji_keyboard_space" + android:layout_width="0dip" + android:layout_weight="0.70" + android:layout_height="match_parent" /> + <ImageButton android:id="@+id/emoji_keyboard_send" android:layout_width="0dip" - android:layout_weight="0.125" + android:layout_weight="0.15" android:layout_height="match_parent" android:src="@drawable/sym_keyboard_return_holo" /> </LinearLayout> diff --git a/java/res/values-fr-rCA/donottranslate.xml b/java/res/values-fr-rCA/donottranslate.xml new file mode 100644 index 000000000..21f18d852 --- /dev/null +++ b/java/res/values-fr-rCA/donottranslate.xml @@ -0,0 +1,31 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2009, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Symbols that are normally preceded by a space (used to add an auto-space before these) --> + <!-- This is similar to French with the exception of "!" "?" and ";" which do not take a space before in Canadian French. Note that ":" does take a space before according to Canadian rules. --> + <string name="symbols_preceded_by_space">([{&:</string> + <!-- Symbols that are normally followed by a space (used to add an auto-space after these) --> + <string name="symbols_followed_by_space">.,;:!?)]}&</string> + <!-- Symbols that separate words --> + <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) --> + <string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string> + <!-- Word connectors --> + <string name="symbols_word_connectors">\'-</string> +</resources> diff --git a/java/res/values-hi/strings.xml b/java/res/values-hi/strings.xml index fe03e5e8e..01bba5c6e 100644 --- a/java/res/values-hi/strings.xml +++ b/java/res/values-hi/strings.xml @@ -166,7 +166,7 @@ <string name="prefs_usability_study_mode" msgid="1261130555134595254">"उपयोगिता अध्ययन मोड"</string> <string name="prefs_key_longpress_timeout_settings" msgid="6102240298932897873">"कुंजी को देर तक दबाने का विलंब"</string> <string name="prefs_keypress_vibration_duration_settings" msgid="7918341459947439226">"कुंजी-स्पर्श कंपन अवधि"</string> - <string name="prefs_keypress_sound_volume_settings" msgid="6027007337036891623">"कुंजी-स्पर्श ध्वनि वॉल्यूम"</string> + <string name="prefs_keypress_sound_volume_settings" msgid="6027007337036891623">"कुंजी-स्पर्श ध्वनि आवाज़"</string> <string name="prefs_read_external_dictionary" msgid="2588931418575013067">"बाहरी शब्दकोश फ़ाइल पढ़ें"</string> <string name="read_external_dictionary_no_files_message" msgid="4947420942224623792">"डाउनलोड फ़ोल्डर में कोई शब्दकोश फ़ाइल नहीं है"</string> <string name="read_external_dictionary_multiple_files_title" msgid="7637749044265808628">"इंस्टॉल करने के लिए कोई शब्दकोश फ़ाइल चुनें"</string> @@ -186,15 +186,15 @@ <string name="setup_step2_instruction" msgid="9141481964870023336">"इसके बाद, \"<xliff:g id="APPLICATION_NAME">%s</xliff:g>\" को अपनी सक्रिय पाठ-इनपुट पद्धति के रूप में चुनें."</string> <string name="setup_step2_action" msgid="1660330307159824337">"इनपुट पद्धतियां स्विच करें"</string> <string name="setup_step3_title" msgid="3154757183631490281">"बधाई हो, आप बिल्कुल तैयार हैं!"</string> - <string name="setup_step3_instruction" msgid="8025981829605426000">"अब आप <xliff:g id="APPLICATION_NAME">%s</xliff:g> के साथ अपने सभी पसंदीदा एप्लिकेशन में लिख सकते हैं."</string> + <string name="setup_step3_instruction" msgid="8025981829605426000">"अब आप <xliff:g id="APPLICATION_NAME">%s</xliff:g> के साथ अपने सभी पसंदीदा एप्स में लिख सकते हैं."</string> <string name="setup_step3_action" msgid="600879797256942259">"अतिरिक्त भाषाएं कॉन्फ़िगर करना"</string> <string name="setup_finish_action" msgid="276559243409465389">"समाप्त"</string> - <string name="show_setup_wizard_icon" msgid="5008028590593710830">"एप्लिकेशन आइकन दिखाएं"</string> - <string name="show_setup_wizard_icon_summary" msgid="4119998322536880213">"लॉन्चर में एप्लिकेशन आइकन प्रदर्शित करें"</string> + <string name="show_setup_wizard_icon" msgid="5008028590593710830">"एप्स आइकन दिखाएं"</string> + <string name="show_setup_wizard_icon_summary" msgid="4119998322536880213">"लॉन्चर में एप्स आइकन प्रदर्शित करें"</string> <string name="app_name" msgid="6320102637491234792">"डिक्शनरी प्रदाता"</string> <string name="dictionary_provider_name" msgid="3027315045397363079">"डिक्शनरी प्रदाता"</string> <string name="dictionary_service_name" msgid="6237472350693511448">"डिक्शनरी सेवा"</string> - <string name="download_description" msgid="6014835283119198591">"डिक्शनरी अपडेट जानकारी"</string> + <string name="download_description" msgid="6014835283119198591">"शब्दकोश की नई जानकारी"</string> <string name="dictionary_settings_title" msgid="8091417676045693313">"एड-ऑन शब्दकोश"</string> <string name="dictionary_install_over_metered_network_prompt" msgid="3587517870006332980">"शब्दकोश उपलब्ध"</string> <string name="dictionary_settings_summary" msgid="5305694987799824349">"डिक्शनरी की सेटिंग"</string> @@ -207,8 +207,8 @@ <string name="cannot_connect_to_dict_service" msgid="9216933695765732398">"शब्दकोश सेवा से कनेक्ट करने में समस्या"</string> <string name="no_dictionaries_available" msgid="8039920716566132611">"डिक्शनरी अनुपलब्ध"</string> <string name="check_for_updates_now" msgid="8087688440916388581">"रीफ़्रेश करें"</string> - <string name="last_update" msgid="730467549913588780">"अंतिम अपडेट"</string> - <string name="message_updating" msgid="4457761393932375219">"अपडेट देखे जा रहे हैं"</string> + <string name="last_update" msgid="730467549913588780">"अंतिम बार का नई जानकारी"</string> + <string name="message_updating" msgid="4457761393932375219">"नई जानकारी देखा जा रहा हैं"</string> <string name="message_loading" msgid="8689096636874758814">"लोड हो रही है..."</string> <string name="main_dict_description" msgid="3072821352793492143">"मुख्य डिक्शनरी"</string> <string name="cancel" msgid="6830980399865683324">"रद्द करें"</string> diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml index 2a5334f95..09782143f 100644 --- a/java/res/values/attrs.xml +++ b/java/res/values/attrs.xml @@ -45,6 +45,8 @@ possible states: normal, pressed, checkable, checkable+pressed, checkable+checked, checkable+checked+pressed. --> <attr name="keyBackground" format="reference" /> + <!-- Image for the functional key used in Emoji layout. --> + <attr name="keyBackgroundEmojiFunctional" format="reference" /> <!-- Horizontal padding of left/right aligned key label to the edge of the key. --> <attr name="keyLabelHorizontalPadding" format="dimension" /> diff --git a/java/res/values/themes-ics.xml b/java/res/values/themes-ics.xml index f3b6b1321..19fb4fd27 100644 --- a/java/res/values/themes-ics.xml +++ b/java/res/values/themes-ics.xml @@ -103,7 +103,7 @@ name="EmojiKeyboardView.ICS" parent="KeyboardView.ICS" > - <item name="keyBackground">@drawable/btn_keyboard_key_functional_ics</item> + <item name="keyBackgroundEmojiFunctional">@drawable/btn_keyboard_key_functional_ics</item> <item name="emojiTabLabelColor">@color/emoji_tab_label_color_ics</item> </style> <style diff --git a/java/res/xml-sw600dp/key_f2.xml b/java/res/xml/key_f2.xml index 674153ae3..473dd210a 100644 --- a/java/res/xml-sw600dp/key_f2.xml +++ b/java/res/xml/key_f2.xml @@ -26,11 +26,13 @@ latin:mode="email|url" > <Key - latin:keyStyle="comKeyStyle" /> + latin:keyStyle="comKeyStyle" + latin:keyWidth="fillRight" /> </case> <default> <Key - latin:keyStyle="emojiKeyStyle" /> + latin:keyStyle="emojiKeyStyle" + latin:keyWidth="fillRight" /> </default> </switch> </merge> diff --git a/java/res/xml/keyboard_layout_set_armenian_phonetic.xml b/java/res/xml/keyboard_layout_set_armenian_phonetic.xml index b374fae3b..35bd43fb9 100644 --- a/java/res/xml/keyboard_layout_set_armenian_phonetic.xml +++ b/java/res/xml/keyboard_layout_set_armenian_phonetic.xml @@ -28,6 +28,9 @@ latin:elementName="symbols" latin:elementKeyboard="@xml/kbd_symbols" /> <Element + latin:elementName="symbolsShifted" + latin:elementKeyboard="@xml/kbd_symbols_shift" /> + <Element latin:elementName="phone" latin:elementKeyboard="@xml/kbd_phone" /> <Element diff --git a/java/res/xml/row_symbols_shift4.xml b/java/res/xml/row_symbols_shift4.xml index 57a2ec09c..72d24a321 100644 --- a/java/res/xml/row_symbols_shift4.xml +++ b/java/res/xml/row_symbols_shift4.xml @@ -24,8 +24,7 @@ latin:keyboardLayout="@xml/key_space_symbols" /> <include latin:keyboardLayout="@xml/keys_comma_period" /> - <Key - latin:keyStyle="enterKeyStyle" - latin:keyWidth="fillRight" /> + <include + latin:keyboardLayout="@xml/key_f2" /> </merge> diff --git a/java/res/xml/rowkeys_nepali_traditional2.xml b/java/res/xml/rowkeys_nepali_traditional2.xml index 2c53b3b8b..45620a9d7 100644 --- a/java/res/xml/rowkeys_nepali_traditional2.xml +++ b/java/res/xml/rowkeys_nepali_traditional2.xml @@ -29,13 +29,13 @@ <Key latin:keyLabel="आ" latin:keyLabelFlags="fontNormal" /> - <!-- U+0919/U+094D/U+0915: "ङ्क" DEVANAGARI LETTER NGA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER KA --> + <!-- U+0919/U+094D: "ङ्" DEVANAGARI LETTER NGA/DEVANAGARI SIGN VIRAMA --> <Key - latin:keyLabel="ङ्क" + latin:keyLabel="ङ्" latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> - <!-- U+0919/U+094D/U+0917: "ङ्ग" DEVANAGARI LETTER NGA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER GA --> + <!-- U+0921/U+094D/U+0921: "ड्ड" DEVANAGARI LETTER DDA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER DDA --> <Key - latin:keyLabel="ङ्ग" + latin:keyLabel="ड्ड" latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> <!-- Because the font rendering system prior to API version 16 can't automatically render dotted circle for incomplete combining letter of some scripts, different diff --git a/java/res/xml/rowkeys_nepali_traditional3_left6.xml b/java/res/xml/rowkeys_nepali_traditional3_left6.xml index d4388e0a2..1cacced83 100644 --- a/java/res/xml/rowkeys_nepali_traditional3_left6.xml +++ b/java/res/xml/rowkeys_nepali_traditional3_left6.xml @@ -25,9 +25,9 @@ <case latin:keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" > - <!-- U+0915/U+094D/U+0915: "क्क" DEVANAGARI LETTER KA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER KA --> + <!-- U+0915/U+094D: "क्" DEVANAGARI LETTER KA/DEVANAGARI SIGN VIRAMA --> <Key - latin:keyLabel="क्क" + latin:keyLabel="क्" latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> <!-- U+0939/U+094D/U+092E: "ह्म" DEVANAGARI LETTER HA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER MA --> <Key diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java index 25ff8d0ce..9996a6d6a 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java @@ -22,9 +22,11 @@ import android.content.Context; import android.content.res.ColorStateList; import android.content.res.Resources; import android.content.res.TypedArray; +import android.os.Build; import android.support.v4.view.PagerAdapter; import android.support.v4.view.ViewPager; import android.util.AttributeSet; +import android.util.Log; import android.util.SparseArray; import android.view.LayoutInflater; import android.view.View; @@ -35,7 +37,7 @@ import android.widget.TabHost; import android.widget.TabHost.OnTabChangeListener; import android.widget.TextView; -import com.android.inputmethod.keyboard.internal.RecentsKeyboard; +import com.android.inputmethod.keyboard.internal.DynamicGridKeyboard; import com.android.inputmethod.keyboard.internal.ScrollKeyboardView; import com.android.inputmethod.keyboard.internal.ScrollViewWithNotifier; import com.android.inputmethod.latin.Constants; @@ -44,6 +46,7 @@ import com.android.inputmethod.latin.SubtypeSwitcher; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.ResourceUtils; +import java.util.ArrayList; import java.util.HashMap; /** @@ -60,51 +63,134 @@ import java.util.HashMap; public final class EmojiKeyboardView extends LinearLayout implements OnTabChangeListener, ViewPager.OnPageChangeListener, View.OnClickListener, ScrollKeyboardView.OnKeyClickListener { + private static final String TAG = EmojiKeyboardView.class.getSimpleName(); private final int mKeyBackgroundId; + private final int mEmojiFunctionalKeyBackgroundId; + private final KeyboardLayoutSet mLayoutSet; private final ColorStateList mTabLabelColor; - private final EmojiKeyboardAdapter mEmojiKeyboardAdapter; + private EmojiKeyboardAdapter mEmojiKeyboardAdapter; private TabHost mTabHost; private ViewPager mEmojiPager; private KeyboardActionListener mKeyboardActionListener = KeyboardActionListener.EMPTY_LISTENER; - private int mCurrentCategory = CATEGORY_UNSPECIFIED; - private static final int CATEGORY_UNSPECIFIED = -1; - private static final int CATEGORY_RECENTS = 0; - private static final int CATEGORY_PEOPLE = 1; - private static final int CATEGORY_OBJECTS = 2; - private static final int CATEGORY_NATURE = 3; - private static final int CATEGORY_PLACES = 4; - private static final int CATEGORY_SYMBOLS = 5; - private static final int CATEGORY_EMOTICONS = 6; - private static final HashMap<String, Integer> sCategoryNameToIdMap = - CollectionUtils.newHashMap(); - private static final String[] sCategoryName = { - "recents", "people", "objects", "nature", "places", "symbols", "emoticons" - }; - private static final int[] sCategoryIcon = new int[] { - R.drawable.ic_emoji_recent_light, - R.drawable.ic_emoji_people_light, - R.drawable.ic_emoji_objects_light, - R.drawable.ic_emoji_nature_light, - R.drawable.ic_emoji_places_light, - R.drawable.ic_emoji_symbols_light, - 0 - }; - private static final String[] sCategoryLabel = { - null, null, null, null, null, null, - ":-)" - }; - private static final int[] sCategoryElementId = { - KeyboardId.ELEMENT_EMOJI_RECENTS, - KeyboardId.ELEMENT_EMOJI_CATEGORY1, - KeyboardId.ELEMENT_EMOJI_CATEGORY2, - KeyboardId.ELEMENT_EMOJI_CATEGORY3, - KeyboardId.ELEMENT_EMOJI_CATEGORY4, - KeyboardId.ELEMENT_EMOJI_CATEGORY5, - KeyboardId.ELEMENT_EMOJI_CATEGORY6, - }; + private static class EmojiCategory { + private int mCurrentCategory = CATEGORY_UNSPECIFIED; + private static final int CATEGORY_UNSPECIFIED = -1; + private static final int CATEGORY_RECENTS = 0; + private static final int CATEGORY_PEOPLE = 1; + private static final int CATEGORY_OBJECTS = 2; + private static final int CATEGORY_NATURE = 3; + private static final int CATEGORY_PLACES = 4; + private static final int CATEGORY_SYMBOLS = 5; + private static final int CATEGORY_EMOTICONS = 6; + private static final String[] sCategoryName = { + "recents", + "people", + "objects", + "nature", + "places", + "symbols", + "emoticons" }; + private static final int[] sCategoryIcon = new int[] { + R.drawable.ic_emoji_recent_light, + R.drawable.ic_emoji_people_light, + R.drawable.ic_emoji_objects_light, + R.drawable.ic_emoji_nature_light, + R.drawable.ic_emoji_places_light, + R.drawable.ic_emoji_symbols_light, + 0 }; + private static final String[] sCategoryLabel = + { null, null, null, null, null, null, ":-)" }; + private static final int[] sCategoryElementId = { + KeyboardId.ELEMENT_EMOJI_RECENTS, + KeyboardId.ELEMENT_EMOJI_CATEGORY1, + KeyboardId.ELEMENT_EMOJI_CATEGORY2, + KeyboardId.ELEMENT_EMOJI_CATEGORY3, + KeyboardId.ELEMENT_EMOJI_CATEGORY4, + KeyboardId.ELEMENT_EMOJI_CATEGORY5, + KeyboardId.ELEMENT_EMOJI_CATEGORY6, }; + private final HashMap<String, Integer> mCategoryNameToIdMap = CollectionUtils.newHashMap(); + private final ArrayList<Integer> mShownCategories = new ArrayList<Integer>(); + + public EmojiCategory() { + for (int i = 0; i < sCategoryName.length; ++i) { + mCategoryNameToIdMap.put(sCategoryName[i], i); + } + mShownCategories.add(CATEGORY_RECENTS); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { + mShownCategories.add(CATEGORY_PEOPLE); + mShownCategories.add(CATEGORY_OBJECTS); + mShownCategories.add(CATEGORY_NATURE); + mShownCategories.add(CATEGORY_PLACES); + // TODO: Restore last saved category + mCurrentCategory = CATEGORY_PEOPLE; + } else { + // TODO: Restore last saved category + mCurrentCategory = CATEGORY_SYMBOLS; + } + mShownCategories.add(CATEGORY_SYMBOLS); + mShownCategories.add(CATEGORY_EMOTICONS); + } + + public String getCategoryName(int category) { + return sCategoryName[category]; + } + + public int getCategoryId(String name) { + return mCategoryNameToIdMap.get(name); + } + + public int getCategoryIcon(int category) { + return sCategoryIcon[category]; + } + + public String getCategoryLabel(int category) { + return sCategoryLabel[category]; + } + + public ArrayList<Integer> getShownCategories() { + return mShownCategories; + } + + public int getCurrentCategory() { + // TODO: Record current category. + return mCurrentCategory; + } + + public void setCurrentCategory(int category) { + mCurrentCategory = category; + } + + public boolean isInRecentTab() { + return mCurrentCategory == CATEGORY_RECENTS; + } + + public int getTabIdFromCategory(int category) { + for (int i = 0; i < mShownCategories.size(); ++i) { + if (mShownCategories.get(i) == category) { + return i; + } + } + Log.w(TAG, "category not found: " + category); + return 0; + } + + public int getRecentTabId() { + return getTabIdFromCategory(CATEGORY_RECENTS); + } + + public int getCategoryFromTabId(int tabId) { + return mShownCategories.get(tabId); + } + + public int getElementIdFromTabId(int tabId) { + return sCategoryElementId[getCategoryFromTabId(tabId)]; + } + } + + private final EmojiCategory mEmojiCategory = new EmojiCategory(); public EmojiKeyboardView(final Context context, final AttributeSet attrs) { this(context, attrs, R.attr.emojiKeyboardViewStyle); @@ -116,6 +202,8 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange R.styleable.KeyboardView, defStyle, R.style.KeyboardView); mKeyBackgroundId = keyboardViewAttr.getResourceId( R.styleable.KeyboardView_keyBackground, 0); + mEmojiFunctionalKeyBackgroundId = keyboardViewAttr.getResourceId( + R.styleable.KeyboardView_keyBackgroundEmojiFunctional, 0); keyboardViewAttr.recycle(); final TypedArray emojiKeyboardViewAttr = context.obtainStyledAttributes(attrs, R.styleable.EmojiKeyboardView, defStyle, R.style.EmojiKeyboardView); @@ -126,13 +214,11 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange context, null /* editorInfo */); final Resources res = context.getResources(); builder.setSubtype(SubtypeSwitcher.getInstance().getEmojiSubtype()); - // TODO: Make Keyboard height variable. builder.setKeyboardGeometry(ResourceUtils.getDefaultKeyboardWidth(res), - (int)(ResourceUtils.getDefaultKeyboardHeight(res) - - res.getDimension(R.dimen.suggestions_strip_height))); + (int)ResourceUtils.getDefaultKeyboardHeight(res) + + res.getDimensionPixelSize(R.dimen.suggestions_strip_height)); builder.setOptions(false, false, false /* lanuageSwitchKeyEnabled */); - final KeyboardLayoutSet layoutSet = builder.build(); - mEmojiKeyboardAdapter = new EmojiKeyboardAdapter(layoutSet, this); + mLayoutSet = builder.build(); // TODO: Save/restore recent keys from/to preferences. } @@ -150,22 +236,20 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange } private void addTab(final TabHost host, final int category) { - final String tabId = sCategoryName[category]; - sCategoryNameToIdMap.put(tabId, category); + final String tabId = mEmojiCategory.getCategoryName(category); final TabHost.TabSpec tspec = host.newTabSpec(tabId); tspec.setContent(R.id.emoji_keyboard_dummy); - if (sCategoryIcon[category] != 0) { + if (mEmojiCategory.getCategoryIcon(category) != 0) { final ImageView iconView = (ImageView)LayoutInflater.from(getContext()).inflate( R.layout.emoji_keyboard_tab_icon, null); - iconView.setImageResource(sCategoryIcon[category]); + iconView.setImageResource(mEmojiCategory.getCategoryIcon(category)); tspec.setIndicator(iconView); } - if (sCategoryLabel[category] != null) { + if (mEmojiCategory.getCategoryLabel(category) != null) { final TextView textView = (TextView)LayoutInflater.from(getContext()).inflate( R.layout.emoji_keyboard_tab_label, null); - textView.setText(sCategoryLabel[category]); + textView.setText(mEmojiCategory.getCategoryLabel(category)); textView.setTextColor(mTabLabelColor); - textView.setBackgroundResource(mKeyBackgroundId); tspec.setIndicator(textView); } host.addTab(tspec); @@ -175,55 +259,57 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange protected void onFinishInflate() { mTabHost = (TabHost)findViewById(R.id.emoji_category_tabhost); mTabHost.setup(); - addTab(mTabHost, CATEGORY_RECENTS); - addTab(mTabHost, CATEGORY_PEOPLE); - addTab(mTabHost, CATEGORY_OBJECTS); - addTab(mTabHost, CATEGORY_NATURE); - addTab(mTabHost, CATEGORY_PLACES); - addTab(mTabHost, CATEGORY_SYMBOLS); - addTab(mTabHost, CATEGORY_EMOTICONS); + for (final int i : mEmojiCategory.getShownCategories()) { + addTab(mTabHost, i); + } mTabHost.setOnTabChangedListener(this); mTabHost.getTabWidget().setStripEnabled(true); + mEmojiKeyboardAdapter = new EmojiKeyboardAdapter(mEmojiCategory, mLayoutSet, this); + mEmojiPager = (ViewPager)findViewById(R.id.emoji_keyboard_pager); mEmojiPager.setAdapter(mEmojiKeyboardAdapter); mEmojiPager.setOnPageChangeListener(this); mEmojiPager.setOffscreenPageLimit(0); - final ViewGroup.LayoutParams lp = mEmojiPager.getLayoutParams(); final Resources res = getResources(); - lp.height = ResourceUtils.getDefaultKeyboardHeight(res) - - res.getDimensionPixelSize(R.dimen.suggestions_strip_height); - mEmojiPager.setLayoutParams(lp); + final EmojiLayoutParams emojiLp = new EmojiLayoutParams(res); + emojiLp.setPagerProps(mEmojiPager); + + setCurrentCategory(mEmojiCategory.getCurrentCategory(), true /* force */); - // TODO: Record current category. - final int category = CATEGORY_PEOPLE; - setCurrentCategory(category, true /* force */); + final LinearLayout actionBar = (LinearLayout)findViewById(R.id.emoji_action_bar); + emojiLp.setActionBarProps(actionBar); // TODO: Implement auto repeat, using View.OnTouchListener? - final View deleteKey = findViewById(R.id.emoji_keyboard_delete); - deleteKey.setBackgroundResource(mKeyBackgroundId); + final ImageView deleteKey = (ImageView)findViewById(R.id.emoji_keyboard_delete); + deleteKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); deleteKey.setTag(Constants.CODE_DELETE); deleteKey.setOnClickListener(this); - final View alphabetKey = findViewById(R.id.emoji_keyboard_alphabet); - alphabetKey.setBackgroundResource(mKeyBackgroundId); + final ImageView alphabetKey = (ImageView)findViewById(R.id.emoji_keyboard_alphabet); + alphabetKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); alphabetKey.setTag(Constants.CODE_SWITCH_ALPHA_SYMBOL); alphabetKey.setOnClickListener(this); - final View sendKey = findViewById(R.id.emoji_keyboard_send); - sendKey.setBackgroundResource(mKeyBackgroundId); + final ImageView spaceKey = (ImageView)findViewById(R.id.emoji_keyboard_space); + spaceKey.setBackgroundResource(mKeyBackgroundId); + spaceKey.setTag(Constants.CODE_SPACE); + spaceKey.setOnClickListener(this); + emojiLp.setKeyProps(spaceKey); + final ImageView sendKey = (ImageView)findViewById(R.id.emoji_keyboard_send); + sendKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); sendKey.setTag(Constants.CODE_ENTER); sendKey.setOnClickListener(this); } @Override public void onTabChanged(final String tabId) { - final int category = sCategoryNameToIdMap.get(tabId); + final int category = mEmojiCategory.getCategoryId(tabId); setCurrentCategory(category, false /* force */); } @Override public void onPageSelected(final int position) { - setCurrentCategory(position, false /* force */); + setCurrentCategory(mEmojiCategory.getCategoryFromTabId(position), false /* force */); } @Override @@ -272,16 +358,17 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange } private void setCurrentCategory(final int category, final boolean force) { - if (mCurrentCategory == category && !force) { + if (mEmojiCategory.getCurrentCategory() == category && !force) { return; } - mCurrentCategory = category; - if (force || mEmojiPager.getCurrentItem() != category) { - mEmojiPager.setCurrentItem(category, true /* smoothScroll */); + mEmojiCategory.setCurrentCategory(category); + final int newTabId = mEmojiCategory.getTabIdFromCategory(category); + if (force || mEmojiPager.getCurrentItem() != newTabId) { + mEmojiPager.setCurrentItem(newTabId, true /* smoothScroll */); } - if (force || mTabHost.getCurrentTab() != category) { - mTabHost.setCurrentTab(category); + if (force || mTabHost.getCurrentTab() != newTabId) { + mTabHost.setCurrentTab(newTabId); } // TODO: Record current category } @@ -289,25 +376,29 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange private static class EmojiKeyboardAdapter extends PagerAdapter { private final ScrollKeyboardView.OnKeyClickListener mListener; private final KeyboardLayoutSet mLayoutSet; - private final RecentsKeyboard mRecentsKeyboard; + private final DynamicGridKeyboard mRecentsKeyboard; private final SparseArray<ScrollKeyboardView> mActiveKeyboardView = CollectionUtils.newSparseArray(); - private int mActivePosition = CATEGORY_UNSPECIFIED; + private final EmojiCategory mEmojiCategory; + private int mActivePosition = 0; - public EmojiKeyboardAdapter(final KeyboardLayoutSet layoutSet, + public EmojiKeyboardAdapter(final EmojiCategory emojiCategory, + final KeyboardLayoutSet layoutSet, final ScrollKeyboardView.OnKeyClickListener listener) { + mEmojiCategory = emojiCategory; mListener = listener; mLayoutSet = layoutSet; - mRecentsKeyboard = new RecentsKeyboard( + mRecentsKeyboard = new DynamicGridKeyboard( layoutSet.getKeyboard(KeyboardId.ELEMENT_EMOJI_RECENTS)); } public void addRecentKey(final Key key) { - if (mActivePosition == CATEGORY_RECENTS) { + if (mEmojiCategory.isInRecentTab()) { return; } mRecentsKeyboard.addRecentKey(key); - final KeyboardView recentKeyboardView = mActiveKeyboardView.get(CATEGORY_RECENTS); + final KeyboardView recentKeyboardView = + mActiveKeyboardView.get(mEmojiCategory.getRecentTabId()); if (recentKeyboardView != null) { recentKeyboardView.invalidateAllKeys(); } @@ -315,7 +406,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange @Override public int getCount() { - return sCategoryName.length; + return mEmojiCategory.getShownCategories().size(); } @Override @@ -333,7 +424,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange @Override public Object instantiateItem(final ViewGroup container, final int position) { - final int elementId = sCategoryElementId[position]; + final int elementId = mEmojiCategory.getElementIdFromTabId(position); final Keyboard keyboard = (elementId == KeyboardId.ELEMENT_EMOJI_RECENTS) ? mRecentsKeyboard : mLayoutSet.getKeyboard(elementId); final LayoutInflater inflater = LayoutInflater.from(container.getContext()); diff --git a/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java new file mode 100644 index 000000000..6486fc9db --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard; + +import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.utils.ResourceUtils; + +import android.content.res.Resources; +import android.support.v4.view.ViewPager; +import android.widget.ImageView; +import android.widget.LinearLayout; + +public class EmojiLayoutParams { + private static final int DEFAULT_KEYBOARD_ROWS = 4; + + public final int mEmojiKeyboardHeight; + public final int mEmojiActionBarHeight; + public final int mKeyVerticalGap; + private final int mKeyHorizontalGap; + private final int mBottomPadding; + private final int mTopPadding; + + public EmojiLayoutParams(Resources res) { + final int defaultKeyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); + final int defaultKeyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); + mKeyVerticalGap = (int) res.getFraction(R.fraction.key_bottom_gap_ics, + (int) defaultKeyboardHeight, (int) defaultKeyboardHeight); + mBottomPadding = (int) res.getFraction(R.fraction.keyboard_bottom_padding_ics, + (int) defaultKeyboardHeight, (int) defaultKeyboardHeight); + mTopPadding = (int) res.getFraction(R.fraction.keyboard_top_padding_ics, + (int) defaultKeyboardHeight, (int) defaultKeyboardHeight); + mKeyHorizontalGap = (int) (res.getFraction(R.fraction.key_horizontal_gap_ics, + defaultKeyboardWidth, defaultKeyboardWidth)); + final int baseheight = defaultKeyboardHeight - mBottomPadding - mTopPadding + + mKeyVerticalGap; + mEmojiActionBarHeight = ((int) baseheight) / DEFAULT_KEYBOARD_ROWS + - (mKeyVerticalGap - mBottomPadding) / 2; + mEmojiKeyboardHeight = defaultKeyboardHeight - mEmojiActionBarHeight; + } + + public void setPagerProps(ViewPager vp) { + final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) vp.getLayoutParams(); + lp.height = mEmojiKeyboardHeight - mKeyVerticalGap / 2; + lp.bottomMargin = mKeyVerticalGap / 2; + vp.setLayoutParams(lp); + } + + public void setActionBarProps(LinearLayout ll) { + final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) ll.getLayoutParams(); + lp.height = mEmojiActionBarHeight; + lp.topMargin = 0; + lp.bottomMargin = mBottomPadding; + ll.setLayoutParams(lp); + } + + public void setKeyProps(ImageView ib) { + final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) ib.getLayoutParams(); + lp.leftMargin = mKeyHorizontalGap / 2; + lp.rightMargin = mKeyHorizontalGap / 2; + ib.setLayoutParams(lp); + } +} diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index d128d3595..cc1ffd183 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -314,15 +314,19 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { mState.onCodeInput(code, mLatinIME.getCurrentAutoCapsState()); } + public boolean isShowingEmojiKeyboard() { + return mEmojiKeyboardView.getVisibility() == View.VISIBLE; + } + public boolean isShowingMoreKeysPanel() { - if (mEmojiKeyboardView.getVisibility() == View.VISIBLE) { + if (isShowingEmojiKeyboard()) { return false; } return mKeyboardView.isShowingMoreKeysPanel(); } public View getVisibleKeyboardView() { - if (mEmojiKeyboardView.getVisibility() == View.VISIBLE) { + if (isShowingEmojiKeyboard()) { return mEmojiKeyboardView; } return mKeyboardView; diff --git a/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java index 629c60460..a226891b4 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java +++ b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java @@ -26,10 +26,10 @@ import java.util.ArrayDeque; import java.util.Random; /** - * This is a Keyboard class to host recently used keys. + * This is a Keyboard class where you can add keys dynamically shown in a grid layout */ // TODO: Save/restore recent keys from/to preferences. -public class RecentsKeyboard extends Keyboard { +public class DynamicGridKeyboard extends Keyboard { private static final int TEMPLATE_KEY_CODE_0 = 0x30; private static final int TEMPLATE_KEY_CODE_1 = 0x31; @@ -42,7 +42,7 @@ public class RecentsKeyboard extends Keyboard { private Key[] mCachedRecentKeys; - public RecentsKeyboard(final Keyboard templateKeyboard) { + public DynamicGridKeyboard(final Keyboard templateKeyboard) { super(templateKeyboard); final Key key0 = getTemplateKey(TEMPLATE_KEY_CODE_0); final Key key1 = getTemplateKey(TEMPLATE_KEY_CODE_1); diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index eb5c4bef9..dacb8483c 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -46,6 +46,7 @@ public final class BinaryDictionary extends Dictionary { private long mNativeDict; private final Locale mLocale; + private final long mDictSize; private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH]; private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS]; private final int[] mSpaceIndices = new int[MAX_RESULTS]; @@ -66,7 +67,7 @@ public final class BinaryDictionary extends Dictionary { if (traverseSession == null) { traverseSession = mDicTraverseSessions.get(traverseSessionId); if (traverseSession == null) { - traverseSession = new DicTraverseSession(mLocale, mNativeDict); + traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); mDicTraverseSessions.put(traverseSessionId, traverseSession); } } @@ -89,6 +90,7 @@ public final class BinaryDictionary extends Dictionary { final boolean isUpdatable) { super(dictType); mLocale = locale; + mDictSize = length; mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance); loadDictionary(filename, offset, length, isUpdatable); } diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 2b6d983c0..566184244 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,9 +21,10 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.LocaleUtils; @@ -228,7 +229,7 @@ final public class BinaryDictionaryGetter { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { try { // Read the version of the file - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f); final FileHeader header = dictDecoder.readHeader(); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); diff --git a/java/src/com/android/inputmethod/latin/DicTraverseSession.java b/java/src/com/android/inputmethod/latin/DicTraverseSession.java index 45b281318..8d295adee 100644 --- a/java/src/com/android/inputmethod/latin/DicTraverseSession.java +++ b/java/src/com/android/inputmethod/latin/DicTraverseSession.java @@ -25,16 +25,16 @@ public final class DicTraverseSession { JniUtils.loadNativeLibrary(); } - private static native long setDicTraverseSessionNative(String locale); + private static native long setDicTraverseSessionNative(String locale, long dictSize); private static native void initDicTraverseSessionNative(long nativeDicTraverseSession, long dictionary, int[] previousWord, int previousWordLength); private static native void releaseDicTraverseSessionNative(long nativeDicTraverseSession); private long mNativeDicTraverseSession; - public DicTraverseSession(Locale locale, long dictionary) { + public DicTraverseSession(Locale locale, long dictionary, long dictSize) { mNativeDicTraverseSession = createNativeDicTraverseSession( - locale != null ? locale.toString() : ""); + locale != null ? locale.toString() : "", dictSize); initSession(dictionary); } @@ -51,8 +51,8 @@ public final class DicTraverseSession { mNativeDicTraverseSession, dictionary, previousWord, previousWordLength); } - private final long createNativeDicTraverseSession(String locale) { - return setDicTraverseSessionNative(locale); + private final long createNativeDicTraverseSession(String locale, long dictSize) { + return setDicTraverseSessionNative(locale, dictSize); } private void closeInternal() { diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index c884e7b1f..2a9076436 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -617,4 +617,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } + + @UsedForTesting + public void shutdownExecutorForTests() { + getExecutor(mFilename).shutdown(); + } + + @UsedForTesting + public boolean isTerminatedForTests() { + return getExecutor(mFilename).isTerminated(); + } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index b7b86b760..d8a47a307 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1234,7 +1234,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen int visibleTopY = extraHeight; // Need to set touchable region only if input view is being shown if (visibleKeyboardView.isShown()) { - if (mSuggestionStripView.getVisibility() == View.VISIBLE) { + // Note that the height of Emoji layout is the same as the height of the main keyboard + // and the suggestion strip + if (mKeyboardSwitcher.isShowingEmojiKeyboard() + || mSuggestionStripView.getVisibility() == View.VISIBLE) { visibleTopY -= suggestionsHeight; } final int touchY = mKeyboardSwitcher.isShowingMoreKeysPanel() ? 0 : visibleTopY; @@ -2671,6 +2674,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // recorrection. This is a temporary, stopgap measure that will be removed later. // TODO: remove this. if (mAppWorkAroundsUtils.isBrokenByRecorrection()) return; + // A simple way to test for support from the TextView. + if (!isSuggestionsStripVisible()) return; // Recorrection is not supported in languages without spaces because we don't know // how to segment them yet. if (!mSettings.getCurrent().mCurrentLanguageHasSpaces) return; diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 21e9811ef..f333b0d86 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -126,8 +126,14 @@ public class BinaryDictEncoderUtils { */ private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { int size = getNodeHeaderSize(ptNode, options); - // If terminal, one byte for the frequency - if (ptNode.isTerminal()) size += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (ptNode.isTerminal()) { + // If terminal, one byte for the frequency or four bytes for the terminal id. + if (options.mHasTerminalId) { + size += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + size += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address size += getShortcutListSize(ptNode.mShortcutTargets); if (null != ptNode.mBigrams) { @@ -345,7 +351,13 @@ public class BinaryDictEncoderUtils { changed = true; } int nodeSize = getNodeHeaderSize(ptNode, formatOptions); - if (ptNode.isTerminal()) nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (ptNode.isTerminal()) { + if (formatOptions.mHasTerminalId) { + nodeSize += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } if (formatOptions.mSupportsDynamicUpdate) { nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; } else if (null != ptNode.mChildren) { @@ -787,7 +799,6 @@ public class BinaryDictEncoderUtils { + FormatSpec.MAX_TERMINAL_FREQUENCY + " : " + ptNode.mFrequency); } - dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); } if (formatOptions.mSupportsDynamicUpdate) { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 5cb40f30c..2c5e93e5c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -148,7 +148,7 @@ public final class BinaryDictIOUtils { * @throws IOException if the file can't be read. * @throws UnsupportedFormatException if the format of the file is not recognized. */ - /* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, + /* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { @@ -169,7 +169,7 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - /* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder, + /* package */ static int getTerminalPosition(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { if (word == null) return FormatSpec.NOT_VALID_WORD; dictDecoder.setPosition(0); @@ -519,7 +519,7 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, new DictDecoder.DictionaryBufferFactory() { @Override public DictBuffer getDictionaryBuffer(File file) diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 5e398bd41..40e852423 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -118,6 +118,14 @@ public interface DictDecoder { public boolean readForwardLinkAndAdvancePosition(); public boolean hasNextPtNodeArray(); + /** + * Opens the dictionary file and makes DictBuffer. + */ + @UsedForTesting + public void openDictBuffer() throws FileNotFoundException, IOException; + @UsedForTesting + public boolean isOpenedDictBuffer(); + // Flags for DictionaryBufferFactory. public static final int USE_READONLY_BYTEBUFFER = 0x01000000; public static final int USE_BYTEARRAY = 0x02000000; diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index bf35f6a8a..96ccd8e49 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -18,8 +18,11 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import java.io.File; + /** * Dictionary File Format Specification. */ @@ -195,9 +198,12 @@ public final class FormatSpec { public static final int MAGIC_NUMBER = 0x9BC13AFE; static final int MINIMUM_SUPPORTED_VERSION = 2; - static final int MAXIMUM_SUPPORTED_VERSION = 3; + static final int MAXIMUM_SUPPORTED_VERSION = 4; static final int NOT_A_VERSION_NUMBER = -1; static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; + static final int FIRST_VERSION_WITH_TERMINAL_ID = 4; + static final int VERSION3 = 3; + static final int VERSION4 = 4; // These options need to be the same numeric values as the one in the native reading code. static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; @@ -248,11 +254,17 @@ public final class FormatSpec { static final int PTNODE_TERMINATOR_SIZE = 1; static final int PTNODE_FLAGS_SIZE = 1; static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_TERMINAL_ID_SIZE = 4; static final int PTNODE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; + // These values are used only by version 4 or later. + static final String TRIE_FILE_EXTENSION = ".trie"; + static final String FREQ_FILE_EXTENSION = ".freq"; + static final int FREQUENCY_AND_FLAGS_SIZE = 2; + static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; @@ -261,6 +273,7 @@ public final class FormatSpec { static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; + static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF; static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; @@ -284,6 +297,7 @@ public final class FormatSpec { public static final class FormatOptions { public final int mVersion; public final boolean mSupportsDynamicUpdate; + public final boolean mHasTerminalId; @UsedForTesting public FormatOptions(final int version) { this(version, false); @@ -297,6 +311,7 @@ public final class FormatSpec { + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); } mSupportsDynamicUpdate = supportsDynamicUpdate; + mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); } } @@ -341,6 +356,28 @@ public final class FormatSpec { } } + /** + * Returns new dictionary decoder. + * + * @param dictFile the dictionary file. + * @param bufferType the flag indicating buffer type which is used by the dictionary decoder. + * @return new dictionary decoder if the dictionary file exists, otherwise null. + */ + public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) { + if (!dictFile.isFile()) return null; + return new Ver3DictDecoder(dictFile, bufferType); + } + + public static DictDecoder getDictDecoder(final File dictFile, + final DictionaryBufferFactory factory) { + if (!dictFile.isFile()) return null; + return new Ver3DictDecoder(dictFile, factory); + } + + public static DictDecoder getDictDecoder(final File dictFile) { + return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER); + } + private FormatSpec() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 3e685a3d7..be653feec 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -111,6 +111,7 @@ public final class FusionDictionary implements Iterable<Word> { ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal. + int mTerminalId; // NOT_A_TERMINAL == mTerminalId indicates this is not a terminal. PtNodeArray mChildren; boolean mIsNotAWord; // Only a shortcut boolean mIsBlacklistEntry; @@ -129,6 +130,7 @@ public final class FusionDictionary implements Iterable<Word> { final boolean isNotAWord, final boolean isBlacklistEntry) { mChars = chars; mFrequency = frequency; + mTerminalId = frequency; mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = null; @@ -156,6 +158,10 @@ public final class FusionDictionary implements Iterable<Word> { mChildren.mData.add(n); } + public int getTerminalId() { + return mTerminalId; + } + public boolean isTerminal() { return NOT_A_TERMINAL != mFrequency; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java index 6dff9b6d2..1a90a4b98 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -173,11 +173,7 @@ public class Ver3DictDecoder implements DictDecoder { private final DictionaryBufferFactory mBufferFactory; private DictBuffer mDictBuffer; - public Ver3DictDecoder(final File file) { - this(file, USE_READONLY_BYTEBUFFER); - } - - public Ver3DictDecoder(final File file, final int factoryFlag) { + /* package */ Ver3DictDecoder(final File file, final int factoryFlag) { mDictionaryBinaryFile = file; mDictBuffer = null; @@ -192,15 +188,21 @@ public class Ver3DictDecoder implements DictDecoder { } } - public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { + /* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { mDictionaryBinaryFile = file; mBufferFactory = factory; } + @Override public void openDictBuffer() throws FileNotFoundException, IOException { mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); } + @Override + public boolean isOpenedDictBuffer() { + return mDictBuffer != null; + } + /* package */ DictBuffer getDictBuffer() { return mDictBuffer; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 48a823d43..222a0f474 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -68,7 +68,7 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { - if (formatOptions.mVersion > 3) { + if (formatOptions.mVersion > FormatSpec.VERSION3) { throw new UnsupportedFormatException( "The given format options has wrong version number : " + formatOptions.mVersion); @@ -200,7 +200,7 @@ public class Ver3DictEncoder implements DictEncoder { mPosition += shortcutShift; } final int shortcutByteSize = mPosition - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { throw new RuntimeException("Shortcut list too large"); } BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java new file mode 100644 index 000000000..75b75ae2e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -0,0 +1,269 @@ +/* +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * An implementation of DictEncoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictEncoder implements DictEncoder { + private final File mDictPlacedDir; + private byte[] mTrieBuf; + private byte[] mFreqBuf; + private int mTriePos; + private OutputStream mTrieOutStream; + private OutputStream mFreqOutStream; + + @UsedForTesting + public Ver4DictEncoder(final File dictPlacedDir) { + mDictPlacedDir = dictPlacedDir; + } + + private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions) + throws FileNotFoundException, IOException { + final FileHeader header = new FileHeader(0, dictOptions, formatOptions); + final String filename = header.getId() + "." + header.getVersion(); + final File mDictDir = new File(mDictPlacedDir, filename); + final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION); + final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION); + if (!mDictDir.isDirectory()) { + if (mDictDir.exists()) mDictDir.delete(); + mDictDir.mkdirs(); + } + if (!trieFile.exists()) trieFile.createNewFile(); + if (!freqFile.exists()) freqFile.createNewFile(); + mTrieOutStream = new FileOutputStream(trieFile); + mFreqOutStream = new FileOutputStream(freqFile); + } + + private void close() throws IOException { + try { + if (mTrieOutStream != null) { + mTrieOutStream.close(); + } + if (mFreqOutStream != null) { + mFreqOutStream.close(); + } + } finally { + mTrieOutStream = null; + mFreqOutStream = null; + } + } + + @Override + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion != FormatSpec.VERSION4) { + throw new UnsupportedFormatException("File header has a wrong version number : " + + formatOptions.mVersion); + } + if (!mDictPlacedDir.isDirectory()) { + throw new UnsupportedFormatException("Given path is not a directory."); + } + + if (mTrieOutStream == null) { + openStreams(formatOptions, dict.mOptions); + } + + BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, formatOptions); + + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + int terminalCount = 0; + for (final PtNodeArray array : flatNodes) { + for (final PtNode node : array.mData) { + if (node.isTerminal()) node.mTerminalId = terminalCount++; + } + } + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mTrieBuf = new byte[bufferSize]; + mFreqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE]; + + MakedictLog.i("Writing file..."); + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); + } + if (MakedictLog.DBG) { + BinaryDictEncoderUtils.showStatistics(flatNodes); + MakedictLog.i("has " + terminalCount + " terminals."); + } + mTrieOutStream.write(mTrieBuf); + mFreqOutStream.write(mFreqBuf); + + MakedictLog.i("Done"); + close(); + } + + @Override + public void setPosition(int position) { + if (mTrieBuf == null || position < 0 || position >- mTrieBuf.length) return; + mTriePos = position; + } + + @Override + public int getPosition() { + return mTriePos; + } + + @Override + public void writePtNodeCount(int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + // ptNodeCount must fit on one byte or two bytes. + // Please see comments in FormatSpec + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); + } + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, ptNodeCount, + countSize); + } + + private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mTriePos, childrenPos, + formatOptions), + FormatSpec.PTNODE_FLAGS_SIZE); + } + + private void writeParentPosition(int parentPos, final PtNode ptNode, + final FormatOptions formatOptions) { + if (parentPos != FormatSpec.NO_PARENT_ADDRESS) { + parentPos -= ptNode.mCachedAddressAfterUpdate; + } + mTriePos = BinaryDictEncoderUtils.writeParentAddress(mTrieBuf, mTriePos, parentPos, + formatOptions); + } + + private void writeCharacters(final int[] characters, final boolean hasSeveralChars) { + mTriePos = CharEncoding.writeCharArray(characters, mTrieBuf, mTriePos); + if (hasSeveralChars) { + mTrieBuf[mTriePos++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + private void writeTerminalId(final int terminalId) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, terminalId, + FormatSpec.PTNODE_TERMINAL_ID_SIZE); + } + + private void writeFrequency(final int frequency, final int terminalId) { + final int freqPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE; + BinaryDictEncoderUtils.writeUIntToBuffer(mFreqBuf, freqPos, frequency, + FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + } + + private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + if (formatOptions.mSupportsDynamicUpdate) { + mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } else { + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } + } + + private void writeShortcuts(ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mTriePos; + mTriePos += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.mFrequency); + mTrieBuf[mTriePos++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos, + target.mWord); + mTriePos += shortcutShift; + } + final int shortcutByteSize = mTriePos - indexOfShortcutByteSize; + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { + throw new RuntimeException("Shortcut list too large : " + shortcutByteSize); + } + BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, indexOfShortcutByteSize, + shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + final int offset = addressOfBigram + - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + mTrieBuf[mTriePos++] = (byte) bigramFlags; + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(int forwardLinkAddress) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + forwardLinkAddress, FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict) { + writePtNodeFlags(ptNode, parentPosition, formatOptions); + writeParentPosition(parentPosition, ptNode, formatOptions); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + if (ptNode.isTerminal()) { + writeTerminalId(ptNode.mTerminalId); + writeFrequency(ptNode.mFrequency, ptNode.mTerminalId); + } + writeChildrenPosition(ptNode, formatOptions); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 5b1d0647b..9364fb034 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -25,14 +25,13 @@ import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.makedict.DictDecoder; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -173,16 +172,19 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi // Load the dictionary from binary file final File dictFile = new File(mContext.getFilesDir(), mFileName); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, DictDecoder.USE_BYTEARRAY); + if (dictDecoder == null) { + // This is an expected condition: we don't have a user history dictionary for this + // language yet. It will be created sometime later. + return; + } + try { dictDecoder.openDictBuffer(); UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); - } catch (FileNotFoundException e) { - // This is an expected condition: we don't have a user history dictionary for this - // language yet. It will be created sometime later. } catch (IOException e) { - Log.e(TAG, "IOException on opening a bytebuffer", e); + Log.d(TAG, "IOException on opening a bytebuffer", e); } finally { if (PROFILE_SAVE_RESTORE) { final long diff = System.currentTimeMillis() - now; diff --git a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java index 3c1db6529..5dc0b5893 100644 --- a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java +++ b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java @@ -31,6 +31,7 @@ public class PrioritizedSerialExecutor { private static final int TASK_QUEUE_CAPACITY = 1000; private final Queue<Runnable> mTasks; private final Queue<Runnable> mPrioritizedTasks; + private boolean mIsShutdown; // The task which is running now. private Runnable mActive; @@ -38,6 +39,7 @@ public class PrioritizedSerialExecutor { public PrioritizedSerialExecutor() { mTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); mPrioritizedTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); + mIsShutdown = false; } /** @@ -56,9 +58,11 @@ public class PrioritizedSerialExecutor { */ public void execute(final Runnable r) { synchronized(mLock) { - mTasks.offer(r); - if (mActive == null) { - scheduleNext(); + if (!mIsShutdown) { + mTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } } } } @@ -69,9 +73,11 @@ public class PrioritizedSerialExecutor { */ public void executePrioritized(final Runnable r) { synchronized(mLock) { - mPrioritizedTasks.offer(r); - if (mActive == null) { - scheduleNext(); + if (!mIsShutdown) { + mPrioritizedTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } } } } @@ -123,4 +129,19 @@ public class PrioritizedSerialExecutor { execute(newTask); } } + + public void shutdown() { + synchronized(mLock) { + mIsShutdown = true; + } + } + + public boolean isTerminated() { + synchronized(mLock) { + if (!mIsShutdown) { + return false; + } + return mPrioritizedTasks.isEmpty() && mTasks.isEmpty() && mActive == null; + } + } } diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index 05f3061a8..ea32a74ff 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -20,13 +20,13 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; @@ -125,7 +125,7 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, + public static void readDictionaryBinary(final DictDecoder dictDecoder, final OnAddWordListener dict) { final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap(); final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); diff --git a/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp b/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp index 72e625836..386643332 100644 --- a/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp +++ b/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp @@ -25,8 +25,9 @@ namespace latinime { class Dictionary; -static jlong latinime_setDicTraverseSession(JNIEnv *env, jclass clazz, jstring localeJStr) { - void *traverseSession = DicTraverseSession::getSessionInstance(env, localeJStr); +static jlong latinime_setDicTraverseSession(JNIEnv *env, jclass clazz, jstring localeJStr, + jlong dictSize) { + void *traverseSession = DicTraverseSession::getSessionInstance(env, localeJStr, dictSize); return reinterpret_cast<jlong>(traverseSession); } @@ -53,7 +54,7 @@ static void latinime_releaseDicTraverseSession(JNIEnv *env, jclass clazz, jlong static const JNINativeMethod sMethods[] = { { const_cast<char *>("setDicTraverseSessionNative"), - const_cast<char *>("(Ljava/lang/String;)J"), + const_cast<char *>("(Ljava/lang/String;J)J"), reinterpret_cast<void *>(latinime_setDicTraverseSession) }, { diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h index 07f1e52c6..4605890c7 100644 --- a/native/jni/src/defines.h +++ b/native/jni/src/defines.h @@ -32,8 +32,6 @@ #define MAX_WORD_LENGTH 48 // Must be equal to BinaryDictionary.MAX_RESULTS in Java #define MAX_RESULTS 18 -// The biggest value among MAX_CACHE_DIC_NODE_SIZE, MAX_CACHE_DIC_NODE_SIZE_FOR_SINGLE_POINT, ... -#define MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY 310 // Must be equal to ProximityInfo.MAX_PROXIMITY_CHARS_SIZE in Java #define MAX_PROXIMITY_CHARS_SIZE 16 #define ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE 2 diff --git a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp index c3d2a2e74..b6be47e90 100644 --- a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp @@ -23,6 +23,11 @@ namespace latinime { +// The biggest value among MAX_CACHE_DIC_NODE_SIZE, MAX_CACHE_DIC_NODE_SIZE_FOR_SINGLE_POINT, ... +const int DicNodesCache::LARGE_PRIORITY_QUEUE_CAPACITY = 310; +// Capacity for reducing memory footprint. +const int DicNodesCache::SMALL_PRIORITY_QUEUE_CAPACITY = 100; + /** * Truncates all of the dicNodes so that they start at the given commit point. * Only called for multi-word typing input. diff --git a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h index f085848aa..8493b6a8b 100644 --- a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h +++ b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h @@ -31,10 +31,11 @@ class DicNode; */ class DicNodesCache { public: - AK_FORCE_INLINE DicNodesCache() - : mDicNodePriorityQueue0(MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY), - mDicNodePriorityQueue1(MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY), - mDicNodePriorityQueue2(MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY), + AK_FORCE_INLINE explicit DicNodesCache(const bool usesLargeCapacityCache) + : mUsesLargeCapacityCache(usesLargeCapacityCache), + mDicNodePriorityQueue0(getCacheCapacity()), + mDicNodePriorityQueue1(getCacheCapacity()), + mDicNodePriorityQueue2(getCacheCapacity()), mDicNodePriorityQueueForTerminal(MAX_RESULTS), mActiveDicNodes(&mDicNodePriorityQueue0), mNextActiveDicNodes(&mDicNodePriorityQueue1), @@ -50,7 +51,8 @@ class DicNodesCache { // We want to use the max capacity for the current active dic node queue. mActiveDicNodes->clearAndResizeToCapacity(); // nextActiveSize is used to limit the next iteration's active dic node size. - mNextActiveDicNodes->clearAndResize(nextActiveSize); + const int nextActiveSizeFittingToTheCapacity = min(nextActiveSize, getCacheCapacity()); + mNextActiveDicNodes->clearAndResize(nextActiveSizeFittingToTheCapacity); mTerminalDicNodes->clearAndResize(terminalSize); // We want to use the max capacity for the cached dic nodes that will be used for the // continuous suggestion. @@ -162,12 +164,21 @@ class DicNodesCache { return tmp; } + AK_FORCE_INLINE int getCacheCapacity() const { + return mUsesLargeCapacityCache ? + LARGE_PRIORITY_QUEUE_CAPACITY : SMALL_PRIORITY_QUEUE_CAPACITY; + } + AK_FORCE_INLINE void resetTemporaryCaches() { mActiveDicNodes->clear(); mNextActiveDicNodes->clear(); mTerminalDicNodes->clear(); } + static const int LARGE_PRIORITY_QUEUE_CAPACITY; + static const int SMALL_PRIORITY_QUEUE_CAPACITY; + + const bool mUsesLargeCapacityCache; // Instances DicNodePriorityQueue mDicNodePriorityQueue0; DicNodePriorityQueue mDicNodePriorityQueue1; diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.cpp b/native/jni/src/suggest/core/session/dic_traverse_session.cpp index e7b386b2d..2c2259214 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.cpp +++ b/native/jni/src/suggest/core/session/dic_traverse_session.cpp @@ -23,6 +23,11 @@ namespace latinime { +// 256K bytes threshold is heuristically used to distinguish dictionaries containing many unigrams +// (e.g. main dictionary) from small dictionaries (e.g. contacts...) +const int DicTraverseSession::DICTIONARY_SIZE_THRESHOLD_TO_USE_LARGE_CACHE_FOR_SUGGESTION = + 256 * 1024; + void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord, int prevWordLength, const SuggestOptions *const suggestOptions) { mDictionary = dictionary; diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h index b25580b96..fe8893590 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.h +++ b/native/jni/src/suggest/core/session/dic_traverse_session.h @@ -37,8 +37,12 @@ class DicTraverseSession { public: // A factory method for DicTraverseSession - static AK_FORCE_INLINE void *getSessionInstance(JNIEnv *env, jstring localeStr) { - return new DicTraverseSession(env, localeStr); + static AK_FORCE_INLINE void *getSessionInstance(JNIEnv *env, jstring localeStr, + jlong dictSize) { + // To deal with the trade-off between accuracy and memory space, large cache is used for + // dictionaries larger that the threshold + return new DicTraverseSession(env, localeStr, + dictSize >= DICTIONARY_SIZE_THRESHOLD_TO_USE_LARGE_CACHE_FOR_SUGGESTION); } static AK_FORCE_INLINE void initSessionInstance(DicTraverseSession *traverseSession, @@ -54,10 +58,10 @@ class DicTraverseSession { delete traverseSession; } - AK_FORCE_INLINE DicTraverseSession(JNIEnv *env, jstring localeStr) + AK_FORCE_INLINE DicTraverseSession(JNIEnv *env, jstring localeStr, bool usesLargeCache) : mPrevWordPos(NOT_A_VALID_WORD_POS), mProximityInfo(0), - mDictionary(0), mSuggestOptions(0), mDicNodesCache(), mMultiBigramMap(), - mInputSize(0), mPartiallyCommited(false), mMaxPointerCount(1), + mDictionary(0), mSuggestOptions(0), mDicNodesCache(usesLargeCache), + mMultiBigramMap(), mInputSize(0), mPartiallyCommited(false), mMaxPointerCount(1), mMultiWordCostMultiplier(1.0f) { // NOTE: mProximityInfoStates is an array of instances. // No need to initialize it explicitly here. @@ -181,6 +185,7 @@ class DicTraverseSession { DISALLOW_IMPLICIT_CONSTRUCTORS(DicTraverseSession); // threshold to start caching static const int CACHE_START_INPUT_LENGTH_THRESHOLD; + static const int DICTIONARY_SIZE_THRESHOLD_TO_USE_LARGE_CACHE_FOR_SUGGESTION; void initializeProximityInfoStates(const int *const inputCodePoints, const int *const inputXs, const int *const inputYs, const int *const times, const int *const pointerIds, const int inputSize, const float maxSpatialDistance, const int maxPointerCount); diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index 936dc9c5d..4ee138125 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -18,6 +18,42 @@ namespace latinime { +const int DynamicBigramListPolicy::BIGRAM_LINK_COUNT_LIMIT = 10000; + +void DynamicBigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability, + bool *const outHasNext, int *const pos) const { + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + *pos -= mBuffer->getOriginalBufferSize(); + } + const BigramListReadWriteUtils::BigramFlags flags = + BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + buffer, flags, pos); + if (usesAdditionalBuffer && originalBigramPos != NOT_A_VALID_WORD_POS) { + originalBigramPos += mBuffer->getOriginalBufferSize(); + } + *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); + *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags); + *outHasNext = BigramListReadWriteUtils::hasNext(flags); + if (usesAdditionalBuffer) { + *pos += mBuffer->getOriginalBufferSize(); + } +} + +void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const { + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + *pos -= mBuffer->getOriginalBufferSize(); + } + BigramListReadWriteUtils::skipExistingBigrams(buffer, pos); + if (usesAdditionalBuffer) { + *pos += mBuffer->getOriginalBufferSize(); + } +} + bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); if (usesAdditionalBuffer) { @@ -28,15 +64,16 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo // The buffer address can be changed after calling buffer writing methods. const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, fromPos); - int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( buffer, flags, fromPos); - if (bigramPos == NOT_A_VALID_WORD_POS) { + if (originalBigramPos == NOT_A_VALID_WORD_POS) { // skip invalid bigram entry. continue; } if (usesAdditionalBuffer) { - bigramPos += mBuffer->getOriginalBufferSize(); + originalBigramPos += mBuffer->getOriginalBufferSize(); } + const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); BigramListReadWriteUtils::BigramFlags newBigramFlags; uint32_t newBigramOffset; int newBigramOffsetFieldSize; @@ -133,11 +170,12 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta if (usesAdditionalBuffer) { bigramOffsetFieldPos += mBuffer->getOriginalBufferSize(); } - int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( buffer, flags, &pos); - if (usesAdditionalBuffer && bigramPos != NOT_A_VALID_WORD_POS) { - bigramPos += mBuffer->getOriginalBufferSize(); + if (usesAdditionalBuffer && originalBigramPos != NOT_A_VALID_WORD_POS) { + originalBigramPos += mBuffer->getOriginalBufferSize(); } + const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); if (bigramPos != targetBigramPos) { continue; } @@ -152,4 +190,26 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta return false; } +int DynamicBigramListPolicy::followBigramLinkAndGetCurrentBigramPtNodePos( + const int originalBigramPos) const { + if (originalBigramPos == NOT_A_VALID_WORD_POS) { + return NOT_A_VALID_WORD_POS; + } + int currentPos = originalBigramPos; + DynamicPatriciaTrieNodeReader nodeReader(mBuffer, this /* bigramsPolicy */, mShortcutPolicy); + nodeReader.fetchNodeInfoFromBuffer(currentPos); + int bigramLinkCount = 0; + while (nodeReader.getBigramLinkedNodePos() != NOT_A_DICT_POS) { + currentPos = nodeReader.getBigramLinkedNodePos(); + nodeReader.fetchNodeInfoFromBuffer(currentPos); + bigramLinkCount++; + if (bigramLinkCount > BIGRAM_LINK_COUNT_LIMIT) { + AKLOGI("Bigram link is invalid. start position: %d", bigramPos); + ASSERT(false); + return NOT_A_VALID_WORD_POS; + } + } + return currentPos; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index b7c05376d..e451e313d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -21,7 +21,9 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" +#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" +#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { @@ -31,43 +33,16 @@ namespace latinime { */ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { public: - DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer) - : mBuffer(buffer) {} + DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer, + const DictionaryShortcutsStructurePolicy *const shortcutPolicy) + : mBuffer(buffer), mShortcutPolicy(shortcutPolicy) {} ~DynamicBigramListPolicy() {} void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, - int *const pos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *pos -= mBuffer->getOriginalBufferSize(); - } - const BigramListReadWriteUtils::BigramFlags flags = - BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); - *outBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( - buffer, flags, pos); - if (usesAdditionalBuffer && *outBigramPos != NOT_A_VALID_WORD_POS) { - *outBigramPos += mBuffer->getOriginalBufferSize(); - } - *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags); - *outHasNext = BigramListReadWriteUtils::hasNext(flags); - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); - } - } + int *const pos) const; - void skipAllBigrams(int *const pos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *pos -= mBuffer->getOriginalBufferSize(); - } - BigramListReadWriteUtils::skipExistingBigrams(buffer, pos); - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); - } - } + void skipAllBigrams(int *const pos) const; // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these // positions after bigram lists. This method skips invalid bigram entries. @@ -81,7 +56,13 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy); + static const int BIGRAM_LINK_COUNT_LIMIT; + BufferWithExtendableBuffer *const mBuffer; + const DictionaryShortcutsStructurePolicy *const mShortcutPolicy; + + // Follow bigram link and return the position of bigram target PtNode that is currently valid. + int followBigramLinkAndGetCurrentBigramPtNodePos(const int originalBigramPos) const; }; } // namespace latinime #endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index 5674cb48e..56ef60ae4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -28,6 +28,7 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(nodePos); const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); int pos = nodePos; + mHeadPos = nodePos; if (usesAdditionalBuffer) { pos -= mBuffer->getOriginalBufferSize(); } @@ -57,10 +58,15 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mChildrenPosFieldPos += mBuffer->getOriginalBufferSize(); } mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( - dictBuf, mFlags, &pos); + dictBuf, &pos); if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { mChildrenPos += mBuffer->getOriginalBufferSize(); } + if (mSiblingPos == NOT_A_VALID_WORD_POS && DynamicPatriciaTrieReadingUtils::isMoved(mFlags)) { + mBigramLinkedNodePos = mChildrenPos; + } else { + mBigramLinkedNodePos = NOT_A_DICT_POS; + } if (usesAdditionalBuffer) { pos += mBuffer->getOriginalBufferSize(); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h index 2ee7c2495..89d38a590 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h @@ -39,11 +39,11 @@ class DynamicPatriciaTrieNodeReader { const DictionaryBigramsStructurePolicy *const bigramsPolicy, const DictionaryShortcutsStructurePolicy *const shortcutsPolicy) : mBuffer(buffer), mBigramsPolicy(bigramsPolicy), - mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), - mParentPos(NOT_A_DICT_POS), mCodePointCount(0), - mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), - mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS), - mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), + mShortcutsPolicy(shortcutsPolicy), mHeadPos(NOT_A_VALID_WORD_POS), mFlags(0), + mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbabilityFieldPos(NOT_A_DICT_POS), + mProbability(NOT_A_PROBABILITY), mChildrenPosFieldPos(NOT_A_DICT_POS), + mChildrenPos(NOT_A_DICT_POS), mBigramLinkedNodePos(NOT_A_DICT_POS), + mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {} ~DynamicPatriciaTrieNodeReader() {} @@ -56,13 +56,14 @@ class DynamicPatriciaTrieNodeReader { AK_FORCE_INLINE void fetchNodeInfoFromBufferAndGetNodeCodePoints(const int nodePos, const int maxCodePointCount, int *const outCodePoints) { - mNodePos = nodePos; mSiblingPos = NOT_A_VALID_WORD_POS; - fetchNodeInfoFromBufferAndProcessMovedNode(mNodePos, maxCodePointCount, outCodePoints); + mBigramLinkedNodePos = NOT_A_DICT_POS; + fetchNodeInfoFromBufferAndProcessMovedNode(nodePos, maxCodePointCount, outCodePoints); } - AK_FORCE_INLINE int getNodePos() const { - return mNodePos; + // HeadPos is different from NodePos when the current PtNode is a moved PtNode. + AK_FORCE_INLINE int getHeadPos() const { + return mHeadPos; } // Flags @@ -114,6 +115,11 @@ class DynamicPatriciaTrieNodeReader { return mChildrenPos; } + // Bigram linked node position. + AK_FORCE_INLINE int getBigramLinkedNodePos() const { + return mBigramLinkedNodePos; + } + // Shortcutlist position AK_FORCE_INLINE int getShortcutPos() const { return mShortcutPos; @@ -135,7 +141,7 @@ class DynamicPatriciaTrieNodeReader { const BufferWithExtendableBuffer *const mBuffer; const DictionaryBigramsStructurePolicy *const mBigramsPolicy; const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy; - int mNodePos; + int mHeadPos; DynamicPatriciaTrieReadingUtils::NodeFlags mFlags; int mParentPos; uint8_t mCodePointCount; @@ -143,6 +149,7 @@ class DynamicPatriciaTrieNodeReader { int mProbability; int mChildrenPosFieldPos; int mChildrenPos; + int mBigramLinkedNodePos; int mShortcutPos; int mBigramPos; int mSiblingPos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index 945677b50..ece1781f1 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -38,7 +38,7 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d readingHelper.initWithNodeArrayPos(dicNode->getChildrenPos()); const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); while (!readingHelper.isEnd()) { - childDicNodes->pushLeavingChild(dicNode, nodeReader->getNodePos(), + childDicNodes->pushLeavingChild(dicNode, nodeReader->getHeadPos(), nodeReader->getChildrenPos(), nodeReader->getProbability(), nodeReader->isTerminal() && !nodeReader->isDeleted(), nodeReader->hasChildren(), nodeReader->isBlacklisted() || nodeReader->isNotAWord(), @@ -122,7 +122,7 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in // All characters are matched. if (length == readingHelper.getTotalCodePointCount()) { // Terminal position is found. - return nodeReader->getNodePos(); + return nodeReader->getHeadPos(); } if (!nodeReader->hasChildren()) { return NOT_A_VALID_WORD_POS; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index cdab0e16a..50c724012 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -36,8 +36,8 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(), mBuffer->getBufferSize() - mHeaderPolicy.getSize()), - mBigramListPolicy(&mBufferWithExtendableBuffer), - mShortcutListPolicy(&mBufferWithExtendableBuffer) {} + mShortcutListPolicy(&mBufferWithExtendableBuffer), + mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy) {} ~DynamicPatriciaTriePolicy() { delete mBuffer; @@ -91,8 +91,8 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; BufferWithExtendableBuffer mBufferWithExtendableBuffer; - DynamicBigramListPolicy mBigramListPolicy; DynamicShortcutListPolicy mShortcutListPolicy; + DynamicBigramListPolicy mBigramListPolicy; }; } // namespace latinime #endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp index 5d979fa51..c7e89fff8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp @@ -29,18 +29,14 @@ const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; /* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( - const uint8_t *const buffer, const NodeFlags flags, int *const pos) { - if ((flags & MASK_MOVED) == FLAG_IS_NOT_MOVED) { - const int base = *pos; - const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); - if (offset == 0) { - // 0 offset means that the node does not have children. - return NOT_A_DICT_POS; - } else { - return base + offset; - } - } else { + const uint8_t *const buffer, int *const pos) { + const int base = *pos; + const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); + if (offset == 0) { + // 0 offset means that the node does not have children. return NOT_A_DICT_POS; + } else { + return base + offset; } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h index 2e604a202..5a2ad9cb9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h @@ -42,8 +42,7 @@ class DynamicPatriciaTrieReadingUtils { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } - static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, - const NodeFlags flags, int *const pos); + static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, int *const pos); /** * Node Flags diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index 7c0b6286c..dbc80f66a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -63,7 +63,7 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( codePointCount - readingHelper->getTotalCodePointCount()); } // Advance to the children nodes. - parentPos = nodeReader->getNodePos(); + parentPos = nodeReader->getHeadPos(); readingHelper->readChildNode(); } if (readingHelper->isError()) { @@ -100,8 +100,9 @@ bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, con } bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( - const DynamicPatriciaTrieNodeReader *const originalNode, const int movedPos) { - int pos = originalNode->getNodePos(); + const DynamicPatriciaTrieNodeReader *const originalNode, const int movedPos, + const int bigramLinkedNodePos) { + int pos = originalNode->getHeadPos(); const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(pos); const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { @@ -113,18 +114,42 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( const PatriciaTrieReadingUtils::NodeFlags updatedFlags = DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, false /* isDeleted */); - int writingPos = originalNode->getNodePos(); + int writingPos = originalNode->getHeadPos(); // Update flags. if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, updatedFlags, &writingPos)) { return false; } // Update moved position, which is stored in the parent offset field. - const int movedPosOffset = movedPos - originalNode->getNodePos(); + const int movedPosOffset = movedPos - originalNode->getHeadPos(); if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( mBuffer, movedPosOffset, &writingPos)) { return false; } + // Update bigram linked node position, which is stored in the children position field. + int childrenPosFieldPos = originalNode->getChildrenPosFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( + mBuffer, bigramLinkedNodePos, &childrenPosFieldPos)) { + return false; + } + if (originalNode->hasChildren()) { + // Update children's parent position. + DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy); + const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); + readingHelper.initWithNodeArrayPos(originalNode->getChildrenPos()); + while (!readingHelper.isEnd()) { + const int childPtNodeWrittenPos = nodeReader->getHeadPos(); + const int parentOffset = movedPos - childPtNodeWrittenPos; + int parentOffsetFieldPos = childPtNodeWrittenPos + 1 /* Flags */; + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( + mBuffer, parentOffset, &parentOffsetFieldPos)) { + // Parent offset cannot be written because of a bug or a broken dictionary; thus, + // we give up to update dictionary. + return false; + } + readingHelper.readNextSiblingNode(); + } + } return true; } @@ -230,7 +255,7 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( } else { // Make the node terminal and write the probability. int movedPos = mBuffer->getTailPosition(); - if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos)) { + if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { return false; } if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), @@ -250,7 +275,7 @@ bool DynamicPatriciaTrieWritingHelper::createChildrenPtNodeArrayAndAChildPtNode( newPtNodeArrayPos, &childrenPosFieldPos)) { return false; } - return createNewPtNodeArrayWithAChildPtNode(parentNode->getNodePos(), codePoints, + return createNewPtNodeArrayWithAChildPtNode(parentNode->getHeadPos(), codePoints, codePointCount, probability); } @@ -287,11 +312,8 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( // Reallocating PtNode: abcde, newNode: abc. // abc (1st, terminal) __ de (2nd) const bool addsExtraChild = newNodeCodePointCount > overlappingCodePointCount; - const int firstPtNodePos = mBuffer->getTailPosition(); - if (!markNodeAsMovedAndSetPosition(reallocatingPtNode, firstPtNodePos)) { - return false; - } - int writingPos = firstPtNodePos; + const int firstPartOfReallocatedPtNodePos = mBuffer->getTailPosition(); + int writingPos = firstPartOfReallocatedPtNodePos; // Write the 1st part of the reallocating node. The children position will be updated later // with actual children position. const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; @@ -307,15 +329,15 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( return false; } // Write the 2nd part of the reallocating node. - if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, - reallocatingPtNode->getNodePos(), + const int secondPartOfReallocatedPtNodePos = writingPos; + if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, firstPartOfReallocatedPtNodePos, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getProbability(), &writingPos)) { return false; } if (addsExtraChild) { - if (!writePtNodeToBuffer(reallocatingPtNode->getNodePos(), + if (!writePtNodeToBuffer(firstPartOfReallocatedPtNodePos, newNodeCodePoints + overlappingCodePointCount, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, &writingPos)) { @@ -326,9 +348,14 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } + // Update original reallocatingPtNode as moved. + if (!markNodeAsMovedAndSetPosition(reallocatingPtNode, firstPartOfReallocatedPtNodePos, + secondPartOfReallocatedPtNodePos)) { + return false; + } // Load node info. Information of the 1st part will be fetched. DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); - nodeReader.fetchNodeInfoFromBuffer(firstPtNodePos); + nodeReader.fetchNodeInfoFromBuffer(firstPartOfReallocatedPtNodePos); // Update children position. int childrenPosFieldPos = nodeReader.getChildrenPosFieldPos(); if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index ada634a54..e1b9d2e75 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -54,7 +54,7 @@ class DynamicPatriciaTrieWritingHelper { DynamicShortcutListPolicy *const mShortcutPolicy; bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, - const int movedPos); + const int movedPos, const int bigramLinkedNodePos); bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h index f727ecf8e..6bafb64ee 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h @@ -172,6 +172,7 @@ class ByteArrayUtils { int codePoint = readCodePointAndAdvancePosition(buffer, pos); while (NOT_A_CODE_POINT != codePoint && length < maxLength) { codePoint = readCodePointAndAdvancePosition(buffer, pos); + length++; } return length; } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 186484570..807c25244 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -25,6 +25,7 @@ import android.util.SparseArray; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -132,18 +133,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Utilities for test /** - * Makes new DictDecoder according to BUFFER_TYPE. - */ - private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) { - if (bufferType == USE_BYTE_BUFFER) { - return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER); - } else if (bufferType == USE_BYTE_ARRAY) { - return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); - } - return null; - } - - /** * Generates a random word. */ private String generateWord(final Random random, final int[] codePointSet) { @@ -276,6 +265,27 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; } + private DictionaryOptions getDictionaryOptions(final String id, final String version) { + final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), + false, false); + options.mAttributes.put("version", version); + options.mAttributes.put("dictionary", id); + return options; + } + + private File setUpDictionaryFile(final String name, final String version) { + File file = null; + try { + file = new File(getContext().getCacheDir(), name + "." + version + + TEST_DICT_FILE_EXTENSION); + file.createNewFile(); + } catch (IOException e) { + // do nothing + } + assertTrue("Failed to create the dictionary file.", file.exists()); + return file; + } + // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List<String> words, @@ -285,9 +295,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { FusionDictionary dict = null; try { - final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType); - dictDecoder.openDictBuffer(); - assertNotNull(dictDecoder.getDictBuffer()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; @@ -306,17 +314,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcuts, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { - File file = null; - try { - file = File.createTempFile("runReadAndWrite", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - assertNotNull(file); + + final String dictName = "runReadAndWrite"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); checkDictionary(dict, words, bigrams, shortcuts); @@ -443,9 +447,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType); - dictDecoder.openDictBuffer(); - assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -470,19 +472,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private String runReadUnigramsAndBigramsBinary(final ArrayList<String> words, final SparseArray<List<Integer>> bigrams, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { - File file = null; - try { - file = File.createTempFile("runReadUnigrams", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - assertNotNull(file); + final String dictName = "runReadUnigrams"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); + getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); addBigrams(dict, words, bigrams); @@ -531,9 +527,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } // Tests for getTerminalPosition - private String getWordFromBinary(final Ver3DictDecoder dictDecoder, final int address) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - if (dictBuffer.position() != 0) dictBuffer.position(0); + private String getWordFromBinary(final DictDecoder dictDecoder, final int address) { + if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0); FileHeader fileHeader = null; try { @@ -548,7 +543,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { address, fileHeader.mFormatOptions).mWord; } - private long runGetTerminalPosition(final Ver3DictDecoder dictDecoder, final String word, + private long runGetTerminalPosition(final DictDecoder dictDecoder, final String word, int index, boolean contained) { final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; long diff = -1; @@ -569,29 +564,23 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testGetTerminalPosition() { - File file = null; - try { - file = File.createTempFile("testGetTerminalPosition", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - // do nothing - } - assertNotNull(file); + final String dictName = "testGetTerminalPosition"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); + getDictionaryOptions(dictName, dictVersion)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer()); + assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer()); try { // too long word @@ -626,14 +615,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testDeleteWord() { - File file = null; - try { - file = File.createTempFile("testDeleteWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - // do nothing - } - assertNotNull(file); + final String dictName = "testDeleteWord"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( @@ -648,7 +632,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer()); + assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer()); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index 8e0c6dfe2..a83749499 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -140,7 +140,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int position = FormatSpec.NOT_VALID_WORD; try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + DictDecoder.USE_READONLY_BYTEBUFFER); position = dictDecoder.getTerminalPosition(word); } catch (IOException e) { } catch (UnsupportedFormatException e) { @@ -149,7 +150,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } /** - * Find a word using the Ver3DictDecoder. + * Find a word using the DictDecoder. * * @param dictDecoder the dict decoder * @param word the word searched @@ -157,21 +158,20 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { * @throws IOException * @throws UnsupportedFormatException */ - private static PtNodeInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder, + private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = dictDecoder.getTerminalPosition(word); - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (position != FormatSpec.NOT_VALID_WORD) { - dictBuffer.position(0); + dictDecoder.setPosition(0); final FileHeader header = dictDecoder.readHeader(); - dictBuffer.position(position); + dictDecoder.setPosition(position); return dictDecoder.readPtNode(position, header.mFormatOptions); } return null; } private PtNodeInfo findWordFromFile(final File file, final String word) { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); PtNodeInfo info = null; try { dictDecoder.openDictBuffer(); @@ -234,7 +234,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private void checkReverseLookup(final File file, final String word, final int position) { try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); final FileHeader fileHeader = dictDecoder.readHeader(); assertEquals(word, BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize, diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index d15e88bdb..bf44a1424 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -46,6 +46,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { }; private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000; + private static final int WAIT_TERMINATING_IN_MILLISECONDS = 100; @Override public void setUp() { @@ -122,8 +123,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { true /* checksContents */); } finally { try { + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); Log.d(TAG, "waiting for writing ..."); - Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } @@ -146,11 +153,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final int numberOfWordsInsertedForEachLanguageSwitch = 100; final File dictFiles[] = new File[numberOfLanguages]; + final String testFilenameSuffixes[] = new String[numberOfLanguages]; try { final Random random = new Random(123456); // Create filename suffixes for this test. - String testFilenameSuffixes[] = new String[numberOfLanguages]; for (int i = 0; i < numberOfLanguages; i++) { testFilenameSuffixes[i] = "testSwitchingLanguages" + i; final String fileName = UserHistoryPredictionDictionary.NAME + "." + @@ -174,7 +181,15 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } finally { try { Log.d(TAG, "waiting for writing ..."); - Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); + for (int i = 0; i < numberOfLanguages; i++) { + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffixes[i], mPrefs); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java index 72b9478d4..3eabe2b3c 100644 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java @@ -143,7 +143,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase } private void readDictFromFile(final File file, final OnAddWordListener listener) { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (FileNotFoundException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index 1417ca44e..fa80385fc 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -18,9 +18,9 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import org.xml.sax.SAXException; @@ -185,7 +185,7 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index 767c147bf..5302e976e 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -267,7 +267,7 @@ public class DictionaryMaker { private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { final File file = new File(binaryFilename); - final DictDecoder dictDecoder = new Ver3DictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 6e82e37bd..1eff497c1 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -18,12 +18,12 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.DictDecoder; import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import junit.framework.TestCase; @@ -69,7 +69,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); - final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile); final FusionDictionary resultDict = dictDecoder.readDictionaryBinary( null /* dict : an optional dictionary to add words to, or null */, false /* deleteDictIfBroken */); |