diff options
136 files changed, 3494 insertions, 1034 deletions
diff --git a/java/res/layout/emoji_keyboard_view.xml b/java/res/layout/emoji_keyboard_view.xml index ccbcfdc6f..5fee419d0 100644 --- a/java/res/layout/emoji_keyboard_view.xml +++ b/java/res/layout/emoji_keyboard_view.xml @@ -76,18 +76,24 @@ android:id="@+id/emoji_action_bar" android:orientation="horizontal" android:layout_width="match_parent" - android:layout_height="@dimen/suggestions_strip_height" + android:layout_height="0dip" + android:layout_weight="1" > <ImageButton android:id="@+id/emoji_keyboard_alphabet" android:layout_width="0dip" - android:layout_weight="0.825" + android:layout_weight="0.15" android:layout_height="match_parent" android:src="@drawable/ic_ime_light" /> <ImageButton + android:id="@+id/emoji_keyboard_space" + android:layout_width="0dip" + android:layout_weight="0.70" + android:layout_height="match_parent" /> + <ImageButton android:id="@+id/emoji_keyboard_send" android:layout_width="0dip" - android:layout_weight="0.125" + android:layout_weight="0.15" android:layout_height="match_parent" android:src="@drawable/sym_keyboard_return_holo" /> </LinearLayout> diff --git a/java/res/values-fr-rCA/donottranslate.xml b/java/res/values-fr-rCA/donottranslate.xml new file mode 100644 index 000000000..21f18d852 --- /dev/null +++ b/java/res/values-fr-rCA/donottranslate.xml @@ -0,0 +1,31 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2009, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Symbols that are normally preceded by a space (used to add an auto-space before these) --> + <!-- This is similar to French with the exception of "!" "?" and ";" which do not take a space before in Canadian French. Note that ":" does take a space before according to Canadian rules. --> + <string name="symbols_preceded_by_space">([{&:</string> + <!-- Symbols that are normally followed by a space (used to add an auto-space after these) --> + <string name="symbols_followed_by_space">.,;:!?)]}&</string> + <!-- Symbols that separate words --> + <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) --> + <string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string> + <!-- Word connectors --> + <string name="symbols_word_connectors">\'-</string> +</resources> diff --git a/java/res/values-land/dimens.xml b/java/res/values-land/dimens.xml index c954e60da..42a746b60 100644 --- a/java/res/values-land/dimens.xml +++ b/java/res/values-land/dimens.xml @@ -75,5 +75,9 @@ <dimen name="gesture_floating_preview_vertical_padding">15dp</dimen> <!-- Emoji keyboard --> - <fraction name="emoji_keyboard_key_width">8.3333%p</fraction> + <fraction name="emoji_keyboard_key_width">10%p</fraction> + <fraction name="emoji_keyboard_row_height">50%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">60%p</fraction> + <integer name="emoji_keyboard_max_key_count">20</integer> + </resources> diff --git a/java/res/values-lo/donottranslate.xml b/java/res/values-lo/donottranslate.xml new file mode 100644 index 000000000..a9893feec --- /dev/null +++ b/java/res/values-lo/donottranslate.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Whether this language uses spaces between words --> + <bool name="current_language_has_spaces">false</bool> +</resources> diff --git a/java/res/values-sw600dp-land/dimens.xml b/java/res/values-sw600dp-land/dimens.xml index cda462f29..730b7d813 100644 --- a/java/res/values-sw600dp-land/dimens.xml +++ b/java/res/values-sw600dp-land/dimens.xml @@ -62,4 +62,11 @@ <dimen name="gesture_floating_preview_text_offset">76dp</dimen> <dimen name="gesture_floating_preview_horizontal_padding">26dp</dimen> <dimen name="gesture_floating_preview_vertical_padding">17dp</dimen> + + <!-- Emoji keyboard --> + <fraction name="emoji_keyboard_key_width">10%p</fraction> + <fraction name="emoji_keyboard_row_height">33%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">95%p</fraction> + <integer name="emoji_keyboard_max_key_count">30</integer> + </resources> diff --git a/java/res/values-sw600dp/dimens.xml b/java/res/values-sw600dp/dimens.xml index 7dfd0b162..2bcf2fa84 100644 --- a/java/res/values-sw600dp/dimens.xml +++ b/java/res/values-sw600dp/dimens.xml @@ -88,4 +88,11 @@ <dimen name="gesture_floating_preview_horizontal_padding">28dp</dimen> <dimen name="gesture_floating_preview_vertical_padding">19dp</dimen> <dimen name="gesture_floating_preview_round_radius">3dp</dimen> + + <!-- Emoji keyboard --> + <fraction name="emoji_keyboard_key_width">12.5%p</fraction> + <fraction name="emoji_keyboard_row_height">33%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">85%p</fraction> + <integer name="emoji_keyboard_max_key_count">24</integer> + </resources> diff --git a/java/res/values-sw768dp-land/dimens.xml b/java/res/values-sw768dp-land/dimens.xml index 0a7048060..1e2e1c651 100644 --- a/java/res/values-sw768dp-land/dimens.xml +++ b/java/res/values-sw768dp-land/dimens.xml @@ -63,4 +63,11 @@ <dimen name="gesture_floating_preview_text_offset">100dp</dimen> <dimen name="gesture_floating_preview_horizontal_padding">32dp</dimen> <dimen name="gesture_floating_preview_vertical_padding">21dp</dimen> + + <!-- Emoji keyboard --> + <fraction name="emoji_keyboard_key_width">7.69%p</fraction> + <fraction name="emoji_keyboard_row_height">33%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">75%p</fraction> + <integer name="emoji_keyboard_max_key_count">39</integer> + </resources> diff --git a/java/res/values-sw768dp/dimens.xml b/java/res/values-sw768dp/dimens.xml index 877e83b59..f62a536d7 100644 --- a/java/res/values-sw768dp/dimens.xml +++ b/java/res/values-sw768dp/dimens.xml @@ -88,4 +88,11 @@ <dimen name="gesture_floating_preview_horizontal_padding">26dp</dimen> <dimen name="gesture_floating_preview_vertical_padding">17dp</dimen> <dimen name="gesture_floating_preview_round_radius">3dp</dimen> + + <!-- Emoji keyboard --> + <fraction name="emoji_keyboard_key_width">10%p</fraction> + <fraction name="emoji_keyboard_row_height">33%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">85%p</fraction> + <integer name="emoji_keyboard_max_key_count">30</integer> + </resources> diff --git a/java/res/values-th/donottranslate.xml b/java/res/values-th/donottranslate.xml index aeeebed15..a9893feec 100644 --- a/java/res/values-th/donottranslate.xml +++ b/java/res/values-th/donottranslate.xml @@ -18,6 +18,6 @@ */ --> <resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> - <!-- Whether this language uses spaces --> + <!-- Whether this language uses spaces between words --> <bool name="current_language_has_spaces">false</bool> </resources> diff --git a/java/res/values/attrs.xml b/java/res/values/attrs.xml index 2a5334f95..09782143f 100644 --- a/java/res/values/attrs.xml +++ b/java/res/values/attrs.xml @@ -45,6 +45,8 @@ possible states: normal, pressed, checkable, checkable+pressed, checkable+checked, checkable+checked+pressed. --> <attr name="keyBackground" format="reference" /> + <!-- Image for the functional key used in Emoji layout. --> + <attr name="keyBackgroundEmojiFunctional" format="reference" /> <!-- Horizontal padding of left/right aligned key label to the edge of the key. --> <attr name="keyLabelHorizontalPadding" format="dimension" /> diff --git a/java/res/xml/kbd_thai_symbols_shift.xml b/java/res/values/config-additional-features.xml index a2d67caf4..47eb7724c 100644 --- a/java/res/xml/kbd_thai_symbols_shift.xml +++ b/java/res/values/config-additional-features.xml @@ -2,7 +2,7 @@ <!-- /* ** -** Copyright 2012, The Android Open Source Project +** Copyright 2013, The Android Open Source Project ** ** Licensed under the Apache License, Version 2.0 (the "License"); ** you may not use this file except in compliance with the License. @@ -18,10 +18,7 @@ */ --> -<Keyboard - xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" - latin:touchPositionCorrectionData="@array/touch_position_correction_data_default" -> - <include - latin:keyboardLayout="@xml/rows_symbols_shift" /> -</Keyboard> +<resources> + <!-- Whether phrase gestures are enabled by default --> + <bool name="config_default_phrase_gesture_enabled">false</bool> +</resources> diff --git a/java/res/values/dimens.xml b/java/res/values/dimens.xml index 7de93e6e3..88e327f26 100644 --- a/java/res/values/dimens.xml +++ b/java/res/values/dimens.xml @@ -117,6 +117,9 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">14.2857%p</fraction> + <fraction name="emoji_keyboard_row_height">33%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">90%p</fraction> + <integer name="emoji_keyboard_max_key_count">21</integer> <!-- Inset used in Accessibility mode to avoid accidental key presses when a finger slides off the screen. --> <dimen name="accessibility_edge_slop">8dp</dimen> @@ -124,4 +127,5 @@ <integer name="user_dictionary_max_word_length" translatable="false">48</integer> <dimen name="language_on_spacebar_horizontal_margin">1dp</dimen> + </resources> diff --git a/java/res/values/donottranslate.xml b/java/res/values/donottranslate.xml index 52ebe161c..82c5ce456 100644 --- a/java/res/values/donottranslate.xml +++ b/java/res/values/donottranslate.xml @@ -31,7 +31,7 @@ <string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"</string> <!-- Word connectors --> <string name="symbols_word_connectors">\'-</string> - <!-- Whether this language uses spaces --> + <!-- Whether this language uses spaces between words --> <bool name="current_language_has_spaces">true</bool> <!-- Always show the suggestion strip --> diff --git a/java/res/values/themes-ics.xml b/java/res/values/themes-ics.xml index f3b6b1321..19fb4fd27 100644 --- a/java/res/values/themes-ics.xml +++ b/java/res/values/themes-ics.xml @@ -103,7 +103,7 @@ name="EmojiKeyboardView.ICS" parent="KeyboardView.ICS" > - <item name="keyBackground">@drawable/btn_keyboard_key_functional_ics</item> + <item name="keyBackgroundEmojiFunctional">@drawable/btn_keyboard_key_functional_ics</item> <item name="emojiTabLabelColor">@color/emoji_tab_label_color_ics</item> </style> <style diff --git a/java/res/xml-sw600dp/rows_lao.xml b/java/res/xml-sw600dp/rows_lao.xml new file mode 100644 index 000000000..cfe8db98e --- /dev/null +++ b/java/res/xml-sw600dp/rows_lao.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <include + latin:keyboardLayout="@xml/key_styles_common" /> + <Row + latin:keyWidth="7.5%p" + > + <include + latin:keyboardLayout="@xml/rowkeys_lao1" /> + <Key + latin:keyStyle="deleteKeyStyle" + latin:keyWidth="fillRight" /> + </Row> + <Row + latin:keyWidth="7.5%p" + > + <include + latin:keyboardLayout="@xml/rowkeys_lao2" /> + </Row> + <Row + latin:keyWidth="7.5%p" + > + <include + latin:keyboardLayout="@xml/rowkeys_lao3" /> + <Key + latin:keyStyle="enterKeyStyle" + latin:keyWidth="fillRight" /> + </Row> + <Row + latin:keyWidth="7.5%p" + > + <Key + latin:keyStyle="shiftKeyStyle" + latin:keyWidth="10.0%p" /> + <include + latin:keyboardLayout="@xml/rowkeys_lao4" /> + <include + latin:keyboardLayout="@xml/keys_exclamation_question" /> + </Row> + <include + latin:keyboardLayout="@xml/row_qwerty4" /> +</merge> diff --git a/java/res/xml/kbd_emoji_category1.xml b/java/res/xml/kbd_emoji_category1.xml index 92b0c3fbf..c11a83017 100644 --- a/java/res/xml/kbd_emoji_category1.xml +++ b/java/res/xml/kbd_emoji_category1.xml @@ -22,6 +22,7 @@ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" latin:keyWidth="@fraction/emoji_keyboard_key_width" latin:keyLetterSize="90%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:codesArray="@array/emoji_faces" diff --git a/java/res/xml/kbd_emoji_category2.xml b/java/res/xml/kbd_emoji_category2.xml index 17d36c52f..d3e5890ea 100644 --- a/java/res/xml/kbd_emoji_category2.xml +++ b/java/res/xml/kbd_emoji_category2.xml @@ -22,6 +22,7 @@ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" latin:keyWidth="@fraction/emoji_keyboard_key_width" latin:keyLetterSize="90%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:codesArray="@array/emoji_objects" diff --git a/java/res/xml/kbd_emoji_category3.xml b/java/res/xml/kbd_emoji_category3.xml index 9000a3a11..0efafa814 100644 --- a/java/res/xml/kbd_emoji_category3.xml +++ b/java/res/xml/kbd_emoji_category3.xml @@ -22,6 +22,7 @@ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" latin:keyWidth="@fraction/emoji_keyboard_key_width" latin:keyLetterSize="90%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:codesArray="@array/emoji_nature" diff --git a/java/res/xml/kbd_emoji_category4.xml b/java/res/xml/kbd_emoji_category4.xml index e79e124e7..e5291207f 100644 --- a/java/res/xml/kbd_emoji_category4.xml +++ b/java/res/xml/kbd_emoji_category4.xml @@ -22,6 +22,7 @@ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" latin:keyWidth="@fraction/emoji_keyboard_key_width" latin:keyLetterSize="90%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:codesArray="@array/emoji_places" diff --git a/java/res/xml/kbd_emoji_category5.xml b/java/res/xml/kbd_emoji_category5.xml index 07b3d908c..1836879d6 100644 --- a/java/res/xml/kbd_emoji_category5.xml +++ b/java/res/xml/kbd_emoji_category5.xml @@ -22,6 +22,7 @@ xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" latin:keyWidth="@fraction/emoji_keyboard_key_width" latin:keyLetterSize="90%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:codesArray="@array/emoji_symbols" diff --git a/java/res/xml/kbd_emoji_category6.xml b/java/res/xml/kbd_emoji_category6.xml index 838f3f52c..b47ebfec0 100644 --- a/java/res/xml/kbd_emoji_category6.xml +++ b/java/res/xml/kbd_emoji_category6.xml @@ -23,6 +23,7 @@ latin:keyWidth="@fraction/emoji_keyboard_key_width" latin:keyLetterSize="90%p" latin:keyLabelSize="60%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:textsArray="@array/emoji_emoticons" diff --git a/java/res/xml/kbd_emoji_recents.xml b/java/res/xml/kbd_emoji_recents.xml index f56b79ab7..73926ecc0 100644 --- a/java/res/xml/kbd_emoji_recents.xml +++ b/java/res/xml/kbd_emoji_recents.xml @@ -21,8 +21,9 @@ <Keyboard xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" latin:keyWidth="@fraction/emoji_keyboard_key_width" - latin:keyLetterSize="90%p" + latin:keyLetterSize="@fraction/emoji_keyboard_key_letter_size" latin:keyLabelSize="60%p" + latin:rowHeight="@fraction/emoji_keyboard_row_height" > <GridRows latin:codesArray="@array/emoji_recents" diff --git a/java/res/xml/kbd_thai_symbols.xml b/java/res/xml/kbd_lao.xml index 4d9861b73..2bba330de 100644 --- a/java/res/xml/kbd_thai_symbols.xml +++ b/java/res/xml/kbd_lao.xml @@ -2,7 +2,7 @@ <!-- /* ** -** Copyright 2012, The Android Open Source Project +** Copyright 2013, The Android Open Source Project ** ** Licensed under the Apache License, Version 2.0 (the "License"); ** you may not use this file except in compliance with the License. @@ -20,8 +20,12 @@ <Keyboard xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" + latin:rowHeight="20%p" + latin:verticalGap="@fraction/key_bottom_gap_5row" + latin:keyLetterSize="@fraction/key_letter_ratio_5row" + latin:keyShiftedLetterHintRatio="@fraction/key_uppercase_letter_ratio_5row" latin:touchPositionCorrectionData="@array/touch_position_correction_data_default" > <include - latin:keyboardLayout="@xml/rows_symbols" /> + latin:keyboardLayout="@xml/rows_lao" /> </Keyboard> diff --git a/java/res/xml-sw600dp/key_f2.xml b/java/res/xml/key_f2.xml index 674153ae3..473dd210a 100644 --- a/java/res/xml-sw600dp/key_f2.xml +++ b/java/res/xml/key_f2.xml @@ -26,11 +26,13 @@ latin:mode="email|url" > <Key - latin:keyStyle="comKeyStyle" /> + latin:keyStyle="comKeyStyle" + latin:keyWidth="fillRight" /> </case> <default> <Key - latin:keyStyle="emojiKeyStyle" /> + latin:keyStyle="emojiKeyStyle" + latin:keyWidth="fillRight" /> </default> </switch> </merge> diff --git a/java/res/xml/key_space_5kw.xml b/java/res/xml/key_space_5kw.xml index 02ee42fd2..b6d38fb33 100644 --- a/java/res/xml/key_space_5kw.xml +++ b/java/res/xml/key_space_5kw.xml @@ -23,7 +23,7 @@ > <switch> <case - latin:languageCode="fa" + latin:languageCode="fa|ne" latin:languageSwitchKeyEnabled="true" > <Key @@ -35,7 +35,7 @@ latin:keyStyle="zwnjKeyStyle" /> </case> <case - latin:languageCode="fa" + latin:languageCode="fa|ne" latin:languageSwitchKeyEnabled="false" > <Key diff --git a/java/res/xml/key_styles_currency.xml b/java/res/xml/key_styles_currency.xml index 60333eeb4..b7677a20d 100644 --- a/java/res/xml/key_styles_currency.xml +++ b/java/res/xml/key_styles_currency.xml @@ -95,14 +95,16 @@ <!-- fa: Persian (Rial and Afgahni) hi: Hindi (Indian Rupee) iw: Hebrew (New Sheqel) + lo: Lao (Kip) mn: Mongolian (Tugrik) + ne: Nepali (Nepalese Rupee) th: Thai (Baht) uk: Ukrainian (Hryvnia) vi: Vietnamese (Dong) --> <!-- TODO: The currency sign of Turkish Lira was created in 2012 and assigned U+20BA for its unicode, although there is no font glyph for it as of November 2012. --> <case - latin:languageCode="fa|hi|iw|mn|th|uk|vi" + latin:languageCode="fa|hi|iw|lo|mn|ne|th|uk|vi" > <!-- U+00A3: "£" POUND SIGN U+20AC: "€" EURO SIGN diff --git a/java/res/xml/keyboard_layout_set_armenian_phonetic.xml b/java/res/xml/keyboard_layout_set_armenian_phonetic.xml index b374fae3b..35bd43fb9 100644 --- a/java/res/xml/keyboard_layout_set_armenian_phonetic.xml +++ b/java/res/xml/keyboard_layout_set_armenian_phonetic.xml @@ -28,6 +28,9 @@ latin:elementName="symbols" latin:elementKeyboard="@xml/kbd_symbols" /> <Element + latin:elementName="symbolsShifted" + latin:elementKeyboard="@xml/kbd_symbols_shift" /> + <Element latin:elementName="phone" latin:elementKeyboard="@xml/kbd_phone" /> <Element diff --git a/java/res/xml/keyboard_layout_set_lao.xml b/java/res/xml/keyboard_layout_set_lao.xml new file mode 100644 index 000000000..2ffde45db --- /dev/null +++ b/java/res/xml/keyboard_layout_set_lao.xml @@ -0,0 +1,58 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<KeyboardLayoutSet + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin"> + <Element + latin:elementName="alphabet" + latin:elementKeyboard="@xml/kbd_lao" + latin:enableProximityCharsCorrection="true" /> + <Element + latin:elementName="alphabetAutomaticShifted" + latin:elementKeyboard="@xml/kbd_lao" + latin:enableProximityCharsCorrection="true" /> + <!-- On these shifted alphabet layouts the proximity characters correction should be disabled + because the letters on these layouts aren't the ones in different case of the above + unshifted layouts. --> + <Element + latin:elementName="alphabetManualShifted" + latin:elementKeyboard="@xml/kbd_lao" /> + <Element + latin:elementName="alphabetShiftLocked" + latin:elementKeyboard="@xml/kbd_lao" /> + <Element + latin:elementName="alphabetShiftLockShifted" + latin:elementKeyboard="@xml/kbd_lao" /> + <Element + latin:elementName="symbols" + latin:elementKeyboard="@xml/kbd_symbols" /> + <Element + latin:elementName="symbolsShifted" + latin:elementKeyboard="@xml/kbd_symbols_shift" /> + <Element + latin:elementName="phone" + latin:elementKeyboard="@xml/kbd_phone" /> + <Element + latin:elementName="phoneSymbols" + latin:elementKeyboard="@xml/kbd_phone_symbols" /> + <Element + latin:elementName="number" + latin:elementKeyboard="@xml/kbd_number" /> +</KeyboardLayoutSet> diff --git a/java/res/xml/keyboard_layout_set_nepali_romanized.xml b/java/res/xml/keyboard_layout_set_nepali_romanized.xml index 82f36cfbd..fbbc6a5a0 100644 --- a/java/res/xml/keyboard_layout_set_nepali_romanized.xml +++ b/java/res/xml/keyboard_layout_set_nepali_romanized.xml @@ -44,6 +44,9 @@ latin:elementName="symbols" latin:elementKeyboard="@xml/kbd_symbols" /> <Element + latin:elementName="symbolsShifted" + latin:elementKeyboard="@xml/kbd_symbols_shift" /> + <Element latin:elementName="phone" latin:elementKeyboard="@xml/kbd_phone" /> <Element diff --git a/java/res/xml/keyboard_layout_set_nepali_traditional.xml b/java/res/xml/keyboard_layout_set_nepali_traditional.xml index 2a6dc8e83..4a3b60153 100644 --- a/java/res/xml/keyboard_layout_set_nepali_traditional.xml +++ b/java/res/xml/keyboard_layout_set_nepali_traditional.xml @@ -44,6 +44,9 @@ latin:elementName="symbols" latin:elementKeyboard="@xml/kbd_symbols" /> <Element + latin:elementName="symbolsShifted" + latin:elementKeyboard="@xml/kbd_symbols_shift" /> + <Element latin:elementName="phone" latin:elementKeyboard="@xml/kbd_phone" /> <Element diff --git a/java/res/xml/keyboard_layout_set_thai.xml b/java/res/xml/keyboard_layout_set_thai.xml index 94713e3d4..b8f99971b 100644 --- a/java/res/xml/keyboard_layout_set_thai.xml +++ b/java/res/xml/keyboard_layout_set_thai.xml @@ -42,10 +42,10 @@ latin:elementKeyboard="@xml/kbd_thai" /> <Element latin:elementName="symbols" - latin:elementKeyboard="@xml/kbd_thai_symbols" /> + latin:elementKeyboard="@xml/kbd_symbols" /> <Element latin:elementName="symbolsShifted" - latin:elementKeyboard="@xml/kbd_thai_symbols_shift" /> + latin:elementKeyboard="@xml/kbd_symbols_shift" /> <Element latin:elementName="phone" latin:elementKeyboard="@xml/kbd_phone" /> diff --git a/java/res/xml/method.xml b/java/res/xml/method.xml index c3d68c6e5..6014646bb 100644 --- a/java/res/xml/method.xml +++ b/java/res/xml/method.xml @@ -55,6 +55,7 @@ ka: Georgian/georgian (kk: Kazakh/east_slavic) # disabled temporarily. waiting for strnig resources. ky: Kyrgyz/east_slavic + lo: Lao/lao lt: Lithuanian/qwerty lv: Latvian/qwerty mk: Macedonian/south_slavic @@ -332,6 +333,13 @@ /> <subtype android:icon="@drawable/ic_subtype_keyboard" android:label="@string/subtype_generic" + android:subtypeId="0x8315772c" + android:imeSubtypeLocale="lo" + android:imeSubtypeMode="keyboard" + android:imeSubtypeExtraValue="KeyboardLayoutSet=lao" + /> + <subtype android:icon="@drawable/ic_subtype_keyboard" + android:label="@string/subtype_generic" android:subtypeId="0x8321bb43" android:imeSubtypeLocale="lt" android:imeSubtypeMode="keyboard" diff --git a/java/res/xml/row_symbols_shift4.xml b/java/res/xml/row_symbols_shift4.xml index 57a2ec09c..72d24a321 100644 --- a/java/res/xml/row_symbols_shift4.xml +++ b/java/res/xml/row_symbols_shift4.xml @@ -24,8 +24,7 @@ latin:keyboardLayout="@xml/key_space_symbols" /> <include latin:keyboardLayout="@xml/keys_comma_period" /> - <Key - latin:keyStyle="enterKeyStyle" - latin:keyWidth="fillRight" /> + <include + latin:keyboardLayout="@xml/key_f2" /> </merge> diff --git a/java/res/xml/rowkeys_lao1.xml b/java/res/xml/rowkeys_lao1.xml new file mode 100644 index 000000000..fa1ad97d8 --- /dev/null +++ b/java/res/xml/rowkeys_lao1.xml @@ -0,0 +1,164 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <switch> + <case + latin:keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" + > + <!-- U+0ED1: "໑" LAO DIGIT ONE --> + <Key + latin:keyLabel="໑" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED2: "໒" LAO DIGIT TWO --> + <Key + latin:keyLabel="໒" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED3: "໓" LAO DIGIT THREE --> + <Key + latin:keyLabel="໓" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED4: "໔" LAO DIGIT FOUR --> + <Key + latin:keyLabel="໔" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ECC: "໌" LAO CANCELLATION MARK --> + <Key + latin:keyLabel="໌" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EBC: "ຼ" LAO SEMIVOWEL SIGN LO --> + <Key + latin:keyLabel="ຼ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED5: "໕" LAO DIGIT FIVE --> + <Key + latin:keyLabel="໕" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED6: "໖" LAO DIGIT SIX --> + <Key + latin:keyLabel="໖" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED7: "໗" LAO DIGIT SEVEN --> + <Key + latin:keyLabel="໗" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED8: "໘" LAO DIGIT EIGHT --> + <Key + latin:keyLabel="໘" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ED9: "໙" LAO DIGIT NINE --> + <Key + latin:keyLabel="໙" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ECD/U+0EC8: "ໍ່" LAO NIGGAHITA/LAO TONE MAI EK --> + <Key + latin:keyLabel="ໍ່" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + </case> + <default> + <!-- U+0EA2: "ຢ" LAO LETTER YO + U+0ED1: "໑" LAO DIGIT ONE --> + <Key + latin:keyLabel="ຢ" + latin:keyHintLabel="1" + latin:additionalMoreKeys="1" + latin:moreKeys="໑" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E9F: "ຟ" LAO LETTER FO SUNG + U+0ED2: "໒" LAO DIGIT TWO --> + <Key + latin:keyLabel="ຟ" + latin:keyHintLabel="2" + latin:additionalMoreKeys="2" + latin:moreKeys="໒" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC2: "ໂ" LAO VOWEL SIGN O + U+0ED3: "໓" LAO DIGIT THREE --> + <Key + latin:keyLabel="ໂ" + latin:keyHintLabel="3" + latin:additionalMoreKeys="3" + latin:moreKeys="໓" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E96: "ຖ" LAO LETTER THO SUNG + U+0ED4: "໔" LAO DIGIT FOUR --> + <Key + latin:keyLabel="ຖ" + latin:keyHintLabel="4" + latin:additionalMoreKeys="4" + latin:moreKeys="໔" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB8: "ຸ" LAO VOWEL SIGN U --> + <Key + latin:keyLabel="ຸ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB9: "ູ" LAO VOWEL SIGN UU --> + <Key + latin:keyLabel="ູ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E84: "ຄ" LAO LETTER KHO TAM + U+0ED5: "໕" LAO DIGIT FIVE --> + <Key + latin:keyLabel="ຄ" + latin:keyHintLabel="5" + latin:additionalMoreKeys="5" + latin:moreKeys="໕" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E95: "ຕ" LAO LETTER TO + U+0ED6: "໖" LAO DIGIT SIX --> + <Key + latin:keyLabel="ຕ" + latin:keyHintLabel="6" + latin:additionalMoreKeys="6" + latin:moreKeys="໖" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E88: "ຈ" LAO LETTER CO + U+0ED7: "໗" LAO DIGIT SEVEN --> + <Key + latin:keyLabel="ຈ" + latin:keyHintLabel="7" + latin:additionalMoreKeys="7" + latin:moreKeys="໗" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E82: "ຂ" LAO LETTER KHO SUNG + U+0ED8: "໘" LAO DIGIT EIGHT --> + <Key + latin:keyLabel="ຂ" + latin:keyHintLabel="8" + latin:additionalMoreKeys="8" + latin:moreKeys="໘" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E8A: "ຊ" LAO LETTER SO TAM + U+0ED9: "໙" LAO DIGIT NINE --> + <Key + latin:keyLabel="ຊ" + latin:keyHintLabel="9" + latin:additionalMoreKeys="9" + latin:moreKeys="໙" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ECD: "ໍ" LAO NIGGAHITA --> + <Key + latin:keyLabel="ໍ" + latin:keyLabelFlags="fontNormal" /> + </default> + </switch> +</merge> diff --git a/java/res/xml/rowkeys_lao2.xml b/java/res/xml/rowkeys_lao2.xml new file mode 100644 index 000000000..fca58ac0e --- /dev/null +++ b/java/res/xml/rowkeys_lao2.xml @@ -0,0 +1,127 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <switch> + <case + latin:keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" + > + <!-- U+0EBB/U+0EC9: "" LAO VOWEL SIGN MAI KON/LAO TONE MAI THO --> + <Key + latin:keyLabel="ົ້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <!-- U+0ED0: "໐" LAO DIGIT ZERO --> + <Key + latin:keyLabel="໐" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB3/U+0EC9: "ຳ້" LAO VOWEL SIGN AM/LAO TONE MAI THO --> + <Key + latin:keyLabel="ຳ້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <Key + latin:keyLabel="_" /> + <Key + latin:keyLabel="+" /> + <!-- U+0EB4/U+0EC9: "ິ້" LAO VOWEL SIGN I/LAO TONE MAI THO --> + <Key + latin:keyLabel="ິ້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <!-- U+0EB5/U+0EC9: "ີ້" LAO VOWEL SIGN II/LAO TONE MAI THO --> + <Key + latin:keyLabel="ີ້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <!-- U+0EA3: "ຣ" LAO LETTER LO LING --> + <Key + latin:keyLabel="ຣ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EDC: "ໜ" LAO HO NO --> + <Key + latin:keyLabel="ໜ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EBD: "ຽ" LAO SEMIVOWEL SIGN NYO --> + <Key + latin:keyLabel="ຽ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EAB/U+0EBC: "" LAO LETTER HO SUNG/LAO SEMIVOWEL SIGN LO --> + <Key + latin:keyLabel="ຫຼ" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <!-- U+201D: "”" RIGHT DOUBLE QUOTATION MARK --> + <Key + latin:keyLabel="”" /> + </case> + <default> + <!-- U+0EBB: "ົ" LAO VOWEL SIGN MAI KON --> + <Key + latin:keyLabel="ົ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC4: "ໄ" LAO VOWEL SIGN AI + U+0ED0: "໐" LAO DIGIT ZERO --> + <Key + latin:keyLabel="ໄ" + latin:keyHintLabel="0" + latin:additionalMoreKeys="0" + latin:moreKeys="໐" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB3: "ຳ" LAO VOWEL SIGN AM --> + <Key + latin:keyLabel="ຳ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E9E: "ພ" LAO LETTER PHO TAM --> + <Key + latin:keyLabel="ພ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB0: "ະ" LAO VOWEL SIGN A --> + <Key + latin:keyLabel="ະ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB4: "ິ" LAO VOWEL SIGN I --> + <Key + latin:keyLabel="ິ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB5: "ີ" LAO VOWEL SIGN II --> + <Key + latin:keyLabel="ີ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EAE: "ຮ" LAO LETTER HO TAM --> + <Key + latin:keyLabel="ຮ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E99: "ນ" LAO LETTER NO --> + <Key + latin:keyLabel="ນ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E8D: "ຍ" LAO LETTER NYO --> + <Key + latin:keyLabel="ຍ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E9A: "ບ" LAO LETTER BO --> + <Key + latin:keyLabel="ບ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EA5: "ລ" LAO LETTER LO LOOT --> + <Key + latin:keyLabel="ລ" + latin:keyLabelFlags="fontNormal" /> + </default> + </switch> +</merge> diff --git a/java/res/xml/rowkeys_lao3.xml b/java/res/xml/rowkeys_lao3.xml new file mode 100644 index 000000000..2a6c2d1dd --- /dev/null +++ b/java/res/xml/rowkeys_lao3.xml @@ -0,0 +1,110 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <switch> + <case + latin:keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" + > + <!-- U+0EB1/U+0EC9: "ັ້" LAO VOWEL SIGN MAI KAN/LAO TONE MAI THO --> + <Key + latin:keyLabel="ັ້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <Key + latin:keyLabel=";" /> + <Key + latin:keyLabel="." /> + <Key + latin:keyLabel="," /> + <Key + latin:keyLabel=":" /> + <!-- U+0ECA: "໊" LAO TONE MAI TI --> + <Key + latin:keyLabel="໊" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0ECB: "໋" LAO TONE MAI CATAWA --> + <Key + latin:keyLabel="໋" + latin:keyLabelFlags="fontNormal" /> + <Key + latin:keyLabel="!" /> + <Key + latin:keyLabel="\?" /> + <Key + latin:keyLabel="%" /> + <Key + latin:keyLabel="=" /> + <!-- U+201C: "“" LEFT DOUBLE QUOTATION MARK --> + <Key + latin:keyLabel="“" /> + </case> + <default> + <!-- U+0EB1: "ັ" LAO VOWEL SIGN MAI KAN --> + <Key + latin:keyLabel="ັ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EAB: "ຫ" LAO LETTER HO SUNG --> + <Key + latin:keyLabel="ຫ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E81: "ກ" LAO LETTER KO --> + <Key + latin:keyLabel="ກ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E94: "ດ" LAO LETTER DO --> + <Key + latin:keyLabel="ດ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC0: "ເ" LAO VOWEL SIGN E --> + <Key + latin:keyLabel="ເ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC9: "້" LAO TONE MAI THO --> + <Key + latin:keyLabel="້" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC8: "່" LAO TONE MAI EK --> + <Key + latin:keyLabel="່" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB2: "າ" LAO VOWEL SIGN AA --> + <Key + latin:keyLabel="າ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EAA: "ສ" LAO LETTER SO SUNG --> + <Key + latin:keyLabel="ສ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EA7: "ວ" LAO LETTER WO --> + <Key + latin:keyLabel="ວ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E87: "ງ" LAO LETTER NGO --> + <Key + latin:keyLabel="ງ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+201C: "“" LEFT DOUBLE QUOTATION MARK --> + <Key + latin:keyLabel="“" /> + </default> + </switch> +</merge> diff --git a/java/res/xml/rowkeys_lao4.xml b/java/res/xml/rowkeys_lao4.xml new file mode 100644 index 000000000..fae9cc923 --- /dev/null +++ b/java/res/xml/rowkeys_lao4.xml @@ -0,0 +1,103 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <switch> + <case + latin:keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" + > + <!-- U+20AD: "₭" KIP SIGN --> + <Key + latin:keyLabel="₭" /> + <Key + latin:keyLabel="(" /> + <!-- U+0EAF: "ຯ" LAO ELLIPSIS --> + <Key + latin:keyLabel="ຯ" + latin:keyLabelFlags="fontNormal" /> + <Key + latin:keyLabel="\@" /> + <!-- U+0EB6/U+0EC9: "ຶ້" LAO VOWEL SIGN Y/LAO TONE MAI THO --> + <Key + latin:keyLabel="ຶ້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <!-- U+0EB7/U+0EC9: "ື້" LAO VOWEL SIGN YY/LAO TONE MAI THO --> + <Key + latin:keyLabel="ື້" + latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> + <!-- U+0EC6: "ໆ" LAO KO LA --> + <Key + latin:keyLabel="ໆ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EDD: "ໝ" LAO HO MO --> + <Key + latin:keyLabel="ໝ" + latin:keyLabelFlags="fontNormal" /> + <Key + latin:keyLabel="$" /> + <Key + latin:keyLabel=")" /> + </case> + <default> + <!-- U+0E9C: "ຜ" LAO LETTER PHO SUNG --> + <Key + latin:keyLabel="ຜ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E9B: "ປ" LAO LETTER PO --> + <Key + latin:keyLabel="ປ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC1: "ແ" LAO VOWEL SIGN EI --> + <Key + latin:keyLabel="ແ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EAD: "ອ" LAO LETTER O --> + <Key + latin:keyLabel="ອ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB6: "ຶ" LAO VOWEL SIGN Y --> + <Key + latin:keyLabel="ຶ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EB7: "ື" LAO VOWEL SIGN YY --> + <Key + latin:keyLabel="ື" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E97: "ທ" LAO LETTER THO TAM --> + <Key + latin:keyLabel="ທ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EA1: "ມ" LAO LETTER MO --> + <Key + latin:keyLabel="ມ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0EC3: "ໃ" LAO VOWEL SIGN AY --> + <Key + latin:keyLabel="ໃ" + latin:keyLabelFlags="fontNormal" /> + <!-- U+0E9D: "ຝ" LAO LETTER FO TAM --> + <Key + latin:keyLabel="ຝ" + latin:keyLabelFlags="fontNormal" /> + </default> + </switch> +</merge> diff --git a/java/res/xml/rowkeys_nepali_romanized3.xml b/java/res/xml/rowkeys_nepali_romanized3.xml index 5660596f0..166d028a3 100644 --- a/java/res/xml/rowkeys_nepali_romanized3.xml +++ b/java/res/xml/rowkeys_nepali_romanized3.xml @@ -48,7 +48,7 @@ latin:keyLabelFlags="fontNormal" /> <!-- U+0923: "ण" DEVANAGARI LETTER NNA --> <Key - latin:keyLabel="श" + latin:keyLabel="ण" latin:keyLabelFlags="fontNormal" /> <!-- Because the font rendering system prior to API version 16 can't automatically render dotted circle for incomplete combining letter of some scripts, different diff --git a/java/res/xml/rowkeys_nepali_traditional2.xml b/java/res/xml/rowkeys_nepali_traditional2.xml index 2c53b3b8b..45620a9d7 100644 --- a/java/res/xml/rowkeys_nepali_traditional2.xml +++ b/java/res/xml/rowkeys_nepali_traditional2.xml @@ -29,13 +29,13 @@ <Key latin:keyLabel="आ" latin:keyLabelFlags="fontNormal" /> - <!-- U+0919/U+094D/U+0915: "ङ्क" DEVANAGARI LETTER NGA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER KA --> + <!-- U+0919/U+094D: "ङ्" DEVANAGARI LETTER NGA/DEVANAGARI SIGN VIRAMA --> <Key - latin:keyLabel="ङ्क" + latin:keyLabel="ङ्" latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> - <!-- U+0919/U+094D/U+0917: "ङ्ग" DEVANAGARI LETTER NGA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER GA --> + <!-- U+0921/U+094D/U+0921: "ड्ड" DEVANAGARI LETTER DDA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER DDA --> <Key - latin:keyLabel="ङ्ग" + latin:keyLabel="ड्ड" latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> <!-- Because the font rendering system prior to API version 16 can't automatically render dotted circle for incomplete combining letter of some scripts, different diff --git a/java/res/xml/rowkeys_nepali_traditional3_left6.xml b/java/res/xml/rowkeys_nepali_traditional3_left6.xml index d4388e0a2..1cacced83 100644 --- a/java/res/xml/rowkeys_nepali_traditional3_left6.xml +++ b/java/res/xml/rowkeys_nepali_traditional3_left6.xml @@ -25,9 +25,9 @@ <case latin:keyboardLayoutSetElement="alphabetManualShifted|alphabetShiftLocked|alphabetShiftLockShifted" > - <!-- U+0915/U+094D/U+0915: "क्क" DEVANAGARI LETTER KA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER KA --> + <!-- U+0915/U+094D: "क्" DEVANAGARI LETTER KA/DEVANAGARI SIGN VIRAMA --> <Key - latin:keyLabel="क्क" + latin:keyLabel="क्" latin:keyLabelFlags="fontNormal|followKeyLetterRatio" /> <!-- U+0939/U+094D/U+092E: "ह्म" DEVANAGARI LETTER HA/DEVANAGARI SIGN VIRAMA/DEVANAGARI LETTER MA --> <Key diff --git a/java/res/xml/rows_lao.xml b/java/res/xml/rows_lao.xml new file mode 100644 index 000000000..321f4112a --- /dev/null +++ b/java/res/xml/rows_lao.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> + +<merge + xmlns:latin="http://schemas.android.com/apk/res/com.android.inputmethod.latin" +> + <include + latin:keyboardLayout="@xml/key_styles_common" /> + <Row + latin:keyWidth="8.3333%p" + > + <include + latin:keyboardLayout="@xml/rowkeys_lao1" /> + </Row> + <Row + latin:keyWidth="8.3333%p" + > + <include + latin:keyboardLayout="@xml/rowkeys_lao2" /> + </Row> + <Row + latin:keyWidth="8.3333%p" + > + <include + latin:keyboardLayout="@xml/rowkeys_lao3" /> + </Row> + <Row + latin:keyWidth="8.3333%p" + > + <Key + latin:keyStyle="shiftKeyStyle" /> + <include + latin:keyboardLayout="@xml/rowkeys_lao4" /> + <Key + latin:keyStyle="deleteKeyStyle" /> + </Row> + <include + latin:keyboardLayout="@xml/row_qwerty4" /> +</merge> diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java index 25ff8d0ce..702ed2075 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java @@ -19,12 +19,18 @@ package com.android.inputmethod.keyboard; import static com.android.inputmethod.latin.Constants.NOT_A_COORDINATE; import android.content.Context; +import android.content.SharedPreferences; import android.content.res.ColorStateList; import android.content.res.Resources; import android.content.res.TypedArray; +import android.graphics.Rect; +import android.os.Build; +import android.preference.PreferenceManager; import android.support.v4.view.PagerAdapter; import android.support.v4.view.ViewPager; import android.util.AttributeSet; +import android.util.Log; +import android.util.Pair; import android.util.SparseArray; import android.view.LayoutInflater; import android.view.View; @@ -35,16 +41,21 @@ import android.widget.TabHost; import android.widget.TabHost.OnTabChangeListener; import android.widget.TextView; -import com.android.inputmethod.keyboard.internal.RecentsKeyboard; +import com.android.inputmethod.keyboard.internal.DynamicGridKeyboard; import com.android.inputmethod.keyboard.internal.ScrollKeyboardView; import com.android.inputmethod.keyboard.internal.ScrollViewWithNotifier; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.R; import com.android.inputmethod.latin.SubtypeSwitcher; +import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.ResourceUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; /** * View class to implement Emoji keyboards. @@ -60,51 +71,287 @@ import java.util.HashMap; public final class EmojiKeyboardView extends LinearLayout implements OnTabChangeListener, ViewPager.OnPageChangeListener, View.OnClickListener, ScrollKeyboardView.OnKeyClickListener { + private static final String TAG = EmojiKeyboardView.class.getSimpleName(); private final int mKeyBackgroundId; + private final int mEmojiFunctionalKeyBackgroundId; + private final KeyboardLayoutSet mLayoutSet; private final ColorStateList mTabLabelColor; - private final EmojiKeyboardAdapter mEmojiKeyboardAdapter; + private EmojiKeyboardAdapter mEmojiKeyboardAdapter; private TabHost mTabHost; private ViewPager mEmojiPager; private KeyboardActionListener mKeyboardActionListener = KeyboardActionListener.EMPTY_LISTENER; - private int mCurrentCategory = CATEGORY_UNSPECIFIED; - private static final int CATEGORY_UNSPECIFIED = -1; - private static final int CATEGORY_RECENTS = 0; - private static final int CATEGORY_PEOPLE = 1; - private static final int CATEGORY_OBJECTS = 2; - private static final int CATEGORY_NATURE = 3; - private static final int CATEGORY_PLACES = 4; - private static final int CATEGORY_SYMBOLS = 5; - private static final int CATEGORY_EMOTICONS = 6; - private static final HashMap<String, Integer> sCategoryNameToIdMap = - CollectionUtils.newHashMap(); - private static final String[] sCategoryName = { - "recents", "people", "objects", "nature", "places", "symbols", "emoticons" - }; - private static final int[] sCategoryIcon = new int[] { - R.drawable.ic_emoji_recent_light, - R.drawable.ic_emoji_people_light, - R.drawable.ic_emoji_objects_light, - R.drawable.ic_emoji_nature_light, - R.drawable.ic_emoji_places_light, - R.drawable.ic_emoji_symbols_light, - 0 - }; - private static final String[] sCategoryLabel = { - null, null, null, null, null, null, - ":-)" - }; - private static final int[] sCategoryElementId = { - KeyboardId.ELEMENT_EMOJI_RECENTS, - KeyboardId.ELEMENT_EMOJI_CATEGORY1, - KeyboardId.ELEMENT_EMOJI_CATEGORY2, - KeyboardId.ELEMENT_EMOJI_CATEGORY3, - KeyboardId.ELEMENT_EMOJI_CATEGORY4, - KeyboardId.ELEMENT_EMOJI_CATEGORY5, - KeyboardId.ELEMENT_EMOJI_CATEGORY6, - }; + private static final int CATEGORY_ID_UNSPECIFIED = -1; + public static final int CATEGORY_ID_RECENTS = 0; + public static final int CATEGORY_ID_PEOPLE = 1; + public static final int CATEGORY_ID_OBJECTS = 2; + public static final int CATEGORY_ID_NATURE = 3; + public static final int CATEGORY_ID_PLACES = 4; + public static final int CATEGORY_ID_SYMBOLS = 5; + public static final int CATEGORY_ID_EMOTICONS = 6; + + private static class CategoryProperties { + public int mCategoryId; + public int mPageCount; + public CategoryProperties(final int categoryId, final int pageCount) { + mCategoryId = categoryId; + mPageCount = pageCount; + } + } + + private static class EmojiCategory { + private static final String[] sCategoryName = { + "recents", + "people", + "objects", + "nature", + "places", + "symbols", + "emoticons" }; + private static final int[] sCategoryIcon = new int[] { + R.drawable.ic_emoji_recent_light, + R.drawable.ic_emoji_people_light, + R.drawable.ic_emoji_objects_light, + R.drawable.ic_emoji_nature_light, + R.drawable.ic_emoji_places_light, + R.drawable.ic_emoji_symbols_light, + 0 }; + private static final String[] sCategoryLabel = + { null, null, null, null, null, null, ":-)" }; + private static final int[] sCategoryElementId = { + KeyboardId.ELEMENT_EMOJI_RECENTS, + KeyboardId.ELEMENT_EMOJI_CATEGORY1, + KeyboardId.ELEMENT_EMOJI_CATEGORY2, + KeyboardId.ELEMENT_EMOJI_CATEGORY3, + KeyboardId.ELEMENT_EMOJI_CATEGORY4, + KeyboardId.ELEMENT_EMOJI_CATEGORY5, + KeyboardId.ELEMENT_EMOJI_CATEGORY6 }; + private final SharedPreferences mPrefs; + private final int mMaxPageKeyCount; + private final KeyboardLayoutSet mLayoutSet; + private final HashMap<String, Integer> mCategoryNameToIdMap = CollectionUtils.newHashMap(); + private final ArrayList<CategoryProperties> mShownCategories = + CollectionUtils.newArrayList(); + private final ConcurrentHashMap<Long, DynamicGridKeyboard> + mCategoryKeyboardMap = new ConcurrentHashMap<Long, DynamicGridKeyboard>(); + + private int mCurrentCategoryId = CATEGORY_ID_UNSPECIFIED; + private int mCurrentCategoryPageId = 0; + + public EmojiCategory(final SharedPreferences prefs, final Resources res, + final KeyboardLayoutSet layoutSet) { + mPrefs = prefs; + mMaxPageKeyCount = res.getInteger(R.integer.emoji_keyboard_max_key_count); + mLayoutSet = layoutSet; + for (int i = 0; i < sCategoryName.length; ++i) { + mCategoryNameToIdMap.put(sCategoryName[i], i); + } + addShownCategoryId(CATEGORY_ID_RECENTS); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { + addShownCategoryId(CATEGORY_ID_PEOPLE); + addShownCategoryId(CATEGORY_ID_OBJECTS); + addShownCategoryId(CATEGORY_ID_NATURE); + addShownCategoryId(CATEGORY_ID_PLACES); + mCurrentCategoryId = CATEGORY_ID_PEOPLE; + } else { + mCurrentCategoryId = CATEGORY_ID_SYMBOLS; + } + addShownCategoryId(CATEGORY_ID_SYMBOLS); + addShownCategoryId(CATEGORY_ID_EMOTICONS); + getKeyboard(CATEGORY_ID_RECENTS, 0 /* cagetoryPageId */) + .loadRecentKeys(mCategoryKeyboardMap.values()); + } + + private void addShownCategoryId(int categoryId) { + // Load a keyboard of categoryId + getKeyboard(categoryId, 0 /* cagetoryPageId */); + final CategoryProperties properties = + new CategoryProperties(categoryId, getCategoryPageCount(categoryId)); + mShownCategories.add(properties); + } + + public String getCategoryName(int categoryId, int categoryPageId) { + return sCategoryName[categoryId] + "-" + categoryPageId; + } + + public int getCategoryId(String name) { + final String[] strings = name.split("-"); + return mCategoryNameToIdMap.get(strings[0]); + } + + public int getCategoryIcon(int categoryId) { + return sCategoryIcon[categoryId]; + } + + public String getCategoryLabel(int categoryId) { + return sCategoryLabel[categoryId]; + } + + public ArrayList<CategoryProperties> getShownCategories() { + return mShownCategories; + } + + public int getCurrentCategoryId() { + return mCurrentCategoryId; + } + + public void setCurrentCategoryId(int categoryId) { + mCurrentCategoryId = categoryId; + } + + public void setCurrentCategoryPageId(int id) { + mCurrentCategoryPageId = id; + } + + public void saveLastTypedCategoryPage() { + Settings.writeEmojiCategoryLastTypedId( + mPrefs, mCurrentCategoryId, mCurrentCategoryPageId); + } + + public boolean isInRecentTab() { + return mCurrentCategoryId == CATEGORY_ID_RECENTS; + } + + public int getTabIdFromCategoryId(int categoryId) { + for (int i = 0; i < mShownCategories.size(); ++i) { + if (mShownCategories.get(i).mCategoryId == categoryId) { + return i; + } + } + Log.w(TAG, "categoryId not found: " + categoryId); + return 0; + } + + // Returns the view pager's page position for the categoryId + public int getPageIdFromCategoryId(int categoryId) { + final int lastSavedCategoryPageId = + Settings.readEmojiCategoryLastTypedId(mPrefs, categoryId); + int sum = 0; + for (int i = 0; i < mShownCategories.size(); ++i) { + final CategoryProperties props = mShownCategories.get(i); + if (props.mCategoryId == categoryId) { + return sum + lastSavedCategoryPageId; + } + sum += props.mPageCount; + } + Log.w(TAG, "categoryId not found: " + categoryId); + return 0; + } + + public int getRecentTabId() { + return getTabIdFromCategoryId(CATEGORY_ID_RECENTS); + } + + private int getCategoryPageCount(int categoryId) { + final Keyboard keyboard = mLayoutSet.getKeyboard(sCategoryElementId[categoryId]); + return (keyboard.getKeys().length - 1) / mMaxPageKeyCount + 1; + } + + // Returns a pair of the category id and the category page id from the view pager's page + // position. The category page id is numbered in each category. And the view page position + // is the position of the current shown page in the view pager which contains all pages of + // all categories. + public Pair<Integer, Integer> getCategoryIdAndPageIdFromPagePosition(int position) { + int sum = 0; + for (CategoryProperties properties : mShownCategories) { + final int temp = sum; + sum += properties.mPageCount; + if (sum > position) { + return new Pair<Integer, Integer>(properties.mCategoryId, position - temp); + } + } + return null; + } + + // Returns a keyboard from the view pager's page position. + public DynamicGridKeyboard getKeyboardFromPagePosition(int position) { + final Pair<Integer, Integer> categoryAndId = + getCategoryIdAndPageIdFromPagePosition(position); + if (categoryAndId != null) { + return getKeyboard(categoryAndId.first, categoryAndId.second); + } + return null; + } + + public DynamicGridKeyboard getKeyboard(int categoryId, int id) { + synchronized(mCategoryKeyboardMap) { + final long key = (((long) categoryId) << Constants.MAX_INT_BIT_COUNT) | id; + final DynamicGridKeyboard kbd; + if (!mCategoryKeyboardMap.containsKey(key)) { + if (categoryId != CATEGORY_ID_RECENTS) { + final Keyboard keyboard = + mLayoutSet.getKeyboard(sCategoryElementId[categoryId]); + final Key[][] sortedKeys = sortKeys(keyboard.getKeys(), mMaxPageKeyCount); + for (int i = 0; i < sortedKeys.length; ++i) { + final DynamicGridKeyboard tempKbd = new DynamicGridKeyboard(mPrefs, + mLayoutSet.getKeyboard(KeyboardId.ELEMENT_EMOJI_RECENTS), + mMaxPageKeyCount, categoryId, i /* categoryPageId */); + for (Key emojiKey : sortedKeys[i]) { + if (emojiKey == null) { + break; + } + tempKbd.addKeyLast(emojiKey); + } + mCategoryKeyboardMap.put((((long) categoryId) + << Constants.MAX_INT_BIT_COUNT) | i, tempKbd); + } + kbd = mCategoryKeyboardMap.get(key); + } else { + kbd = new DynamicGridKeyboard(mPrefs, + mLayoutSet.getKeyboard(KeyboardId.ELEMENT_EMOJI_RECENTS), + mMaxPageKeyCount, categoryId, 0 /* categoryPageId */); + mCategoryKeyboardMap.put(key, kbd); + } + } else { + kbd = mCategoryKeyboardMap.get(key); + } + return kbd; + } + } + + public int getTotalPageCountOfAllCategories() { + int sum = 0; + for (CategoryProperties properties : mShownCategories) { + sum += properties.mPageCount; + } + return sum; + } + + private Key[][] sortKeys(Key[] inKeys, int maxPageCount) { + Key[] keys = Arrays.copyOf(inKeys, inKeys.length); + Arrays.sort(keys, 0, keys.length, new Comparator<Key>() { + @Override + public int compare(Key lhs, Key rhs) { + final Rect lHitBox = lhs.getHitBox(); + final Rect rHitBox = rhs.getHitBox(); + if (lHitBox.top < rHitBox.top) { + return -1; + } else if (lHitBox.top > rHitBox.top) { + return 1; + } + if (lHitBox.left < rHitBox.left) { + return -1; + } else if (lHitBox.left > rHitBox.left) { + return 1; + } + if (lhs.getCode() == rhs.getCode()) { + return 0; + } + return lhs.getCode() < rhs.getCode() ? -1 : 1; + } + }); + final int pageCount = (keys.length - 1) / maxPageCount + 1; + final Key[][] retval = new Key[pageCount][maxPageCount]; + for (int i = 0; i < keys.length; ++i) { + retval[i / maxPageCount][i % maxPageCount] = keys[i]; + } + return retval; + } + } + + private final EmojiCategory mEmojiCategory; public EmojiKeyboardView(final Context context, final AttributeSet attrs) { this(context, attrs, R.attr.emojiKeyboardViewStyle); @@ -116,6 +363,8 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange R.styleable.KeyboardView, defStyle, R.style.KeyboardView); mKeyBackgroundId = keyboardViewAttr.getResourceId( R.styleable.KeyboardView_keyBackground, 0); + mEmojiFunctionalKeyBackgroundId = keyboardViewAttr.getResourceId( + R.styleable.KeyboardView_keyBackgroundEmojiFunctional, 0); keyboardViewAttr.recycle(); final TypedArray emojiKeyboardViewAttr = context.obtainStyledAttributes(attrs, R.styleable.EmojiKeyboardView, defStyle, R.style.EmojiKeyboardView); @@ -125,15 +374,14 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder( context, null /* editorInfo */); final Resources res = context.getResources(); + final EmojiLayoutParams emojiLp = new EmojiLayoutParams(res); builder.setSubtype(SubtypeSwitcher.getInstance().getEmojiSubtype()); - // TODO: Make Keyboard height variable. builder.setKeyboardGeometry(ResourceUtils.getDefaultKeyboardWidth(res), - (int)(ResourceUtils.getDefaultKeyboardHeight(res) - - res.getDimension(R.dimen.suggestions_strip_height))); + emojiLp.mEmojiKeyboardHeight); builder.setOptions(false, false, false /* lanuageSwitchKeyEnabled */); - final KeyboardLayoutSet layoutSet = builder.build(); - mEmojiKeyboardAdapter = new EmojiKeyboardAdapter(layoutSet, this); - // TODO: Save/restore recent keys from/to preferences. + mLayoutSet = builder.build(); + mEmojiCategory = new EmojiCategory(PreferenceManager.getDefaultSharedPreferences(context), + context.getResources(), builder.build()); } @Override @@ -149,23 +397,21 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange setMeasuredDimension(width, height); } - private void addTab(final TabHost host, final int category) { - final String tabId = sCategoryName[category]; - sCategoryNameToIdMap.put(tabId, category); + private void addTab(final TabHost host, final int categoryId) { + final String tabId = mEmojiCategory.getCategoryName(categoryId, 0 /* categoryPageId */); final TabHost.TabSpec tspec = host.newTabSpec(tabId); tspec.setContent(R.id.emoji_keyboard_dummy); - if (sCategoryIcon[category] != 0) { + if (mEmojiCategory.getCategoryIcon(categoryId) != 0) { final ImageView iconView = (ImageView)LayoutInflater.from(getContext()).inflate( R.layout.emoji_keyboard_tab_icon, null); - iconView.setImageResource(sCategoryIcon[category]); + iconView.setImageResource(mEmojiCategory.getCategoryIcon(categoryId)); tspec.setIndicator(iconView); } - if (sCategoryLabel[category] != null) { + if (mEmojiCategory.getCategoryLabel(categoryId) != null) { final TextView textView = (TextView)LayoutInflater.from(getContext()).inflate( R.layout.emoji_keyboard_tab_label, null); - textView.setText(sCategoryLabel[category]); + textView.setText(mEmojiCategory.getCategoryLabel(categoryId)); textView.setTextColor(mTabLabelColor); - textView.setBackgroundResource(mKeyBackgroundId); tspec.setIndicator(textView); } host.addTab(tspec); @@ -175,55 +421,60 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange protected void onFinishInflate() { mTabHost = (TabHost)findViewById(R.id.emoji_category_tabhost); mTabHost.setup(); - addTab(mTabHost, CATEGORY_RECENTS); - addTab(mTabHost, CATEGORY_PEOPLE); - addTab(mTabHost, CATEGORY_OBJECTS); - addTab(mTabHost, CATEGORY_NATURE); - addTab(mTabHost, CATEGORY_PLACES); - addTab(mTabHost, CATEGORY_SYMBOLS); - addTab(mTabHost, CATEGORY_EMOTICONS); + for (final CategoryProperties properties : mEmojiCategory.getShownCategories()) { + addTab(mTabHost, properties.mCategoryId); + } mTabHost.setOnTabChangedListener(this); mTabHost.getTabWidget().setStripEnabled(true); + mEmojiKeyboardAdapter = new EmojiKeyboardAdapter(mEmojiCategory, mLayoutSet, this); + mEmojiPager = (ViewPager)findViewById(R.id.emoji_keyboard_pager); mEmojiPager.setAdapter(mEmojiKeyboardAdapter); mEmojiPager.setOnPageChangeListener(this); mEmojiPager.setOffscreenPageLimit(0); - final ViewGroup.LayoutParams lp = mEmojiPager.getLayoutParams(); final Resources res = getResources(); - lp.height = ResourceUtils.getDefaultKeyboardHeight(res) - - res.getDimensionPixelSize(R.dimen.suggestions_strip_height); - mEmojiPager.setLayoutParams(lp); + final EmojiLayoutParams emojiLp = new EmojiLayoutParams(res); + emojiLp.setPagerProps(mEmojiPager); + + setCurrentCategoryId(mEmojiCategory.getCurrentCategoryId(), true /* force */); - // TODO: Record current category. - final int category = CATEGORY_PEOPLE; - setCurrentCategory(category, true /* force */); + final LinearLayout actionBar = (LinearLayout)findViewById(R.id.emoji_action_bar); + emojiLp.setActionBarProps(actionBar); // TODO: Implement auto repeat, using View.OnTouchListener? - final View deleteKey = findViewById(R.id.emoji_keyboard_delete); - deleteKey.setBackgroundResource(mKeyBackgroundId); + final ImageView deleteKey = (ImageView)findViewById(R.id.emoji_keyboard_delete); + deleteKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); deleteKey.setTag(Constants.CODE_DELETE); deleteKey.setOnClickListener(this); - final View alphabetKey = findViewById(R.id.emoji_keyboard_alphabet); - alphabetKey.setBackgroundResource(mKeyBackgroundId); + final ImageView alphabetKey = (ImageView)findViewById(R.id.emoji_keyboard_alphabet); + alphabetKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); alphabetKey.setTag(Constants.CODE_SWITCH_ALPHA_SYMBOL); alphabetKey.setOnClickListener(this); - final View sendKey = findViewById(R.id.emoji_keyboard_send); - sendKey.setBackgroundResource(mKeyBackgroundId); + final ImageView spaceKey = (ImageView)findViewById(R.id.emoji_keyboard_space); + spaceKey.setBackgroundResource(mKeyBackgroundId); + spaceKey.setTag(Constants.CODE_SPACE); + spaceKey.setOnClickListener(this); + emojiLp.setKeyProps(spaceKey); + final ImageView sendKey = (ImageView)findViewById(R.id.emoji_keyboard_send); + sendKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); sendKey.setTag(Constants.CODE_ENTER); sendKey.setOnClickListener(this); } @Override public void onTabChanged(final String tabId) { - final int category = sCategoryNameToIdMap.get(tabId); - setCurrentCategory(category, false /* force */); + final int categoryId = mEmojiCategory.getCategoryId(tabId); + setCurrentCategoryId(categoryId, false /* force */); } @Override public void onPageSelected(final int position) { - setCurrentCategory(position, false /* force */); + final Pair<Integer, Integer> newPos = + mEmojiCategory.getCategoryIdAndPageIdFromPagePosition(position); + setCurrentCategoryId(newPos.first /* categoryId */, false /* force */); + mEmojiCategory.setCurrentCategoryPageId(newPos.second /* categoryPageId */); } @Override @@ -254,7 +505,11 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange @Override public void onKeyClick(final Key key) { - mEmojiKeyboardAdapter.addRecentKey(key); + // TODO: Save emoticons to recents + if (mEmojiCategory.getCurrentCategoryId() != CATEGORY_ID_EMOTICONS) { + mEmojiKeyboardAdapter.addRecentKey(key); + } + mEmojiCategory.saveLastTypedCategoryPage(); final int code = key.getCode(); if (code == Constants.CODE_OUTPUT_TEXT) { mKeyboardActionListener.onTextInput(key.getOutputText()); @@ -271,43 +526,46 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange mKeyboardActionListener = listener; } - private void setCurrentCategory(final int category, final boolean force) { - if (mCurrentCategory == category && !force) { + private void setCurrentCategoryId(final int categoryId, final boolean force) { + if (mEmojiCategory.getCurrentCategoryId() == categoryId && !force) { return; } - mCurrentCategory = category; - if (force || mEmojiPager.getCurrentItem() != category) { - mEmojiPager.setCurrentItem(category, true /* smoothScroll */); + mEmojiCategory.setCurrentCategoryId(categoryId); + final int newTabId = mEmojiCategory.getTabIdFromCategoryId(categoryId); + final int newCategoryPageId = mEmojiCategory.getPageIdFromCategoryId(categoryId); + if (force || mEmojiCategory.getCategoryIdAndPageIdFromPagePosition( + mEmojiPager.getCurrentItem()).first != categoryId) { + mEmojiPager.setCurrentItem(newCategoryPageId, false /* smoothScroll */); } - if (force || mTabHost.getCurrentTab() != category) { - mTabHost.setCurrentTab(category); + if (force || mTabHost.getCurrentTab() != newTabId) { + mTabHost.setCurrentTab(newTabId); } - // TODO: Record current category } private static class EmojiKeyboardAdapter extends PagerAdapter { private final ScrollKeyboardView.OnKeyClickListener mListener; - private final KeyboardLayoutSet mLayoutSet; - private final RecentsKeyboard mRecentsKeyboard; + private final DynamicGridKeyboard mRecentsKeyboard; private final SparseArray<ScrollKeyboardView> mActiveKeyboardView = CollectionUtils.newSparseArray(); - private int mActivePosition = CATEGORY_UNSPECIFIED; + private final EmojiCategory mEmojiCategory; + private int mActivePosition = 0; - public EmojiKeyboardAdapter(final KeyboardLayoutSet layoutSet, + public EmojiKeyboardAdapter(final EmojiCategory emojiCategory, + final KeyboardLayoutSet layoutSet, final ScrollKeyboardView.OnKeyClickListener listener) { + mEmojiCategory = emojiCategory; mListener = listener; - mLayoutSet = layoutSet; - mRecentsKeyboard = new RecentsKeyboard( - layoutSet.getKeyboard(KeyboardId.ELEMENT_EMOJI_RECENTS)); + mRecentsKeyboard = mEmojiCategory.getKeyboard(CATEGORY_ID_RECENTS, 0); } public void addRecentKey(final Key key) { - if (mActivePosition == CATEGORY_RECENTS) { + if (mEmojiCategory.isInRecentTab()) { return; } - mRecentsKeyboard.addRecentKey(key); - final KeyboardView recentKeyboardView = mActiveKeyboardView.get(CATEGORY_RECENTS); + mRecentsKeyboard.addKeyFirst(key); + final KeyboardView recentKeyboardView = + mActiveKeyboardView.get(mEmojiCategory.getRecentTabId()); if (recentKeyboardView != null) { recentKeyboardView.invalidateAllKeys(); } @@ -315,7 +573,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange @Override public int getCount() { - return sCategoryName.length; + return mEmojiCategory.getTotalPageCountOfAllCategories(); } @Override @@ -333,9 +591,8 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange @Override public Object instantiateItem(final ViewGroup container, final int position) { - final int elementId = sCategoryElementId[position]; - final Keyboard keyboard = (elementId == KeyboardId.ELEMENT_EMOJI_RECENTS) - ? mRecentsKeyboard : mLayoutSet.getKeyboard(elementId); + final Keyboard keyboard = + mEmojiCategory.getKeyboardFromPagePosition(position); final LayoutInflater inflater = LayoutInflater.from(container.getContext()); final View view = inflater.inflate( R.layout.emoji_keyboard_page, container, false /* attachToRoot */); diff --git a/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java new file mode 100644 index 000000000..5570d594d --- /dev/null +++ b/java/src/com/android/inputmethod/keyboard/EmojiLayoutParams.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.keyboard; + +import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.utils.ResourceUtils; + +import android.content.res.Resources; +import android.support.v4.view.ViewPager; +import android.widget.ImageView; +import android.widget.LinearLayout; + +public class EmojiLayoutParams { + private static final int DEFAULT_KEYBOARD_ROWS = 4; + + public final int mEmojiPagerHeight; + private final int mEmojiPagerBottomMargin; + public final int mEmojiKeyboardHeight; + public final int mEmojiActionBarHeight; + public final int mKeyVerticalGap; + private final int mKeyHorizontalGap; + private final int mBottomPadding; + private final int mTopPadding; + + public EmojiLayoutParams(Resources res) { + final int defaultKeyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res); + final int defaultKeyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res); + mKeyVerticalGap = (int) res.getFraction(R.fraction.key_bottom_gap_ics, + (int) defaultKeyboardHeight, (int) defaultKeyboardHeight); + mBottomPadding = (int) res.getFraction(R.fraction.keyboard_bottom_padding_ics, + (int) defaultKeyboardHeight, (int) defaultKeyboardHeight); + mTopPadding = (int) res.getFraction(R.fraction.keyboard_top_padding_ics, + (int) defaultKeyboardHeight, (int) defaultKeyboardHeight); + mKeyHorizontalGap = (int) (res.getFraction(R.fraction.key_horizontal_gap_ics, + defaultKeyboardWidth, defaultKeyboardWidth)); + final int baseheight = defaultKeyboardHeight - mBottomPadding - mTopPadding + + mKeyVerticalGap; + mEmojiActionBarHeight = ((int) baseheight) / DEFAULT_KEYBOARD_ROWS + - (mKeyVerticalGap - mBottomPadding) / 2; + mEmojiPagerHeight = defaultKeyboardHeight - mEmojiActionBarHeight; + mEmojiPagerBottomMargin = mKeyVerticalGap / 2; + mEmojiKeyboardHeight = mEmojiPagerHeight - mEmojiPagerBottomMargin - 1; + } + + public void setPagerProps(ViewPager vp) { + final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) vp.getLayoutParams(); + lp.height = mEmojiPagerHeight - mEmojiPagerBottomMargin; + lp.bottomMargin = mEmojiPagerBottomMargin; + vp.setLayoutParams(lp); + } + + public void setActionBarProps(LinearLayout ll) { + final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) ll.getLayoutParams(); + lp.height = mEmojiActionBarHeight; + lp.topMargin = 0; + lp.bottomMargin = mBottomPadding; + ll.setLayoutParams(lp); + } + + public void setKeyProps(ImageView ib) { + final LinearLayout.LayoutParams lp = (LinearLayout.LayoutParams) ib.getLayoutParams(); + lp.leftMargin = mKeyHorizontalGap / 2; + lp.rightMargin = mKeyHorizontalGap / 2; + ib.setLayoutParams(lp); + } +} diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index d128d3595..cc1ffd183 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -314,15 +314,19 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { mState.onCodeInput(code, mLatinIME.getCurrentAutoCapsState()); } + public boolean isShowingEmojiKeyboard() { + return mEmojiKeyboardView.getVisibility() == View.VISIBLE; + } + public boolean isShowingMoreKeysPanel() { - if (mEmojiKeyboardView.getVisibility() == View.VISIBLE) { + if (isShowingEmojiKeyboard()) { return false; } return mKeyboardView.isShowingMoreKeysPanel(); } public View getVisibleKeyboardView() { - if (mEmojiKeyboardView.getVisibility() == View.VISIBLE) { + if (isShowingEmojiKeyboard()) { return mEmojiKeyboardView; } return mKeyboardView; diff --git a/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java index 629c60460..f203eb7d7 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/RecentsKeyboard.java +++ b/java/src/com/android/inputmethod/keyboard/internal/DynamicGridKeyboard.java @@ -16,33 +16,41 @@ package com.android.inputmethod.keyboard.internal; +import android.content.SharedPreferences; import android.text.TextUtils; +import com.android.inputmethod.keyboard.EmojiKeyboardView; import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import java.util.ArrayDeque; -import java.util.Random; +import java.util.Collection; /** - * This is a Keyboard class to host recently used keys. + * This is a Keyboard class where you can add keys dynamically shown in a grid layout */ -// TODO: Save/restore recent keys from/to preferences. -public class RecentsKeyboard extends Keyboard { +public class DynamicGridKeyboard extends Keyboard { private static final int TEMPLATE_KEY_CODE_0 = 0x30; private static final int TEMPLATE_KEY_CODE_1 = 0x31; + // Recent codes are saved as an integer array, so we use comma as a separater. + private static final String RECENT_KEY_SEPARATOR = Constants.STRING_COMMA; + private final SharedPreferences mPrefs; private final int mLeftPadding; private final int mHorizontalStep; private final int mVerticalStep; private final int mColumnsNum; - private final int mMaxRecentKeyCount; - private final ArrayDeque<RecentKey> mRecentKeys = CollectionUtils.newArrayDeque(); + private final int mMaxKeyCount; + private final boolean mIsRecents; + private final ArrayDeque<GridKey> mGridKeys = CollectionUtils.newArrayDeque(); - private Key[] mCachedRecentKeys; + private Key[] mCachedGridKeys; - public RecentsKeyboard(final Keyboard templateKeyboard) { + public DynamicGridKeyboard(final SharedPreferences prefs, final Keyboard templateKeyboard, + final int maxKeyCount, final int categoryId, final int categoryPageId) { super(templateKeyboard); final Key key0 = getTemplateKey(TEMPLATE_KEY_CODE_0); final Key key1 = getTemplateKey(TEMPLATE_KEY_CODE_1); @@ -50,8 +58,9 @@ public class RecentsKeyboard extends Keyboard { mHorizontalStep = Math.abs(key1.getX() - key0.getX()); mVerticalStep = key0.getHeight() + mVerticalGap; mColumnsNum = mBaseWidth / mHorizontalStep; - final int rowsNum = mBaseHeight / mVerticalStep; - mMaxRecentKeyCount = mColumnsNum * rowsNum; + mMaxKeyCount = maxKeyCount; + mIsRecents = categoryId == EmojiKeyboardView.CATEGORY_ID_RECENTS; + mPrefs = prefs; } private Key getTemplateKey(final int code) { @@ -63,32 +72,67 @@ public class RecentsKeyboard extends Keyboard { throw new RuntimeException("Can't find template key: code=" + code); } - private final Random random = new Random(); + public void addKeyFirst(final Key usedKey) { + addKey(usedKey, true); + if (mIsRecents) { + saveRecentKeys(); + } + } + + public void addKeyLast(final Key usedKey) { + addKey(usedKey, false); + } - public void addRecentKey(final Key usedKey) { - synchronized (mRecentKeys) { - mCachedRecentKeys = null; - final RecentKey key = (usedKey instanceof RecentKey) - ? (RecentKey)usedKey : new RecentKey(usedKey); - while (mRecentKeys.remove(key)) { + private void addKey(final Key usedKey, final boolean addFirst) { + synchronized (mGridKeys) { + mCachedGridKeys = null; + final GridKey key = new GridKey(usedKey); + while (mGridKeys.remove(key)) { // Remove duplicate keys. } - mRecentKeys.addFirst(key); - while (mRecentKeys.size() > mMaxRecentKeyCount) { - mRecentKeys.removeLast(); + if (addFirst) { + mGridKeys.addFirst(key); + } else { + mGridKeys.addLast(key); + } + while (mGridKeys.size() > mMaxKeyCount) { + mGridKeys.removeLast(); } int index = 0; - for (final RecentKey recentKey : mRecentKeys) { + for (final GridKey gridKey : mGridKeys) { final int keyX = getKeyX(index); final int keyY = getKeyY(index); - final int x = keyX+random.nextInt(recentKey.getWidth()); - final int y = keyY+random.nextInt(recentKey.getHeight()); - recentKey.updateCorrdinates(keyX, keyY); + gridKey.updateCorrdinates(keyX, keyY); index++; } } } + private void saveRecentKeys() { + final StringBuilder sb = new StringBuilder(); + for (final Key key : mGridKeys) { + sb.append(key.getCode()).append(RECENT_KEY_SEPARATOR); + } + Settings.writeEmojiRecentKeys(mPrefs, sb.toString()); + } + + public void loadRecentKeys(Collection<DynamicGridKeyboard> keyboards) { + final String str = Settings.readEmojiRecentKeys(mPrefs); + for (String s : str.split(RECENT_KEY_SEPARATOR)) { + if (TextUtils.isEmpty(s)) { + continue; + } + final int code = Integer.valueOf(s); + for (DynamicGridKeyboard kbd : keyboards) { + final Key key = kbd.getKey(code); + if (key != null) { + addKeyLast(key); + break; + } + } + } + } + private int getKeyX(final int index) { final int column = index % mColumnsNum; return column * mHorizontalStep + mLeftPadding; @@ -101,26 +145,26 @@ public class RecentsKeyboard extends Keyboard { @Override public Key[] getKeys() { - synchronized (mRecentKeys) { - if (mCachedRecentKeys != null) { - return mCachedRecentKeys; + synchronized (mGridKeys) { + if (mCachedGridKeys != null) { + return mCachedGridKeys; } - mCachedRecentKeys = mRecentKeys.toArray(new Key[mRecentKeys.size()]); - return mCachedRecentKeys; + mCachedGridKeys = mGridKeys.toArray(new Key[mGridKeys.size()]); + return mCachedGridKeys; } } @Override public Key[] getNearestKeys(final int x, final int y) { - // TODO: Calculate the nearest key index in mRecentKeys from x and y. + // TODO: Calculate the nearest key index in mGridKeys from x and y. return getKeys(); } - static final class RecentKey extends Key { + static final class GridKey extends Key { private int mCurrentX; private int mCurrentY; - public RecentKey(final Key originalKey) { + public GridKey(final Key originalKey) { super(originalKey); } @@ -151,7 +195,7 @@ public class RecentsKeyboard extends Keyboard { @Override public String toString() { - return "RecentKey: " + super.toString(); + return "GridKey: " + super.toString(); } } } diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index 7008b0619..a72595f7c 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -383,7 +383,7 @@ public final class KeyboardTextsSet { // Label for "switch to more symbol" modifier key. Must be short to fit on key! /* 124 */ "= \\ <", // Label for "switch to more symbol" modifier key on tablets. Must be short to fit on key! - /* 125 */ "~ [ {", + /* 125 */ "~ [ <", // Label for "Tab" key. Must be short to fit on key! /* 126 */ "Tab", // Label for "switch to phone numeric" key. Must be short to fit on key! @@ -2056,6 +2056,25 @@ public final class KeyboardTextsSet { /* 45 */ "\u0410\u0411\u0412", }; + /* Language lo: Lao */ + private static final String[] LANGUAGE_lo = { + /* 0~ */ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + /* ~44 */ + // Label for "switch to alphabetic" key. + // U+0E81: "ກ" LAO LETTER KO + // U+0E82: "ຂ" LAO LETTER KHO SUNG + // U+0E84: "ຄ" LAO LETTER KHO TAM + /* 45 */ "\u0E81\u0E82\u0E84", + /* 46~ */ + null, null, null, null, null, + /* ~50 */ + // U+20AD: "₭" KIP SIGN + /* 51 */ "\u20AD", + }; + /* Language lt: Lithuanian */ private static final String[] LANGUAGE_lt = { // U+0105: "ą" LATIN SMALL LETTER A WITH OGONEK @@ -2347,6 +2366,63 @@ public final class KeyboardTextsSet { /* 47 */ "!text/double_9qm_rqm", }; + /* Language ne: Nepali */ + private static final String[] LANGUAGE_ne = { + /* 0~ */ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + /* ~44 */ + // Label for "switch to alphabetic" key. + // U+0915: "क" DEVANAGARI LETTER KA + // U+0916: "ख" DEVANAGARI LETTER KHA + // U+0917: "ग" DEVANAGARI LETTER GA + /* 45 */ "\u0915\u0916\u0917", + /* 46~ */ + null, null, null, null, null, + /* ~50 */ + // U+0930/U+0941/U+002E "रु." NEPALESE RUPEE SIGN + /* 51 */ "\u0930\u0941.", + /* 52~ */ + null, null, null, null, null, null, null, null, null, null, null, + /* ~62 */ + // U+0967: "१" DEVANAGARI DIGIT ONE + /* 63 */ "\u0967", + // U+0968: "२" DEVANAGARI DIGIT TWO + /* 64 */ "\u0968", + // U+0969: "३" DEVANAGARI DIGIT THREE + /* 65 */ "\u0969", + // U+096A: "४" DEVANAGARI DIGIT FOUR + /* 66 */ "\u096A", + // U+096B: "५" DEVANAGARI DIGIT FIVE + /* 67 */ "\u096B", + // U+096C: "६" DEVANAGARI DIGIT SIX + /* 68 */ "\u096C", + // U+096D: "७" DEVANAGARI DIGIT SEVEN + /* 69 */ "\u096D", + // U+096E: "८" DEVANAGARI DIGIT EIGHT + /* 70 */ "\u096E", + // U+096F: "९" DEVANAGARI DIGIT NINE + /* 71 */ "\u096F", + // U+0966: "०" DEVANAGARI DIGIT ZERO + /* 72 */ "\u0966", + // Label for "switch to symbols" key. + /* 73 */ "?\u0967\u0968\u0969", + // Label for "switch to symbols with microphone" key. This string shouldn't include the "mic" + // part because it'll be appended by the code. + /* 74 */ "\u0967\u0968\u0969", + /* 75 */ "1", + /* 76 */ "2", + /* 77 */ "3", + /* 78 */ "4", + /* 79 */ "5", + /* 80 */ "6", + /* 81 */ "7", + /* 82 */ "8", + /* 83 */ "9", + /* 84 */ "0", + }; + /* Language nl: Dutch */ private static final String[] LANGUAGE_nl = { // U+00E1: "á" LATIN SMALL LETTER A WITH ACUTE @@ -3332,11 +3408,13 @@ public final class KeyboardTextsSet { "ka", LANGUAGE_ka, /* Georgian */ "kk", LANGUAGE_kk, /* Kazakh */ "ky", LANGUAGE_ky, /* Kirghiz */ + "lo", LANGUAGE_lo, /* Lao */ "lt", LANGUAGE_lt, /* Lithuanian */ "lv", LANGUAGE_lv, /* Latvian */ "mk", LANGUAGE_mk, /* Macedonian */ "mn", LANGUAGE_mn, /* Mongolian */ "nb", LANGUAGE_nb, /* Norwegian Bokmål */ + "ne", LANGUAGE_ne, /* Nepali */ "nl", LANGUAGE_nl, /* Dutch */ "pl", LANGUAGE_pl, /* Polish */ "pt", LANGUAGE_pt, /* Portuguese */ diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index e8b06570f..b49cd80ab 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin; import android.text.TextUtils; import android.util.SparseArray; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.settings.NativeSuggestOptions; @@ -40,14 +41,20 @@ public final class BinaryDictionary extends Dictionary { private static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; // Must be equal to MAX_RESULTS in native/jni/src/defines.h private static final int MAX_RESULTS = 18; + // Required space count for auto commit. + // TODO: Remove this heuristic. + private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3; private long mNativeDict; private final Locale mLocale; + private final long mDictSize; + private final String mDictFilePath; private final int[] mInputCodePoints = new int[MAX_WORD_LENGTH]; private final int[] mOutputCodePoints = new int[MAX_WORD_LENGTH * MAX_RESULTS]; private final int[] mSpaceIndices = new int[MAX_RESULTS]; private final int[] mOutputScores = new int[MAX_RESULTS]; private final int[] mOutputTypes = new int[MAX_RESULTS]; + private final int[] mOutputAutoCommitFirstWordConfidence = new int[MAX_RESULTS]; private final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions(); @@ -62,7 +69,7 @@ public final class BinaryDictionary extends Dictionary { if (traverseSession == null) { traverseSession = mDicTraverseSessions.get(traverseSessionId); if (traverseSession == null) { - traverseSession = new DicTraverseSession(mLocale, mNativeDict); + traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); mDicTraverseSessions.put(traverseSessionId, traverseSession); } } @@ -85,6 +92,8 @@ public final class BinaryDictionary extends Dictionary { final boolean isUpdatable) { super(dictType); mLocale = locale; + mDictSize = length; + mDictFilePath = filename; mNativeSuggestOptions.setUseFullEditDistance(useFullEditDistance); loadDictionary(filename, offset, length, isUpdatable); } @@ -95,6 +104,9 @@ public final class BinaryDictionary extends Dictionary { private static native long openNative(String sourceDir, long dictOffset, long dictSize, boolean isUpdatable); + private static native void flushNative(long dict, String filePath); + private static native boolean needsToRunGCNative(long dict); + private static native void flushWithGCNative(long dict, String filePath); private static native void closeNative(long dict); private static native int getProbabilityNative(long dict, int[] word); private static native boolean isValidBigramNative(long dict, int[] word0, int[] word1); @@ -102,7 +114,8 @@ public final class BinaryDictionary extends Dictionary { long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint, int[] suggestOptions, int[] prevWordCodePointArray, - int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes); + int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes, + int[] outputAutoCommitFirstWordConfidence); private static native float calcNormalizedScoreNative(int[] before, int[] after, int score); private static native int editDistanceNative(int[] before, int[] after); private static native void addUnigramWordNative(long dict, int[] word, int probability); @@ -155,7 +168,7 @@ public final class BinaryDictionary extends Dictionary { ips.getYCoordinates(), ips.getTimes(), ips.getPointerIds(), mInputCodePoints, inputSize, 0 /* commitPoint */, mNativeSuggestOptions.getOptions(), prevWordCodePointArray, mOutputCodePoints, mOutputScores, mSpaceIndices, - mOutputTypes); + mOutputTypes, mOutputAutoCommitFirstWordConfidence); final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); for (int j = 0; j < count; ++j) { final int start = j * MAX_WORD_LENGTH; @@ -179,7 +192,8 @@ public final class BinaryDictionary extends Dictionary { // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), score, kind, this /* sourceDict */, - mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */)); + mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, + mOutputAutoCommitFirstWordConfidence[0])); } } return suggestions; @@ -253,6 +267,40 @@ public final class BinaryDictionary extends Dictionary { removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); } + @UsedForTesting + public void flush() { + if (!isValidDictionary()) return; + flushNative(mNativeDict, mDictFilePath); + } + + @UsedForTesting + public void flushWithGC() { + if (!isValidDictionary()) return; + flushWithGCNative(mNativeDict, mDictFilePath); + } + + @UsedForTesting + public boolean needsToRunGC() { + if (!isValidDictionary()) return false; + return needsToRunGCNative(mNativeDict); + } + + @Override + public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { + // TODO: actually use the confidence rather than use this completely broken heuristic + final String word = candidate.mWord; + final int length = word.length(); + int remainingSpaces = SPACE_COUNT_FOR_AUTO_COMMIT; + for (int i = 0; i < length; ++i) { + // This is okay because no low-surrogate and no high-surrogate can ever match the + // space character, so we don't need to take care of iterating on code points. + if (Constants.CODE_SPACE == word.charAt(i)) { + if (0 >= --remainingSpaces) return true; + } + } + return false; + } + @Override public void close() { synchronized (mDicTraverseSessions) { diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java index 2b6d983c0..566184244 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionaryGetter.java @@ -21,9 +21,10 @@ import android.content.SharedPreferences; import android.content.res.AssetFileDescriptor; import android.util.Log; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.DictionaryInfoUtils; import com.android.inputmethod.latin.utils.LocaleUtils; @@ -228,7 +229,7 @@ final public class BinaryDictionaryGetter { private static boolean hackCanUseDictionaryFile(final Locale locale, final File f) { try { // Read the version of the file - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(f); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(f); final FileHeader header = dictDecoder.readHeader(); final String version = header.mDictionaryOptions.mAttributes.get(VERSION_KEY); diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index 8aec03f71..029ba02ed 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -220,7 +220,11 @@ public final class Constants { } } + public static final int MAX_INT_BIT_COUNT = 32; + public static final String STRING_COMMA = ","; + private Constants() { // This utility class is not publicly instantiable. } + } diff --git a/java/src/com/android/inputmethod/latin/DicTraverseSession.java b/java/src/com/android/inputmethod/latin/DicTraverseSession.java index 45b281318..8d295adee 100644 --- a/java/src/com/android/inputmethod/latin/DicTraverseSession.java +++ b/java/src/com/android/inputmethod/latin/DicTraverseSession.java @@ -25,16 +25,16 @@ public final class DicTraverseSession { JniUtils.loadNativeLibrary(); } - private static native long setDicTraverseSessionNative(String locale); + private static native long setDicTraverseSessionNative(String locale, long dictSize); private static native void initDicTraverseSessionNative(long nativeDicTraverseSession, long dictionary, int[] previousWord, int previousWordLength); private static native void releaseDicTraverseSessionNative(long nativeDicTraverseSession); private long mNativeDicTraverseSession; - public DicTraverseSession(Locale locale, long dictionary) { + public DicTraverseSession(Locale locale, long dictionary, long dictSize) { mNativeDicTraverseSession = createNativeDicTraverseSession( - locale != null ? locale.toString() : ""); + locale != null ? locale.toString() : "", dictSize); initSession(dictionary); } @@ -51,8 +51,8 @@ public final class DicTraverseSession { mNativeDicTraverseSession, dictionary, previousWord, previousWordLength); } - private final long createNativeDicTraverseSession(String locale) { - return setDicTraverseSessionNative(locale); + private final long createNativeDicTraverseSession(String locale, long dictSize) { + return setDicTraverseSessionNative(locale, dictSize); } private void closeInternal() { diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 8a3a88438..fa79f5af7 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -137,7 +137,10 @@ public abstract class Dictionary { } /** - * Whether we think this suggestion should trigger an auto-commit. + * Whether we think this suggestion should trigger an auto-commit. prevWord is the word + * before the suggestion, so that we can use n-gram frequencies. + * @param candidate The candidate suggestion, in whole (not only the first part). + * @return whether we should auto-commit or not. */ public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { // If we don't have support for auto-commit, or if we don't know, we return false to diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index c884e7b1f..2a9076436 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -617,4 +617,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } + + @UsedForTesting + public void shutdownExecutorForTests() { + getExecutor(mFilename).shutdown(); + } + + @UsedForTesting + public boolean isTerminatedForTests() { + return getExecutor(mFilename).isTerminated(); + } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index ba7d1a2b0..d491f988a 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -344,7 +344,8 @@ public class ExpandableDictionary extends Dictionary { // in the future. suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; } if (null != node.mShortcutTargets) { @@ -353,7 +354,8 @@ public class ExpandableDictionary extends Dictionary { final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; } } @@ -604,7 +606,8 @@ public class ExpandableDictionary extends Dictionary { suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, Constants.DICTIONARY_MAX_WORD_LENGTH - index), freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } } } diff --git a/java/src/com/android/inputmethod/latin/InputPointers.java b/java/src/com/android/inputmethod/latin/InputPointers.java index e96a46e12..2e638aaf3 100644 --- a/java/src/com/android/inputmethod/latin/InputPointers.java +++ b/java/src/com/android/inputmethod/latin/InputPointers.java @@ -105,6 +105,17 @@ public final class InputPointers { mTimes.append(times, startPos, length); } + /** + * Shift to the left by elementCount, discarding elementCount pointers at the start. + * @param elementCount how many elements to shift. + */ + public void shift(final int elementCount) { + mXCoordinates.shift(elementCount); + mYCoordinates.shift(elementCount); + mPointerIds.shift(elementCount); + mTimes.shift(elementCount); + } + public void reset() { final int defaultCapacity = mDefaultCapacity; mXCoordinates.reset(defaultCapacity); diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 6c83ac7ed..d3a18d410 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -95,7 +95,6 @@ import com.android.inputmethod.latin.utils.InputTypeUtils; import com.android.inputmethod.latin.utils.IntentUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LatinImeLoggerUtils; -import com.android.inputmethod.latin.utils.PositionalInfoForUserDictPendingAddition; import com.android.inputmethod.latin.utils.RecapitalizeStatus; import com.android.inputmethod.latin.utils.StaticInnerHandlerWrapper; import com.android.inputmethod.latin.utils.TargetPackageInfoGetterTask; @@ -185,8 +184,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean mIsUserDictionaryAvailable; private LastComposedWord mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD; - private PositionalInfoForUserDictPendingAddition - mPositionalInfoForUserDictPendingAddition = null; private final WordComposer mWordComposer = new WordComposer(); private final RichInputConnection mConnection = new RichInputConnection(this); private final RecapitalizeStatus mRecapitalizeStatus = new RecapitalizeStatus(); @@ -898,19 +895,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen currentSettingsValues.mGestureTrailEnabled, currentSettingsValues.mGestureFloatingPreviewTextEnabled); - // If we have a user dictionary addition in progress, we should check now if we should - // replace the previously committed string with the word that has actually been added - // to the user dictionary. - if (null != mPositionalInfoForUserDictPendingAddition - && mPositionalInfoForUserDictPendingAddition.tryReplaceWithActualWord( - mConnection, editorInfo, mLastSelectionEnd, currentLocale)) { - mPositionalInfoForUserDictPendingAddition = null; - } - // If tryReplaceWithActualWord returns false, we don't know what word was - // added to the user dictionary yet, so we keep the data and defer processing. The word will - // be replaced when the user dictionary reports back with the actual word, which ends - // up calling #onWordAddedToUserDictionary() in this class. - initPersonalizationDebugSettings(currentSettingsValues); if (TRACE) Debug.startMethodTracing("/data/trace/latinime"); @@ -1250,7 +1234,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen int visibleTopY = extraHeight; // Need to set touchable region only if input view is being shown if (visibleKeyboardView.isShown()) { - if (mSuggestionStripView.getVisibility() == View.VISIBLE) { + // Note that the height of Emoji layout is the same as the height of the main keyboard + // and the suggestion strip + if (mKeyboardSwitcher.isShowingEmojiKeyboard() + || mSuggestionStripView.getVisibility() == View.VISIBLE) { visibleTopY -= suggestionsHeight; } final int touchY = mKeyboardSwitcher.isShowingMoreKeysPanel() ? 0 : visibleTopY; @@ -1416,7 +1403,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen public void addWordToUserDictionary(final String word) { if (TextUtils.isEmpty(word)) { // Probably never supposed to happen, but just in case. - mPositionalInfoForUserDictPendingAddition = null; return; } final String wordToEdit; @@ -1428,22 +1414,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mUserDictionary.addWordToUserDictionary(wordToEdit); } - public void onWordAddedToUserDictionary(final String newSpelling) { - // If word was added but not by us, bail out - if (null == mPositionalInfoForUserDictPendingAddition) return; - if (mWordComposer.isComposingWord()) { - // We are late... give up and return - mPositionalInfoForUserDictPendingAddition = null; - return; - } - mPositionalInfoForUserDictPendingAddition.setActualWordBeingAdded(newSpelling); - if (mPositionalInfoForUserDictPendingAddition.tryReplaceWithActualWord( - mConnection, getCurrentInputEditorInfo(), mLastSelectionEnd, - mSubtypeSwitcher.getCurrentSubtypeLocale())) { - mPositionalInfoForUserDictPendingAddition = null; - } - } - private void onSettingsKeyPressed() { if (isShowingOptionDialog()) return; showSubtypeSelectorAndSettings(); @@ -1877,10 +1847,18 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void onUpdateBatchInput(final InputPointers batchPointers) { - final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate(); - if (null != candidate) { - if (candidate.mSourceDict.shouldAutoCommit(candidate)) { - // TODO: implement auto-commit + if (mSettings.getCurrent().mPhraseGestureEnabled) { + final SuggestedWordInfo candidate = mSuggestedWords.getAutoCommitCandidate(); + if (null != candidate) { + if (candidate.mSourceDict.shouldAutoCommit(candidate)) { + final String[] commitParts = candidate.mWord.split(" ", 2); + batchPointers.shift(candidate.mIndexOfTouchPointOfSecondWord); + promotePhantomSpace(); + mConnection.commitText(commitParts[0], 0); + mSpaceState = SPACE_STATE_PHANTOM; + mKeyboardSwitcher.updateShiftState(); + mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); + } } } mInputUpdater.onUpdateBatchInput(batchPointers); @@ -1893,12 +1871,24 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (TextUtils.isEmpty(batchInputText)) { return; } - mWordComposer.setBatchInputWord(batchInputText); mConnection.beginBatchEdit(); if (SPACE_STATE_PHANTOM == mSpaceState) { promotePhantomSpace(); } - mConnection.setComposingText(batchInputText, 1); + if (mSettings.getCurrent().mPhraseGestureEnabled) { + // Find the last space + final int indexOfLastSpace = batchInputText.lastIndexOf(Constants.CODE_SPACE) + 1; + if (0 != indexOfLastSpace) { + mConnection.commitText(batchInputText.substring(0, indexOfLastSpace), 1); + showSuggestionStrip(suggestedWords.getSuggestedWordsForLastWordOfPhraseGesture()); + } + final String lastWord = batchInputText.substring(indexOfLastSpace); + mWordComposer.setBatchInputWord(lastWord); + mConnection.setComposingText(lastWord, 1); + } else { + mWordComposer.setBatchInputWord(batchInputText); + mConnection.setComposingText(batchInputText, 1); + } mExpectingUpdateSelection = true; mConnection.endBatchEdit(); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { @@ -2704,6 +2694,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // recorrection. This is a temporary, stopgap measure that will be removed later. // TODO: remove this. if (mAppWorkAroundsUtils.isBrokenByRecorrection()) return; + // A simple way to test for support from the TextView. + if (!isSuggestionsStripVisible()) return; // Recorrection is not supported in languages without spaces because we don't know // how to segment them yet. if (!mSettings.getCurrent().mCurrentLanguageHasSpaces) return; @@ -2730,7 +2722,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen suggestions.add(new SuggestedWordInfo(s, SuggestionStripView.MAX_SUGGESTIONS - i, SuggestedWordInfo.KIND_RESUMED, Dictionary.DICTIONARY_RESUMED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE + /* autoCommitFirstWordConfidence */)); } } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 18ba15872..7815f4d41 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -48,8 +48,9 @@ public final class Suggest { // Session id for // {@link #getSuggestedWords(WordComposer,String,ProximityInfo,boolean,int)}. + // We are sharing the same ID between typing and gesture to save RAM footprint. public static final int SESSION_TYPING = 0; - public static final int SESSION_GESTURE = 1; + public static final int SESSION_GESTURE = 0; // TODO: rename this to CORRECTION_OFF public static final int CORRECTION_NONE = 0; @@ -326,7 +327,8 @@ public final class Suggest { suggestionsContainer.add(0, new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } SuggestedWordInfo.removeDups(suggestionsContainer); @@ -473,7 +475,8 @@ public final class Suggest { sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE); } return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind, - wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord); + wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); } public void close() { diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index b27fd81e9..fed4cdbbb 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -114,7 +114,8 @@ public final class SuggestedWords { final SuggestedWordInfo suggestedWordInfo = new SuggestedWordInfo(text.toString(), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_APP_DEFINED, Dictionary.DICTIONARY_APPLICATION_DEFINED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); result.add(suggestedWordInfo); } return result; @@ -128,7 +129,8 @@ public final class SuggestedWords { final HashSet<String> alreadySeen = CollectionUtils.newHashSet(); suggestionsList.add(new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); alreadySeen.add(typedWord.toString()); final int previousSize = previousSuggestions.size(); for (int index = 1; index < previousSize; index++) { @@ -151,6 +153,7 @@ public final class SuggestedWords { public static final class SuggestedWordInfo { public static final int NOT_AN_INDEX = -1; + public static final int NOT_A_CONFIDENCE = -1; public static final int MAX_SCORE = Integer.MAX_VALUE; public static final int KIND_MASK_KIND = 0xFF; // Mask to get only the kind public static final int KIND_TYPED = 0; // What user typed @@ -180,16 +183,30 @@ public final class SuggestedWords { // passed to native code to get suggestions for a gesture that corresponds to the first // letter of the second word. public final int mIndexOfTouchPointOfSecondWord; + // For auto-commit. This is a measure of how confident we are that we can commit the + // first word of this suggestion. + public final int mAutoCommitFirstWordConfidence; private String mDebugString = ""; + /** + * Create a new suggested word info. + * @param word The string to suggest. + * @param score A measure of how likely this suggestion is. + * @param kind The kind of suggestion, as one of the above KIND_* constants. + * @param sourceDict What instance of Dictionary produced this suggestion. + * @param indexOfTouchPointOfSecondWord See mIndexOfTouchPointOfSecondWord. + * @param autoCommitFirstWordConfidence See mAutoCommitFirstWordConfidence. + */ public SuggestedWordInfo(final String word, final int score, final int kind, - final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord) { + final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord, + final int autoCommitFirstWordConfidence) { mWord = word; mScore = score; mKind = kind; mSourceDict = sourceDict; mCodePointCount = StringUtils.codePointCount(mWord); mIndexOfTouchPointOfSecondWord = indexOfTouchPointOfSecondWord; + mAutoCommitFirstWordConfidence = autoCommitFirstWordConfidence; } public boolean isEligibleForAutoCommit() { @@ -259,4 +276,24 @@ public final class SuggestedWords { false /* willAutoCorrect */, mIsPunctuationSuggestions, mIsObsoleteSuggestions, mIsPrediction); } + + // Creates a new SuggestedWordInfo from the currently suggested words that removes all but the + // last word of all suggestions, separated by a space. This is necessary because when we commit + // a multiple-word suggestion, the IME only retains the last word as the composing word, and + // we should only suggest replacements for this last word. + // TODO: make this work with languages without spaces. + public SuggestedWords getSuggestedWordsForLastWordOfPhraseGesture() { + final ArrayList<SuggestedWordInfo> newSuggestions = CollectionUtils.newArrayList(); + for (int i = 0; i < mSuggestedWordInfoList.size(); ++i) { + final SuggestedWordInfo info = mSuggestedWordInfoList.get(i); + final int indexOfLastSpace = info.mWord.lastIndexOf(Constants.CODE_SPACE) + 1; + final String lastWord = info.mWord.substring(indexOfLastSpace); + newSuggestions.add(new SuggestedWordInfo(lastWord, info.mScore, info.mKind, + info.mSourceDict, SuggestedWordInfo.NOT_AN_INDEX, + SuggestedWordInfo.NOT_A_CONFIDENCE)); + } + return new SuggestedWords(newSuggestions, mTypedWordValid, + mWillAutoCorrect, mIsPunctuationSuggestions, mIsObsoleteSuggestions, + mIsPrediction); + } } diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index b2bb61596..a241b5505 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -103,14 +103,6 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { @Override public void onChange(final boolean self, final Uri uri) { setRequiresReload(true); - // We want to report back to Latin IME in case the user just entered the word. - // If the user changed the word in the dialog box, then we want to replace - // what was entered in the text field. - if (null == uri || !(context instanceof LatinIME)) return; - final long changedRowId = ContentUris.parseId(uri); - if (-1 == changedRowId) return; // Unknown content... Not sure why we're here - final String changedWord = getChangedWordForUri(uri); - ((LatinIME)context).onWordAddedToUserDictionary(changedWord); } }; cres.registerContentObserver(Words.CONTENT_URI, true, mObserver); @@ -118,19 +110,6 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { loadDictionary(); } - private String getChangedWordForUri(final Uri uri) { - final Cursor cursor = mContext.getContentResolver().query(uri, - PROJECTION_QUERY, null, null, null); - if (cursor == null) return null; - try { - if (!cursor.moveToFirst()) return null; - final int indexWord = cursor.getColumnIndex(Words.WORD); - return cursor.getString(indexWord); - } finally { - cursor.close(); - } - } - @Override public synchronized void close() { if (mObserver != null) { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index ceb8fa81f..5b319ad90 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -342,13 +342,11 @@ public final class BinaryDictDecoderUtils { * @param formatOptions file format options. * @return the word with its frequency, as a weighted string. */ - /* package for tests */ static WeightedString getWordAtPosition( - final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, - final FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + /* package for tests */ static WeightedString getWordAtPosition(final DictDecoder dictDecoder, + final int headerSize, final int pos, final FormatOptions formatOptions) { final WeightedString result; - final int originalPos = dictBuffer.position(); - dictBuffer.position(pos); + final int originalPos = dictDecoder.getPosition(); + dictDecoder.setPosition(pos); if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions); @@ -357,14 +355,13 @@ public final class BinaryDictDecoderUtils { formatOptions); } - dictBuffer.position(originalPos); + dictDecoder.setPosition(originalPos); return result; } @SuppressWarnings("unused") - private static WeightedString getWordAtPositionWithParentAddress( - final Ver3DictDecoder dictDecoder, final int pos, final FormatOptions options) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + private static WeightedString getWordAtPositionWithParentAddress(final DictDecoder dictDecoder, + final int pos, final FormatOptions options) { int currentPos = pos; int frequency = Integer.MIN_VALUE; final StringBuilder builder = new StringBuilder(); @@ -373,7 +370,7 @@ public final class BinaryDictDecoderUtils { PtNodeInfo currentInfo; int loopCounter = 0; do { - dictBuffer.position(currentPos); + dictDecoder.setPosition(currentPos); currentInfo = dictDecoder.readPtNode(currentPos, options); if (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)) { currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; @@ -392,11 +389,10 @@ public final class BinaryDictDecoderUtils { } private static WeightedString getWordAtPositionWithoutParentAddress( - final Ver3DictDecoder dictDecoder, final int headerSize, final int pos, + final DictDecoder dictDecoder, final int headerSize, final int pos, final FormatOptions options) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - dictBuffer.position(headerSize); - final int count = readPtNodeCount(dictBuffer); + dictDecoder.setPosition(headerSize); + final int count = dictDecoder.readPtNodeCount(); int groupPos = headerSize + BinaryDictIOUtils.getPtNodeCountSize(count); final StringBuilder builder = new StringBuilder(); WeightedString result = null; @@ -414,8 +410,8 @@ public final class BinaryDictDecoderUtils { if (info.mChildrenAddress > pos) { if (null == last) continue; builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - dictBuffer.position(last.mChildrenAddress); - i = readPtNodeCount(dictBuffer); + dictDecoder.setPosition(last.mChildrenAddress); + i = dictDecoder.readPtNodeCount(); groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); last = null; continue; @@ -424,8 +420,8 @@ public final class BinaryDictDecoderUtils { } if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - dictBuffer.position(last.mChildrenAddress); - i = readPtNodeCount(dictBuffer); + dictDecoder.setPosition(last.mChildrenAddress); + i = dictDecoder.readPtNodeCount(); groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); last = null; continue; @@ -449,17 +445,16 @@ public final class BinaryDictDecoderUtils { * @param options file format options. * @return the read node array with all his children already read. */ - private static PtNodeArray readNodeArray(final Ver3DictDecoder dictDecoder, + private static PtNodeArray readNodeArray(final DictDecoder dictDecoder, final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, final Map<Integer, PtNode> reversePtNodeMap, final FormatOptions options) throws IOException { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); final ArrayList<PtNode> nodeArrayContents = new ArrayList<PtNode>(); - final int nodeArrayOriginPos = dictBuffer.position(); + final int nodeArrayOriginPos = dictDecoder.getPosition(); do { // Scan the linked-list node. - final int nodeArrayHeadPos = dictBuffer.position(); - final int count = readPtNodeCount(dictBuffer); + final int nodeArrayHeadPos = dictDecoder.getPosition(); + final int count = dictDecoder.readPtNodeCount(); int groupOffsetPos = nodeArrayHeadPos + BinaryDictIOUtils.getPtNodeCountSize(count); for (int i = count; i > 0; --i) { // Scan the array of PtNode. PtNodeInfo info = dictDecoder.readPtNode(groupOffsetPos, options); @@ -480,11 +475,11 @@ public final class BinaryDictDecoderUtils { if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); if (null == children) { - final int currentPosition = dictBuffer.position(); - dictBuffer.position(info.mChildrenAddress); + final int currentPosition = dictDecoder.getPosition(); + dictDecoder.setPosition(info.mChildrenAddress); children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, reversePtNodeMap, options); - dictBuffer.position(currentPosition); + dictDecoder.setPosition(currentPosition); } nodeArrayContents.add( new PtNode(info.mCharacters, shortcutTargets, bigrams, @@ -503,15 +498,10 @@ public final class BinaryDictDecoderUtils { // reach the end of the array. if (options.mSupportsDynamicUpdate) { - final int nextAddress = dictBuffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < dictBuffer.limit()) { - dictBuffer.position(nextAddress); - } else { - break; - } + final boolean hasValidForwardLink = dictDecoder.readForwardLinkAndAdvancePosition(); + if (!hasValidForwardLink) break; } - } while (options.mSupportsDynamicUpdate && - dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); + } while (options.mSupportsDynamicUpdate && dictDecoder.hasNextPtNodeArray()); final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 5a213415a..f333b0d86 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -126,8 +126,14 @@ public class BinaryDictEncoderUtils { */ private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { int size = getNodeHeaderSize(ptNode, options); - // If terminal, one byte for the frequency - if (ptNode.isTerminal()) size += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (ptNode.isTerminal()) { + // If terminal, one byte for the frequency or four bytes for the terminal id. + if (options.mHasTerminalId) { + size += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + size += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address size += getShortcutListSize(ptNode.mShortcutTargets); if (null != ptNode.mBigrams) { @@ -198,6 +204,27 @@ public class BinaryDictEncoderUtils { } } + static int writeUIntToBuffer(final byte[] buffer, int position, final int value, + final int size) { + switch(size) { + case 4: + buffer[position++] = (byte) ((value >> 24) & 0xFF); + /* fall through */ + case 3: + buffer[position++] = (byte) ((value >> 16) & 0xFF); + /* fall through */ + case 2: + buffer[position++] = (byte) ((value >> 8) & 0xFF); + /* fall through */ + case 1: + buffer[position++] = (byte) (value & 0xFF); + break; + default: + /* nop */ + } + return position; + } + // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time @@ -324,7 +351,13 @@ public class BinaryDictEncoderUtils { changed = true; } int nodeSize = getNodeHeaderSize(ptNode, formatOptions); - if (ptNode.isTerminal()) nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (ptNode.isTerminal()) { + if (formatOptions.mHasTerminalId) { + nodeSize += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } if (formatOptions.mSupportsDynamicUpdate) { nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; } else if (null != ptNode.mChildren) { @@ -733,7 +766,7 @@ public class BinaryDictEncoderUtils { } /** - * Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached. + * Write a PtNodeArray. The PtNodeArray is expected to have its final position cached. * * @param dict the dictionary the node array is a part of (for relative offsets). * @param dictEncoder the dictionary encoder. @@ -741,7 +774,7 @@ public class BinaryDictEncoderUtils { * @param formatOptions file format options. */ @SuppressWarnings("unused") - /* package */ static void writePlacedNode(final FusionDictionary dict, + /* package */ static void writePlacedPtNodeArray(final FusionDictionary dict, final DictEncoder dictEncoder, final PtNodeArray ptNodeArray, final FormatOptions formatOptions) { // TODO: Make the code in common with BinaryDictIOUtils#writePtNode @@ -766,14 +799,7 @@ public class BinaryDictEncoderUtils { + FormatSpec.MAX_TERMINAL_FREQUENCY + " : " + ptNode.mFrequency); } - - dictEncoder.writePtNodeFlags(ptNode, parentPosition, formatOptions); - dictEncoder.writeParentPosition(parentPosition, ptNode, formatOptions); - dictEncoder.writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); - dictEncoder.writeFrequency(ptNode.mFrequency); - dictEncoder.writeChildrenPosition(ptNode, formatOptions); - dictEncoder.writeShortcuts(ptNode.mShortcutTargets); - dictEncoder.writeBigrams(ptNode.mBigrams, dict); + dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); } if (formatOptions.mSupportsDynamicUpdate) { dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 106f02519..2c5e93e5c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -61,12 +61,11 @@ public final class BinaryDictIOUtils { /** * Retrieves all node arrays without recursive call. */ - private static void readUnigramsAndBigramsBinaryInner( - final Ver3DictDecoder dictDecoder, final int headerSize, - final Map<Integer, String> words, final Map<Integer, Integer> frequencies, + private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder, + final int headerSize, final Map<Integer, String> words, + final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; Stack<Position> stack = new Stack<Position>(); @@ -83,11 +82,11 @@ public final class BinaryDictIOUtils { p.mNumOfPtNode + ", position=" + p.mPosition + ", length=" + p.mLength); } - if (dictBuffer.position() != p.mAddress) dictBuffer.position(p.mAddress); + if (dictDecoder.getPosition() != p.mAddress) dictDecoder.setPosition(p.mAddress); if (index != p.mLength) index = p.mLength; if (p.mNumOfPtNode == Position.NOT_READ_PTNODE_COUNT) { - p.mNumOfPtNode = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + p.mNumOfPtNode = dictDecoder.readPtNodeCount(); p.mAddress += getPtNodeCountSize(p.mNumOfPtNode); p.mPosition = 0; } @@ -114,11 +113,12 @@ public final class BinaryDictIOUtils { if (p.mPosition == p.mNumOfPtNode) { if (formatOptions.mSupportsDynamicUpdate) { - final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); - if (forwardLinkAddress != FormatSpec.NO_FORWARD_LINK_ADDRESS) { + final boolean hasValidForwardLinkAddress = + dictDecoder.readForwardLinkAndAdvancePosition(); + if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) { // The node array has a forward link. p.mNumOfPtNode = Position.NOT_READ_PTNODE_COUNT; - p.mAddress = forwardLinkAddress; + p.mAddress = dictDecoder.getPosition(); } else { stack.pop(); } @@ -127,7 +127,7 @@ public final class BinaryDictIOUtils { } } else { // The Ptnode array has more PtNodes. - p.mAddress = dictBuffer.position(); + p.mAddress = dictDecoder.getPosition(); } if (!isMovedPtNode && hasChildrenAddress(info.mChildrenAddress)) { @@ -148,7 +148,7 @@ public final class BinaryDictIOUtils { * @throws IOException if the file can't be read. * @throws UnsupportedFormatException if the format of the file is not recognized. */ - /* package */ static void readUnigramsAndBigramsBinary(final Ver3DictDecoder dictDecoder, + /* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { @@ -169,11 +169,10 @@ public final class BinaryDictIOUtils { * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - /* package */ static int getTerminalPosition(final Ver3DictDecoder dictDecoder, + /* package */ static int getTerminalPosition(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (word == null) return FormatSpec.NOT_VALID_WORD; - if (dictBuffer.position() != 0) dictBuffer.position(0); + dictDecoder.setPosition(0); final FileHeader header = dictDecoder.readHeader(); int wordPos = 0; @@ -182,10 +181,10 @@ public final class BinaryDictIOUtils { if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; do { - final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + final int ptNodeCount = dictDecoder.readPtNodeCount(); boolean foundNextPtNode = false; for (int i = 0; i < ptNodeCount; ++i) { - final int ptNodePos = dictBuffer.position(); + final int ptNodePos = dictDecoder.getPosition(); final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos, header.mFormatOptions); final boolean isMovedNode = isMovedPtNode(currentInfo.mFlags, @@ -219,7 +218,7 @@ public final class BinaryDictIOUtils { return FormatSpec.NOT_VALID_WORD; } foundNextPtNode = true; - dictBuffer.position(currentInfo.mChildrenAddress); + dictDecoder.setPosition(currentInfo.mChildrenAddress); break; } } @@ -233,11 +232,11 @@ public final class BinaryDictIOUtils { return FormatSpec.NOT_VALID_WORD; } - final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); - if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { + final boolean hasValidForwardLinkAddress = + dictDecoder.readForwardLinkAndAdvancePosition(); + if (!hasValidForwardLinkAddress || !dictDecoder.hasNextPtNodeArray()) { return FormatSpec.NOT_VALID_WORD; } - dictBuffer.position(forwardLinkAddress); } while(true); } return FormatSpec.NOT_VALID_WORD; @@ -520,7 +519,7 @@ public final class BinaryDictIOUtils { final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, new DictDecoder.DictionaryBufferFactory() { @Override public DictBuffer getDictionaryBuffer(File file) diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 11a3f0b3a..40e852423 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -54,10 +54,13 @@ public interface DictDecoder { * which words from the buffer should be added. If it is null, a new dictionary is created. * * @param dict an optional dictionary to add words to, or null. + * @param deleteDictIfBroken a flag indicating whether this method should remove the broken + * dictionary or not. * @return the created (or merged) dictionary. */ @UsedForTesting - public FusionDictionary readDictionaryBinary(final FusionDictionary dict) + public FusionDictionary readDictionaryBinary(final FusionDictionary dict, + final boolean deleteDictIfBroken) throws FileNotFoundException, IOException, UnsupportedFormatException; /** @@ -88,6 +91,41 @@ public interface DictDecoder { final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException; + /** + * Sets the position of the buffer to the given value. + * + * @param newPos the new position + */ + public void setPosition(final int newPos); + + /** + * Gets the position of the buffer. + * + * @return the position + */ + public int getPosition(); + + /** + * Reads and returns the PtNode count out of a buffer and forwards the pointer. + */ + public int readPtNodeCount(); + + /** + * Reads the forward link and advances the position. + * + * @return if this method advances the position then true else false. + */ + public boolean readForwardLinkAndAdvancePosition(); + public boolean hasNextPtNodeArray(); + + /** + * Opens the dictionary file and makes DictBuffer. + */ + @UsedForTesting + public void openDictBuffer() throws FileNotFoundException, IOException; + @UsedForTesting + public boolean isOpenedDictBuffer(); + // Flags for DictionaryBufferFactory. public static final int USE_READONLY_BYTEBUFFER = 0x01000000; public static final int USE_BYTEARRAY = 0x02000000; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java index d1589a30e..ea5d492d8 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -18,10 +18,8 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.IOException; -import java.util.ArrayList; /** * An interface of binary dictionary encoder. @@ -33,28 +31,8 @@ public interface DictEncoder { public void setPosition(final int position); public int getPosition(); public void writePtNodeCount(final int ptNodeCount); - public void writePtNodeFlags(final PtNode ptNode, final int parentAddress, - final FormatOptions formatOptions); - public void writeParentPosition(final int parentPosition, final PtNode ptNode, - final FormatOptions formatOptions); - public void writeCharacters(final int[] characters, final boolean hasSeveralChars); - public void writeFrequency(final int frequency); - public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions); - - /** - * Write a shortcut attributes list to memory. - * - * @param shortcuts the shortcut attributes list. - */ - public void writeShortcuts(final ArrayList<WeightedString> shortcuts); - - /** - * Write a bigram attributes list to memory. - * - * @param bigrams the bigram attributes list. - * @param dict the dictionary the node array is a part of (for relative offsets). - */ - public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict); - public void writeForwardLinkAddress(final int forwardLinkAddress); + + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index bf35f6a8a..96ccd8e49 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -18,8 +18,11 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import java.io.File; + /** * Dictionary File Format Specification. */ @@ -195,9 +198,12 @@ public final class FormatSpec { public static final int MAGIC_NUMBER = 0x9BC13AFE; static final int MINIMUM_SUPPORTED_VERSION = 2; - static final int MAXIMUM_SUPPORTED_VERSION = 3; + static final int MAXIMUM_SUPPORTED_VERSION = 4; static final int NOT_A_VERSION_NUMBER = -1; static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; + static final int FIRST_VERSION_WITH_TERMINAL_ID = 4; + static final int VERSION3 = 3; + static final int VERSION4 = 4; // These options need to be the same numeric values as the one in the native reading code. static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; @@ -248,11 +254,17 @@ public final class FormatSpec { static final int PTNODE_TERMINATOR_SIZE = 1; static final int PTNODE_FLAGS_SIZE = 1; static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_TERMINAL_ID_SIZE = 4; static final int PTNODE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; + // These values are used only by version 4 or later. + static final String TRIE_FILE_EXTENSION = ".trie"; + static final String FREQ_FILE_EXTENSION = ".freq"; + static final int FREQUENCY_AND_FLAGS_SIZE = 2; + static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; @@ -261,6 +273,7 @@ public final class FormatSpec { static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; + static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF; static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; @@ -284,6 +297,7 @@ public final class FormatSpec { public static final class FormatOptions { public final int mVersion; public final boolean mSupportsDynamicUpdate; + public final boolean mHasTerminalId; @UsedForTesting public FormatOptions(final int version) { this(version, false); @@ -297,6 +311,7 @@ public final class FormatSpec { + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); } mSupportsDynamicUpdate = supportsDynamicUpdate; + mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); } } @@ -341,6 +356,28 @@ public final class FormatSpec { } } + /** + * Returns new dictionary decoder. + * + * @param dictFile the dictionary file. + * @param bufferType the flag indicating buffer type which is used by the dictionary decoder. + * @return new dictionary decoder if the dictionary file exists, otherwise null. + */ + public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) { + if (!dictFile.isFile()) return null; + return new Ver3DictDecoder(dictFile, bufferType); + } + + public static DictDecoder getDictDecoder(final File dictFile, + final DictionaryBufferFactory factory) { + if (!dictFile.isFile()) return null; + return new Ver3DictDecoder(dictFile, factory); + } + + public static DictDecoder getDictDecoder(final File dictFile) { + return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER); + } + private FormatSpec() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 3e685a3d7..be653feec 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -111,6 +111,7 @@ public final class FusionDictionary implements Iterable<Word> { ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal. + int mTerminalId; // NOT_A_TERMINAL == mTerminalId indicates this is not a terminal. PtNodeArray mChildren; boolean mIsNotAWord; // Only a shortcut boolean mIsBlacklistEntry; @@ -129,6 +130,7 @@ public final class FusionDictionary implements Iterable<Word> { final boolean isNotAWord, final boolean isBlacklistEntry) { mChars = chars; mFrequency = frequency; + mTerminalId = frequency; mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = null; @@ -156,6 +158,10 @@ public final class FusionDictionary implements Iterable<Word> { mChildren.mData.add(n); } + public int getTerminalId() { + return mTerminalId; + } + public boolean isTerminal() { return NOT_A_TERMINAL != mFrequency; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java index 1a5023ef6..1a90a4b98 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -173,11 +173,7 @@ public class Ver3DictDecoder implements DictDecoder { private final DictionaryBufferFactory mBufferFactory; private DictBuffer mDictBuffer; - public Ver3DictDecoder(final File file) { - this(file, USE_READONLY_BYTEBUFFER); - } - - public Ver3DictDecoder(final File file, final int factoryFlag) { + /* package */ Ver3DictDecoder(final File file, final int factoryFlag) { mDictionaryBinaryFile = file; mDictBuffer = null; @@ -192,15 +188,21 @@ public class Ver3DictDecoder implements DictDecoder { } } - public Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { + /* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { mDictionaryBinaryFile = file; mBufferFactory = factory; } + @Override public void openDictBuffer() throws FileNotFoundException, IOException { mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); } + @Override + public boolean isOpenedDictBuffer() { + return mDictBuffer != null; + } + /* package */ DictBuffer getDictBuffer() { return mDictBuffer; } @@ -306,7 +308,8 @@ public class Ver3DictDecoder implements DictDecoder { } @Override - public FusionDictionary readDictionaryBinary(final FusionDictionary dict) + public FusionDictionary readDictionaryBinary(final FusionDictionary dict, + final boolean deleteDictIfBroken) throws FileNotFoundException, IOException, UnsupportedFormatException { if (mDictBuffer == null) { openDictBuffer(); @@ -315,13 +318,13 @@ public class Ver3DictDecoder implements DictDecoder { return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); } catch (IOException e) { Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); - if (!mDictionaryBinaryFile.delete()) { + if (deleteDictIfBroken && !mDictionaryBinaryFile.delete()) { Log.e(TAG, "Failed to delete the broken dictionary."); } throw e; } catch (UnsupportedFormatException e) { Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); - if (!mDictionaryBinaryFile.delete()) { + if (deleteDictIfBroken && !mDictionaryBinaryFile.delete()) { Log.e(TAG, "Failed to delete the broken dictionary."); } throw e; @@ -347,4 +350,33 @@ public class Ver3DictDecoder implements DictDecoder { BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); } + @Override + public void setPosition(int newPos) { + mDictBuffer.position(newPos); + } + + @Override + public int getPosition() { + return mDictBuffer.position(); + } + + @Override + public int readPtNodeCount() { + return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); + } + + @Override + public boolean readForwardLinkAndAdvancePosition() { + final int nextAddress = mDictBuffer.readUnsignedInt24(); + if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { + mDictBuffer.position(nextAddress); + return true; + } + return false; + } + + @Override + public boolean hasNextPtNodeArray() { + return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 3f26ff378..222a0f474 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -68,7 +68,7 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { - if (formatOptions.mVersion > 3) { + if (formatOptions.mVersion > FormatSpec.VERSION3) { throw new UnsupportedFormatException( "The given format options has wrong version number : " + formatOptions.mVersion); @@ -103,7 +103,7 @@ public class Ver3DictEncoder implements DictEncoder { MakedictLog.i("Writing file..."); for (PtNodeArray nodeArray : flatNodes) { - BinaryDictEncoderUtils.writePlacedNode(dict, this, nodeArray, formatOptions); + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); } if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes); mOutStream.write(mBuffer, 0, mPosition); @@ -126,26 +126,23 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writePtNodeCount(final int ptNodeCount) { final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); - if (1 == countSize) { - mBuffer[mPosition++] = (byte) ptNodeCount; - } else if (2 == countSize) { - mBuffer[mPosition++] = (byte) ((ptNodeCount >> 8) & 0xFF); - mBuffer[mPosition++] = (byte) (ptNodeCount & 0xFF); - } else { + if (countSize != 1 && countSize != 2) { throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); } + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, ptNodeCount, + countSize); } - @Override - public void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); - mBuffer[mPosition++] = BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, - childrenPos, formatOptions); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, childrenPos, + formatOptions), + FormatSpec.PTNODE_FLAGS_SIZE); } - @Override - public void writeParentPosition(final int parentPosition, final PtNode ptNode, + private void writeParentPosition(final int parentPosition, final PtNode ptNode, final FormatOptions formatOptions) { if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) { mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, @@ -156,22 +153,20 @@ public class Ver3DictEncoder implements DictEncoder { } } - @Override - public void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) { + private void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) { mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition); if (hasSeveralChars) { mBuffer[mPosition++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; } } - @Override - public void writeFrequency(final int frequency) { + private void writeFrequency(final int frequency) { if (frequency >= 0) { - mBuffer[mPosition++] = (byte) frequency; + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, frequency, + FormatSpec.PTNODE_FREQUENCY_SIZE); } } - @Override public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); if (formatOptions.mSupportsDynamicUpdate) { @@ -183,8 +178,12 @@ public class Ver3DictEncoder implements DictEncoder { } } - @Override - public void writeShortcuts(final ArrayList<WeightedString> shortcuts) { + /** + * Write a shortcut attributes list to mBuffer. + * + * @param shortcuts the shortcut attributes list. + */ + private void writeShortcuts(final ArrayList<WeightedString> shortcuts) { if (null == shortcuts || shortcuts.isEmpty()) return; final int indexOfShortcutByteSize = mPosition; @@ -195,20 +194,27 @@ public class Ver3DictEncoder implements DictEncoder { final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency); - mBuffer[mPosition++] = (byte)shortcutFlags; + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord); mPosition += shortcutShift; } final int shortcutByteSize = mPosition - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { throw new RuntimeException("Shortcut list too large"); } - mBuffer[indexOfShortcutByteSize] = (byte)((shortcutByteSize >> 8) & 0xFF); - mBuffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); - } - - @Override - public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict) { + BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, + FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + /** + * Write a bigram attributes list to mBuffer. + * + * @param bigrams the bigram attributes list. + * @param dict the dictionary the node array is a part of (for relative offsets). + */ + private void writeBigrams(final ArrayList<WeightedString> bigrams, + final FusionDictionary dict) { if (bigrams == null) return; final Iterator<WeightedString> bigramIterator = bigrams.iterator(); @@ -220,9 +226,10 @@ public class Ver3DictEncoder implements DictEncoder { final int unigramFrequencyForThisWord = target.mFrequency; final int offset = addressOfBigram - (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); - mBuffer[mPosition++] = (byte) bigramFlags; + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, Math.abs(offset)); } @@ -230,8 +237,19 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writeForwardLinkAddress(final int forwardLinkAddress) { - mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 16) & 0xFF); - mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 8) & 0xFF); - mBuffer[mPosition++] = (byte) (forwardLinkAddress & 0xFF); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, forwardLinkAddress, + FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict) { + writePtNodeFlags(ptNode, parentPosition, formatOptions); + writeParentPosition(parentPosition, ptNode, formatOptions); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + writeFrequency(ptNode.mFrequency); + writeChildrenPosition(ptNode, formatOptions); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java new file mode 100644 index 000000000..75b75ae2e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -0,0 +1,269 @@ +/* +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * An implementation of DictEncoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictEncoder implements DictEncoder { + private final File mDictPlacedDir; + private byte[] mTrieBuf; + private byte[] mFreqBuf; + private int mTriePos; + private OutputStream mTrieOutStream; + private OutputStream mFreqOutStream; + + @UsedForTesting + public Ver4DictEncoder(final File dictPlacedDir) { + mDictPlacedDir = dictPlacedDir; + } + + private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions) + throws FileNotFoundException, IOException { + final FileHeader header = new FileHeader(0, dictOptions, formatOptions); + final String filename = header.getId() + "." + header.getVersion(); + final File mDictDir = new File(mDictPlacedDir, filename); + final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION); + final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION); + if (!mDictDir.isDirectory()) { + if (mDictDir.exists()) mDictDir.delete(); + mDictDir.mkdirs(); + } + if (!trieFile.exists()) trieFile.createNewFile(); + if (!freqFile.exists()) freqFile.createNewFile(); + mTrieOutStream = new FileOutputStream(trieFile); + mFreqOutStream = new FileOutputStream(freqFile); + } + + private void close() throws IOException { + try { + if (mTrieOutStream != null) { + mTrieOutStream.close(); + } + if (mFreqOutStream != null) { + mFreqOutStream.close(); + } + } finally { + mTrieOutStream = null; + mFreqOutStream = null; + } + } + + @Override + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion != FormatSpec.VERSION4) { + throw new UnsupportedFormatException("File header has a wrong version number : " + + formatOptions.mVersion); + } + if (!mDictPlacedDir.isDirectory()) { + throw new UnsupportedFormatException("Given path is not a directory."); + } + + if (mTrieOutStream == null) { + openStreams(formatOptions, dict.mOptions); + } + + BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, formatOptions); + + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + int terminalCount = 0; + for (final PtNodeArray array : flatNodes) { + for (final PtNode node : array.mData) { + if (node.isTerminal()) node.mTerminalId = terminalCount++; + } + } + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mTrieBuf = new byte[bufferSize]; + mFreqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE]; + + MakedictLog.i("Writing file..."); + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); + } + if (MakedictLog.DBG) { + BinaryDictEncoderUtils.showStatistics(flatNodes); + MakedictLog.i("has " + terminalCount + " terminals."); + } + mTrieOutStream.write(mTrieBuf); + mFreqOutStream.write(mFreqBuf); + + MakedictLog.i("Done"); + close(); + } + + @Override + public void setPosition(int position) { + if (mTrieBuf == null || position < 0 || position >- mTrieBuf.length) return; + mTriePos = position; + } + + @Override + public int getPosition() { + return mTriePos; + } + + @Override + public void writePtNodeCount(int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + // ptNodeCount must fit on one byte or two bytes. + // Please see comments in FormatSpec + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); + } + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, ptNodeCount, + countSize); + } + + private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mTriePos, childrenPos, + formatOptions), + FormatSpec.PTNODE_FLAGS_SIZE); + } + + private void writeParentPosition(int parentPos, final PtNode ptNode, + final FormatOptions formatOptions) { + if (parentPos != FormatSpec.NO_PARENT_ADDRESS) { + parentPos -= ptNode.mCachedAddressAfterUpdate; + } + mTriePos = BinaryDictEncoderUtils.writeParentAddress(mTrieBuf, mTriePos, parentPos, + formatOptions); + } + + private void writeCharacters(final int[] characters, final boolean hasSeveralChars) { + mTriePos = CharEncoding.writeCharArray(characters, mTrieBuf, mTriePos); + if (hasSeveralChars) { + mTrieBuf[mTriePos++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + private void writeTerminalId(final int terminalId) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, terminalId, + FormatSpec.PTNODE_TERMINAL_ID_SIZE); + } + + private void writeFrequency(final int frequency, final int terminalId) { + final int freqPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE; + BinaryDictEncoderUtils.writeUIntToBuffer(mFreqBuf, freqPos, frequency, + FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + } + + private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + if (formatOptions.mSupportsDynamicUpdate) { + mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } else { + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } + } + + private void writeShortcuts(ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mTriePos; + mTriePos += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.mFrequency); + mTrieBuf[mTriePos++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos, + target.mWord); + mTriePos += shortcutShift; + } + final int shortcutByteSize = mTriePos - indexOfShortcutByteSize; + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { + throw new RuntimeException("Shortcut list too large : " + shortcutByteSize); + } + BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, indexOfShortcutByteSize, + shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + final int offset = addressOfBigram + - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + mTrieBuf[mTriePos++] = (byte) bigramFlags; + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(int forwardLinkAddress) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + forwardLinkAddress, FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict) { + writePtNodeFlags(ptNode, parentPosition, formatOptions); + writeParentPosition(parentPosition, ptNode, formatOptions); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + if (ptNode.isTerminal()) { + writeTerminalId(ptNode.mTerminalId); + writeFrequency(ptNode.mFrequency, ptNode.mTerminalId); + } + writeChildrenPosition(ptNode, formatOptions); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 5b1d0647b..9364fb034 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -25,14 +25,13 @@ import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.makedict.DictDecoder; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils; import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -173,16 +172,19 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi // Load the dictionary from binary file final File dictFile = new File(mContext.getFilesDir(), mFileName); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(dictFile, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile, DictDecoder.USE_BYTEARRAY); + if (dictDecoder == null) { + // This is an expected condition: we don't have a user history dictionary for this + // language yet. It will be created sometime later. + return; + } + try { dictDecoder.openDictBuffer(); UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener); - } catch (FileNotFoundException e) { - // This is an expected condition: we don't have a user history dictionary for this - // language yet. It will be created sometime later. } catch (IOException e) { - Log.e(TAG, "IOException on opening a bytebuffer", e); + Log.d(TAG, "IOException on opening a bytebuffer", e); } finally { if (PROFILE_SAVE_RESTORE) { final long diff = System.currentTimeMillis() - now; diff --git a/java/src/com/android/inputmethod/latin/settings/Settings.java b/java/src/com/android/inputmethod/latin/settings/Settings.java index 8732a59f8..1a0fecc62 100644 --- a/java/src/com/android/inputmethod/latin/settings/Settings.java +++ b/java/src/com/android/inputmethod/latin/settings/Settings.java @@ -81,6 +81,7 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang public static final String PREF_GESTURE_FLOATING_PREVIEW_TEXT = "pref_gesture_floating_preview_text"; public static final String PREF_SHOW_SETUP_WIZARD_ICON = "pref_show_setup_wizard_icon"; + public static final String PREF_PHRASE_GESTURE_ENABLED = "pref_gesture_space_aware"; public static final String PREF_INPUT_LANGUAGE = "input_language"; public static final String PREF_SELECTED_LANGUAGES = "selected_languages"; @@ -97,6 +98,10 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang public static final String PREF_SEND_FEEDBACK = "send_feedback"; public static final String PREF_ABOUT_KEYBOARD = "about_keyboard"; + // Emoji + public static final String PREF_EMOJI_RECENT_KEYS = "emoji_recent_keys"; + public static final String PREF_EMOJI_CATEGORY_LAST_TYPED_ID = "emoji_category_last_typed_id"; + private Resources mRes; private SharedPreferences mPrefs; private SettingsValues mSettingsValues; @@ -216,6 +221,12 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang && prefs.getBoolean(Settings.PREF_GESTURE_INPUT, true); } + public static boolean readPhraseGestureEnabled(final SharedPreferences prefs, + final Resources res) { + return prefs.getBoolean(Settings.PREF_PHRASE_GESTURE_ENABLED, + res.getBoolean(R.bool.config_default_phrase_gesture_enabled)); + } + public static boolean readFromBuildConfigIfToShowKeyPreviewPopupSettingsOption( final Resources res) { return res.getBoolean(R.bool.config_enable_show_option_of_key_preview_popup); @@ -363,4 +374,24 @@ public final class Settings implements SharedPreferences.OnSharedPreferenceChang final String tokenStr = mPrefs.getString(PREF_LAST_USED_PERSONALIZATION_TOKEN, null); return StringUtils.hexStringToByteArray(tokenStr); } + + public static void writeEmojiRecentKeys(final SharedPreferences prefs, String str) { + prefs.edit().putString(PREF_EMOJI_RECENT_KEYS, str).apply(); + } + + public static String readEmojiRecentKeys(final SharedPreferences prefs) { + return prefs.getString(PREF_EMOJI_RECENT_KEYS, ""); + } + + public static void writeEmojiCategoryLastTypedId( + final SharedPreferences prefs, final int category, final int id) { + final String key = PREF_EMOJI_CATEGORY_LAST_TYPED_ID + category; + prefs.edit().putInt(key, id).apply(); + } + + public static int readEmojiCategoryLastTypedId( + final SharedPreferences prefs, final int category) { + final String key = PREF_EMOJI_CATEGORY_LAST_TYPED_ID + category; + return prefs.getInt(key, 0); + } } diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 072bb8731..ee322e91b 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -76,6 +76,7 @@ public final class SettingsValues { public final boolean mGestureTrailEnabled; public final boolean mGestureFloatingPreviewTextEnabled; public final boolean mSlidingKeyInputPreviewEnabled; + public final boolean mPhraseGestureEnabled; public final int mKeyLongpressTimeout; public final Locale mLocale; @@ -159,6 +160,7 @@ public final class SettingsValues { mGestureTrailEnabled = prefs.getBoolean(Settings.PREF_GESTURE_PREVIEW_TRAIL, true); mGestureFloatingPreviewTextEnabled = prefs.getBoolean( Settings.PREF_GESTURE_FLOATING_PREVIEW_TEXT, true); + mPhraseGestureEnabled = Settings.readPhraseGestureEnabled(prefs, res); mCorrectionEnabled = mAutoCorrectEnabled && !mInputAttributes.mInputTypeNoAutoCorrect; final String showSuggestionsSetting = prefs.getString( Settings.PREF_SHOW_SUGGESTIONS_SETTING, @@ -211,6 +213,7 @@ public final class SettingsValues { mGestureInputEnabled = true; mGestureTrailEnabled = true; mGestureFloatingPreviewTextEnabled = true; + mPhraseGestureEnabled = true; mCorrectionEnabled = mAutoCorrectEnabled && !mInputAttributes.mInputTypeNoAutoCorrect; mSuggestionVisibility = 0; mIsInternal = false; @@ -295,7 +298,8 @@ public final class SettingsValues { puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, Dictionary.DICTIONARY_HARDCODED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } } return new SuggestedWords(puncList, diff --git a/java/src/com/android/inputmethod/latin/utils/PositionalInfoForUserDictPendingAddition.java b/java/src/com/android/inputmethod/latin/utils/PositionalInfoForUserDictPendingAddition.java deleted file mode 100644 index 1fc7eccc6..000000000 --- a/java/src/com/android/inputmethod/latin/utils/PositionalInfoForUserDictPendingAddition.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.utils; - -import android.view.inputmethod.EditorInfo; - -import com.android.inputmethod.latin.RichInputConnection; - -import java.util.Locale; - -/** - * Holder class for data about a word already committed but that may still be edited. - * - * When the user chooses to add a word to the user dictionary by pressing the appropriate - * suggestion, a dialog is presented to give a chance to edit the word before it is actually - * registered as a user dictionary word. If the word is actually modified, the IME needs to - * go back and replace the word that was committed with the amended version. - * The word we need to replace with will only be known after it's actually committed, so - * the IME needs to take a note of what it has to replace and where it is. - * This class encapsulates this data. - */ -public final class PositionalInfoForUserDictPendingAddition { - final private String mOriginalWord; - final private int mCursorPos; // Position of the cursor after the word - final private EditorInfo mEditorInfo; // On what binding this has been added - final private int mCapitalizedMode; - private String mActualWordBeingAdded; - - public PositionalInfoForUserDictPendingAddition(final String word, final int cursorPos, - final EditorInfo editorInfo, final int capitalizedMode) { - mOriginalWord = word; - mCursorPos = cursorPos; - mEditorInfo = editorInfo; - mCapitalizedMode = capitalizedMode; - } - - public void setActualWordBeingAdded(final String actualWordBeingAdded) { - mActualWordBeingAdded = actualWordBeingAdded; - } - - /** - * Try to replace the string at the remembered position with the actual word being added. - * - * After the user validated the word being added, the IME has to replace the old version - * (which has been committed in the text view) with the amended version if it's different. - * This method tries to do that, but may fail because the IME is not yet ready to do so - - * for example, it is still waiting for the new string, or it is waiting to return to the text - * view in which the amendment should be made. In these cases, we should keep the data - * and wait until all conditions are met. - * This method returns true if the replacement has been successfully made and this data - * can be forgotten; it returns false if the replacement can't be made yet and we need to - * keep this until a later time. - * The IME knows about the actual word being added through a callback called by the - * user dictionary facility of the device. When this callback comes, the keyboard may still - * be connected to the edition dialog, or it may have already returned to the original text - * field. Replacement has to work in both cases. - * Accordingly, this method is called at two different points in time : upon getting the - * event that a new word was added to the user dictionary, and upon starting up in a - * new text field. - * @param connection The RichInputConnection through which to contact the editor. - * @param editorInfo Information pertaining to the editor we are currently in. - * @param currentCursorPosition The current cursor position, for checking purposes. - * @param locale The locale for changing case, if necessary - * @return true if the edit has been successfully made, false if we need to try again later - */ - public boolean tryReplaceWithActualWord(final RichInputConnection connection, - final EditorInfo editorInfo, final int currentCursorPosition, final Locale locale) { - // If we still don't know the actual word being added, we need to try again later. - if (null == mActualWordBeingAdded) return false; - // The entered text and the registered text were the same anyway : we can - // return success right away even if focus has not returned yet to the text field we - // want to amend. - if (mActualWordBeingAdded.equals(mOriginalWord)) return true; - // Not the same text field : we need to try again later. This happens when the addition - // is reported by the user dictionary provider before the focus has moved back to the - // original text view, so the IME is still in the text view of the dialog and has no way to - // edit the original text view at this time. - if (!mEditorInfo.packageName.equals(editorInfo.packageName) - || mEditorInfo.fieldId != editorInfo.fieldId) { - return false; - } - // Same text field, but not the same cursor position : we give up, so we return success - // so that it won't be tried again - if (currentCursorPosition != mCursorPos) return true; - // We have made all the checks : do the replacement and report success - // If this was auto-capitalized, we need to restore the case before committing - final String wordWithCaseFixed = CapsModeUtils.applyAutoCapsMode(mActualWordBeingAdded, - mCapitalizedMode, locale); - connection.setComposingRegion(currentCursorPosition - mOriginalWord.length(), - currentCursorPosition); - connection.commitText(wordWithCaseFixed, wordWithCaseFixed.length()); - return true; - } -} diff --git a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java index 3c1db6529..5dc0b5893 100644 --- a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java +++ b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java @@ -31,6 +31,7 @@ public class PrioritizedSerialExecutor { private static final int TASK_QUEUE_CAPACITY = 1000; private final Queue<Runnable> mTasks; private final Queue<Runnable> mPrioritizedTasks; + private boolean mIsShutdown; // The task which is running now. private Runnable mActive; @@ -38,6 +39,7 @@ public class PrioritizedSerialExecutor { public PrioritizedSerialExecutor() { mTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); mPrioritizedTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); + mIsShutdown = false; } /** @@ -56,9 +58,11 @@ public class PrioritizedSerialExecutor { */ public void execute(final Runnable r) { synchronized(mLock) { - mTasks.offer(r); - if (mActive == null) { - scheduleNext(); + if (!mIsShutdown) { + mTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } } } } @@ -69,9 +73,11 @@ public class PrioritizedSerialExecutor { */ public void executePrioritized(final Runnable r) { synchronized(mLock) { - mPrioritizedTasks.offer(r); - if (mActive == null) { - scheduleNext(); + if (!mIsShutdown) { + mPrioritizedTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } } } } @@ -123,4 +129,19 @@ public class PrioritizedSerialExecutor { execute(newTask); } } + + public void shutdown() { + synchronized(mLock) { + mIsShutdown = true; + } + } + + public boolean isTerminated() { + synchronized(mLock) { + if (!mIsShutdown) { + return false; + } + return mPrioritizedTasks.isEmpty() && mTasks.isEmpty() && mActive == null; + } + } } diff --git a/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java b/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java index 4c7739a7a..7c6fe93ac 100644 --- a/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java +++ b/java/src/com/android/inputmethod/latin/utils/ResizableIntArray.java @@ -132,6 +132,15 @@ public final class ResizableIntArray { } } + /** + * Shift to the left by elementCount, discarding elementCount pointers at the start. + * @param elementCount how many elements to shift. + */ + public void shift(final int elementCount) { + System.arraycopy(mArray, elementCount, mArray, 0, mLength - elementCount); + mLength -= elementCount; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder(); diff --git a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java index 05f3061a8..ea32a74ff 100644 --- a/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java @@ -20,13 +20,13 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.PendingAttribute; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.personalization.UserHistoryDictionaryBigramList; import java.io.IOException; @@ -125,7 +125,7 @@ public final class UserHistoryDictIOUtils { /** * Reads dictionary from file. */ - public static void readDictionaryBinary(final Ver3DictDecoder dictDecoder, + public static void readDictionaryBinary(final DictDecoder dictDecoder, final OnAddWordListener dict) { final TreeMap<Integer, String> unigrams = CollectionUtils.newTreeMap(); final TreeMap<Integer, Integer> frequencies = CollectionUtils.newTreeMap(); diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 8da1859c4..a63fab6dc 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -46,8 +46,7 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s sourceDirChars[sourceDirUtf8Length] = '\0'; DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy = DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy( - sourceDirChars, static_cast<int>(sourceDirUtf8Length), - static_cast<int>(dictOffset), static_cast<int>(dictSize), + sourceDirChars, static_cast<int>(dictOffset), static_cast<int>(dictSize), isUpdatable == JNI_TRUE); if (!dictionaryStructureWithBufferPolicy) { return 0; @@ -59,6 +58,35 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return reinterpret_cast<jlong>(dictionary); } +static void latinime_BinaryDictionary_flush(JNIEnv *env, jclass clazz, jlong dict, + jstring filePath) { + Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); + if (!dictionary) return; + const jsize filePathUtf8Length = env->GetStringUTFLength(filePath); + char filePathChars[filePathUtf8Length + 1]; + env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars); + filePathChars[filePathUtf8Length] = '\0'; + dictionary->flush(filePathChars); +} + +static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz, + jlong dict) { + Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); + if (!dictionary) return false; + return dictionary->needsToRunGC(); +} + +static void latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict, + jstring filePath) { + Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); + if (!dictionary) return; + const jsize filePathUtf8Length = env->GetStringUTFLength(filePath); + char filePathChars[filePathUtf8Length + 1]; + env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars); + filePathChars[filePathUtf8Length] = '\0'; + dictionary->flushWithGC(filePathChars); +} + static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return; @@ -70,7 +98,8 @@ static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, j jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray, jintArray inputCodePointsArray, jint inputSize, jint commitPoint, jintArray suggestOptions, jintArray prevWordCodePointsForBigrams, jintArray outputCodePointsArray, - jintArray scoresArray, jintArray spaceIndicesArray, jintArray outputTypesArray) { + jintArray scoresArray, jintArray spaceIndicesArray, jintArray outputTypesArray, + jintArray outputAutoCommitFirstWordConfidence) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return 0; ProximityInfo *pInfo = reinterpret_cast<ProximityInfo *>(proximityInfo); @@ -252,8 +281,23 @@ static const JNINativeMethod sMethods[] = { reinterpret_cast<void *>(latinime_BinaryDictionary_close) }, { + const_cast<char *>("flushNative"), + const_cast<char *>("(JLjava/lang/String;)V"), + reinterpret_cast<void *>(latinime_BinaryDictionary_flush) + }, + { + const_cast<char *>("needsToRunGCNative"), + const_cast<char *>("(J)Z"), + reinterpret_cast<void *>(latinime_BinaryDictionary_needsToRunGC) + }, + { + const_cast<char *>("flushWithGCNative"), + const_cast<char *>("(JLjava/lang/String;)V"), + reinterpret_cast<void *>(latinime_BinaryDictionary_flushWithGC) + }, + { const_cast<char *>("getSuggestionsNative"), - const_cast<char *>("(JJJ[I[I[I[I[III[I[I[I[I[I[I)I"), + const_cast<char *>("(JJJ[I[I[I[I[III[I[I[I[I[I[I[I)I"), reinterpret_cast<void *>(latinime_BinaryDictionary_getSuggestions) }, { diff --git a/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp b/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp index 72e625836..386643332 100644 --- a/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp +++ b/native/jni/com_android_inputmethod_latin_DicTraverseSession.cpp @@ -25,8 +25,9 @@ namespace latinime { class Dictionary; -static jlong latinime_setDicTraverseSession(JNIEnv *env, jclass clazz, jstring localeJStr) { - void *traverseSession = DicTraverseSession::getSessionInstance(env, localeJStr); +static jlong latinime_setDicTraverseSession(JNIEnv *env, jclass clazz, jstring localeJStr, + jlong dictSize) { + void *traverseSession = DicTraverseSession::getSessionInstance(env, localeJStr, dictSize); return reinterpret_cast<jlong>(traverseSession); } @@ -53,7 +54,7 @@ static void latinime_releaseDicTraverseSession(JNIEnv *env, jclass clazz, jlong static const JNINativeMethod sMethods[] = { { const_cast<char *>("setDicTraverseSessionNative"), - const_cast<char *>("(Ljava/lang/String;)J"), + const_cast<char *>("(Ljava/lang/String;J)J"), reinterpret_cast<void *>(latinime_setDicTraverseSession) }, { diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h index 34a646f80..89dfa39b3 100644 --- a/native/jni/src/defines.h +++ b/native/jni/src/defines.h @@ -292,7 +292,6 @@ static inline void prof_out(void) { // of the binary dictionary where a {key,value} string pair scheme is used. #define LARGEST_INT_DIGIT_COUNT 11 -#define NOT_A_VALID_WORD_POS (-99) #define NOT_A_CODE_POINT (-1) #define NOT_A_DISTANCE (-1) #define NOT_A_COORDINATE (-1) @@ -322,13 +321,6 @@ static inline void prof_out(void) { #define MAX_POINTER_COUNT 1 #define MAX_POINTER_COUNT_G 2 -// Queue IDs and size for DicNodesCache -#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_ACTIVE 0 -#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_NEXT_ACTIVE 1 -#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_TERMINAL 2 -#define DIC_NODES_CACHE_INITIAL_QUEUE_ID_CACHE_FOR_CONTINUOUS_SUGGESTION 3 -#define DIC_NODES_CACHE_PRIORITY_QUEUES_SIZE 4 - template<typename T> AK_FORCE_INLINE const T &min(const T &a, const T &b) { return a < b ? a : b; } template<typename T> AK_FORCE_INLINE const T &max(const T &a, const T &b) { return a > b ? a : b; } diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h index cdd9f59aa..41ef9d2b2 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node.h +++ b/native/jni/src/suggest/core/dicnode/dic_node.h @@ -112,7 +112,7 @@ class DicNode { mIsUsed = true; mIsCachedForNextSuggestion = false; mDicNodeProperties.init( - NOT_A_VALID_WORD_POS /* pos */, rootGroupPos, NOT_A_CODE_POINT /* nodeCodePoint */, + NOT_A_DICT_POS /* pos */, rootGroupPos, NOT_A_CODE_POINT /* nodeCodePoint */, NOT_A_PROBABILITY /* probability */, false /* isTerminal */, true /* hasChildren */, false /* isBlacklistedOrNotAWord */, 0 /* depth */, 0 /* terminalDepth */); @@ -125,7 +125,7 @@ class DicNode { mIsUsed = true; mIsCachedForNextSuggestion = dicNode->mIsCachedForNextSuggestion; mDicNodeProperties.init( - NOT_A_VALID_WORD_POS /* pos */, rootGroupPos, NOT_A_CODE_POINT /* nodeCodePoint */, + NOT_A_DICT_POS /* pos */, rootGroupPos, NOT_A_CODE_POINT /* nodeCodePoint */, NOT_A_PROBABILITY /* probability */, false /* isTerminal */, true /* hasChildren */, false /* isBlacklistedOrNotAWord */, 0 /* depth */, 0 /* terminalDepth */); @@ -143,7 +143,7 @@ class DicNode { dicNode->mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength(), dicNode->getOutputWordBuf(), dicNode->mDicNodeProperties.getDepth(), - dicNode->mDicNodeState.mDicNodeStatePrevWord.mPrevSpacePositions, + dicNode->mDicNodeState.mDicNodeStatePrevWord.getSecondWordFirstInputIndex(), mDicNodeState.mDicNodeStateInput.getInputIndex(0) /* lastInputIndex */); PROF_NODE_COPY(&dicNode->mProfiler, mProfiler); } @@ -234,7 +234,7 @@ class DicNode { } bool isFirstWord() const { - return mDicNodeState.mDicNodeStatePrevWord.getPrevWordNodePos() == NOT_A_VALID_WORD_POS; + return mDicNodeState.mDicNodeStatePrevWord.getPrevWordNodePos() == NOT_A_DICT_POS; } bool isCompletion(const int inputSize) const { @@ -321,8 +321,13 @@ class DicNode { DUMP_WORD_AND_SCORE("OUTPUT"); } - void outputSpacePositionsResult(int *spaceIndices) const { - mDicNodeState.mDicNodeStatePrevWord.outputSpacePositions(spaceIndices); + int getSecondWordFirstInputIndex(const ProximityInfoState *const pInfoState) const { + const int inputIndex = mDicNodeState.mDicNodeStatePrevWord.getSecondWordFirstInputIndex(); + if (inputIndex == NOT_AN_INDEX) { + return NOT_AN_INDEX; + } else { + return pInfoState->getInputIndexOfSampledPoint(inputIndex); + } } bool hasMultipleWords() const { @@ -573,7 +578,11 @@ class DicNode { } } - AK_FORCE_INLINE void updateInputIndexG(DicNode_InputStateG *inputStateG) { + AK_FORCE_INLINE void updateInputIndexG(const DicNode_InputStateG *const inputStateG) { + if (mDicNodeState.mDicNodeStatePrevWord.getPrevWordCount() == 1 && isFirstLetter()) { + mDicNodeState.mDicNodeStatePrevWord.setSecondWordFirstInputIndex( + inputStateG->mInputIndex); + } mDicNodeState.mDicNodeStateInput.updateInputIndexG(inputStateG->mPointerId, inputStateG->mInputIndex, inputStateG->mPrevCodePoint, inputStateG->mTerminalDiffCost, inputStateG->mRawLength); diff --git a/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h b/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h index 2a486b804..7461f0cc6 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h +++ b/native/jni/src/suggest/core/dicnode/dic_node_priority_queue.h @@ -24,20 +24,16 @@ #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_release_listener.h" -// The biggest value among MAX_CACHE_DIC_NODE_SIZE, MAX_CACHE_DIC_NODE_SIZE_FOR_SINGLE_POINT, ... -#define MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY 310 - namespace latinime { class DicNodePriorityQueue : public DicNodeReleaseListener { public: - AK_FORCE_INLINE DicNodePriorityQueue() - : MAX_CAPACITY(MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY), - mMaxSize(MAX_DIC_NODE_PRIORITY_QUEUE_CAPACITY), mDicNodesBuf(), mUnusedNodeIndices(), - mNextUnusedNodeId(0), mDicNodesQueue() { - mDicNodesBuf.resize(MAX_CAPACITY + 1); - mUnusedNodeIndices.resize(MAX_CAPACITY + 1); - reset(); + AK_FORCE_INLINE explicit DicNodePriorityQueue(const int capacity) + : mCapacity(capacity), mMaxSize(capacity), mDicNodesBuf(), + mUnusedNodeIndices(), mNextUnusedNodeId(0), mDicNodesQueue() { + mDicNodesBuf.resize(mCapacity + 1); + mUnusedNodeIndices.resize(mCapacity + 1); + clearAndResizeToCapacity(); } // Non virtual inline destructor -- never inherit this class @@ -52,11 +48,12 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { } AK_FORCE_INLINE void setMaxSize(const int maxSize) { - mMaxSize = min(maxSize, MAX_CAPACITY); + ASSERT(maxSize <= mCapacity); + mMaxSize = min(maxSize, mCapacity); } - AK_FORCE_INLINE void reset() { - clearAndResize(MAX_CAPACITY); + AK_FORCE_INLINE void clearAndResizeToCapacity() { + clearAndResize(mCapacity); } AK_FORCE_INLINE void clear() { @@ -64,27 +61,19 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { } AK_FORCE_INLINE void clearAndResize(const int maxSize) { + ASSERT(maxSize <= mCapacity); while (!mDicNodesQueue.empty()) { mDicNodesQueue.pop(); } setMaxSize(maxSize); - for (int i = 0; i < MAX_CAPACITY + 1; ++i) { + for (int i = 0; i < mCapacity + 1; ++i) { mDicNodesBuf[i].remove(); mDicNodesBuf[i].setReleaseListener(this); - mUnusedNodeIndices[i] = i == MAX_CAPACITY ? NOT_A_NODE_ID : static_cast<int>(i) + 1; + mUnusedNodeIndices[i] = i == mCapacity ? NOT_A_NODE_ID : static_cast<int>(i) + 1; } mNextUnusedNodeId = 0; } - AK_FORCE_INLINE DicNode *newDicNode(DicNode *dicNode) { - DicNode *newNode = searchEmptyDicNode(); - if (newNode) { - DicNodeUtils::initByCopy(dicNode, newNode); - return newNode; - } - return 0; - } - // Copy AK_FORCE_INLINE DicNode *copyPush(DicNode *dicNode) { return copyPush(dicNode, mMaxSize); @@ -111,12 +100,12 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { } mUnusedNodeIndices[index] = mNextUnusedNodeId; mNextUnusedNodeId = index; - ASSERT(index >= 0 && index < (MAX_CAPACITY + 1)); + ASSERT(index >= 0 && index < (mCapacity + 1)); } AK_FORCE_INLINE void dump() const { AKLOGI("\n\n\n\n\n==========================="); - for (int i = 0; i < MAX_CAPACITY + 1; ++i) { + for (int i = 0; i < mCapacity + 1; ++i) { if (mDicNodesBuf[i].isUsed()) { mDicNodesBuf[i].dump("QUEUE: "); } @@ -125,7 +114,7 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { } private: - DISALLOW_COPY_AND_ASSIGN(DicNodePriorityQueue); + DISALLOW_IMPLICIT_CONSTRUCTORS(DicNodePriorityQueue); static const int NOT_A_NODE_ID = -1; AK_FORCE_INLINE static bool compareDicNode(DicNode *left, DicNode *right) { @@ -139,7 +128,7 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { }; typedef std::priority_queue<DicNode *, std::vector<DicNode *>, DicNodeComparator> DicNodesQueue; - const int MAX_CAPACITY; + const int mCapacity; int mMaxSize; std::vector<DicNode> mDicNodesBuf; // of each element of mDicNodesBuf respectively std::vector<int> mUnusedNodeIndices; @@ -163,13 +152,12 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { } AK_FORCE_INLINE DicNode *searchEmptyDicNode() { - // TODO: Currently O(n) but should be improved to O(1) - if (MAX_CAPACITY == 0) { + if (mCapacity == 0) { return 0; } if (mNextUnusedNodeId == NOT_A_NODE_ID) { AKLOGI("No unused node found."); - for (int i = 0; i < MAX_CAPACITY + 1; ++i) { + for (int i = 0; i < mCapacity + 1; ++i) { AKLOGI("Dump node availability, %d, %d, %d", i, mDicNodesBuf[i].isUsed(), mUnusedNodeIndices[i]); } @@ -185,7 +173,7 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { const int index = static_cast<int>(dicNode - &mDicNodesBuf[0]); mNextUnusedNodeId = mUnusedNodeIndices[index]; mUnusedNodeIndices[index] = NOT_A_NODE_ID; - ASSERT(index >= 0 && index < (MAX_CAPACITY + 1)); + ASSERT(index >= 0 && index < (mCapacity + 1)); } AK_FORCE_INLINE DicNode *pushPoolNodeWithMaxSize(DicNode *dicNode, const int maxSize) { @@ -209,6 +197,15 @@ class DicNodePriorityQueue : public DicNodeReleaseListener { AK_FORCE_INLINE DicNode *copyPush(DicNode *dicNode, const int maxSize) { return pushPoolNodeWithMaxSize(newDicNode(dicNode), maxSize); } + + AK_FORCE_INLINE DicNode *newDicNode(DicNode *dicNode) { + DicNode *newNode = searchEmptyDicNode(); + if (newNode) { + DicNodeUtils::initByCopy(dicNode, newNode); + } + return newNode; + } + }; } // namespace latinime #endif // LATINIME_DIC_NODE_PRIORITY_QUEUE_H diff --git a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp index e81591992..ec65114c7 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_node_utils.cpp @@ -89,7 +89,7 @@ namespace latinime { const int unigramProbability = node->getProbability(); const int wordPos = node->getPos(); const int prevWordPos = node->getPrevWordPos(); - if (NOT_A_VALID_WORD_POS == wordPos || NOT_A_VALID_WORD_POS == prevWordPos) { + if (NOT_A_DICT_POS == wordPos || NOT_A_DICT_POS == prevWordPos) { // Note: Normally wordPos comes from the dictionary and should never equal // NOT_A_VALID_WORD_POS. return dictionaryStructurePolicy->getProbability(unigramProbability, diff --git a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp index c3d2a2e74..b6be47e90 100644 --- a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp +++ b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.cpp @@ -23,6 +23,11 @@ namespace latinime { +// The biggest value among MAX_CACHE_DIC_NODE_SIZE, MAX_CACHE_DIC_NODE_SIZE_FOR_SINGLE_POINT, ... +const int DicNodesCache::LARGE_PRIORITY_QUEUE_CAPACITY = 310; +// Capacity for reducing memory footprint. +const int DicNodesCache::SMALL_PRIORITY_QUEUE_CAPACITY = 100; + /** * Truncates all of the dicNodes so that they start at the given commit point. * Only called for multi-word typing input. diff --git a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h index 7aab0906e..8493b6a8b 100644 --- a/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h +++ b/native/jni/src/suggest/core/dicnode/dic_nodes_cache.h @@ -31,25 +31,32 @@ class DicNode; */ class DicNodesCache { public: - AK_FORCE_INLINE DicNodesCache() - : mActiveDicNodes(&mDicNodePriorityQueues[DIC_NODES_CACHE_INITIAL_QUEUE_ID_ACTIVE]), - mNextActiveDicNodes(&mDicNodePriorityQueues[ - DIC_NODES_CACHE_INITIAL_QUEUE_ID_NEXT_ACTIVE]), - mTerminalDicNodes(&mDicNodePriorityQueues[DIC_NODES_CACHE_INITIAL_QUEUE_ID_TERMINAL]), - mCachedDicNodesForContinuousSuggestion(&mDicNodePriorityQueues[ - DIC_NODES_CACHE_INITIAL_QUEUE_ID_CACHE_FOR_CONTINUOUS_SUGGESTION]), - mInputIndex(0), mLastCachedInputIndex(0) { - } + AK_FORCE_INLINE explicit DicNodesCache(const bool usesLargeCapacityCache) + : mUsesLargeCapacityCache(usesLargeCapacityCache), + mDicNodePriorityQueue0(getCacheCapacity()), + mDicNodePriorityQueue1(getCacheCapacity()), + mDicNodePriorityQueue2(getCacheCapacity()), + mDicNodePriorityQueueForTerminal(MAX_RESULTS), + mActiveDicNodes(&mDicNodePriorityQueue0), + mNextActiveDicNodes(&mDicNodePriorityQueue1), + mCachedDicNodesForContinuousSuggestion(&mDicNodePriorityQueue2), + mTerminalDicNodes(&mDicNodePriorityQueueForTerminal), + mInputIndex(0), mLastCachedInputIndex(0) {} AK_FORCE_INLINE virtual ~DicNodesCache() {} AK_FORCE_INLINE void reset(const int nextActiveSize, const int terminalSize) { mInputIndex = 0; mLastCachedInputIndex = 0; - mActiveDicNodes->reset(); - mNextActiveDicNodes->clearAndResize(nextActiveSize); + // We want to use the max capacity for the current active dic node queue. + mActiveDicNodes->clearAndResizeToCapacity(); + // nextActiveSize is used to limit the next iteration's active dic node size. + const int nextActiveSizeFittingToTheCapacity = min(nextActiveSize, getCacheCapacity()); + mNextActiveDicNodes->clearAndResize(nextActiveSizeFittingToTheCapacity); mTerminalDicNodes->clearAndResize(terminalSize); - mCachedDicNodesForContinuousSuggestion->reset(); + // We want to use the max capacity for the cached dic nodes that will be used for the + // continuous suggestion. + mCachedDicNodesForContinuousSuggestion->clearAndResizeToCapacity(); } AK_FORCE_INLINE void continueSearch() { @@ -157,21 +164,35 @@ class DicNodesCache { return tmp; } + AK_FORCE_INLINE int getCacheCapacity() const { + return mUsesLargeCapacityCache ? + LARGE_PRIORITY_QUEUE_CAPACITY : SMALL_PRIORITY_QUEUE_CAPACITY; + } + AK_FORCE_INLINE void resetTemporaryCaches() { mActiveDicNodes->clear(); mNextActiveDicNodes->clear(); mTerminalDicNodes->clear(); } - DicNodePriorityQueue mDicNodePriorityQueues[DIC_NODES_CACHE_PRIORITY_QUEUES_SIZE]; + static const int LARGE_PRIORITY_QUEUE_CAPACITY; + static const int SMALL_PRIORITY_QUEUE_CAPACITY; + + const bool mUsesLargeCapacityCache; + // Instances + DicNodePriorityQueue mDicNodePriorityQueue0; + DicNodePriorityQueue mDicNodePriorityQueue1; + DicNodePriorityQueue mDicNodePriorityQueue2; + DicNodePriorityQueue mDicNodePriorityQueueForTerminal; + // Active dicNodes currently being expanded. DicNodePriorityQueue *mActiveDicNodes; // Next dicNodes to be expanded. DicNodePriorityQueue *mNextActiveDicNodes; - // Current top terminal dicNodes. - DicNodePriorityQueue *mTerminalDicNodes; // Cached dicNodes used for continuous suggestion. DicNodePriorityQueue *mCachedDicNodesForContinuousSuggestion; + // Current top terminal dicNodes. + DicNodePriorityQueue *mTerminalDicNodes; int mInputIndex; int mLastCachedInputIndex; }; diff --git a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h index 9bc96877e..b8986203d 100644 --- a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h +++ b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h @@ -22,6 +22,7 @@ #include "defines.h" #include "suggest/core/dicnode/dic_node_utils.h" +#include "suggest/core/layout/proximity_info_state.h" namespace latinime { @@ -29,9 +30,8 @@ class DicNodeStatePrevWord { public: AK_FORCE_INLINE DicNodeStatePrevWord() : mPrevWordCount(0), mPrevWordLength(0), mPrevWordStart(0), mPrevWordProbability(0), - mPrevWordNodePos(NOT_A_VALID_WORD_POS) { + mPrevWordNodePos(NOT_A_DICT_POS), mSecondWordFirstInputIndex(NOT_AN_INDEX) { memset(mPrevWord, 0, sizeof(mPrevWord)); - memset(mPrevSpacePositions, 0, sizeof(mPrevSpacePositions)); } virtual ~DicNodeStatePrevWord() {} @@ -41,8 +41,8 @@ class DicNodeStatePrevWord { mPrevWordCount = 0; mPrevWordStart = 0; mPrevWordProbability = -1; - mPrevWordNodePos = NOT_A_VALID_WORD_POS; - memset(mPrevSpacePositions, 0, sizeof(mPrevSpacePositions)); + mPrevWordNodePos = NOT_A_DICT_POS; + mSecondWordFirstInputIndex = NOT_AN_INDEX; } void init(const int prevWordNodePos) { @@ -51,7 +51,7 @@ class DicNodeStatePrevWord { mPrevWordStart = 0; mPrevWordProbability = -1; mPrevWordNodePos = prevWordNodePos; - memset(mPrevSpacePositions, 0, sizeof(mPrevSpacePositions)); + mSecondWordFirstInputIndex = NOT_AN_INDEX; } // Init by copy @@ -61,14 +61,14 @@ class DicNodeStatePrevWord { mPrevWordStart = prevWord->mPrevWordStart; mPrevWordProbability = prevWord->mPrevWordProbability; mPrevWordNodePos = prevWord->mPrevWordNodePos; + mSecondWordFirstInputIndex = prevWord->mSecondWordFirstInputIndex; memcpy(mPrevWord, prevWord->mPrevWord, prevWord->mPrevWordLength * sizeof(mPrevWord[0])); - memcpy(mPrevSpacePositions, prevWord->mPrevSpacePositions, sizeof(mPrevSpacePositions)); } void init(const int16_t prevWordCount, const int16_t prevWordProbability, const int prevWordNodePos, const int *const src0, const int16_t length0, - const int *const src1, const int16_t length1, const int *const prevSpacePositions, - const int lastInputIndex) { + const int *const src1, const int16_t length1, + const int prevWordSecondWordFirstInputIndex, const int lastInputIndex) { mPrevWordCount = min(prevWordCount, static_cast<int16_t>(MAX_RESULTS)); mPrevWordProbability = prevWordProbability; mPrevWordNodePos = prevWordNodePos; @@ -80,8 +80,7 @@ class DicNodeStatePrevWord { mPrevWord[twoWordsLen] = KEYCODE_SPACE; mPrevWordStart = length0; mPrevWordLength = static_cast<int16_t>(twoWordsLen + 1); - memcpy(mPrevSpacePositions, prevSpacePositions, sizeof(mPrevSpacePositions)); - mPrevSpacePositions[mPrevWordCount - 1] = lastInputIndex; + mSecondWordFirstInputIndex = prevWordSecondWordFirstInputIndex; } void truncate(const int offset) { @@ -96,11 +95,12 @@ class DicNodeStatePrevWord { mPrevWordLength = newPrevWordLength; } - void outputSpacePositions(int *spaceIndices) const { - // Convert uint16_t to int - for (int i = 0; i < MAX_RESULTS; i++) { - spaceIndices[i] = mPrevSpacePositions[i]; - } + void setSecondWordFirstInputIndex(const int inputIndex) { + mSecondWordFirstInputIndex = inputIndex; + } + + int getSecondWordFirstInputIndex() const { + return mSecondWordFirstInputIndex; } // TODO: remove @@ -138,8 +138,6 @@ class DicNodeStatePrevWord { // TODO: Move to private int mPrevWord[MAX_WORD_LENGTH]; - // TODO: Move to private - int mPrevSpacePositions[MAX_RESULTS]; private: // Caution!!! @@ -150,6 +148,7 @@ class DicNodeStatePrevWord { int16_t mPrevWordStart; int16_t mPrevWordProbability; int mPrevWordNodePos; + int mSecondWordFirstInputIndex; }; } // namespace latinime #endif // LATINIME_DIC_NODE_STATE_PREVWORD_H diff --git a/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp index cf1cd8815..425b07624 100644 --- a/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp @@ -116,7 +116,7 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos); while (bigramsIt.hasNext()) { bigramsIt.next(); - if (bigramsIt.getBigramPos() == NOT_A_VALID_WORD_POS) { + if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) { continue; } const int codePointCount = mDictionaryStructurePolicy-> @@ -146,7 +146,7 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in if (0 >= prevWordLength) return NOT_A_DICT_POS; int pos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(prevWord, prevWordLength, forceLowerCaseSearch); - if (NOT_A_VALID_WORD_POS == pos) return NOT_A_DICT_POS; + if (NOT_A_DICT_POS == pos) return NOT_A_DICT_POS; return mDictionaryStructurePolicy->getBigramsPositionOfNode(pos); } @@ -157,7 +157,7 @@ bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *w if (NOT_A_DICT_POS == pos) return false; int nextWordPos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(word1, length1, false /* forceLowerCaseSearch */); - if (NOT_A_VALID_WORD_POS == nextWordPos) return false; + if (NOT_A_DICT_POS == nextWordPos) return false; BinaryDictionaryBigramsIterator bigramsIt( mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos); diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index 02ece639c..033572201 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -87,7 +87,7 @@ int Dictionary::getBigrams(const int *word, int length, int *outWords, int *freq int Dictionary::getProbability(const int *word, int length) const { int pos = getDictionaryStructurePolicy()->getTerminalNodePositionOfWord(word, length, false /* forceLowerCaseSearch */); - if (NOT_A_VALID_WORD_POS == pos) { + if (NOT_A_DICT_POS == pos) { return NOT_A_PROBABILITY; } return getDictionaryStructurePolicy()->getUnigramProbabilityOfPtNode(pos); @@ -112,6 +112,18 @@ void Dictionary::removeBigramWords(const int *const word0, const int length0, mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1); } +void Dictionary::flush(const char *const filePath) { + mDictionaryStructureWithBufferPolicy->flush(filePath); +} + +void Dictionary::flushWithGC(const char *const filePath) { + mDictionaryStructureWithBufferPolicy->flushWithGC(filePath); +} + +bool Dictionary::needsToRunGC() { + return mDictionaryStructureWithBufferPolicy->needsToRunGC(); +} + void Dictionary::logDictionaryInfo(JNIEnv *const env) const { const int BUFFER_SIZE = 16; int dictionaryIdCodePointBuffer[BUFFER_SIZE]; diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 0afe5a54b..06e84bbfe 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -77,6 +77,12 @@ class Dictionary { void removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1); + void flush(const char *const filePath); + + void flushWithGC(const char *const filePath); + + bool needsToRunGC(); + const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const { return mDictionaryStructureWithBufferPolicy; } diff --git a/native/jni/src/suggest/core/dictionary/multi_bigram_map.h b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h index 9efe5f6f9..aecf41386 100644 --- a/native/jni/src/suggest/core/dictionary/multi_bigram_map.h +++ b/native/jni/src/suggest/core/dictionary/multi_bigram_map.h @@ -73,7 +73,7 @@ class MultiBigramMap { bigramsListPos); while (bigramsIt.hasNext()) { bigramsIt.next(); - if (bigramsIt.getBigramPos() == NOT_A_VALID_WORD_POS) { + if (bigramsIt.getBigramPos() == NOT_A_DICT_POS) { continue; } mBigramMap[bigramsIt.getBigramPos()] = bigramsIt.getProbability(); diff --git a/native/jni/src/suggest/core/layout/proximity_info_state.cpp b/native/jni/src/suggest/core/layout/proximity_info_state.cpp index 7780efdfd..fbabd92f2 100644 --- a/native/jni/src/suggest/core/layout/proximity_info_state.cpp +++ b/native/jni/src/suggest/core/layout/proximity_info_state.cpp @@ -36,8 +36,8 @@ void ProximityInfoState::initInputParams(const int pointerId, const float maxPoi const int *const xCoordinates, const int *const yCoordinates, const int *const times, const int *const pointerIds, const bool isGeometric) { ASSERT(isGeometric || (inputSize < MAX_WORD_LENGTH)); - mIsContinuousSuggestionPossible = - ProximityInfoStateUtils::checkAndReturnIsContinuousSuggestionPossible( + mIsContinuousSuggestionPossible = (mHasBeenUpdatedByGeometricInput != isGeometric) ? + false : ProximityInfoStateUtils::checkAndReturnIsContinuousSuggestionPossible( inputSize, xCoordinates, yCoordinates, times, mSampledInputSize, &mSampledInputXs, &mSampledInputYs, &mSampledTimes, &mSampledInputIndice); if (DEBUG_DICT) { @@ -155,6 +155,7 @@ void ProximityInfoState::initInputParams(const int pointerId, const float maxPoi if (DEBUG_GEO_FULL) { AKLOGI("ProximityState init finished: %d points out of %d", mSampledInputSize, inputSize); } + mHasBeenUpdatedByGeometricInput = isGeometric; } // This function basically converts from a length to an edit distance. Accordingly, it's obviously diff --git a/native/jni/src/suggest/core/layout/proximity_info_state.h b/native/jni/src/suggest/core/layout/proximity_info_state.h index dbcd54488..c94060fa9 100644 --- a/native/jni/src/suggest/core/layout/proximity_info_state.h +++ b/native/jni/src/suggest/core/layout/proximity_info_state.h @@ -46,10 +46,11 @@ class ProximityInfoState { : mProximityInfo(0), mMaxPointToKeyLength(0.0f), mAverageSpeed(0.0f), mHasTouchPositionCorrectionData(false), mMostCommonKeyWidthSquare(0), mKeyCount(0), mCellHeight(0), mCellWidth(0), mGridHeight(0), mGridWidth(0), - mIsContinuousSuggestionPossible(false), mSampledInputXs(), mSampledInputYs(), - mSampledTimes(), mSampledInputIndice(), mSampledLengthCache(), - mBeelineSpeedPercentiles(), mSampledNormalizedSquaredLengthCache(), mSpeedRates(), - mDirections(), mCharProbabilities(), mSampledNearKeySets(), mSampledSearchKeySets(), + mIsContinuousSuggestionPossible(false), mHasBeenUpdatedByGeometricInput(false), + mSampledInputXs(), mSampledInputYs(), mSampledTimes(), mSampledInputIndice(), + mSampledLengthCache(), mBeelineSpeedPercentiles(), + mSampledNormalizedSquaredLengthCache(), mSpeedRates(), mDirections(), + mCharProbabilities(), mSampledNearKeySets(), mSampledSearchKeySets(), mSampledSearchKeyVectors(), mTouchPositionCorrectionEnabled(false), mSampledInputSize(0), mMostProbableStringProbability(0.0f) { memset(mInputProximities, 0, sizeof(mInputProximities)); @@ -129,6 +130,10 @@ class ProximityInfoState { return mSampledInputYs[index]; } + int getInputIndexOfSampledPoint(const int sampledIndex) const { + return mSampledInputIndice[sampledIndex]; + } + bool hasSpaceProximity(const int index) const; int getLengthCache(const int index) const { @@ -204,6 +209,7 @@ class ProximityInfoState { int mGridHeight; int mGridWidth; bool mIsContinuousSuggestionPossible; + bool mHasBeenUpdatedByGeometricInput; std::vector<int> mSampledInputXs; std::vector<int> mSampledInputYs; diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index c8cbbcfdf..24d1b8ba1 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -74,6 +74,12 @@ class DictionaryStructureWithBufferPolicy { virtual bool removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1) = 0; + virtual void flush(const char *const filePath) = 0; + + virtual void flushWithGC(const char *const filePath) = 0; + + virtual bool needsToRunGC() const = 0; + protected: DictionaryStructureWithBufferPolicy() {} diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.cpp b/native/jni/src/suggest/core/session/dic_traverse_session.cpp index 0ca583f90..50f2bbd8d 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.cpp +++ b/native/jni/src/suggest/core/session/dic_traverse_session.cpp @@ -23,6 +23,11 @@ namespace latinime { +// 256K bytes threshold is heuristically used to distinguish dictionaries containing many unigrams +// (e.g. main dictionary) from small dictionaries (e.g. contacts...) +const int DicTraverseSession::DICTIONARY_SIZE_THRESHOLD_TO_USE_LARGE_CACHE_FOR_SUGGESTION = + 256 * 1024; + void DicTraverseSession::init(const Dictionary *const dictionary, const int *prevWord, int prevWordLength, const SuggestOptions *const suggestOptions) { mDictionary = dictionary; @@ -30,13 +35,13 @@ void DicTraverseSession::init(const Dictionary *const dictionary, const int *pre ->getMultiWordCostMultiplier(); mSuggestOptions = suggestOptions; if (!prevWord) { - mPrevWordPos = NOT_A_VALID_WORD_POS; + mPrevWordPos = NOT_A_DICT_POS; return; } // TODO: merge following similar calls to getTerminalPosition into one case-insensitive call. mPrevWordPos = getDictionaryStructurePolicy()->getTerminalNodePositionOfWord( prevWord, prevWordLength, false /* forceLowerCaseSearch */); - if (mPrevWordPos == NOT_A_VALID_WORD_POS) { + if (mPrevWordPos == NOT_A_DICT_POS) { // Check bigrams for lower-cased previous word if original was not found. Useful for // auto-capitalized words like "The [current_word]". mPrevWordPos = getDictionaryStructurePolicy()->getTerminalNodePositionOfWord( @@ -59,8 +64,9 @@ const DictionaryStructureWithBufferPolicy *DicTraverseSession::getDictionaryStru return mDictionary->getDictionaryStructurePolicy(); } -void DicTraverseSession::resetCache(const int nextActiveCacheSize, const int maxWords) { - mDicNodesCache.reset(nextActiveCacheSize, maxWords); +void DicTraverseSession::resetCache(const int thresholdForNextActiveDicNodes, const int maxWords) { + mDicNodesCache.reset(thresholdForNextActiveDicNodes /* nextActiveSize */, + maxWords /* terminalSize */); mMultiBigramMap.clear(); mPartiallyCommited = false; } diff --git a/native/jni/src/suggest/core/session/dic_traverse_session.h b/native/jni/src/suggest/core/session/dic_traverse_session.h index 23de5cc65..e0b1c67d9 100644 --- a/native/jni/src/suggest/core/session/dic_traverse_session.h +++ b/native/jni/src/suggest/core/session/dic_traverse_session.h @@ -37,8 +37,12 @@ class DicTraverseSession { public: // A factory method for DicTraverseSession - static AK_FORCE_INLINE void *getSessionInstance(JNIEnv *env, jstring localeStr) { - return new DicTraverseSession(env, localeStr); + static AK_FORCE_INLINE void *getSessionInstance(JNIEnv *env, jstring localeStr, + jlong dictSize) { + // To deal with the trade-off between accuracy and memory space, large cache is used for + // dictionaries larger that the threshold + return new DicTraverseSession(env, localeStr, + dictSize >= DICTIONARY_SIZE_THRESHOLD_TO_USE_LARGE_CACHE_FOR_SUGGESTION); } static AK_FORCE_INLINE void initSessionInstance(DicTraverseSession *traverseSession, @@ -54,10 +58,10 @@ class DicTraverseSession { delete traverseSession; } - AK_FORCE_INLINE DicTraverseSession(JNIEnv *env, jstring localeStr) - : mPrevWordPos(NOT_A_VALID_WORD_POS), mProximityInfo(0), - mDictionary(0), mSuggestOptions(0), mDicNodesCache(), mMultiBigramMap(), - mInputSize(0), mPartiallyCommited(false), mMaxPointerCount(1), + AK_FORCE_INLINE DicTraverseSession(JNIEnv *env, jstring localeStr, bool usesLargeCache) + : mPrevWordPos(NOT_A_DICT_POS), mProximityInfo(0), + mDictionary(0), mSuggestOptions(0), mDicNodesCache(usesLargeCache), + mMultiBigramMap(), mInputSize(0), mPartiallyCommited(false), mMaxPointerCount(1), mMultiWordCostMultiplier(1.0f) { // NOTE: mProximityInfoStates is an array of instances. // No need to initialize it explicitly here. @@ -73,7 +77,7 @@ class DicTraverseSession { const int inputSize, const int *const inputXs, const int *const inputYs, const int *const times, const int *const pointerIds, const float maxSpatialDistance, const int maxPointerCount); - void resetCache(const int nextActiveCacheSize, const int maxWords); + void resetCache(const int thresholdForNextActiveDicNodes, const int maxWords); const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const; @@ -109,7 +113,9 @@ class DicTraverseSession { if (usedPointerCount != 1) { return false; } - *pointerId = usedPointerId; + if (pointerId) { + *pointerId = usedPointerId; + } return true; } @@ -181,6 +187,7 @@ class DicTraverseSession { DISALLOW_IMPLICIT_CONSTRUCTORS(DicTraverseSession); // threshold to start caching static const int CACHE_START_INPUT_LENGTH_THRESHOLD; + static const int DICTIONARY_SIZE_THRESHOLD_TO_USE_LARGE_CACHE_FOR_SUGGESTION; void initializeProximityInfoStates(const int *const inputCodePoints, const int *const inputXs, const int *const inputYs, const int *const times, const int *const pointerIds, const int inputSize, const float maxSpatialDistance, const int maxPointerCount); diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index e788e914a..0c925be25 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -117,7 +117,7 @@ void Suggest::initializeSearch(DicTraverseSession *traverseSession, int commitPo * Outputs the final list of suggestions (i.e., terminal nodes). */ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequencies, - int *outputCodePoints, int *spaceIndices, int *outputTypes) const { + int *outputCodePoints, int *outputIndicesToPartialCommit, int *outputTypes) const { #if DEBUG_EVALUATE_MOST_PROBABLE_STRING const int terminalSize = 0; #else @@ -139,6 +139,7 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen SCORING->getMostProbableString(traverseSession, terminalSize, languageWeight, &outputCodePoints[0], &outputTypes[0], &frequencies[0]); if (hasMostProbableString) { + outputIndicesToPartialCommit[outputWordIndex] = NOT_AN_INDEX; ++outputWordIndex; } @@ -160,6 +161,9 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen || (traverseSession->getInputSize() >= MIN_LEN_FOR_MULTI_WORD_AUTOCORRECT && terminals[0].hasMultipleWords())) : false; + // TODO: have partial commit work even with multiple pointers. + const bool outputSecondWordFirstLetterInputIndex = + traverseSession->isOnlyOnePointerUsed(0 /* pointerId */); // Output suggestion results here for (int terminalIndex = 0; terminalIndex < terminalSize && outputWordIndex < MAX_RESULTS; ++terminalIndex) { @@ -194,18 +198,21 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen terminalDicNode->isExactMatch() || (forceCommitMultiWords && terminalDicNode->hasMultipleWords()) || (isValidWord && SCORING->doesAutoCorrectValidWord())); - maxScore = max(maxScore, finalScore); - - // TODO: Implement a smarter auto-commit method for handling multi-word suggestions. - // Index for top typing suggestion should be 0. - if (isValidWord && outputWordIndex == 0) { - terminalDicNode->outputSpacePositionsResult(spaceIndices); + if (maxScore < finalScore && isValidWord) { + maxScore = finalScore; } // Don't output invalid words. However, we still need to submit their shortcuts if any. if (isValidWord) { outputTypes[outputWordIndex] = Dictionary::KIND_CORRECTION | outputTypeFlags; frequencies[outputWordIndex] = finalScore; + if (outputSecondWordFirstLetterInputIndex) { + outputIndicesToPartialCommit[outputWordIndex] = + terminalDicNode->getSecondWordFirstInputIndex( + traverseSession->getProximityInfoState(0)); + } else { + outputIndicesToPartialCommit[outputWordIndex] = NOT_AN_INDEX; + } // Populate the outputChars array with the suggested word. const int startIndex = outputWordIndex * MAX_WORD_LENGTH; terminalDicNode->outputResult(&outputCodePoints[startIndex]); @@ -220,8 +227,19 @@ int Suggest::outputSuggestions(DicTraverseSession *traverseSession, int *frequen // Shortcut is not supported for multiple words suggestions. // TODO: Check shortcuts during traversal for multiple words suggestions. const bool sameAsTyped = TRAVERSAL->sameAsTyped(traverseSession, terminalDicNode); - outputWordIndex = ShortcutUtils::outputShortcuts(&shortcutIt, outputWordIndex, - finalScore, outputCodePoints, frequencies, outputTypes, sameAsTyped); + const int updatedOutputWordIndex = ShortcutUtils::outputShortcuts(&shortcutIt, + outputWordIndex, finalScore, outputCodePoints, frequencies, outputTypes, + sameAsTyped); + const int secondWordFirstInputIndex = terminalDicNode->getSecondWordFirstInputIndex( + traverseSession->getProximityInfoState(0)); + for (int i = outputWordIndex; i < updatedOutputWordIndex; ++i) { + if (outputSecondWordFirstLetterInputIndex) { + outputIndicesToPartialCommit[i] = secondWordFirstInputIndex; + } else { + outputIndicesToPartialCommit[i] = NOT_AN_INDEX; + } + } + outputWordIndex = updatedOutputWordIndex; } DicNode::managedDelete(terminalDicNode); } diff --git a/native/jni/src/suggest/core/suggest.h b/native/jni/src/suggest/core/suggest.h index 875cbe4e0..b24019632 100644 --- a/native/jni/src/suggest/core/suggest.h +++ b/native/jni/src/suggest/core/suggest.h @@ -55,7 +55,7 @@ class Suggest : public SuggestInterface { void createNextWordDicNode(DicTraverseSession *traverseSession, DicNode *dicNode, const bool spaceSubstitution) const; int outputSuggestions(DicTraverseSession *traverseSession, int *frequencies, - int *outputCodePoints, int *outputIndices, int *outputTypes) const; + int *outputCodePoints, int *outputIndicesToPartialCommit, int *outputTypes) const; void initializeSearch(DicTraverseSession *traverseSession, int commitPoint) const; void expandCurrentDicNodes(DicTraverseSession *traverseSession) const; void processTerminalDicNode(DicTraverseSession *traverseSession, DicNode *dicNode) const; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp index d57547445..09e832f07 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp @@ -38,6 +38,22 @@ const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F; const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; +/* static */ BigramListReadWriteUtils::BigramFlags + BigramListReadWriteUtils::getFlagsAndForwardPointer(const uint8_t *const bigramsBuf, + int *const pos) { + return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); +} + +/* static */ void BigramListReadWriteUtils::skipExistingBigrams(const uint8_t *const bigramsBuf, + int *const pos) { + BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); + while (hasNext(flags)) { + *pos += attributeAddressSize(flags); + flags = getFlagsAndForwardPointer(bigramsBuf, pos); + } + *pos += attributeAddressSize(flags); +} + /* static */ int BigramListReadWriteUtils::getBigramAddressAndForwardPointer( const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) { int offset = 0; @@ -54,7 +70,7 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; break; } if (offset == 0) { - return NOT_A_VALID_WORD_POS; + return NOT_A_DICT_POS; } if (isOffsetNegative(flags)) { return origin - offset; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h index ee2b722a4..884bcd7a9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h @@ -21,7 +21,6 @@ #include <stdint.h> #include "defines.h" -#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { @@ -29,10 +28,7 @@ class BigramListReadWriteUtils { public: typedef uint8_t BigramFlags; - static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer( - const uint8_t *const bigramsBuf, int *const pos) { - return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); - } + static BigramFlags getFlagsAndForwardPointer(const uint8_t *const bigramsBuf, int *const pos); static AK_FORCE_INLINE int getProbabilityFromFlags(const BigramFlags flags) { return flags & MASK_ATTRIBUTE_PROBABILITY; @@ -43,15 +39,7 @@ public: } // Bigrams reading methods - static AK_FORCE_INLINE void skipExistingBigrams(const uint8_t *const bigramsBuf, - int *const pos) { - BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); - while (hasNext(flags)) { - *pos += attributeAddressSize(flags); - flags = getFlagsAndForwardPointer(bigramsBuf, pos); - } - *pos += attributeAddressSize(flags); - } + static void skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos); static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos); @@ -74,12 +62,17 @@ public: return flags | FLAG_ATTRIBUTE_HAS_NEXT; } + static AK_FORCE_INLINE BigramFlags setProbabilityInFlags(const BigramFlags flags, + const int probability) { + return (flags & (~MASK_ATTRIBUTE_PROBABILITY)) | (probability & MASK_ATTRIBUTE_PROBABILITY); + } + // Returns true if the bigram entry is valid and put entry values into out*. static AK_FORCE_INLINE bool createBigramEntryAndGetFlagsAndOffsetAndOffsetFieldSize( const int entryPos, const int targetPos, const int probability, const bool hasNext, BigramFlags *const outBigramFlags, uint32_t *const outOffset, int *const outOffsetFieldSize) { - if (targetPos == NOT_A_VALID_WORD_POS) { + if (targetPos == NOT_A_DICT_POS) { return false; } BigramFlags flags = probability & MASK_ATTRIBUTE_PROBABILITY; diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index 936dc9c5d..4c44d22fd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -18,25 +18,64 @@ namespace latinime { -bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) { +const int DynamicBigramListPolicy::BIGRAM_LINK_COUNT_LIMIT = 10000; + +void DynamicBigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability, + bool *const outHasNext, int *const pos) const { + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + *pos -= mBuffer->getOriginalBufferSize(); + } + const BigramListReadWriteUtils::BigramFlags flags = + BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + buffer, flags, pos); + if (usesAdditionalBuffer && originalBigramPos != NOT_A_DICT_POS) { + originalBigramPos += mBuffer->getOriginalBufferSize(); + } + *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); + *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags); + *outHasNext = BigramListReadWriteUtils::hasNext(flags); + if (usesAdditionalBuffer) { + *pos += mBuffer->getOriginalBufferSize(); + } +} + +void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const { + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); + if (usesAdditionalBuffer) { + *pos -= mBuffer->getOriginalBufferSize(); + } + BigramListReadWriteUtils::skipExistingBigrams(buffer, pos); + if (usesAdditionalBuffer) { + *pos += mBuffer->getOriginalBufferSize(); + } +} + +bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos, + int *outBigramsCount) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); } + *outBigramsCount = 0; BigramListReadWriteUtils::BigramFlags flags; do { // The buffer address can be changed after calling buffer writing methods. const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, fromPos); - int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( buffer, flags, fromPos); - if (bigramPos == NOT_A_VALID_WORD_POS) { + if (originalBigramPos == NOT_A_DICT_POS) { // skip invalid bigram entry. continue; } if (usesAdditionalBuffer) { - bigramPos += mBuffer->getOriginalBufferSize(); + originalBigramPos += mBuffer->getOriginalBufferSize(); } + const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); BigramListReadWriteUtils::BigramFlags newBigramFlags; uint32_t newBigramOffset; int newBigramOffsetFieldSize; @@ -54,6 +93,7 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo toPos)) { return false; } + (*outBigramsCount)++; } while(BigramListReadWriteUtils::hasNext(flags)); if (usesAdditionalBuffer) { *fromPos += mBuffer->getOriginalBufferSize(); @@ -61,8 +101,8 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo return true; } -bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int probability, - int *const pos) { +bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramPos, + const int probability, int *const pos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); if (usesAdditionalBuffer) { *pos -= mBuffer->getOriginalBufferSize(); @@ -76,7 +116,17 @@ bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int prob // The buffer address can be changed after calling buffer writing methods. const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); - BigramListReadWriteUtils::getBigramAddressAndForwardPointer(buffer, flags, pos); + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + buffer, flags, pos); + if (usesAdditionalBuffer && originalBigramPos != NOT_A_DICT_POS) { + originalBigramPos += mBuffer->getOriginalBufferSize(); + } + if (followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos) == bigramPos) { + // Update this bigram entry. + const BigramListReadWriteUtils::BigramFlags updatedFlags = + BigramListReadWriteUtils::setProbabilityInFlags(flags, probability); + return mBuffer->writeUintAndAdvancePosition(updatedFlags, 1 /* size */, &entryPos); + } if (BigramListReadWriteUtils::hasNext(flags)) { continue; } @@ -87,33 +137,35 @@ bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int prob if (!mBuffer->writeUintAndAdvancePosition(updatedFlags, 1 /* size */, &entryPos)) { return false; } - // Then, add a new entry after the last entry. - BigramListReadWriteUtils::BigramFlags newBigramFlags; - uint32_t newBigramOffset; - int newBigramOffsetFieldSize; - if(!BigramListReadWriteUtils::createBigramEntryAndGetFlagsAndOffsetAndOffsetFieldSize( - *pos, bigramPos, BigramListReadWriteUtils::getProbabilityFromFlags(flags), - BigramListReadWriteUtils::hasNext(flags), &newBigramFlags, &newBigramOffset, - &newBigramOffsetFieldSize)) { - continue; - } - int newEntryPos = *pos; if (usesAdditionalBuffer) { - newEntryPos += mBuffer->getOriginalBufferSize(); - } - // Write bigram flags. - if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */, - &newEntryPos)) { - return false; - } - // Write bigram positon offset. - if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, - &newEntryPos)) { - return false; + *pos += mBuffer->getOriginalBufferSize(); } + // Then, add a new entry after the last entry. + return writeNewBigramEntry(bigramPos, probability, pos); } while(BigramListReadWriteUtils::hasNext(flags)); - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); + // We return directly from the while loop. + ASSERT(false); + return false; +} + +bool DynamicBigramListPolicy::writeNewBigramEntry(const int bigramPos, const int probability, + int *const writingPos) { + BigramListReadWriteUtils::BigramFlags newBigramFlags; + uint32_t newBigramOffset; + int newBigramOffsetFieldSize; + if(!BigramListReadWriteUtils::createBigramEntryAndGetFlagsAndOffsetAndOffsetFieldSize( + *writingPos, bigramPos, probability, false /* hasNext */, &newBigramFlags, + &newBigramOffset, &newBigramOffsetFieldSize)) { + return false; + } + // Write bigram flags. + if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */, writingPos)) { + return false; + } + // Write bigram positon offset. + if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, + writingPos)) { + return false; } return true; } @@ -133,11 +185,12 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta if (usesAdditionalBuffer) { bigramOffsetFieldPos += mBuffer->getOriginalBufferSize(); } - int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( + int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( buffer, flags, &pos); - if (usesAdditionalBuffer && bigramPos != NOT_A_VALID_WORD_POS) { - bigramPos += mBuffer->getOriginalBufferSize(); + if (usesAdditionalBuffer && originalBigramPos != NOT_A_DICT_POS) { + originalBigramPos += mBuffer->getOriginalBufferSize(); } + const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); if (bigramPos != targetBigramPos) { continue; } @@ -152,4 +205,26 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta return false; } +int DynamicBigramListPolicy::followBigramLinkAndGetCurrentBigramPtNodePos( + const int originalBigramPos) const { + if (originalBigramPos == NOT_A_DICT_POS) { + return NOT_A_DICT_POS; + } + int currentPos = originalBigramPos; + DynamicPatriciaTrieNodeReader nodeReader(mBuffer, this /* bigramsPolicy */, mShortcutPolicy); + nodeReader.fetchNodeInfoFromBuffer(currentPos); + int bigramLinkCount = 0; + while (nodeReader.getBigramLinkedNodePos() != NOT_A_DICT_POS) { + currentPos = nodeReader.getBigramLinkedNodePos(); + nodeReader.fetchNodeInfoFromBuffer(currentPos); + bigramLinkCount++; + if (bigramLinkCount > BIGRAM_LINK_COUNT_LIMIT) { + AKLOGI("Bigram link is invalid. start position: %d", bigramPos); + ASSERT(false); + return NOT_A_DICT_POS; + } + } + return currentPos; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index b7c05376d..dafb62d80 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -21,7 +21,9 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" +#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" +#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { @@ -31,49 +33,26 @@ namespace latinime { */ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { public: - DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer) - : mBuffer(buffer) {} + DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer, + const DictionaryShortcutsStructurePolicy *const shortcutPolicy) + : mBuffer(buffer), mShortcutPolicy(shortcutPolicy) {} ~DynamicBigramListPolicy() {} void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext, - int *const pos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *pos -= mBuffer->getOriginalBufferSize(); - } - const BigramListReadWriteUtils::BigramFlags flags = - BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); - *outBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( - buffer, flags, pos); - if (usesAdditionalBuffer && *outBigramPos != NOT_A_VALID_WORD_POS) { - *outBigramPos += mBuffer->getOriginalBufferSize(); - } - *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags); - *outHasNext = BigramListReadWriteUtils::hasNext(flags); - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); - } - } - - void skipAllBigrams(int *const pos) const { - const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); - if (usesAdditionalBuffer) { - *pos -= mBuffer->getOriginalBufferSize(); - } - BigramListReadWriteUtils::skipExistingBigrams(buffer, pos); - if (usesAdditionalBuffer) { - *pos += mBuffer->getOriginalBufferSize(); - } - } + int *const pos) const; + + void skipAllBigrams(int *const pos) const; // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these - // positions after bigram lists. This method skips invalid bigram entries. - bool copyAllBigrams(int *const fromPos, int *const toPos); + // positions after bigram lists. This method skips invalid bigram entries and write the valid + // bigram entry count to outBigramsCount. + bool copyAllBigrams(int *const fromPos, int *const toPos, int *outBigramsCount); + + bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos); - bool addBigramEntry(const int bigramPos, const int probability, int *const pos); + bool writeNewBigramEntry(const int bigramPos, const int probability, + int *const writingPos); // Return if targetBigramPos is found or not. bool removeBigram(const int bigramListPos, const int targetBigramPos); @@ -81,7 +60,13 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy); + static const int BIGRAM_LINK_COUNT_LIMIT; + BufferWithExtendableBuffer *const mBuffer; + const DictionaryShortcutsStructurePolicy *const mShortcutPolicy; + + // Follow bigram link and return the position of bigram target PtNode that is currently valid. + int followBigramLinkAndGetCurrentBigramPtNodePos(const int originalBigramPos) const; }; } // namespace latinime #endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp index 434858d67..ff80dd2f6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp @@ -27,12 +27,12 @@ namespace latinime { /* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory - ::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength, - const int bufOffset, const int size, const bool isUpdatable) { + ::newDictionaryStructureWithBufferPolicy(const char *const path, const int bufOffset, + const int size, const bool isUpdatable) { // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of // impl classes of DictionaryStructureWithBufferPolicy. - const MmappedBuffer *const mmapedBuffer = MmappedBuffer::openBuffer(path, pathLength, bufOffset, - size, isUpdatable); + const MmappedBuffer *const mmapedBuffer = MmappedBuffer::openBuffer(path, bufOffset, size, + isUpdatable); if (!mmapedBuffer) { return 0; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h index 1cb7a89c4..8cebc3b16 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h @@ -27,8 +27,7 @@ namespace latinime { class DictionaryStructureWithBufferPolicyFactory { public: static DictionaryStructureWithBufferPolicy *newDictionaryStructureWithBufferPolicy( - const char *const path, const int pathLength, const int bufOffset, const int size, - const bool isUpdatable); + const char *const path, const int bufOffset, const int size, const bool isUpdatable); private: DISALLOW_IMPLICIT_CONSTRUCTORS(DictionaryStructureWithBufferPolicyFactory); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index 5674cb48e..737098423 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -25,9 +25,17 @@ namespace latinime { void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(const int nodePos, const int maxCodePointCount, int *const outCodePoints) { + if (nodePos < 0 || nodePos >= mBuffer->getTailPosition()) { + AKLOGE("Fetching PtNode info form invalid dictionary position: %d, dictionary size: %d", + nodePos, mBuffer->getTailPosition()); + ASSERT(false); + invalidatePtNodeInfo(); + return; + } const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(nodePos); const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); int pos = nodePos; + mHeadPos = nodePos; if (usesAdditionalBuffer) { pos -= mBuffer->getOriginalBufferSize(); } @@ -57,10 +65,17 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mChildrenPosFieldPos += mBuffer->getOriginalBufferSize(); } mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( - dictBuf, mFlags, &pos); + dictBuf, &pos); if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { mChildrenPos += mBuffer->getOriginalBufferSize(); } + if (mSiblingPos == NOT_A_DICT_POS) { + if (DynamicPatriciaTrieReadingUtils::isMoved(mFlags)) { + mBigramLinkedNodePos = mChildrenPos; + } else { + mBigramLinkedNodePos = NOT_A_DICT_POS; + } + } if (usesAdditionalBuffer) { pos += mBuffer->getOriginalBufferSize(); } @@ -77,7 +92,7 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mBigramPos = NOT_A_DICT_POS; } // Update siblingPos if needed. - if (mSiblingPos == NOT_A_VALID_WORD_POS) { + if (mSiblingPos == NOT_A_DICT_POS) { // Sibling position is the tail position of current node. mSiblingPos = pos; } @@ -88,4 +103,19 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c } } +void DynamicPatriciaTrieNodeReader::invalidatePtNodeInfo() { + mHeadPos = NOT_A_DICT_POS; + mFlags = 0; + mParentPos = NOT_A_DICT_POS; + mCodePointCount = 0; + mProbabilityFieldPos = NOT_A_DICT_POS; + mProbability = NOT_A_PROBABILITY; + mChildrenPosFieldPos = NOT_A_DICT_POS; + mChildrenPos = NOT_A_DICT_POS; + mBigramLinkedNodePos = NOT_A_DICT_POS; + mShortcutPos = NOT_A_DICT_POS; + mBigramPos = NOT_A_DICT_POS; + mSiblingPos = NOT_A_DICT_POS; +} + } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h index 2ee7c2495..6ef5f5813 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h @@ -39,12 +39,12 @@ class DynamicPatriciaTrieNodeReader { const DictionaryBigramsStructurePolicy *const bigramsPolicy, const DictionaryShortcutsStructurePolicy *const shortcutsPolicy) : mBuffer(buffer), mBigramsPolicy(bigramsPolicy), - mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), - mParentPos(NOT_A_DICT_POS), mCodePointCount(0), - mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), - mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS), - mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), - mSiblingPos(NOT_A_VALID_WORD_POS) {} + mShortcutsPolicy(shortcutsPolicy), mHeadPos(NOT_A_DICT_POS), mFlags(0), + mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbabilityFieldPos(NOT_A_DICT_POS), + mProbability(NOT_A_PROBABILITY), mChildrenPosFieldPos(NOT_A_DICT_POS), + mChildrenPos(NOT_A_DICT_POS), mBigramLinkedNodePos(NOT_A_DICT_POS), + mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), + mSiblingPos(NOT_A_DICT_POS) {} ~DynamicPatriciaTrieNodeReader() {} @@ -56,13 +56,14 @@ class DynamicPatriciaTrieNodeReader { AK_FORCE_INLINE void fetchNodeInfoFromBufferAndGetNodeCodePoints(const int nodePos, const int maxCodePointCount, int *const outCodePoints) { - mNodePos = nodePos; - mSiblingPos = NOT_A_VALID_WORD_POS; - fetchNodeInfoFromBufferAndProcessMovedNode(mNodePos, maxCodePointCount, outCodePoints); + mSiblingPos = NOT_A_DICT_POS; + mBigramLinkedNodePos = NOT_A_DICT_POS; + fetchNodeInfoFromBufferAndProcessMovedNode(nodePos, maxCodePointCount, outCodePoints); } - AK_FORCE_INLINE int getNodePos() const { - return mNodePos; + // HeadPos is different from NodePos when the current PtNode is a moved PtNode. + AK_FORCE_INLINE int getHeadPos() const { + return mHeadPos; } // Flags @@ -114,6 +115,11 @@ class DynamicPatriciaTrieNodeReader { return mChildrenPos; } + // Bigram linked node position. + AK_FORCE_INLINE int getBigramLinkedNodePos() const { + return mBigramLinkedNodePos; + } + // Shortcutlist position AK_FORCE_INLINE int getShortcutPos() const { return mShortcutPos; @@ -135,7 +141,7 @@ class DynamicPatriciaTrieNodeReader { const BufferWithExtendableBuffer *const mBuffer; const DictionaryBigramsStructurePolicy *const mBigramsPolicy; const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy; - int mNodePos; + int mHeadPos; DynamicPatriciaTrieReadingUtils::NodeFlags mFlags; int mParentPos; uint8_t mCodePointCount; @@ -143,12 +149,15 @@ class DynamicPatriciaTrieNodeReader { int mProbability; int mChildrenPosFieldPos; int mChildrenPos; + int mBigramLinkedNodePos; int mShortcutPos; int mBigramPos; int mSiblingPos; void fetchNodeInfoFromBufferAndProcessMovedNode(const int nodePos, const int maxCodePointCount, int *const outCodePoints); + + void invalidatePtNodeInfo(); }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_NODE_READER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index 945677b50..3cfbfd85b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -38,7 +38,7 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d readingHelper.initWithNodeArrayPos(dicNode->getChildrenPos()); const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); while (!readingHelper.isEnd()) { - childDicNodes->pushLeavingChild(dicNode, nodeReader->getNodePos(), + childDicNodes->pushLeavingChild(dicNode, nodeReader->getHeadPos(), nodeReader->getChildrenPos(), nodeReader->getProbability(), nodeReader->isTerminal() && !nodeReader->isDeleted(), nodeReader->hasChildren(), nodeReader->isBlacklisted() || nodeReader->isNotAWord(), @@ -116,23 +116,23 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in if (!readingHelper.isMatchedCodePoint( j, searchCodePoints[matchedCodePointCount + j])) { // Different code point is found. The given word is not included in the dictionary. - return NOT_A_VALID_WORD_POS; + return NOT_A_DICT_POS; } } // All characters are matched. if (length == readingHelper.getTotalCodePointCount()) { // Terminal position is found. - return nodeReader->getNodePos(); + return nodeReader->getHeadPos(); } if (!nodeReader->hasChildren()) { - return NOT_A_VALID_WORD_POS; + return NOT_A_DICT_POS; } // Advance to the children nodes. readingHelper.readChildNode(); } // If we already traversed the tree further than the word is long, there means // there was no match (or we would have found it). - return NOT_A_VALID_WORD_POS; + return NOT_A_DICT_POS; } int DynamicPatriciaTriePolicy::getProbability(const int unigramProbability, @@ -149,7 +149,7 @@ int DynamicPatriciaTriePolicy::getProbability(const int unigramProbability, } int DynamicPatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int nodePos) const { - if (nodePos == NOT_A_VALID_WORD_POS) { + if (nodePos == NOT_A_DICT_POS) { return NOT_A_PROBABILITY; } DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, @@ -162,7 +162,7 @@ int DynamicPatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int nodePos) } int DynamicPatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) const { - if (nodePos == NOT_A_VALID_WORD_POS) { + if (nodePos == NOT_A_DICT_POS) { return NOT_A_DICT_POS; } DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, @@ -175,7 +175,7 @@ int DynamicPatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) cons } int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const { - if (nodePos == NOT_A_VALID_WORD_POS) { + if (nodePos == NOT_A_DICT_POS) { return NOT_A_DICT_POS; } DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer, @@ -209,12 +209,12 @@ bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int } const int word0Pos = getTerminalNodePositionOfWord(word0, length0, false /* forceLowerCaseSearch */); - if (word0Pos == NOT_A_VALID_WORD_POS) { + if (word0Pos == NOT_A_DICT_POS) { return false; } const int word1Pos = getTerminalNodePositionOfWord(word1, length1, false /* forceLowerCaseSearch */); - if (word1Pos == NOT_A_VALID_WORD_POS) { + if (word1Pos == NOT_A_DICT_POS) { return false; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, @@ -230,12 +230,12 @@ bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const } const int word0Pos = getTerminalNodePositionOfWord(word0, length0, false /* forceLowerCaseSearch */); - if (word0Pos == NOT_A_VALID_WORD_POS) { + if (word0Pos == NOT_A_DICT_POS) { return false; } const int word1Pos = getTerminalNodePositionOfWord(word1, length1, false /* forceLowerCaseSearch */); - if (word1Pos == NOT_A_VALID_WORD_POS) { + if (word1Pos == NOT_A_DICT_POS) { return false; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, @@ -243,4 +243,29 @@ bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const return writingHelper.removeBigramWords(word0Pos, word1Pos); } +void DynamicPatriciaTriePolicy::flush(const char *const filePath) { + if (!mBuffer->isUpdatable()) { + AKLOGI("Warning: flush() is called for non-updatable dictionary."); + return; + } + // TODO: Implement. +} + +void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) { + if (!mBuffer->isUpdatable()) { + AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); + return; + } + // TODO: Implement. +} + +bool DynamicPatriciaTriePolicy::needsToRunGC() const { + if (!mBuffer->isUpdatable()) { + AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); + return false; + } + // TODO: Implement. + return false; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index cdab0e16a..2cbb0ff3b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -36,8 +36,8 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()), mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(), mBuffer->getBufferSize() - mHeaderPolicy.getSize()), - mBigramListPolicy(&mBufferWithExtendableBuffer), - mShortcutListPolicy(&mBufferWithExtendableBuffer) {} + mShortcutListPolicy(&mBufferWithExtendableBuffer), + mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy) {} ~DynamicPatriciaTriePolicy() { delete mBuffer; @@ -85,14 +85,20 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { bool removeBigramWords(const int *const word0, const int length0, const int *const word1, const int length1); + void flush(const char *const filePath); + + void flushWithGC(const char *const filePath); + + bool needsToRunGC() const; + private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; BufferWithExtendableBuffer mBufferWithExtendableBuffer; - DynamicBigramListPolicy mBigramListPolicy; DynamicShortcutListPolicy mShortcutListPolicy; + DynamicBigramListPolicy mBigramListPolicy; }; } // namespace latinime #endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h index db1c392bb..120fd7699 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h @@ -72,8 +72,7 @@ class DynamicPatriciaTrieReadingHelper { // Initialize reading state with the head position of a node. AK_FORCE_INLINE void initWithNodePos(const int nodePos) { - // TODO: Consolidate NOT_A_VALID_WORD_POS and NOT_A_DICT_POS - if (nodePos == NOT_A_VALID_WORD_POS || nodePos == NOT_A_DICT_POS) { + if (nodePos == NOT_A_DICT_POS) { mPos = NOT_A_DICT_POS; } else { mIsError = false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp index 5d979fa51..8428c0b15 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp @@ -28,19 +28,26 @@ const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; +/* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, + const int pos) { + int linkAddressPos = pos; + return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); +} + +/* static */ int DptReadingUtils::getParentPosAndAdvancePosition(const uint8_t *const buffer, + int *const pos) { + return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); +} + /* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( - const uint8_t *const buffer, const NodeFlags flags, int *const pos) { - if ((flags & MASK_MOVED) == FLAG_IS_NOT_MOVED) { - const int base = *pos; - const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); - if (offset == 0) { - // 0 offset means that the node does not have children. - return NOT_A_DICT_POS; - } else { - return base + offset; - } - } else { + const uint8_t *const buffer, int *const pos) { + const int base = *pos; + const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); + if (offset == 0) { + // 0 offset means that the node does not have children. return NOT_A_DICT_POS; + } else { + return base + offset; } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h index 62d73bb02..db5f9b1bd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" -#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { @@ -28,22 +27,15 @@ class DynamicPatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; - static AK_FORCE_INLINE int getForwardLinkPosition(const uint8_t *const buffer, const int pos) { - int linkAddressPos = pos; - return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); - } + static int getForwardLinkPosition(const uint8_t *const buffer, const int pos); static AK_FORCE_INLINE bool isValidForwardLinkPosition(const int forwardLinkAddress) { return forwardLinkAddress != 0; } - static AK_FORCE_INLINE int getParentPosAndAdvancePosition(const uint8_t *const buffer, - int *const pos) { - return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); - } + static int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos); - static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, - const NodeFlags flags, int *const pos); + static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, int *const pos); /** * Node Flags @@ -59,9 +51,9 @@ class DynamicPatriciaTrieReadingUtils { static AK_FORCE_INLINE NodeFlags updateAndGetFlags(const NodeFlags originalFlags, const bool isMoved, const bool isDeleted) { NodeFlags flags = originalFlags; - flags = isMoved ? ((flags & (!MASK_MOVED)) | FLAG_IS_MOVED) : flags; - flags = isDeleted ? ((flags & (!MASK_MOVED)) | FLAG_IS_DELETED) : flags; - flags = (!isMoved && !isDeleted) ? ((flags & (!MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags; + flags = isMoved ? ((flags & (~MASK_MOVED)) | FLAG_IS_MOVED) : flags; + flags = isDeleted ? ((flags & (~MASK_MOVED)) | FLAG_IS_DELETED) : flags; + flags = (!isMoved && !isDeleted) ? ((flags & (~MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags; return flags; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index 99a983f21..311d31e5d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -26,10 +26,12 @@ namespace latinime { +const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; + bool DynamicPatriciaTrieWritingHelper::addUnigramWord( DynamicPatriciaTrieReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability) { - int parentPos = NOT_A_VALID_WORD_POS; + int parentPos = NOT_A_DICT_POS; while (!readingHelper->isEnd()) { const int matchedCodePointCount = readingHelper->getPrevTotalCodePointCount(); if (!readingHelper->isMatchedCodePoint(0 /* index */, @@ -63,7 +65,7 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( codePointCount - readingHelper->getTotalCodePointCount()); } // Advance to the children nodes. - parentPos = nodeReader->getNodePos(); + parentPos = nodeReader->getHeadPos(); readingHelper->readChildNode(); } if (readingHelper->isError()) { @@ -79,29 +81,60 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos, const int probability) { + int mMergedNodeCodePoints[MAX_WORD_LENGTH]; DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); - nodeReader.fetchNodeInfoFromBuffer(word0Pos); - if (nodeReader.isDeleted()) { + nodeReader.fetchNodeInfoFromBufferAndGetNodeCodePoints(word0Pos, MAX_WORD_LENGTH, + mMergedNodeCodePoints); + // Move node to add bigram entry. + const int newNodePos = mBuffer->getTailPosition(); + if (!markNodeAsMovedAndSetPosition(&nodeReader, newNodePos, newNodePos)) { return false; } - // TODO: Implement. - return false; + int writingPos = newNodePos; + // Write a new PtNode using original PtNode's info to the tail of the dictionary. + if (!writePtNodeToBufferByCopyingPtNodeInfo(&nodeReader, nodeReader.getParentPos(), + mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(), + &writingPos)) { + return false; + } + nodeReader.fetchNodeInfoFromBuffer(newNodePos); + if (nodeReader.getBigramsPos() != NOT_A_DICT_POS) { + // Insert a new bigram entry into the existing bigram list. + int bigramListPos = nodeReader.getBigramsPos(); + return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos); + } else { + // The PtNode doesn't have a bigram list. + // First, Write a bigram entry at the tail position of the PtNode. + if (!mBigramPolicy->writeNewBigramEntry(word1Pos, probability, &writingPos)) { + return false; + } + // Then, Mark as the PtNode having bigram list in the flags. + const PatriciaTrieReadingUtils::NodeFlags updatedFlags = + PatriciaTrieReadingUtils::createAndGetFlags(nodeReader.isBlacklisted(), + nodeReader.isNotAWord(), nodeReader.getProbability() != NOT_A_PROBABILITY, + nodeReader.getShortcutPos() != NOT_A_DICT_POS, true /* hasBigrams */, + nodeReader.getCodePointCount() > 1, CHILDREN_POSITION_FIELD_SIZE); + writingPos = newNodePos; + // Write updated flags into the moved PtNode's flags field. + return DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, updatedFlags, + &writingPos); + } } // Remove a bigram relation from word0Pos to word1Pos. bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, const int word1Pos) { DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); nodeReader.fetchNodeInfoFromBuffer(word0Pos); - if (nodeReader.isDeleted() || nodeReader.getBigramsPos() == NOT_A_DICT_POS) { + if (nodeReader.getBigramsPos() == NOT_A_DICT_POS) { return false; } - // TODO: Implement. - return false; + return mBigramPolicy->removeBigram(nodeReader.getBigramsPos(), word1Pos); } bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( - const DynamicPatriciaTrieNodeReader *const originalNode, const int movedPos) { - int pos = originalNode->getNodePos(); + const DynamicPatriciaTrieNodeReader *const originalNode, const int movedPos, + const int bigramLinkedNodePos) { + int pos = originalNode->getHeadPos(); const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(pos); const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { @@ -113,18 +146,42 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( const PatriciaTrieReadingUtils::NodeFlags updatedFlags = DynamicPatriciaTrieReadingUtils::updateAndGetFlags(originalFlags, true /* isMoved */, false /* isDeleted */); - int writingPos = originalNode->getNodePos(); + int writingPos = originalNode->getHeadPos(); // Update flags. if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, updatedFlags, &writingPos)) { return false; } // Update moved position, which is stored in the parent offset field. - const int movedPosOffset = movedPos - originalNode->getNodePos(); + const int movedPosOffset = movedPos - originalNode->getHeadPos(); if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( mBuffer, movedPosOffset, &writingPos)) { return false; } + // Update bigram linked node position, which is stored in the children position field. + int childrenPosFieldPos = originalNode->getChildrenPosFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( + mBuffer, bigramLinkedNodePos, &childrenPosFieldPos)) { + return false; + } + if (originalNode->hasChildren()) { + // Update children's parent position. + DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy); + const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); + readingHelper.initWithNodeArrayPos(originalNode->getChildrenPos()); + while (!readingHelper.isEnd()) { + const int childPtNodeWrittenPos = nodeReader->getHeadPos(); + const int parentOffset = movedPos - childPtNodeWrittenPos; + int parentOffsetFieldPos = childPtNodeWrittenPos + 1 /* Flags */; + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( + mBuffer, parentOffset, &parentOffsetFieldPos)) { + // Parent offset cannot be written because of a bug or a broken dictionary; thus, + // we give up to update dictionary. + return false; + } + readingHelper.readNextSiblingNode(); + } + } return true; } @@ -135,13 +192,9 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos) { const int nodePos = *writingPos; - // Create node flags and write them. - const PatriciaTrieReadingUtils::NodeFlags nodeFlags = - PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, - probability != NOT_A_PROBABILITY, originalShortcutListPos != NOT_A_DICT_POS, - originalBigramListPos != NOT_A_DICT_POS, codePointCount > 1, - 3 /* childrenPositionFieldSize */); - if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags, + // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the + // PtNode writing. + if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */, writingPos)) { return false; } @@ -154,7 +207,7 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo // Write code points if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints, codePointCount, writingPos)) { - return false;; + return false; } // Write probability when the probability is a valid probability, which means this node is // terminal. @@ -177,12 +230,25 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo } } // Copy bigram list when the originalBigramListPos is valid dictionary position. + int bigramCount = 0; if (originalBigramListPos != NOT_A_DICT_POS) { int fromPos = originalBigramListPos; - if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos)) { + if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) { return false; } } + // Create node flags and write them. + PatriciaTrieReadingUtils::NodeFlags nodeFlags = + PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, + probability != NOT_A_PROBABILITY /* isTerminal */, + originalShortcutListPos != NOT_A_DICT_POS /* hasShortcutTargets */, + bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */, + CHILDREN_POSITION_FIELD_SIZE); + int flagsFieldPos = nodePos; + if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags, + &flagsFieldPos)) { + return false; + } return true; } @@ -230,7 +296,7 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( } else { // Make the node terminal and write the probability. int movedPos = mBuffer->getTailPosition(); - if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos)) { + if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { return false; } if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), @@ -250,7 +316,7 @@ bool DynamicPatriciaTrieWritingHelper::createChildrenPtNodeArrayAndAChildPtNode( newPtNodeArrayPos, &childrenPosFieldPos)) { return false; } - return createNewPtNodeArrayWithAChildPtNode(parentNode->getNodePos(), codePoints, + return createNewPtNodeArrayWithAChildPtNode(parentNode->getHeadPos(), codePoints, codePointCount, probability); } @@ -287,11 +353,8 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( // Reallocating PtNode: abcde, newNode: abc. // abc (1st, terminal) __ de (2nd) const bool addsExtraChild = newNodeCodePointCount > overlappingCodePointCount; - const int firstPtNodePos = mBuffer->getTailPosition(); - if (!markNodeAsMovedAndSetPosition(reallocatingPtNode, firstPtNodePos)) { - return false; - } - int writingPos = firstPtNodePos; + const int firstPartOfReallocatedPtNodePos = mBuffer->getTailPosition(); + int writingPos = firstPartOfReallocatedPtNodePos; // Write the 1st part of the reallocating node. The children position will be updated later // with actual children position. const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; @@ -307,15 +370,15 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( return false; } // Write the 2nd part of the reallocating node. - if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, - reallocatingPtNode->getNodePos(), + const int secondPartOfReallocatedPtNodePos = writingPos; + if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, firstPartOfReallocatedPtNodePos, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getProbability(), &writingPos)) { return false; } if (addsExtraChild) { - if (!writePtNodeToBuffer(reallocatingPtNode->getNodePos(), + if (!writePtNodeToBuffer(firstPartOfReallocatedPtNodePos, newNodeCodePoints + overlappingCodePointCount, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, &writingPos)) { @@ -326,9 +389,14 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } + // Update original reallocatingPtNode as moved. + if (!markNodeAsMovedAndSetPosition(reallocatingPtNode, firstPartOfReallocatedPtNodePos, + secondPartOfReallocatedPtNodePos)) { + return false; + } // Load node info. Information of the 1st part will be fetched. DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); - nodeReader.fetchNodeInfoFromBuffer(firstPtNodePos); + nodeReader.fetchNodeInfoFromBuffer(firstPartOfReallocatedPtNodePos); // Update children position. int childrenPosFieldPos = nodeReader.getChildrenPosFieldPos(); if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index ada634a54..20e35abcf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -49,12 +49,14 @@ class DynamicPatriciaTrieWritingHelper { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); + static const int CHILDREN_POSITION_FIELD_SIZE; + BufferWithExtendableBuffer *const mBuffer; DynamicBigramListPolicy *const mBigramPolicy; DynamicShortcutListPolicy *const mShortcutPolicy; bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, - const int movedPos); + const int movedPos, const int bigramLinkedNodePos); bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp index d5a83a938..e6cff431b 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp @@ -219,7 +219,7 @@ int PatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCount( } // This function gets the position of the terminal node of the exact matching word in the -// dictionary. If no match is found, it returns NOT_A_VALID_WORD_POS. +// dictionary. If no match is found, it returns NOT_A_DICT_POS. int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, const int length, const bool forceLowerCaseSearch) const { int pos = getRootPosition(); @@ -228,7 +228,7 @@ int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, while (true) { // If we already traversed the tree further than the word is long, there means // there was no match (or we would have found it). - if (wordPos >= length) return NOT_A_VALID_WORD_POS; + if (wordPos >= length) return NOT_A_DICT_POS; int ptNodeCount = PatriciaTrieReadingUtils::getPtNodeArraySizeAndAdvancePosition(mDictRoot, &pos); const int wChar = forceLowerCaseSearch @@ -236,7 +236,7 @@ int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, while (true) { // If there are no more PtNodes in this array, it means we could not // find a matching character for this depth, therefore there is no match. - if (0 >= ptNodeCount) return NOT_A_VALID_WORD_POS; + if (0 >= ptNodeCount) return NOT_A_DICT_POS; const int ptNodePos = pos; const PatriciaTrieReadingUtils::NodeFlags flags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(mDictRoot, &pos); @@ -245,7 +245,7 @@ int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, if (character == wChar) { // This is the correct PtNode. Only one PtNode may start with the same char within // a PtNode array, so either we found our match in this array, or there is - // no match and we can return NOT_A_VALID_WORD_POS. So we will check all the + // no match and we can return NOT_A_DICT_POS. So we will check all the // characters in this PtNode indeed does match. if (PatriciaTrieReadingUtils::hasMultipleChars(flags)) { character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition(mDictRoot, @@ -256,8 +256,8 @@ int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, // character that does not match, as explained above, it means the word is // not in the dictionary (by virtue of this PtNode being the only one to // match the word on the first character, but not matching the whole word). - if (wordPos >= length) return NOT_A_VALID_WORD_POS; - if (inWord[wordPos] != character) return NOT_A_VALID_WORD_POS; + if (wordPos >= length) return NOT_A_DICT_POS; + if (inWord[wordPos] != character) return NOT_A_DICT_POS; character = PatriciaTrieReadingUtils::getCodePointAndAdvancePosition( mDictRoot, &pos); } @@ -274,7 +274,7 @@ int PatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const inWord, PatriciaTrieReadingUtils::readProbabilityAndAdvancePosition(mDictRoot, &pos); } if (!PatriciaTrieReadingUtils::hasChildrenInFlags(flags)) { - return NOT_A_VALID_WORD_POS; + return NOT_A_DICT_POS; } // We have children and we are still shorter than the word we are searching for, so // we need to traverse children. Put the pointer on the children position, and @@ -320,7 +320,7 @@ int PatriciaTriePolicy::getProbability(const int unigramProbability, } int PatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int nodePos) const { - if (nodePos == NOT_A_VALID_WORD_POS) { + if (nodePos == NOT_A_DICT_POS) { return NOT_A_PROBABILITY; } int pos = nodePos; @@ -342,7 +342,7 @@ int PatriciaTriePolicy::getUnigramProbabilityOfPtNode(const int nodePos) const { } int PatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) const { - if (nodePos == NOT_A_VALID_WORD_POS) { + if (nodePos == NOT_A_DICT_POS) { return NOT_A_DICT_POS; } int pos = nodePos; @@ -362,7 +362,7 @@ int PatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) const { } int PatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const { - if (nodePos == NOT_A_VALID_WORD_POS) { + if (nodePos == NOT_A_DICT_POS) { return NOT_A_DICT_POS; } int pos = nodePos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index 75d976205..cee3e4ab2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -96,6 +96,22 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { return false; } + void flush(const char *const filePath) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: flush() is called for non-updatable dictionary."); + } + + void flushWithGC(const char *const filePath) { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); + } + + bool needsToRunGC() const { + // This method should not be called for non-updatable dictionary. + AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); + return false; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy); diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp index 576a158bc..1316b425f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp @@ -42,6 +42,63 @@ const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_NOT_A_WORD = 0x02; // Flag for blacklist const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_BLACKLISTED = 0x01; +/* static */ int PtReadingUtils::getPtNodeArraySizeAndAdvancePosition( + const uint8_t *const buffer, int *const pos) { + const uint8_t firstByte = ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); + if (firstByte < 0x80) { + return firstByte; + } else { + return ((firstByte & 0x7F) << 8) ^ ByteArrayUtils::readUint8AndAdvancePosition( + buffer, pos); + } +} + +/* static */ PtReadingUtils::NodeFlags PtReadingUtils::getFlagsAndAdvancePosition( + const uint8_t *const buffer, int *const pos) { + return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); +} + +/* static */ int PtReadingUtils::getCodePointAndAdvancePosition(const uint8_t *const buffer, + int *const pos) { + return ByteArrayUtils::readCodePointAndAdvancePosition(buffer, pos); +} + +// Returns the number of read characters. +/* static */ int PtReadingUtils::getCharsAndAdvancePosition(const uint8_t *const buffer, + const NodeFlags flags, const int maxLength, int *const outBuffer, int *const pos) { + int length = 0; + if (hasMultipleChars(flags)) { + length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer, + pos); + } else { + if (maxLength > 0) { + outBuffer[0] = getCodePointAndAdvancePosition(buffer, pos); + length = 1; + } + } + return length; +} + +// Returns the number of skipped characters. +/* static */ int PtReadingUtils::skipCharacters(const uint8_t *const buffer, const NodeFlags flags, + const int maxLength, int *const pos) { + if (hasMultipleChars(flags)) { + return ByteArrayUtils::advancePositionToBehindString(buffer, maxLength, pos); + } else { + if (maxLength > 0) { + getCodePointAndAdvancePosition(buffer, pos); + return 1; + } else { + return 0; + } + } +} + +/* static */ int PtReadingUtils::readProbabilityAndAdvancePosition(const uint8_t *const buffer, + int *const pos) { + return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); +} + /* static */ int PtReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, const NodeFlags flags, int *const pos) { const int base = *pos; diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h index 2b0646db2..8420ee95a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.h @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" -#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { @@ -28,62 +27,21 @@ class PatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; - static AK_FORCE_INLINE int getPtNodeArraySizeAndAdvancePosition( - const uint8_t *const buffer, int *const pos) { - const uint8_t firstByte = ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); - if (firstByte < 0x80) { - return firstByte; - } else { - return ((firstByte & 0x7F) << 8) ^ ByteArrayUtils::readUint8AndAdvancePosition( - buffer, pos); - } - } + static int getPtNodeArraySizeAndAdvancePosition(const uint8_t *const buffer, int *const pos); - static AK_FORCE_INLINE NodeFlags getFlagsAndAdvancePosition(const uint8_t *const buffer, - int *const pos) { - return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); - } + static NodeFlags getFlagsAndAdvancePosition(const uint8_t *const buffer, int *const pos); - static AK_FORCE_INLINE int getCodePointAndAdvancePosition(const uint8_t *const buffer, - int *const pos) { - return ByteArrayUtils::readCodePointAndAdvancePosition(buffer, pos); - } + static int getCodePointAndAdvancePosition(const uint8_t *const buffer, int *const pos); // Returns the number of read characters. - static AK_FORCE_INLINE int getCharsAndAdvancePosition(const uint8_t *const buffer, - const NodeFlags flags, const int maxLength, int *const outBuffer, int *const pos) { - int length = 0; - if (hasMultipleChars(flags)) { - length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer, - pos); - } else { - if (maxLength > 0) { - outBuffer[0] = getCodePointAndAdvancePosition(buffer, pos); - length = 1; - } - } - return length; - } + static int getCharsAndAdvancePosition(const uint8_t *const buffer, const NodeFlags flags, + const int maxLength, int *const outBuffer, int *const pos); // Returns the number of skipped characters. - static AK_FORCE_INLINE int skipCharacters(const uint8_t *const buffer, const NodeFlags flags, - const int maxLength, int *const pos) { - if (hasMultipleChars(flags)) { - return ByteArrayUtils::advancePositionToBehindString(buffer, maxLength, pos); - } else { - if (maxLength > 0) { - getCodePointAndAdvancePosition(buffer, pos); - return 1; - } else { - return 0; - } - } - } + static int skipCharacters(const uint8_t *const buffer, const NodeFlags flags, + const int maxLength, int *const pos); - static AK_FORCE_INLINE int readProbabilityAndAdvancePosition(const uint8_t *const buffer, - int *const pos) { - return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); - } + static int readProbabilityAndAdvancePosition(const uint8_t *const buffer, int *const pos); static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, const NodeFlags flags, int *const pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.cpp index e70bb5071..847dcdee5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.cpp @@ -16,6 +16,8 @@ #include "suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h" +#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" + namespace latinime { // Flag for presence of more attributes @@ -28,4 +30,22 @@ const int ShortcutListReadingUtils::SHORTCUT_LIST_SIZE_FIELD_SIZE = 2; // The numeric value of the shortcut probability that means 'whitelist'. const int ShortcutListReadingUtils::WHITELIST_SHORTCUT_PROBABILITY = 15; +/* static */ ShortcutListReadingUtils::ShortcutFlags + ShortcutListReadingUtils::getFlagsAndForwardPointer(const uint8_t *const dictRoot, + int *const pos) { + return ByteArrayUtils::readUint8AndAdvancePosition(dictRoot, pos); +} + +/* static */ int ShortcutListReadingUtils::getShortcutListSizeAndForwardPointer( + const uint8_t *const dictRoot, int *const pos) { + // readUint16andAdvancePosition() returns an offset *including* the uint16 field itself. + return ByteArrayUtils::readUint16AndAdvancePosition(dictRoot, pos) + - SHORTCUT_LIST_SIZE_FIELD_SIZE; +} + +/* static */ int ShortcutListReadingUtils::readShortcutTarget( + const uint8_t *const dictRoot, const int maxLength, int *const outWord, int *const pos) { + return ByteArrayUtils::readStringAndAdvancePosition(dictRoot, maxLength, outWord, pos); +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h index 5f4f240f5..a83ed5a50 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/shortcut_list_reading_utils.h @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" -#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { @@ -28,10 +27,7 @@ class ShortcutListReadingUtils { public: typedef uint8_t ShortcutFlags; - static AK_FORCE_INLINE ShortcutFlags getFlagsAndForwardPointer( - const uint8_t *const dictRoot, int *const pos) { - return ByteArrayUtils::readUint8AndAdvancePosition(dictRoot, pos); - } + static ShortcutFlags getFlagsAndForwardPointer(const uint8_t *const dictRoot, int *const pos); static AK_FORCE_INLINE int getProbabilityFromFlags(const ShortcutFlags flags) { return flags & MASK_ATTRIBUTE_PROBABILITY; @@ -43,12 +39,7 @@ class ShortcutListReadingUtils { // This method returns the size of the shortcut list region excluding the shortcut list size // field at the beginning. - static AK_FORCE_INLINE int getShortcutListSizeAndForwardPointer( - const uint8_t *const dictRoot, int *const pos) { - // readUint16andAdvancePosition() returns an offset *including* the uint16 field itself. - return ByteArrayUtils::readUint16AndAdvancePosition(dictRoot, pos) - - SHORTCUT_LIST_SIZE_FIELD_SIZE; - } + static int getShortcutListSizeAndForwardPointer(const uint8_t *const dictRoot, int *const pos); static AK_FORCE_INLINE int getShortcutListSizeFieldSize() { return SHORTCUT_LIST_SIZE_FIELD_SIZE; @@ -63,11 +54,8 @@ class ShortcutListReadingUtils { return getProbabilityFromFlags(flags) == WHITELIST_SHORTCUT_PROBABILITY; } - static AK_FORCE_INLINE int readShortcutTarget( - const uint8_t *const dictRoot, const int maxLength, int *const outWord, - int *const pos) { - return ByteArrayUtils::readStringAndAdvancePosition(dictRoot, maxLength, outWord, pos); - } + static int readShortcutTarget(const uint8_t *const dictRoot, const int maxLength, + int *const outWord, int *const pos); private: DISALLOW_IMPLICIT_CONSTRUCTORS(ShortcutListReadingUtils); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp index dfdaebd18..0fed275e9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp @@ -76,7 +76,7 @@ bool BufferWithExtendableBuffer::checkAndPrepareWriting(const int pos, const int } } mUsedAdditionalBufferSize += size; - } else if (pos + size >= tailPosition) { + } else if (pos + size > tailPosition) { // The access will beyond the tail of used region. return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.cpp index a84cfb9d5..1833e8832 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.cpp @@ -18,7 +18,8 @@ namespace latinime { -const uint8_t ByteArrayUtils::MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; +const uint8_t ByteArrayUtils::MINIMUM_ONE_BYTE_CHARACTER_VALUE = 0x20; +const uint8_t ByteArrayUtils::MAXIMUM_ONE_BYTE_CHARACTER_VALUE = 0xFF; const uint8_t ByteArrayUtils::CHARACTER_ARRAY_TERMINATOR = 0x1F; } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h index f727ecf8e..0c1576818 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h @@ -135,7 +135,7 @@ class ByteArrayUtils { static AK_FORCE_INLINE int readCodePointAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const uint8_t firstByte = readUint8(buffer, *pos); - if (firstByte < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + if (firstByte < MINIMUM_ONE_BYTE_CHARACTER_VALUE) { if (firstByte == CHARACTER_ARRAY_TERMINATOR) { *pos += 1; return NOT_A_CODE_POINT; @@ -172,6 +172,7 @@ class ByteArrayUtils { int codePoint = readCodePointAndAdvancePosition(buffer, pos); while (NOT_A_CODE_POINT != codePoint && length < maxLength) { codePoint = readCodePointAndAdvancePosition(buffer, pos); + length++; } return length; } @@ -186,7 +187,8 @@ class ByteArrayUtils { const int codePoint = codePoints[i]; if (codePoint == NOT_A_CODE_POINT || codePoint == CHARACTER_ARRAY_TERMINATOR) { break; - } else if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + } else if (codePoint < MINIMUM_ONE_BYTE_CHARACTER_VALUE + || codePoint > MAXIMUM_ONE_BYTE_CHARACTER_VALUE) { // three bytes character. writeUint24AndAdvancePosition(buffer, codePoint, pos); } else { @@ -206,7 +208,8 @@ class ByteArrayUtils { const int codePoint = codePoints[i]; if (codePoint == NOT_A_CODE_POINT || codePoint == CHARACTER_ARRAY_TERMINATOR) { break; - } else if (codePoint < MINIMAL_ONE_BYTE_CHARACTER_VALUE) { + } else if (codePoint < MINIMUM_ONE_BYTE_CHARACTER_VALUE + || codePoint > MAXIMUM_ONE_BYTE_CHARACTER_VALUE) { // three bytes character. byteCount += 3; } else { @@ -224,7 +227,8 @@ class ByteArrayUtils { private: DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils); - static const uint8_t MINIMAL_ONE_BYTE_CHARACTER_VALUE; + static const uint8_t MINIMUM_ONE_BYTE_CHARACTER_VALUE; + static const uint8_t MAXIMUM_ONE_BYTE_CHARACTER_VALUE; static const uint8_t CHARACTER_ARRAY_TERMINATOR; static AK_FORCE_INLINE void writeUint32AndAdvancePosition(uint8_t *const buffer, diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h index 6febd7832..6b69116eb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/mmapped_buffer.h @@ -29,8 +29,8 @@ namespace latinime { class MmappedBuffer { public: - static MmappedBuffer* openBuffer(const char *const path, const int pathLength, - const int bufferOffset, const int bufferSize, const bool isUpdatable) { + static MmappedBuffer* openBuffer(const char *const path, const int bufferOffset, + const int bufferSize, const bool isUpdatable) { const int openMode = isUpdatable ? O_RDWR : O_RDONLY; const int mmapFd = open(path, openMode); if (mmapFd < 0) { diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index e2e148903..4d231cde7 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; +import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; @@ -28,8 +29,10 @@ import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Locale; +import java.util.Random; @LargeTest public class BinaryDictionaryTests extends AndroidTestCase { @@ -84,4 +87,216 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.isValidDictionary()); binaryDictionary.close(); } + + public void testAddUnigramWord() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int probability = 100; + binaryDictionary.addUnigramWord("aaa", probability); + // Reallocate and create. + binaryDictionary.addUnigramWord("aab", probability); + // Insert into children. + binaryDictionary.addUnigramWord("aac", probability); + // Make terminal. + binaryDictionary.addUnigramWord("aa", probability); + // Create children. + binaryDictionary.addUnigramWord("aaaa", probability); + // Reallocate and make termianl. + binaryDictionary.addUnigramWord("a", probability); + + final int updatedProbability = 200; + // Update. + binaryDictionary.addUnigramWord("aaa", updatedProbability); + + assertEquals(probability, binaryDictionary.getFrequency("aab")); + assertEquals(probability, binaryDictionary.getFrequency("aac")); + assertEquals(probability, binaryDictionary.getFrequency("aa")); + assertEquals(probability, binaryDictionary.getFrequency("aaaa")); + assertEquals(probability, binaryDictionary.getFrequency("a")); + assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa")); + + dictFile.delete(); + } + + public void testRandomlyAddUnigramWord() { + final int wordCount = 1000; + final int codePointSetSize = 50; + final int seed = 123456789; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final HashMap<String, Integer> probabilityMap = new HashMap<String, Integer>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + probabilityMap.put(word, random.nextInt() & 0xFF); + } + for (String word : probabilityMap.keySet()) { + binaryDictionary.addUnigramWord(word, probabilityMap.get(word)); + } + for (String word : probabilityMap.keySet()) { + assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word)); + } + dictFile.delete(); + } + + public void testAddBigramWords() { + // TODO: Add a test to check the frequency of the bigram score which uses current value + // calculated in the native code + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int bigramProbability = 10; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + + assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); + + assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + + dictFile.delete(); + } + + public void testRandomlyAddBigramWords() { + // TODO: Add a test to check the frequency of the bigram score which uses current value + // calculated in the native code + final int wordCount = 100; + final int bigramCount = 1000; + final int codePointSetSize = 50; + final int seed = 11111; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final int unigramProbability = 100; + final int bigramProbability = 10; + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + final boolean[][] bigramRelations = new boolean[wordCount][wordCount]; + for (int i = 0; i < bigramCount; i++) { + final int word0Index = random.nextInt(wordCount); + final int word1Index = random.nextInt(wordCount); + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + + bigramRelations[word0Index][word1Index] = true; + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + + for (int i = 0; i < words.size(); i++) { + for (int j = 0; j < words.size(); j++) { + assertEquals(bigramRelations[i][j], + binaryDictionary.isValidBigram(words.get(i), words.get(j))); + } + } + + dictFile.delete(); + } + + public void testRemoveBigramWords() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int bigramProbability = 10; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + + assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); + assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); + + binaryDictionary.removeBigramWords("aaa", "abb"); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb")); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); + + + binaryDictionary.removeBigramWords("aaa", "bcc"); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc")); + binaryDictionary.removeBigramWords("abb", "aaa"); + assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa")); + binaryDictionary.removeBigramWords("abb", "bcc"); + assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc")); + + binaryDictionary.removeBigramWords("aaa", "abb"); + // Test remove non-existing bigram operation. + binaryDictionary.removeBigramWords("aaa", "abb"); + binaryDictionary.removeBigramWords("bcc", "aaa"); + + dictFile.delete(); + } } diff --git a/tests/src/com/android/inputmethod/latin/InputPointersTests.java b/tests/src/com/android/inputmethod/latin/InputPointersTests.java index f0b6acc75..5095f9606 100644 --- a/tests/src/com/android/inputmethod/latin/InputPointersTests.java +++ b/tests/src/com/android/inputmethod/latin/InputPointersTests.java @@ -244,4 +244,20 @@ public class InputPointersTests extends AndroidTestCase { expecteds[i + expectedPos], actuals[i + actualPos]); } } + + public void testShift() { + final InputPointers src = new InputPointers(DEFAULT_CAPACITY); + final int limit = 100; + final int shiftAmount = 20; + for (int i = 0; i < limit; i++) { + src.addPointer(i, i * 2, i * 3, i * 4); + } + src.shift(shiftAmount); + for (int i = 0; i < limit - shiftAmount; ++i) { + assertEquals("xCoordinates at " + i, i + shiftAmount, src.getXCoordinates()[i]); + assertEquals("yCoordinates at " + i, (i + shiftAmount) * 2, src.getYCoordinates()[i]); + assertEquals("pointerIds at " + i, (i + shiftAmount) * 3, src.getPointerIds()[i]); + assertEquals("times at " + i, (i + shiftAmount) * 4, src.getTimes()[i]); + } + } } diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java index 2603b35f5..234bb1b31 100644 --- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java +++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java @@ -259,7 +259,8 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { protected void pickSuggestionManually(final int index, final String suggestion) { mLatinIME.pickSuggestionManually(index, new SuggestedWordInfo(suggestion, 1, SuggestedWordInfo.KIND_CORRECTION, null /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } // Helper to avoid writing the try{}catch block each time diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java index a5f3685da..4cf83339a 100644 --- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java +++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java @@ -35,11 +35,13 @@ public class SuggestedWordsTests extends AndroidTestCase { final ArrayList<SuggestedWordInfo> list = CollectionUtils.newArrayList(); list.add(new SuggestedWordInfo(TYPED_WORD, TYPED_WORD_FREQ, SuggestedWordInfo.KIND_TYPED, null /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); for (int i = 0; i < NUMBER_OF_ADDED_SUGGESTIONS; ++i) { list.add(new SuggestedWordInfo("" + i, 1, SuggestedWordInfo.KIND_CORRECTION, null /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } final SuggestedWords words = new SuggestedWords( diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 72ec5a302..8bc0095a5 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -25,6 +25,7 @@ import android.util.SparseArray; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -86,7 +87,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { Log.e(TAG, "Testing dictionary: seed is " + seed); final Random random = new Random(seed); sWords.clear(); - final int[] codePointSet = generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, random); + final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, + random); generateWords(maxUnigrams, random, codePointSet); for (int i = 0; i < sWords.size(); ++i) { @@ -112,63 +114,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } - private int[] generateCodePointSet(final int codePointSetSize, final Random random) { - final int[] codePointSet = new int[codePointSetSize]; - for (int i = codePointSet.length - 1; i >= 0; ) { - final int r = Math.abs(random.nextInt()); - if (r < 0) continue; - // Don't insert 0~0x20, but insert any other code point. - // Code points are in the range 0~0x10FFFF. - final int candidateCodePoint = 0x20 + r % (Character.MAX_CODE_POINT - 0x20); - // Code points between MIN_ and MAX_SURROGATE are not valid on their own. - if (candidateCodePoint >= Character.MIN_SURROGATE - && candidateCodePoint <= Character.MAX_SURROGATE) continue; - codePointSet[i] = candidateCodePoint; - --i; - } - return codePointSet; - } - - // Utilities for test - - /** - * Makes new DictDecoder according to BUFFER_TYPE. - */ - private Ver3DictDecoder getDictDecoder(final File file, final int bufferType) { - if (bufferType == USE_BYTE_BUFFER) { - return new Ver3DictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER); - } else if (bufferType == USE_BYTE_ARRAY) { - return new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); - } - return null; - } - - /** - * Generates a random word. - */ - private String generateWord(final Random random, final int[] codePointSet) { - StringBuilder builder = new StringBuilder(); - // 8 * 4 = 32 chars max, but we do it the following way so as to bias the random toward - // longer words. This should be closer to natural language, and more importantly, it will - // exercise the algorithms in dicttool much more. - final int count = 1 + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5) - + (Math.abs(random.nextInt()) % 5); - while (builder.length() < count) { - builder.appendCodePoint(codePointSet[Math.abs(random.nextInt()) % codePointSet.length]); - } - return builder.toString(); - } - private void generateWords(final int number, final Random random, final int[] codePointSet) { final Set<String> wordSet = CollectionUtils.newHashSet(); while (wordSet.size() < number) { - wordSet.add(generateWord(random, codePointSet)); + wordSet.add(CodePointUtils.generateWord(random, codePointSet)); } sWords.addAll(wordSet); } @@ -276,6 +225,27 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; } + private DictionaryOptions getDictionaryOptions(final String id, final String version) { + final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), + false, false); + options.mAttributes.put("version", version); + options.mAttributes.put("dictionary", id); + return options; + } + + private File setUpDictionaryFile(final String name, final String version) { + File file = null; + try { + file = new File(getContext().getCacheDir(), name + "." + version + + TEST_DICT_FILE_EXTENSION); + file.createNewFile(); + } catch (IOException e) { + // do nothing + } + assertTrue("Failed to create the dictionary file.", file.exists()); + return file; + } + // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List<String> words, @@ -285,11 +255,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { FusionDictionary dict = null; try { - final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType); - dictDecoder.openDictBuffer(); - assertNotNull(dictDecoder.getDictBuffer()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); - dict = dictDecoder.readDictionaryBinary(null); + dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException while reading dictionary", e); @@ -306,17 +274,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcuts, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { - File file = null; - try { - file = File.createTempFile("runReadAndWrite", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - assertNotNull(file); + + final String dictName = "runReadAndWrite"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); checkDictionary(dict, words, bigrams, shortcuts); @@ -443,9 +407,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final Ver3DictDecoder dictDecoder = getDictDecoder(file, bufferType); - dictDecoder.openDictBuffer(); - assertNotNull("Can't get buffer.", dictDecoder.getDictBuffer()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -470,19 +432,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private String runReadUnigramsAndBigramsBinary(final ArrayList<String> words, final SparseArray<List<Integer>> bigrams, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { - File file = null; - try { - file = File.createTempFile("runReadUnigrams", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - Log.e(TAG, "IOException", e); - } - assertNotNull(file); + final String dictName = "runReadUnigrams"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); + getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); addBigrams(dict, words, bigrams); @@ -531,9 +487,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } // Tests for getTerminalPosition - private String getWordFromBinary(final Ver3DictDecoder dictDecoder, final int address) { - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); - if (dictBuffer.position() != 0) dictBuffer.position(0); + private String getWordFromBinary(final DictDecoder dictDecoder, final int address) { + if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0); FileHeader fileHeader = null; try { @@ -548,7 +503,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { address, fileHeader.mFormatOptions).mWord; } - private long runGetTerminalPosition(final Ver3DictDecoder dictDecoder, final String word, + private long runGetTerminalPosition(final DictDecoder dictDecoder, final String word, int index, boolean contained) { final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; long diff = -1; @@ -569,29 +524,23 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testGetTerminalPosition() { - File file = null; - try { - file = File.createTempFile("testGetTerminalPosition", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - // do nothing - } - assertNotNull(file); + final String dictName = "testGetTerminalPosition"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions( - new HashMap<String, String>(), false, false)); + getDictionaryOptions(dictName, dictVersion)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer()); + assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer()); try { // too long word @@ -617,23 +566,19 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Test a word that isn't contained within the dictionary. final Random random = new Random((int)System.currentTimeMillis()); - final int[] codePointSet = generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, random); + final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, + random); for (int i = 0; i < 1000; ++i) { - final String word = generateWord(random, codePointSet); + final String word = CodePointUtils.generateWord(random, codePointSet); if (sWords.indexOf(word) != -1) continue; runGetTerminalPosition(dictDecoder, word, i, false); } } public void testDeleteWord() { - File file = null; - try { - file = File.createTempFile("testDeleteWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - // do nothing - } - assertNotNull(file); + final String dictName = "testDeleteWord"; + final String dictVersion = Long.toString(System.currentTimeMillis()); + final File file = setUpDictionaryFile(dictName, dictVersion); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( @@ -648,7 +593,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertNotNull("Can't get the buffer", dictDecoder.getDictBuffer()); + assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer()); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index 8e0c6dfe2..a83749499 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -140,7 +140,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int position = FormatSpec.NOT_VALID_WORD; try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + DictDecoder.USE_READONLY_BYTEBUFFER); position = dictDecoder.getTerminalPosition(word); } catch (IOException e) { } catch (UnsupportedFormatException e) { @@ -149,7 +150,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } /** - * Find a word using the Ver3DictDecoder. + * Find a word using the DictDecoder. * * @param dictDecoder the dict decoder * @param word the word searched @@ -157,21 +158,20 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { * @throws IOException * @throws UnsupportedFormatException */ - private static PtNodeInfo findWordByBinaryDictReader(final Ver3DictDecoder dictDecoder, + private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = dictDecoder.getTerminalPosition(word); - final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); if (position != FormatSpec.NOT_VALID_WORD) { - dictBuffer.position(0); + dictDecoder.setPosition(0); final FileHeader header = dictDecoder.readHeader(); - dictBuffer.position(position); + dictDecoder.setPosition(position); return dictDecoder.readPtNode(position, header.mFormatOptions); } return null; } private PtNodeInfo findWordFromFile(final File file, final String word) { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); PtNodeInfo info = null; try { dictDecoder.openDictBuffer(); @@ -234,7 +234,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private void checkReverseLookup(final File file, final String word, final int position) { try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); final FileHeader fileHeader = dictDecoder.readHeader(); assertEquals(word, BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mHeaderSize, diff --git a/tests/src/com/android/inputmethod/latin/makedict/CodePointUtils.java b/tests/src/com/android/inputmethod/latin/makedict/CodePointUtils.java new file mode 100644 index 000000000..36b958af8 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/CodePointUtils.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import java.util.Random; + +// Utility methods related with code points used for tests. +public class CodePointUtils { + private CodePointUtils() { + // This utility class is not publicly instantiable. + } + + public static int[] generateCodePointSet(final int codePointSetSize, final Random random) { + final int[] codePointSet = new int[codePointSetSize]; + for (int i = codePointSet.length - 1; i >= 0; ) { + final int r = Math.abs(random.nextInt()); + if (r < 0) continue; + // Don't insert 0~0x20, but insert any other code point. + // Code points are in the range 0~0x10FFFF. + final int candidateCodePoint = 0x20 + r % (Character.MAX_CODE_POINT - 0x20); + // Code points between MIN_ and MAX_SURROGATE are not valid on their own. + if (candidateCodePoint >= Character.MIN_SURROGATE + && candidateCodePoint <= Character.MAX_SURROGATE) continue; + codePointSet[i] = candidateCodePoint; + --i; + } + return codePointSet; + } + + /** + * Generates a random word. + */ + public static String generateWord(final Random random, final int[] codePointSet) { + StringBuilder builder = new StringBuilder(); + // 8 * 4 = 32 chars max, but we do it the following way so as to bias the random toward + // longer words. This should be closer to natural language, and more importantly, it will + // exercise the algorithms in dicttool much more. + final int count = 1 + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5) + + (Math.abs(random.nextInt()) % 5); + while (builder.length() < count) { + builder.appendCodePoint(codePointSet[Math.abs(random.nextInt()) % codePointSet.length]); + } + return builder.toString(); + } +} diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index d15e88bdb..bf44a1424 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -46,6 +46,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { }; private static final int MIN_USER_HISTORY_DICTIONARY_FILE_SIZE = 1000; + private static final int WAIT_TERMINATING_IN_MILLISECONDS = 100; @Override public void setUp() { @@ -122,8 +123,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { true /* checksContents */); } finally { try { + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); Log.d(TAG, "waiting for writing ..."); - Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } @@ -146,11 +153,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final int numberOfWordsInsertedForEachLanguageSwitch = 100; final File dictFiles[] = new File[numberOfLanguages]; + final String testFilenameSuffixes[] = new String[numberOfLanguages]; try { final Random random = new Random(123456); // Create filename suffixes for this test. - String testFilenameSuffixes[] = new String[numberOfLanguages]; for (int i = 0; i < numberOfLanguages; i++) { testFilenameSuffixes[i] = "testSwitchingLanguages" + i; final String fileName = UserHistoryPredictionDictionary.NAME + "." + @@ -174,7 +181,15 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } finally { try { Log.d(TAG, "waiting for writing ..."); - Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); + for (int i = 0; i < numberOfLanguages; i++) { + final UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffixes[i], mPrefs); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: " + e); } diff --git a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java index cfff61ef8..cad80d5ce 100644 --- a/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/ResizableIntArrayTests.java @@ -340,4 +340,18 @@ public class ResizableIntArrayTests extends AndroidTestCase { expecteds[i + expectedPos], actuals[i + actualPos]); } } + + public void testShift() { + final ResizableIntArray src = new ResizableIntArray(DEFAULT_CAPACITY); + final int limit = DEFAULT_CAPACITY * 10; + final int shiftAmount = 20; + for (int i = 0; i < limit; ++i) { + src.add(i, i); + assertEquals("length after add at " + i, i + 1, src.getLength()); + } + src.shift(shiftAmount); + for (int i = 0; i < limit - shiftAmount; ++i) { + assertEquals("value at " + i, i + shiftAmount, src.get(i)); + } + } } diff --git a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java index 72b9478d4..3eabe2b3c 100644 --- a/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtilsTests.java @@ -143,7 +143,7 @@ public class UserHistoryDictIOUtilsTests extends AndroidTestCase } private void readDictFromFile(final File file, final OnAddWordListener listener) { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (FileNotFoundException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index 465b17766..fa80385fc 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -18,9 +18,9 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import org.xml.sax.SAXException; @@ -185,14 +185,14 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); } - return dictDecoder.readDictionaryBinary(null); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } } } catch (IOException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index 709b8196c..5302e976e 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -267,8 +267,8 @@ public class DictionaryMaker { private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { final File file = new File(binaryFilename); - final DictDecoder dictDecoder = new Ver3DictDecoder(file); - return dictDecoder.readDictionaryBinary(null); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } /** diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 47e220617..1eff497c1 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -18,12 +18,12 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.DictDecoder; import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import junit.framework.TestCase; @@ -69,9 +69,10 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); - final DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile); final FusionDictionary resultDict = dictDecoder.readDictionaryBinary( - null /* dict : an optional dictionary to add words to, or null */); + null /* dict : an optional dictionary to add words to, or null */, + false /* deleteDictIfBroken */); assertEquals("Dictionary can't be read back correctly", FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(), TEST_FREQ); diff --git a/tools/make-keyboard-text/res/values-lo/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-lo/donottranslate-more-keys.xml new file mode 100644 index 000000000..1d8ffa8cf --- /dev/null +++ b/tools/make-keyboard-text/res/values-lo/donottranslate-more-keys.xml @@ -0,0 +1,28 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Label for "switch to alphabetic" key. + U+0E81: "ກ" LAO LETTER KO + U+0E82: "ຂ" LAO LETTER KHO SUNG + U+0E84: "ຄ" LAO LETTER KHO TAM --> + <string name="label_to_alpha_key">ກຂຄ</string> + <!-- U+20AD: "₭" KIP SIGN --> + <string name="keylabel_for_currency">₭</string> +</resources> diff --git a/tools/make-keyboard-text/res/values-ne/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-ne/donottranslate-more-keys.xml new file mode 100644 index 000000000..9205e5309 --- /dev/null +++ b/tools/make-keyboard-text/res/values-ne/donottranslate-more-keys.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Label for "switch to alphabetic" key. + U+0915: "क" DEVANAGARI LETTER KA + U+0916: "ख" DEVANAGARI LETTER KHA + U+0917: "ग" DEVANAGARI LETTER GA --> + <string name="label_to_alpha_key">कखग</string> + <!-- U+0967: "१" DEVANAGARI DIGIT ONE --> + <string name="keylabel_for_symbols_1">१</string> + <!-- U+0968: "२" DEVANAGARI DIGIT TWO --> + <string name="keylabel_for_symbols_2">२</string> + <!-- U+0969: "३" DEVANAGARI DIGIT THREE --> + <string name="keylabel_for_symbols_3">३</string> + <!-- U+096A: "४" DEVANAGARI DIGIT FOUR --> + <string name="keylabel_for_symbols_4">४</string> + <!-- U+096B: "५" DEVANAGARI DIGIT FIVE --> + <string name="keylabel_for_symbols_5">५</string> + <!-- U+096C: "६" DEVANAGARI DIGIT SIX --> + <string name="keylabel_for_symbols_6">६</string> + <!-- U+096D: "७" DEVANAGARI DIGIT SEVEN --> + <string name="keylabel_for_symbols_7">७</string> + <!-- U+096E: "८" DEVANAGARI DIGIT EIGHT --> + <string name="keylabel_for_symbols_8">८</string> + <!-- U+096F: "९" DEVANAGARI DIGIT NINE --> + <string name="keylabel_for_symbols_9">९</string> + <!-- U+0966: "०" DEVANAGARI DIGIT ZERO --> + <string name="keylabel_for_symbols_0">०</string> + <!-- Label for "switch to symbols" key. --> + <string name="label_to_symbol_key">\?१२३</string> + <!-- Label for "switch to symbols with microphone" key. This string shouldn't include the "mic" + part because it'll be appended by the code. --> + <string name="label_to_symbol_with_microphone_key">१२३</string> + <string name="additional_more_keys_for_symbols_1">1</string> + <string name="additional_more_keys_for_symbols_2">2</string> + <string name="additional_more_keys_for_symbols_3">3</string> + <string name="additional_more_keys_for_symbols_4">4</string> + <string name="additional_more_keys_for_symbols_5">5</string> + <string name="additional_more_keys_for_symbols_6">6</string> + <string name="additional_more_keys_for_symbols_7">7</string> + <string name="additional_more_keys_for_symbols_8">8</string> + <string name="additional_more_keys_for_symbols_9">9</string> + <string name="additional_more_keys_for_symbols_0">0</string> + <!-- U+0930/U+0941/U+002E "रु." NEPALESE RUPEE SIGN --> + <string name="keylabel_for_currency">रु.</string> +</resources> diff --git a/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml index a4c2f126d..cc09f7fe5 100644 --- a/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values/donottranslate-more-keys.xml @@ -190,7 +190,7 @@ <!-- Label for "switch to more symbol" modifier key. Must be short to fit on key! --> <string name="label_to_more_symbol_key">= \\ <</string> <!-- Label for "switch to more symbol" modifier key on tablets. Must be short to fit on key! --> - <string name="label_to_more_symbol_for_tablet_key">~ [ {</string> + <string name="label_to_more_symbol_for_tablet_key">~ [ <</string> <!-- Label for "Tab" key. Must be short to fit on key! --> <string name="label_tab_key">Tab</string> <!-- Label for "switch to phone numeric" key. Must be short to fit on key! --> |