diff options
176 files changed, 2738 insertions, 741 deletions
diff --git a/dictionaries/cs_wordlist.combined.gz b/dictionaries/cs_wordlist.combined.gz Binary files differindex d69ef6451..7829d6573 100644 --- a/dictionaries/cs_wordlist.combined.gz +++ b/dictionaries/cs_wordlist.combined.gz diff --git a/dictionaries/da_wordlist.combined.gz b/dictionaries/da_wordlist.combined.gz Binary files differindex 919d28e1d..e7140195b 100644 --- a/dictionaries/da_wordlist.combined.gz +++ b/dictionaries/da_wordlist.combined.gz diff --git a/dictionaries/de_wordlist.combined.gz b/dictionaries/de_wordlist.combined.gz Binary files differindex f5cce9d5a..6a4bd445c 100644 --- a/dictionaries/de_wordlist.combined.gz +++ b/dictionaries/de_wordlist.combined.gz diff --git a/dictionaries/en_GB_wordlist.combined.gz b/dictionaries/en_GB_wordlist.combined.gz Binary files differindex d28ef485b..839f3efca 100644 --- a/dictionaries/en_GB_wordlist.combined.gz +++ b/dictionaries/en_GB_wordlist.combined.gz diff --git a/dictionaries/en_US_wordlist.combined.gz b/dictionaries/en_US_wordlist.combined.gz Binary files differindex b656f880d..5595c75c0 100644 --- a/dictionaries/en_US_wordlist.combined.gz +++ b/dictionaries/en_US_wordlist.combined.gz diff --git a/dictionaries/en_wordlist.combined.gz b/dictionaries/en_wordlist.combined.gz Binary files differindex 8aa40e944..69c39d5d9 100644 --- a/dictionaries/en_wordlist.combined.gz +++ b/dictionaries/en_wordlist.combined.gz diff --git a/dictionaries/es_wordlist.combined.gz b/dictionaries/es_wordlist.combined.gz Binary files differindex 53b86076d..0a48b6d11 100644 --- a/dictionaries/es_wordlist.combined.gz +++ b/dictionaries/es_wordlist.combined.gz diff --git a/dictionaries/fi_wordlist.combined.gz b/dictionaries/fi_wordlist.combined.gz Binary files differindex 272011659..eefbfe51a 100644 --- a/dictionaries/fi_wordlist.combined.gz +++ b/dictionaries/fi_wordlist.combined.gz diff --git a/dictionaries/fr_wordlist.combined.gz b/dictionaries/fr_wordlist.combined.gz Binary files differindex 1d988d6fe..1a1832079 100644 --- a/dictionaries/fr_wordlist.combined.gz +++ b/dictionaries/fr_wordlist.combined.gz diff --git a/dictionaries/hr_wordlist.combined.gz b/dictionaries/hr_wordlist.combined.gz Binary files differindex 7694a2ade..864f67651 100644 --- a/dictionaries/hr_wordlist.combined.gz +++ b/dictionaries/hr_wordlist.combined.gz diff --git a/dictionaries/it_wordlist.combined.gz b/dictionaries/it_wordlist.combined.gz Binary files differindex 3b84cd741..dfb175259 100644 --- a/dictionaries/it_wordlist.combined.gz +++ b/dictionaries/it_wordlist.combined.gz diff --git a/dictionaries/lt_wordlist.combined.gz b/dictionaries/lt_wordlist.combined.gz Binary files differindex 316a5af01..029722d95 100644 --- a/dictionaries/lt_wordlist.combined.gz +++ b/dictionaries/lt_wordlist.combined.gz diff --git a/dictionaries/lv_wordlist.combined.gz b/dictionaries/lv_wordlist.combined.gz Binary files differindex b036ac21a..41e1c28bd 100644 --- a/dictionaries/lv_wordlist.combined.gz +++ b/dictionaries/lv_wordlist.combined.gz diff --git a/dictionaries/nb_wordlist.combined.gz b/dictionaries/nb_wordlist.combined.gz Binary files differindex b6e0d42d3..b699912b7 100644 --- a/dictionaries/nb_wordlist.combined.gz +++ b/dictionaries/nb_wordlist.combined.gz diff --git a/dictionaries/nl_wordlist.combined.gz b/dictionaries/nl_wordlist.combined.gz Binary files differindex 48ab0f473..89c238830 100644 --- a/dictionaries/nl_wordlist.combined.gz +++ b/dictionaries/nl_wordlist.combined.gz diff --git a/dictionaries/pl_wordlist.combined.gz b/dictionaries/pl_wordlist.combined.gz Binary files differindex bf02298c1..2b53f69cb 100644 --- a/dictionaries/pl_wordlist.combined.gz +++ b/dictionaries/pl_wordlist.combined.gz diff --git a/dictionaries/pt_BR_wordlist.combined.gz b/dictionaries/pt_BR_wordlist.combined.gz Binary files differindex 221ea7508..2d22447e4 100644 --- a/dictionaries/pt_BR_wordlist.combined.gz +++ b/dictionaries/pt_BR_wordlist.combined.gz diff --git a/dictionaries/pt_PT_wordlist.combined.gz b/dictionaries/pt_PT_wordlist.combined.gz Binary files differindex 6a041d9cf..1504165d0 100644 --- a/dictionaries/pt_PT_wordlist.combined.gz +++ b/dictionaries/pt_PT_wordlist.combined.gz diff --git a/dictionaries/sl_wordlist.combined.gz b/dictionaries/sl_wordlist.combined.gz Binary files differindex 41a576bde..55e1bb1c8 100644 --- a/dictionaries/sl_wordlist.combined.gz +++ b/dictionaries/sl_wordlist.combined.gz diff --git a/dictionaries/sr_wordlist.combined.gz b/dictionaries/sr_wordlist.combined.gz Binary files differindex dec6ae89e..8488a08b4 100644 --- a/dictionaries/sr_wordlist.combined.gz +++ b/dictionaries/sr_wordlist.combined.gz diff --git a/dictionaries/sv_wordlist.combined.gz b/dictionaries/sv_wordlist.combined.gz Binary files differindex 0471772e7..63425206e 100644 --- a/dictionaries/sv_wordlist.combined.gz +++ b/dictionaries/sv_wordlist.combined.gz diff --git a/dictionaries/tr_wordlist.combined.gz b/dictionaries/tr_wordlist.combined.gz Binary files differindex fae79ca21..0251778c7 100644 --- a/dictionaries/tr_wordlist.combined.gz +++ b/dictionaries/tr_wordlist.combined.gz diff --git a/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png Binary files differindex 87211a502..fa2cb8542 100644 --- a/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png +++ b/java/res/drawable-hdpi/btn_keyboard_key_dark_active_holo.9.png diff --git a/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png Binary files differindex c1e16a651..8a6336a57 100644 --- a/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png +++ b/java/res/drawable-hdpi/sym_keyboard_voice_holo_dark.png diff --git a/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png Binary files differindex 26d068490..edf1379ab 100644 --- a/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png +++ b/java/res/drawable-hdpi/sym_keyboard_voice_off_holo_dark.png diff --git a/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png Binary files differindex f98653ea1..8e9a34957 100644 --- a/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png +++ b/java/res/drawable-mdpi/btn_keyboard_key_dark_active_holo.9.png diff --git a/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png Binary files differindex 16be37d05..0795fcc9b 100644 --- a/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png +++ b/java/res/drawable-mdpi/sym_keyboard_voice_holo_dark.png diff --git a/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png Binary files differindex 95d718a46..f76da5797 100644 --- a/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png +++ b/java/res/drawable-mdpi/sym_keyboard_voice_off_holo_dark.png diff --git a/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png Binary files differindex 738316d66..a2f6ac0e2 100644 --- a/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png +++ b/java/res/drawable-xhdpi/btn_keyboard_key_dark_active_holo.9.png diff --git a/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png Binary files differindex 944a8524d..b2bb9b803 100644 --- a/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png +++ b/java/res/drawable-xhdpi/sym_keyboard_voice_holo_dark.png diff --git a/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png Binary files differindex 2016caf40..23e75bfe7 100644 --- a/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png +++ b/java/res/drawable-xhdpi/sym_keyboard_voice_off_holo_dark.png diff --git a/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png b/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png Binary files differindex b35c29fe6..17f0a7a58 100644 --- a/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png +++ b/java/res/drawable-xxhdpi/btn_keyboard_key_dark_active_holo.9.png diff --git a/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png b/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png Binary files differindex 6809f0711..f04cadf6f 100644 --- a/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png +++ b/java/res/drawable-xxhdpi/sym_keyboard_voice_holo_dark.png diff --git a/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png b/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png Binary files differindex 6bd506a11..e74d523bc 100644 --- a/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png +++ b/java/res/drawable-xxhdpi/sym_keyboard_voice_off_holo_dark.png diff --git a/java/res/raw/main_de.dict b/java/res/raw/main_de.dict Binary files differindex 5d35e64a2..69796bbaa 100644 --- a/java/res/raw/main_de.dict +++ b/java/res/raw/main_de.dict diff --git a/java/res/raw/main_en.dict b/java/res/raw/main_en.dict Binary files differindex 6564d47fa..bef6b1005 100644 --- a/java/res/raw/main_en.dict +++ b/java/res/raw/main_en.dict diff --git a/java/res/raw/main_es.dict b/java/res/raw/main_es.dict Binary files differindex f5906c2e2..261ab8c87 100644 --- a/java/res/raw/main_es.dict +++ b/java/res/raw/main_es.dict diff --git a/java/res/raw/main_fr.dict b/java/res/raw/main_fr.dict Binary files differindex 10adad092..18f529887 100644 --- a/java/res/raw/main_fr.dict +++ b/java/res/raw/main_fr.dict diff --git a/java/res/raw/main_it.dict b/java/res/raw/main_it.dict Binary files differindex 523f645c7..e161c2475 100644 --- a/java/res/raw/main_it.dict +++ b/java/res/raw/main_it.dict diff --git a/java/res/raw/main_pt_br.dict b/java/res/raw/main_pt_br.dict Binary files differindex f9ae9b561..21bbe7c67 100644 --- a/java/res/raw/main_pt_br.dict +++ b/java/res/raw/main_pt_br.dict diff --git a/java/res/values-af/strings.xml b/java/res/values-af/strings.xml index 01a57e7e0..a3c95b290 100644 --- a/java/res/values-af/strings.xml +++ b/java/res/values-af/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Koppel \'n kopstuk om te hoor hoe wagwoordsleutels hardop gesê word."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Huidige teks is %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Geen teks ingevoer nie"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Sleutelkode %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift aan (tik om te deaktiveer)"</string> diff --git a/java/res/values-am/strings.xml b/java/res/values-am/strings.xml index 774f2c552..89e37b274 100644 --- a/java/res/values-am/strings.xml +++ b/java/res/values-am/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"የይለፍቃል ቁልፎች ጮክ በለው ሲነገሩ ለመስማት የጆሮ ማዳመጫ ሰካ::"</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"የአሁኑ ፅሁፍ %s ነው"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"ምንም ፅሁፍ አልገባም"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"የቁልፍ ኮድ%d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"ቀይር"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"ቅያር በርቷል (ለማሰናክል ንካ)"</string> @@ -211,7 +215,7 @@ <string name="message_updating" msgid="4457761393932375219">"ዝማኔዎችን በመፈለግ ላይ"</string> <string name="message_loading" msgid="8689096636874758814">"በመጫን ላይ..."</string> <string name="main_dict_description" msgid="3072821352793492143">"ዋና መዝገበ-ቃላት"</string> - <string name="cancel" msgid="6830980399865683324">"ይቅር"</string> + <string name="cancel" msgid="6830980399865683324">"ሰርዝ"</string> <string name="install_dict" msgid="180852772562189365">"ጫን"</string> <string name="cancel_download_dict" msgid="7843340278507019303">"ሰርዝ"</string> <string name="delete_dict" msgid="756853268088330054">"ሰርዝ"</string> diff --git a/java/res/values-ar/strings.xml b/java/res/values-ar/strings.xml index 7e39b597e..9c3ff51c8 100644 --- a/java/res/values-ar/strings.xml +++ b/java/res/values-ar/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"يمكنك توصيل سماعة رأس لسماع مفاتيح كلمة المرور منطوقة بصوت عالٍ."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"النص الحالي هو %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"لم يتم إدخال نص"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"رمز المفتاح %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"العالي"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift يعمل (انقر للتعطيل)"</string> diff --git a/java/res/values-be/strings.xml b/java/res/values-be/strings.xml index db40c8a72..9e591dea9 100644 --- a/java/res/values-be/strings.xml +++ b/java/res/values-be/strings.xml @@ -87,6 +87,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Каб праслухаць паролi, падключыце гарнiтуру."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Бягучы тэкст %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Тэкст не ўведзены"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Клавішны код %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Зрух"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift уключаны (націснiце, каб адключыць)"</string> diff --git a/java/res/values-bg/strings.xml b/java/res/values-bg/strings.xml index d3879cca5..c21b53490 100644 --- a/java/res/values-bg/strings.xml +++ b/java/res/values-bg/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Включете слушалки, за да чуете клавишите за паролата на висок глас."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Текущият текст е %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Няма въведен текст"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Код на клавишa %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"„Shift“ е включен (докоснете за деактивиране)"</string> diff --git a/java/res/values-ca/strings.xml b/java/res/values-ca/strings.xml index 03e2f3fdf..d791452d4 100644 --- a/java/res/values-ca/strings.xml +++ b/java/res/values-ca/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Connecta un auricular per escoltar les claus de la contrasenya en veu alta."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"El text actual és %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"No s\'ha introduït cap text"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Clau de codi %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Maj"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Maj activat (pica per desactivar)"</string> diff --git a/java/res/values-cs/strings.xml b/java/res/values-cs/strings.xml index a707f26a7..2cd159d0f 100644 --- a/java/res/values-cs/strings.xml +++ b/java/res/values-cs/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Chcete-li slyšet, které klávesy jste při zadávání hesla stiskli, připojte sluchátka."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Aktuální text je %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Není zadán žádný text"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Kód klávesy %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Klávesa Shift je zapnutá (vypnete ji klepnutím)."</string> diff --git a/java/res/values-da/strings.xml b/java/res/values-da/strings.xml index 187689559..6c9d9d786 100644 --- a/java/res/values-da/strings.xml +++ b/java/res/values-da/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Tilslut et headset for at høre indtastningen blive læst højt ved angivelse af adgangskode."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Nuværende tekst er %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Der er ingen indtastet tekst"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Tastekode %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift-tast"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Skift er slået til (tryk for at deaktivere)"</string> diff --git a/java/res/values-de/strings.xml b/java/res/values-de/strings.xml index 5f3e619ff..1a4e38542 100644 --- a/java/res/values-de/strings.xml +++ b/java/res/values-de/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Schließen Sie ein Headset an, um das Passwort gesprochen zu hören."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Aktueller Text lautet %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Kein Text eingegeben"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Tastencode %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Umschalttaste"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Umschalttaste aktiviert (zum Deaktivieren berühren)"</string> diff --git a/java/res/values-el/strings.xml b/java/res/values-el/strings.xml index 27eb865ea..c2ee2d39b 100644 --- a/java/res/values-el/strings.xml +++ b/java/res/values-el/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Συνδέστε ένα σετ ακουστικών για να ακούσετε τα πλήκτρα του κωδικού πρόσβασης να εκφωνούνται δυνατά."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Το τρέχον κείμενο είναι %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Δεν υπάρχει κείμενο"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Κωδικός πλήκτρου %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Το Shift είναι ενεργοποιημένο (πατήστε για απενεργοποίηση)"</string> diff --git a/java/res/values-en-rGB/strings.xml b/java/res/values-en-rGB/strings.xml index 6337aa019..0586bcca4 100644 --- a/java/res/values-en-rGB/strings.xml +++ b/java/res/values-en-rGB/strings.xml @@ -77,13 +77,17 @@ <string name="label_go_key" msgid="1635148082137219148">"Go"</string> <string name="label_next_key" msgid="362972844525672568">"Next"</string> <string name="label_previous_key" msgid="1211868118071386787">"Prev"</string> - <string name="label_done_key" msgid="2441578748772529288">"Finished"</string> + <string name="label_done_key" msgid="2441578748772529288">"Done"</string> <string name="label_send_key" msgid="2815056534433717444">"Send"</string> <string name="label_pause_key" msgid="181098308428035340">"Pause"</string> <string name="label_wait_key" msgid="6402152600878093134">"Wait"</string> <string name="spoken_use_headphones" msgid="896961781287283493">"Plug in a headset to hear password keys spoken aloud."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Current text is %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"No text entered"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Key code %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift on (tap to disable)"</string> diff --git a/java/res/values-es-rUS/strings.xml b/java/res/values-es-rUS/strings.xml index b1a421475..42c3932f6 100644 --- a/java/res/values-es-rUS/strings.xml +++ b/java/res/values-es-rUS/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Enchufa tus auriculares para escuchar en voz alta qué teclas presionas al ingresar una contraseña."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"El texto actual es %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"No se ingresó texto."</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Clave de código %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Mayús"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Se activó el modo Mayúscula (toca para desactivarlo)."</string> diff --git a/java/res/values-es/strings.xml b/java/res/values-es/strings.xml index 9ff9430de..4d72799f4 100644 --- a/java/res/values-es/strings.xml +++ b/java/res/values-es/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Conecta un auricular para escuchar las contraseñas en voz alta."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"El texto actual es %s."</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"No se ha introducido texto."</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Código del teclado: %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Mayús"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Mayúsculas activadas (tocar para inhabilitar)"</string> diff --git a/java/res/values-et-rEE/strings-appname.xml b/java/res/values-et-rEE/strings-appname.xml new file mode 100644 index 000000000..bbc13d293 --- /dev/null +++ b/java/res/values-et-rEE/strings-appname.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + --> + +<resources xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="english_ime_name" msgid="5940510615957428904">"Androidi klaviatuur (AOSP)"</string> + <string name="spell_checker_service_name" msgid="1254221805440242662">"Androidi õigekirjakontroll (AOSP)"</string> + <string name="english_ime_settings" msgid="5760361067176802794">"Androidi klaviatuuri seaded (AOSP)"</string> + <string name="android_spell_checker_settings" msgid="6123949487832861885">"Androidi õigekirjakontrolli seaded (AOSP)"</string> +</resources> diff --git a/java/res/values-et-rEE/strings.xml b/java/res/values-et-rEE/strings.xml index 62197d772..5706fa105 100644 --- a/java/res/values-et-rEE/strings.xml +++ b/java/res/values-et-rEE/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Ühendage peakomplekt, et kuulata paroole."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Praegune tekst on %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Teksti ei ole sisestatud"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Klahvi kood: %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Tõstuklahv"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Tõstuklahv sees (puudutage keelamiseks)"</string> diff --git a/java/res/values-fa/strings.xml b/java/res/values-fa/strings.xml index 5d8951619..c03e440e4 100644 --- a/java/res/values-fa/strings.xml +++ b/java/res/values-fa/strings.xml @@ -86,6 +86,10 @@ <!-- no translation found for spoken_current_text_is (2485723011272583845) --> <skip /> <string name="spoken_no_text_entered" msgid="7479685225597344496">"متنی وارد نشده است"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <!-- String.format failed for translation --> <!-- no translation found for spoken_description_unknown (3197434010402179157) --> <skip /> diff --git a/java/res/values-fi/strings.xml b/java/res/values-fi/strings.xml index 4cdf6210e..744e60420 100644 --- a/java/res/values-fi/strings.xml +++ b/java/res/values-fi/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Liitä kuulokkeet, niin kuulet mitä näppäimiä painat kirjoittaessasi salasanaa."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Nykyinen teksti on %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Ei kirjoitettua tekstiä"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Näppäimen koodi %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Vaihto"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Vaihto päällä (poista käytöstä napauttamalla)"</string> diff --git a/java/res/values-fr/strings.xml b/java/res/values-fr/strings.xml index e096a1fb8..5fedd4ebd 100644 --- a/java/res/values-fr/strings.xml +++ b/java/res/values-fr/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Branchez des écouteurs pour entendre l\'énoncé à haute voix des touches lors de la saisie du mot de passe."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Le texte actuel est %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Aucun texte saisi"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Code touche %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Maj"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Touche Maj activée (appuyer pour désactiver)"</string> diff --git a/java/res/values-hi/strings.xml b/java/res/values-hi/strings.xml index dcbbcd97c..3b3228a9f 100644 --- a/java/res/values-hi/strings.xml +++ b/java/res/values-hi/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"ज़ोर से बोली गई पासवर्ड कुंजियां सुनने के लिए हेडसेट प्लग इन करें."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"वर्तमान पाठ %s है"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"कोई पाठ दर्ज नहीं किया गया"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"कुंजी कोड %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"शिफ़्ट"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift चालू (अक्षम करने के लिए टैप करें)"</string> diff --git a/java/res/values-hr/strings.xml b/java/res/values-hr/strings.xml index 0dca64ef0..caef9541d 100644 --- a/java/res/values-hr/strings.xml +++ b/java/res/values-hr/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Priključite slušalice da biste čuli tipke zaporke izgovorene naglas."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Trenutačni tekst je %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nije unesen tekst"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Kôd tipke %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Uključena tipka Shift (dotaknite da onemogućite)"</string> diff --git a/java/res/values-hu/strings.xml b/java/res/values-hu/strings.xml index 2abed7401..3b0ee4798 100644 --- a/java/res/values-hu/strings.xml +++ b/java/res/values-hu/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Csatlakoztasson egy headsetet, ha hallani szeretné a jelszót felolvasva."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"A jelenlegi szöveg: %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Szöveg nincs megadva"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Billentyűkód: %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift be van kapcsolva (érintse meg a kikapcsoláshoz)"</string> diff --git a/java/res/values-in/strings.xml b/java/res/values-in/strings.xml index c73c26c1a..bfbd2a700 100644 --- a/java/res/values-in/strings.xml +++ b/java/res/values-in/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Pasang headset untuk mendengar tombol sandi yang diucapkan dengan keras."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Teks saat ini adalah %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Tidak ada teks yang dimasukkan"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Kode tombol %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift hidup (ketuk untuk mematikan)"</string> diff --git a/java/res/values-it/strings.xml b/java/res/values-it/strings.xml index eb8e1e959..6989643af 100644 --- a/java/res/values-it/strings.xml +++ b/java/res/values-it/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Collega gli auricolari per ascoltare la pronuncia dei tasti premuti per la password."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Il testo attuale è %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nessun testo inserito"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Codice tasto %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Maiuscolo"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Maiuscolo attivo (tocca per disattivare)"</string> diff --git a/java/res/values-iw/strings.xml b/java/res/values-iw/strings.xml index e71bd3190..b3ee01410 100644 --- a/java/res/values-iw/strings.xml +++ b/java/res/values-iw/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"חבר אוזניות כדי לשמוע הקראה של מפתחות סיסמה."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"הטקסט הנוכחי הוא %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"לא הוזן טקסט"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"קוד מקש %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift פועל (הקש כדי להשבית)"</string> diff --git a/java/res/values-ja/strings.xml b/java/res/values-ja/strings.xml index 4d3f155bb..37f06377e 100644 --- a/java/res/values-ja/strings.xml +++ b/java/res/values-ja/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"パスワードのキーが音声出力されるのでヘッドセットを接続してください。"</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"現在のテキスト:%s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"テキストが入力されていません"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"キーコード:%d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift有効（タップして解除）"</string> diff --git a/java/res/values-ka-rGE/strings-appname.xml b/java/res/values-ka-rGE/strings-appname.xml new file mode 100644 index 000000000..703c66aac --- /dev/null +++ b/java/res/values-ka-rGE/strings-appname.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + --> + +<resources xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="english_ime_name" msgid="5940510615957428904">"Android-ის კლავიატურა (AOSP)"</string> + <string name="spell_checker_service_name" msgid="1254221805440242662">"Android-ის მართლწერის შემმოწმებელი (AOSP)"</string> + <string name="english_ime_settings" msgid="5760361067176802794">"Android-ის კლავიატურის პარამეტრები (AOSP)"</string> + <string name="android_spell_checker_settings" msgid="6123949487832861885">"Android-ის მართლწერის შემმოწმებლის პარამეტრები (AOSP)"</string> +</resources> diff --git a/java/res/values-ka-rGE/strings.xml b/java/res/values-ka-rGE/strings.xml index 043c14c94..193f5115d 100644 --- a/java/res/values-ka-rGE/strings.xml +++ b/java/res/values-ka-rGE/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"შეაერთეთ ყურსაცვამი, რათა მოისმინოთ აკრეფილი პაროლის კლავიშების სახელები."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"მიმდინარე ტექსტი არის %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"ტექსტი არ შეყვანილა"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"კლავიატურის კოდი %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift ჩართულია (შეეხეთ გამოსართავად)"</string> diff --git a/java/res/values-kk/strings-appname.xml b/java/res/values-kk/strings-appname.xml new file mode 100644 index 000000000..1e201fa9e --- /dev/null +++ b/java/res/values-kk/strings-appname.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + --> + +<resources xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="english_ime_name" msgid="5940510615957428904">"Android пернетақтасы (AOSP)"</string> + <string name="spell_checker_service_name" msgid="1254221805440242662">"Android емлені тексеру құралы (AOSP)"</string> + <string name="english_ime_settings" msgid="5760361067176802794">"Android пернетақтасының параметрлері (AOSP)"</string> + <string name="android_spell_checker_settings" msgid="6123949487832861885">"Android емлені тексеру құралының параметрлері (AOSP)"</string> +</resources> diff --git a/java/res/values-ko/strings.xml b/java/res/values-ko/strings.xml index a74b4f1f4..1497812e9 100644 --- a/java/res/values-ko/strings.xml +++ b/java/res/values-ko/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"비밀번호 키를 음성으로 들으려면 헤드셋을 연결하세요."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"입력한 텍스트: %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"입력한 텍스트 없음"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"키 코드 %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"시프트 키"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift 사용(사용하지 않으려면 탭하세요.)"</string> diff --git a/java/res/values-land/dimens.xml b/java/res/values-land/dimens.xml index 42a746b60..b874d4881 100644 --- a/java/res/values-land/dimens.xml +++ b/java/res/values-land/dimens.xml @@ -77,7 +77,7 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">10%p</fraction> <fraction name="emoji_keyboard_row_height">50%p</fraction> - <fraction name="emoji_keyboard_key_letter_size">60%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">54%p</fraction> <integer name="emoji_keyboard_max_key_count">20</integer> </resources> diff --git a/java/res/values-lt/strings.xml b/java/res/values-lt/strings.xml index 6110858dd..88328db7d 100644 --- a/java/res/values-lt/strings.xml +++ b/java/res/values-lt/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Prijunkite ausines, kad išgirstumėte sakomus slaptažodžio klavišus."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Dabartinis tekstas yra %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nėra įvesto teksto"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Klavišo kodas %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Antrojo lygio klavišas"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Įjungtas antrasis lygis (palieskite, kad išjungtumėte)"</string> diff --git a/java/res/values-lv/strings.xml b/java/res/values-lv/strings.xml index ecd4e511d..069c48701 100644 --- a/java/res/values-lv/strings.xml +++ b/java/res/values-lv/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Pievienojiet austiņas, lai dzirdētu paroles rakstzīmes."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Pašreizējais teksts ir %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nav ievadīts teksts"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Taustiņu kods %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Pārslēgšanas taustiņš"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Pārslēgšanas taustiņš iespējots (pieskarieties, lai atspējotu)"</string> diff --git a/java/res/values-mn-rMN/strings-appname.xml b/java/res/values-mn-rMN/strings-appname.xml new file mode 100644 index 000000000..6c27e3d34 --- /dev/null +++ b/java/res/values-mn-rMN/strings-appname.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + --> + +<resources xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="english_ime_name" msgid="5940510615957428904">"Андройд Гар (AOSP)"</string> + <string name="spell_checker_service_name" msgid="1254221805440242662">"Андройд Алдаа Шалгагч (AOSP)"</string> + <string name="english_ime_settings" msgid="5760361067176802794">"Андройд Гарын Тохиргоо (AOSP)"</string> + <string name="android_spell_checker_settings" msgid="6123949487832861885">"Андройд Алдаа Шалгагчийн Тохиргоо (AOSP)"</string> +</resources> diff --git a/java/res/values-mn-rMN/strings.xml b/java/res/values-mn-rMN/strings.xml index 92c2befa2..3b039f120 100644 --- a/java/res/values-mn-rMN/strings.xml +++ b/java/res/values-mn-rMN/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Нууц үгний товчнуудыг чангаар уншихыг сонсохын тулд чихэвчээ залгана уу."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Одоогийн текст %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст оруулаагүй"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Товчийн код %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Сэлгэх"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Сэлгэхийг идэвхжүүлсэн (товшиж идэвхгүйжүүлнэ үү)"</string> diff --git a/java/res/values-ms-rMY/strings-appname.xml b/java/res/values-ms-rMY/strings-appname.xml new file mode 100644 index 000000000..76d1d294c --- /dev/null +++ b/java/res/values-ms-rMY/strings-appname.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + --> + +<resources xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="english_ime_name" msgid="5940510615957428904">"Papan Kekunci Android (AOSP)"</string> + <string name="spell_checker_service_name" msgid="1254221805440242662">"Penyemak Ejaan Android (AOSP)"</string> + <string name="english_ime_settings" msgid="5760361067176802794">"Tetapan Papan Kekunci Android (AOSP)"</string> + <string name="android_spell_checker_settings" msgid="6123949487832861885">"Tetapan Penyemak Ejaan Android (AOSP)"</string> +</resources> diff --git a/java/res/values-ms-rMY/strings.xml b/java/res/values-ms-rMY/strings.xml index 49efdf18f..ee241d64b 100644 --- a/java/res/values-ms-rMY/strings.xml +++ b/java/res/values-ms-rMY/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Pasangkan set kepala untuk mendengar kekunci kata laluan disebut dengan kuat."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Teks semasa adalah %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Tiada teks dimasukkan"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Kod kunci %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Kunci anjak dihidupkan (ketik untuk melumpuhkan)"</string> diff --git a/java/res/values-nb/strings.xml b/java/res/values-nb/strings.xml index 9b30ea329..ae43d1078 100644 --- a/java/res/values-nb/strings.xml +++ b/java/res/values-nb/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Koble til hodetelefoner for å høre opplesing av bokstavene i passordet."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Gjeldende tekst er %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Ingen tekst er skrevet inn"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Tastaturkode %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift er på (trykk for å deaktivere)"</string> diff --git a/java/res/values-nl/strings.xml b/java/res/values-nl/strings.xml index c61b090b6..de20da4f8 100644 --- a/java/res/values-nl/strings.xml +++ b/java/res/values-nl/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Sluit een headset aan om wachtwoordtoetsen hardop te laten voorlezen."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Huidige tekst is %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Geen tekst ingevoerd"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Toetscode %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift aan (tik om uit te schakelen)"</string> diff --git a/java/res/values-pl/strings.xml b/java/res/values-pl/strings.xml index 31eace5cb..390837596 100644 --- a/java/res/values-pl/strings.xml +++ b/java/res/values-pl/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Podłącz zestaw słuchawkowy, aby usłyszeć znaki hasła wypowiadane na głos."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Aktualny tekst: %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nie wprowadzono tekstu"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Kod klawisza: %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift włączony (kliknij, by wyłączyć)"</string> diff --git a/java/res/values-pt-rPT/strings.xml b/java/res/values-pt-rPT/strings.xml index cd89a7765..4c81479e5 100644 --- a/java/res/values-pt-rPT/strings.xml +++ b/java/res/values-pt-rPT/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Ligar auscultadores com microfone integrado para ouvir as teclas da palavra-passe."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"O texto atual é %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nenhum texto digitado"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Código da tecla %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift ativado (tocar para desativar)"</string> diff --git a/java/res/values-pt/strings.xml b/java/res/values-pt/strings.xml index 820f577ad..ef533ae83 100644 --- a/java/res/values-pt/strings.xml +++ b/java/res/values-pt/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Conecte um fone de ouvido para ouvir as chaves de senha em voz alta."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"O texto atual é %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nenhum texto digitado"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Código de tecla %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift ativado (toque para desativar)"</string> diff --git a/java/res/values-rm/strings.xml b/java/res/values-rm/strings.xml index 1c509ad73..c68d30c0c 100644 --- a/java/res/values-rm/strings.xml +++ b/java/res/values-rm/strings.xml @@ -139,6 +139,10 @@ <skip /> <!-- no translation found for spoken_no_text_entered (7479685225597344496) --> <skip /> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <!-- no translation found for spoken_description_unknown (3197434010402179157) --> <skip /> <!-- no translation found for spoken_description_shift (244197883292549308) --> diff --git a/java/res/values-ro/strings.xml b/java/res/values-ro/strings.xml index 3caff1337..930b68bd6 100644 --- a/java/res/values-ro/strings.xml +++ b/java/res/values-ro/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Conectaţi un set căşti-microfon pentru a auzi tastele apăsate când introduceţi parola."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Textul curent este %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nu a fost introdus text"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Tasta cu codul %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Tasta Shift este activată (apăsaţi pentru a o dezactiva)"</string> diff --git a/java/res/values-ru/strings.xml b/java/res/values-ru/strings.xml index affe88fbe..8bfb011c5 100644 --- a/java/res/values-ru/strings.xml +++ b/java/res/values-ru/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Подключите гарнитуру, чтобы услышать пароль."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Введенный текст: %s."</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст не введен"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Код клавиши:%d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Клавиша верхнего регистра"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Верхний регистр включен (нажмите, чтобы отключить)"</string> diff --git a/java/res/values-sk/strings.xml b/java/res/values-sk/strings.xml index 85d64c6c0..5c25f28ce 100644 --- a/java/res/values-sk/strings.xml +++ b/java/res/values-sk/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Ak si chcete pri zadávaní hesla vypočuť nahlas vyslovené klávesy, pripojte náhlavnú súpravu."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Aktuálny text je %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Nie je zadaný žiadny text"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Kód klávesu %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Kláves Shift je zapnutý (zakážete ho klepnutím)"</string> diff --git a/java/res/values-sl/strings.xml b/java/res/values-sl/strings.xml index 7648d73b2..dde01dd68 100644 --- a/java/res/values-sl/strings.xml +++ b/java/res/values-sl/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Priključite slušalke, če želite slišati izgovorjene tipke gesla."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Trenutno besedilo je %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Ni vnesenega besedila"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Koda tipke %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift je vklopljen (dotaknite se, da onemogočite)"</string> diff --git a/java/res/values-sr/strings.xml b/java/res/values-sr/strings.xml index db06bce7e..e7f6904c1 100644 --- a/java/res/values-sr/strings.xml +++ b/java/res/values-sr/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Укључите слушалице да бисте чули наглас изговорене тастере за лозинку."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Тренутни текст је %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст није унет"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Кôд тастера %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift је укључен (додирните да бисте га онемогућили)"</string> diff --git a/java/res/values-sv/strings.xml b/java/res/values-sv/strings.xml index c6635165c..0fcca4837 100644 --- a/java/res/values-sv/strings.xml +++ b/java/res/values-sv/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Anslut hörlurar om du vill att lösenordet ska läsas upp."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Nuvarande text är %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Ingen text har angetts"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Nyckelkod %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Skift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Skift på (knacka lätt för att inaktivera)"</string> diff --git a/java/res/values-sw/strings.xml b/java/res/values-sw/strings.xml index 188bbca9c..fbf868f24 100644 --- a/java/res/values-sw/strings.xml +++ b/java/res/values-sw/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Chomeka plagi ya kifaa cha kichwa cha kusikiza ili kusikiliza msimbo wa nenosiri inayozungumwa kwa sauti ya juu."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Maandishi ya sasa ni %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Hakuna maandishi yaliyoingizwa"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Msimbo wa kitufe %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Badilisha"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift imewashwa (gonga ili kulemaza)"</string> diff --git a/java/res/values-sw600dp-land/dimens.xml b/java/res/values-sw600dp-land/dimens.xml index 730b7d813..d067265e3 100644 --- a/java/res/values-sw600dp-land/dimens.xml +++ b/java/res/values-sw600dp-land/dimens.xml @@ -66,7 +66,7 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">10%p</fraction> <fraction name="emoji_keyboard_row_height">33%p</fraction> - <fraction name="emoji_keyboard_key_letter_size">95%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">85%p</fraction> <integer name="emoji_keyboard_max_key_count">30</integer> </resources> diff --git a/java/res/values-sw600dp/dimens.xml b/java/res/values-sw600dp/dimens.xml index 2bcf2fa84..591355d3c 100644 --- a/java/res/values-sw600dp/dimens.xml +++ b/java/res/values-sw600dp/dimens.xml @@ -92,7 +92,7 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">12.5%p</fraction> <fraction name="emoji_keyboard_row_height">33%p</fraction> - <fraction name="emoji_keyboard_key_letter_size">85%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">76%p</fraction> <integer name="emoji_keyboard_max_key_count">24</integer> </resources> diff --git a/java/res/values-sw768dp-land/dimens.xml b/java/res/values-sw768dp-land/dimens.xml index 1e2e1c651..664630b4f 100644 --- a/java/res/values-sw768dp-land/dimens.xml +++ b/java/res/values-sw768dp-land/dimens.xml @@ -67,7 +67,7 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">7.69%p</fraction> <fraction name="emoji_keyboard_row_height">33%p</fraction> - <fraction name="emoji_keyboard_key_letter_size">75%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">68%p</fraction> <integer name="emoji_keyboard_max_key_count">39</integer> </resources> diff --git a/java/res/values-sw768dp/dimens.xml b/java/res/values-sw768dp/dimens.xml index f62a536d7..1fd933abf 100644 --- a/java/res/values-sw768dp/dimens.xml +++ b/java/res/values-sw768dp/dimens.xml @@ -92,7 +92,7 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">10%p</fraction> <fraction name="emoji_keyboard_row_height">33%p</fraction> - <fraction name="emoji_keyboard_key_letter_size">85%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">76%p</fraction> <integer name="emoji_keyboard_max_key_count">30</integer> </resources> diff --git a/java/res/values-th/strings.xml b/java/res/values-th/strings.xml index 632f39a74..f84b32c62 100644 --- a/java/res/values-th/strings.xml +++ b/java/res/values-th/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"เสียบชุดหูฟังเพื่อฟังเสียงเมื่อพิมพ์รหัสผ่าน"</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"ข้อความปัจจุบันคือ %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"ไม่มีข้อความ"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"รหัสคีย์ %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift เปิดอยู่ (แตะเพื่อปิดใช้งาน)"</string> diff --git a/java/res/values-tl/strings.xml b/java/res/values-tl/strings.xml index 3ec962b01..af1c68e01 100644 --- a/java/res/values-tl/strings.xml +++ b/java/res/values-tl/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Mag-plug in ng headset upang marinig ang mga password key na binabanggit nang malakas."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Ang kasalukuyang teksto ay %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Walang tekstong inilagay"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Code ng key %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Naka-on ang shift (i-tap upang huwag paganahin)"</string> diff --git a/java/res/values-tr/strings.xml b/java/res/values-tr/strings.xml index 4d0a6174c..4c8e1a81d 100644 --- a/java/res/values-tr/strings.xml +++ b/java/res/values-tr/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Şifre tuşlarının sesli okunmasını dinlemek için mikrofonlu kulaklık takın."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Mevcut metin: %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Hiç metin girilmedi"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Tuş kodu: %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Üst Karakter"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Üst karakter açık (devre dışı bırakmak için hafifçe vurun)"</string> diff --git a/java/res/values-uk/strings.xml b/java/res/values-uk/strings.xml index df62f3286..287bcb397 100644 --- a/java/res/values-uk/strings.xml +++ b/java/res/values-uk/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Підключіть гарнітуру, щоб прослухати відтворені вголос символи пароля."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Поточний текст – %s."</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Текст не введено"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Код клавіші – %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Клавіша Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift увімкнено (швидко торкніться, щоб вимкнути)"</string> diff --git a/java/res/values-vi/strings.xml b/java/res/values-vi/strings.xml index e08d1c6b9..8764df5e0 100644 --- a/java/res/values-vi/strings.xml +++ b/java/res/values-vi/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Cắm tai nghe để nghe mật khẩu."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Ký tự hiện tại là %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Không có ký tự nào được nhập"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Mã phím %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift đang bật (bấm để tắt)"</string> diff --git a/java/res/values-zh-rCN/strings.xml b/java/res/values-zh-rCN/strings.xml index 5980cbf13..f7ceff44f 100644 --- a/java/res/values-zh-rCN/strings.xml +++ b/java/res/values-zh-rCN/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"需要插入耳机才能听到密码的按键声。"</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"当前文本为%s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"未输入文字"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"键码为 %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift 模式已启用（点按即可停用）"</string> diff --git a/java/res/values-zh-rTW/strings.xml b/java/res/values-zh-rTW/strings.xml index b14a01a0a..b773ad21c 100644 --- a/java/res/values-zh-rTW/strings.xml +++ b/java/res/values-zh-rTW/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"連接耳機即可聽取系統朗讀密碼按鍵。"</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"目前文字為 %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"未輸入文字"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"按鍵代碼 %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift 鍵"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"Shift 鍵已開啟 (輕按即可停用)"</string> diff --git a/java/res/values-zu/strings.xml b/java/res/values-zu/strings.xml index 9ed88a00f..8f722cce3 100644 --- a/java/res/values-zu/strings.xml +++ b/java/res/values-zu/strings.xml @@ -84,6 +84,10 @@ <string name="spoken_use_headphones" msgid="896961781287283493">"Plaka ku-headset ukuze uzwe okhiye bephasiwedi ezindlebeni zakho bezwakala kakhulu."</string> <string name="spoken_current_text_is" msgid="2485723011272583845">"Umbhalo wamanje ngu %s"</string> <string name="spoken_no_text_entered" msgid="7479685225597344496">"Awukho umbhalo ofakiwe"</string> + <!-- no translation found for spoken_auto_correct (5381764628886369268) --> + <skip /> + <!-- no translation found for spoken_auto_correct_obscured (1186884531440481089) --> + <skip /> <string name="spoken_description_unknown" msgid="3197434010402179157">"Ikhodi yokhiye %d"</string> <string name="spoken_description_shift" msgid="244197883292549308">"Shift"</string> <string name="spoken_description_shift_shifted" msgid="1681877323344195035">"U-Shift uvuliwe (thepha ukuwuvimbela)"</string> diff --git a/java/res/values/dimens.xml b/java/res/values/dimens.xml index 4e3b2f567..2d626dbcf 100644 --- a/java/res/values/dimens.xml +++ b/java/res/values/dimens.xml @@ -118,7 +118,7 @@ <!-- Emoji keyboard --> <fraction name="emoji_keyboard_key_width">14.2857%p</fraction> <fraction name="emoji_keyboard_row_height">33%p</fraction> - <fraction name="emoji_keyboard_key_letter_size">90%p</fraction> + <fraction name="emoji_keyboard_key_letter_size">81%p</fraction> <integer name="emoji_keyboard_max_key_count">21</integer> <dimen name="emoji_category_page_id_height">3dp</dimen> diff --git a/java/res/values/strings.xml b/java/res/values/strings.xml index 69da1e862..06f4b4789 100644 --- a/java/res/values/strings.xml +++ b/java/res/values/strings.xml @@ -174,6 +174,11 @@ <!-- Spoken description when there is no text entered --> <string name="spoken_no_text_entered">No text entered</string> + <!-- Spoken description to let the user know what auto-correction will be performed when a key is pressed. --> + <string name="spoken_auto_correct"><xliff:g id="key" example="Space">%1$s</xliff:g> corrects <xliff:g id="original">%2$s</xliff:g> to <xliff:g id="corrected">%3$s</xliff:g></string> + <!-- Spoken description used during obscured (e.g. password) entry to let the user know that auto-correction will be performed when a key is pressed. --> + <string name="spoken_auto_correct_obscured"><xliff:g id="key" example="Space">%1$s</xliff:g> has auto-correction</string> + <!-- Spoken description for unknown keyboard keys. --> <string name="spoken_description_unknown">Key code %d</string> <!-- Spoken description for the "Shift" keyboard key when "Shift" is off. --> diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java index 7639432aa..c628c5b09 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibilityEntityProvider.java @@ -35,6 +35,8 @@ import android.view.inputmethod.EditorInfo; import com.android.inputmethod.keyboard.Key; import com.android.inputmethod.keyboard.Keyboard; import com.android.inputmethod.keyboard.KeyboardView; +import com.android.inputmethod.latin.settings.Settings; +import com.android.inputmethod.latin.settings.SettingsValues; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.CoordinateUtils; @@ -285,9 +287,15 @@ public final class AccessibilityEntityProvider extends AccessibilityNodeProvider private String getKeyDescription(final Key key) { final EditorInfo editorInfo = mInputMethodService.getCurrentInputEditorInfo(); final boolean shouldObscure = mAccessibilityUtils.shouldObscureInput(editorInfo); - final String keyDescription = mKeyCodeDescriptionMapper.getDescriptionForKey( + final SettingsValues currentSettings = Settings.getInstance().getCurrent(); + final String keyCodeDescription = mKeyCodeDescriptionMapper.getDescriptionForKey( mKeyboardView.getContext(), mKeyboardView.getKeyboard(), key, shouldObscure); - return keyDescription; + if (currentSettings.isWordSeparator(key.getCode())) { + return mAccessibilityUtils.getAutoCorrectionDescription( + keyCodeDescription, shouldObscure); + } else { + return keyCodeDescription; + } } /** diff --git a/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java b/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java index 8929dc7e9..10fb9fef4 100644 --- a/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java +++ b/java/src/com/android/inputmethod/accessibility/AccessibilityUtils.java @@ -23,6 +23,7 @@ import android.os.Build; import android.os.SystemClock; import android.provider.Settings; import android.support.v4.view.accessibility.AccessibilityEventCompat; +import android.text.TextUtils; import android.util.Log; import android.view.MotionEvent; import android.view.View; @@ -34,6 +35,7 @@ import android.view.inputmethod.EditorInfo; import com.android.inputmethod.compat.SettingsSecureCompatUtils; import com.android.inputmethod.latin.R; +import com.android.inputmethod.latin.SuggestedWords; import com.android.inputmethod.latin.utils.InputTypeUtils; public final class AccessibilityUtils { @@ -48,6 +50,12 @@ public final class AccessibilityUtils { private AccessibilityManager mAccessibilityManager; private AudioManager mAudioManager; + /** The most recent auto-correction. */ + private String mAutoCorrectionWord; + + /** The most recent typed word for auto-correction. */ + private String mTypedWord; + /* * Setting this constant to {@code false} will disable all keyboard * accessibility code, regardless of whether Accessibility is turned on in @@ -142,6 +150,51 @@ public final class AccessibilityUtils { } /** + * Sets the current auto-correction word and typed word. These may be used + * to provide the user with a spoken description of what auto-correction + * will occur when a key is typed. + * + * @param suggestedWords the list of suggested auto-correction words + * @param typedWord the currently typed word + */ + public void setAutoCorrection(final SuggestedWords suggestedWords, final String typedWord) { + if (suggestedWords != null && suggestedWords.mWillAutoCorrect) { + mAutoCorrectionWord = suggestedWords.getWord(SuggestedWords.INDEX_OF_AUTO_CORRECTION); + mTypedWord = typedWord; + } else { + mAutoCorrectionWord = null; + mTypedWord = null; + } + } + + /** + * Obtains a description for an auto-correction key, taking into account the + * currently typed word and auto-correction. + * + * @param keyCodeDescription spoken description of the key that will insert + * an auto-correction + * @param shouldObscure whether the key should be obscured + * @return a description including a description of the auto-correction, if + * needed + */ + public String getAutoCorrectionDescription( + final String keyCodeDescription, final boolean shouldObscure) { + if (!TextUtils.isEmpty(mAutoCorrectionWord)) { + if (!TextUtils.equals(mAutoCorrectionWord, mTypedWord)) { + if (shouldObscure) { + // This should never happen, but just in case... + return mContext.getString(R.string.spoken_auto_correct_obscured, + keyCodeDescription); + } + return mContext.getString(R.string.spoken_auto_correct, keyCodeDescription, + mTypedWord, mAutoCorrectionWord); + } + } + + return keyCodeDescription; + } + + /** * Sends the specified text to the {@link AccessibilityManager} to be * spoken. * diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java index 4e61edac2..eb48d01f6 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java @@ -153,7 +153,9 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange mCategoryNameToIdMap.put(sCategoryName[i], i); } addShownCategoryId(CATEGORY_ID_RECENTS); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR2) { + if (Build.VERSION.SDK_INT > Build.VERSION_CODES.JELLY_BEAN_MR2 + || android.os.Build.VERSION.CODENAME.equalsIgnoreCase("KeyLimePie") + || android.os.Build.VERSION.CODENAME.equalsIgnoreCase("KitKat")) { addShownCategoryId(CATEGORY_ID_PEOPLE); addShownCategoryId(CATEGORY_ID_OBJECTS); addShownCategoryId(CATEGORY_ID_NATURE); @@ -716,12 +718,14 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange @Override public void run() { + int repeatCount = 1; int timeCount = 0; while (timeCount < MAX_REPEAT_COUNT_TIME && !mAborted) { if (timeCount > mKeyRepeatStartTimeout) { - pressDelete(); + pressDelete(repeatCount); } timeCount += mKeyRepeatInterval; + ++repeatCount; try { Thread.sleep(mKeyRepeatInterval); } catch (InterruptedException e) { @@ -734,9 +738,9 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange } } - public void pressDelete() { + public void pressDelete(int repeatCount) { mKeyboardActionListener.onPressKey( - Constants.CODE_DELETE, 0 /* repeatCount */, true /* isSinglePointer */); + Constants.CODE_DELETE, repeatCount, true /* isSinglePointer */); mKeyboardActionListener.onCodeInput( Constants.CODE_DELETE, NOT_A_COORDINATE, NOT_A_COORDINATE); mKeyboardActionListener.onReleaseKey( @@ -752,7 +756,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange switch(event.getAction()) { case MotionEvent.ACTION_DOWN: v.setBackgroundColor(mDeleteKeyPressedBackgroundColor); - pressDelete(); + pressDelete(0 /* repeatCount */); startRepeat(); return true; case MotionEvent.ACTION_UP: diff --git a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java index cc1ffd183..74edd87cf 100644 --- a/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java +++ b/java/src/com/android/inputmethod/keyboard/KeyboardSwitcher.java @@ -315,7 +315,7 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { } public boolean isShowingEmojiKeyboard() { - return mEmojiKeyboardView.getVisibility() == View.VISIBLE; + return mEmojiKeyboardView != null && mEmojiKeyboardView.getVisibility() == View.VISIBLE; } public boolean isShowingMoreKeysPanel() { diff --git a/java/src/com/android/inputmethod/keyboard/PointerTracker.java b/java/src/com/android/inputmethod/keyboard/PointerTracker.java index d4d0d8718..ee4ac950c 100644 --- a/java/src/com/android/inputmethod/keyboard/PointerTracker.java +++ b/java/src/com/android/inputmethod/keyboard/PointerTracker.java @@ -346,6 +346,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { // true if this pointer is in a sliding key input from a modifier key, // so that further modifier keys should be ignored. boolean mIsInSlidingKeyInputFromModifier; + // if not a NOT_A_CODE, the key of this code is repeating + private int mCurrentRepeatingKeyCode = Constants.NOT_A_CODE; // true if a sliding key input is allowed. private boolean mIsAllowedSlidingKeyInput; @@ -937,9 +939,10 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (!sShouldHandleGesture) { return; } - // A gesture should start only from a non-modifier key. + // A gesture should start only from a non-modifier key. Note that the gesture detection is + // disabled when the key is repeating. mIsDetectingGesture = (mKeyboard != null) && mKeyboard.mId.isAlphabetKeyboard() - && key != null && !key.isModifier() && !key.isRepeatable(); + && key != null && !key.isModifier(); if (mIsDetectingGesture) { if (getActivePointerTrackerCount() == 1) { sGestureFirstDownTime = eventTime; @@ -1247,6 +1250,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { mIsDetectingGesture = false; final Key currentKey = mCurrentKey; mCurrentKey = null; + final int currentRepeatingKeyCode = mCurrentRepeatingKeyCode; + mCurrentRepeatingKeyCode = Constants.NOT_A_CODE; // Release the last pressed key. setReleasedKeyGraphics(currentKey); @@ -1272,8 +1277,8 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (mIsTrackingForActionDisabled) { return; } - if (currentKey != null && currentKey.isRepeatable() && !isInSlidingKeyInput) { - // Repeatable key has been registered in {@link #onDownEventInternal(int,int,long)}. + if (currentKey != null && currentKey.isRepeatable() + && (currentKey.getCode() == currentRepeatingKeyCode) && !isInSlidingKeyInput) { return; } detectAndSendKey(currentKey, mKeyX, mKeyY, eventTime); @@ -1412,7 +1417,6 @@ public final class PointerTracker implements PointerTrackerQueue.Element { if (!key.isRepeatable()) return; // Don't start key repeat when we are in sliding input mode. if (mIsInSlidingKeyInput) return; - detectAndSendKey(key, key.getX(), key.getY(), SystemClock.uptimeMillis()); final int startRepeatCount = 1; mTimerProxy.startKeyRepeatTimer(this, startRepeatCount, sParams.mKeyRepeatStartTimeout); } @@ -1420,8 +1424,11 @@ public final class PointerTracker implements PointerTrackerQueue.Element { public void onKeyRepeat(final int code, final int repeatCount) { final Key key = getKey(); if (key == null || key.getCode() != code) { + mCurrentRepeatingKeyCode = Constants.NOT_A_CODE; return; } + mCurrentRepeatingKeyCode = code; + mIsDetectingGesture = false; final int nextRepeatCount = repeatCount + 1; mTimerProxy.startKeyRepeatTimer(this, nextRepeatCount, sParams.mKeyRepeatInterval); callListenerOnPressAndCheckKeyboardLayoutChange(key, repeatCount); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java index b3491d807..9f9fdaa6f 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardState.java @@ -585,7 +585,7 @@ public final class KeyboardState { } } - private static boolean isSpaceCharacter(final int c) { + private static boolean isSpaceOrEnter(final int c) { return c == Constants.CODE_SPACE || c == Constants.CODE_ENTER; } @@ -614,7 +614,12 @@ public final class KeyboardState { } break; case SWITCH_STATE_SYMBOL_BEGIN: - if (!isSpaceCharacter(code) && (Constants.isLetterCode(code) + if (mIsEmojiMode) { + // When in the Emoji keyboard, we don't want to switch back to the main layout even + // after the user hits an emoji letter followed by an enter or a space. + break; + } + if (!isSpaceOrEnter(code) && (Constants.isLetterCode(code) || code == Constants.CODE_OUTPUT_TEXT)) { mSwitchState = SWITCH_STATE_SYMBOL; } @@ -622,7 +627,7 @@ public final class KeyboardState { case SWITCH_STATE_SYMBOL: // Switch back to alpha keyboard mode if user types one or more non-space/enter // characters followed by a space/enter. - if (isSpaceCharacter(code)) { + if (isSpaceOrEnter(code)) { toggleAlphabetAndSymbols(); mPrevSymbolsKeyboardWasShifted = false; } diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java index 845a9b987..4a0ce3735 100644 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java @@ -25,6 +25,7 @@ import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import java.io.File; import java.io.IOException; +import java.util.Map; // TODO: Quit extending Dictionary after implementing dynamic binary dictionary. abstract public class AbstractDictionaryWriter extends Dictionary { @@ -50,16 +51,16 @@ abstract public class AbstractDictionaryWriter extends Dictionary { abstract public void removeBigramWords(final String word0, final String word1); - abstract protected void writeDictionary(final DictEncoder dictEncoder) - throws IOException, UnsupportedFormatException; + abstract protected void writeDictionary(final DictEncoder dictEncoder, + final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException; - public void write(final String fileName) { + public void write(final String fileName, final Map<String, String> attributeMap) { final String tempFileName = fileName + ".temp"; final File file = new File(mContext.getFilesDir(), fileName); final File tempFile = new File(mContext.getFilesDir(), tempFileName); try { final DictEncoder dictEncoder = new Ver3DictEncoder(tempFile); - writeDictionary(dictEncoder); + writeDictionary(dictEncoder, attributeMap); tempFile.renameTo(file); } catch (IOException e) { Log.e(TAG, "IO exception while writing file", e); diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 61ccfcfad..29c6c0451 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -31,10 +31,12 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; +import java.util.Map; /** * Implements a static, compacted, binary dictionary of standard words. */ +// TODO: All methods which should be locked need to have a suffix "Locked". public final class BinaryDictionary extends Dictionary { private static final String TAG = BinaryDictionary.class.getSimpleName(); @@ -46,6 +48,11 @@ public final class BinaryDictionary extends Dictionary { // TODO: Remove this heuristic. private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3; + @UsedForTesting + public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; + @UsedForTesting + public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; + private long mNativeDict; private final Locale mLocale; private final long mDictSize; @@ -103,10 +110,12 @@ public final class BinaryDictionary extends Dictionary { JniUtils.loadNativeLibrary(); } + private static native boolean createEmptyDictFileNative(String filePath, long dictVersion, + String[] attributeKeyStringArray, String[] attributeValueStringArray); private static native long openNative(String sourceDir, long dictOffset, long dictSize, boolean isUpdatable); private static native void flushNative(long dict, String filePath); - private static native boolean needsToRunGCNative(long dict); + private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); private static native void flushWithGCNative(long dict, String filePath); private static native void closeNative(long dict); private static native int getProbabilityNative(long dict, int[] word); @@ -125,6 +134,21 @@ public final class BinaryDictionary extends Dictionary { private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); private static native int calculateProbabilityNative(long dict, int unigramProbability, int bigramProbability); + private static native String getPropertyNative(long dict, String query); + + @UsedForTesting + public static boolean createEmptyDictFile(final String filePath, final long dictVersion, + final Map<String, String> attributeMap) { + final String[] keyArray = new String[attributeMap.size()]; + final String[] valueArray = new String[attributeMap.size()]; + int index = 0; + for (final String key : attributeMap.keySet()) { + keyArray[index] = key; + valueArray[index] = attributeMap.get(key); + index++; + } + return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray); + } // TODO: Move native dict into session private final void loadDictionary(final String path, final long startOffset, @@ -246,7 +270,7 @@ public final class BinaryDictionary extends Dictionary { } private void runGCIfRequired() { - if (needsToRunGCNative(mNativeDict)) { + if (needsToRunGC(true /* mindsBlockByGC */)) { flushWithGC(); } } @@ -283,27 +307,34 @@ public final class BinaryDictionary extends Dictionary { removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); } - public void flush() { - if (!isValidDictionary()) return; - flushNative(mNativeDict, mDictFilePath); - closeNative(mNativeDict); + private void reopen() { + close(); final File dictFile = new File(mDictFilePath); mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, dictFile.length(), true /* isUpdatable */); } + public void flush() { + if (!isValidDictionary()) return; + flushNative(mNativeDict, mDictFilePath); + reopen(); + } + public void flushWithGC() { if (!isValidDictionary()) return; flushWithGCNative(mNativeDict, mDictFilePath); - closeNative(mNativeDict); - final File dictFile = new File(mDictFilePath); - mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, - dictFile.length(), true /* isUpdatable */); + reopen(); } - public boolean needsToRunGC() { + /** + * Checks whether GC is needed to run or not. + * @param mindsBlockByGC Whether to mind operations blocked by GC. We don't need to care about + * the blocking in some situations such as in idle time or just before closing. + * @return whether GC is needed to run or not. + */ + public boolean needsToRunGC(final boolean mindsBlockByGC) { if (!isValidDictionary()) return false; - return needsToRunGCNative(mNativeDict); + return needsToRunGCNative(mNativeDict, mindsBlockByGC); } @UsedForTesting @@ -312,6 +343,12 @@ public final class BinaryDictionary extends Dictionary { return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability); } + @UsedForTesting + public String getPropertyForTests(String query) { + if (!isValidDictionary()) return ""; + return getPropertyNative(mNativeDict, query); + } + @Override public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { // TODO: actually use the confidence rather than use this completely broken heuristic @@ -338,21 +375,23 @@ public final class BinaryDictionary extends Dictionary { traverseSession.close(); } } + mDicTraverseSessions.clear(); } - closeInternal(); + closeInternalLocked(); } - private synchronized void closeInternal() { + private synchronized void closeInternalLocked() { if (mNativeDict != 0) { closeNative(mNativeDict); mNativeDict = 0; } } + // TODO: Manage BinaryDictionary instances without using WeakReference or something. @Override protected void finalize() throws Throwable { try { - closeInternal(); + closeInternalLocked(); } finally { super.finalize(); } diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java index 5a453dde5..84abfa66d 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/DictionaryWriter.java @@ -31,6 +31,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Map; /** * An in memory dictionary for memorizing entries and writing a binary dictionary. @@ -84,8 +85,11 @@ public class DictionaryWriter extends AbstractDictionaryWriter { } @Override - protected void writeDictionary(final DictEncoder dictEncoder) - throws IOException, UnsupportedFormatException { + protected void writeDictionary(final DictEncoder dictEncoder, + final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException { + for (final Map.Entry<String, String> entry : attributeMap.entrySet()) { + mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue()); + } dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS); } diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 712520212..2d1ca51e2 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -22,12 +22,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; @@ -35,9 +30,9 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -69,8 +64,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */); + private static final int DICTIONARY_FORMAT_VERSION = 3; + + private static final String SUPPORTS_DYNAMIC_UPDATE = + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE; /** * A static map of update controllers, each of which records the time of accesses to a single @@ -235,6 +232,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } + protected Map<String, String> getHeaderAttributeMap() { + HashMap<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + SUPPORTS_DYNAMIC_UPDATE); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename); + return attributeMap; + } + protected void clear() { getExecutor(mFilename).execute(new Runnable() { @Override @@ -242,17 +247,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { mBinaryDictionary.close(); final File file = new File(mContext.getFilesDir(), mFilename); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), - false, false)); - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - try { - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } + BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); } else { mDictionaryWriter.clear(); } @@ -501,31 +497,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); loadDictionaryAsync(); - mDictionaryWriter.write(mFilename); + mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } else { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { final File file = new File(mContext.getFilesDir(), mFilename); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), - false, false)); - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - try { - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } + BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); } else { - if (mBinaryDictionary.needsToRunGC()) { + if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { mBinaryDictionary.flushWithGC(); } else { mBinaryDictionary.flush(); } } } else { - mDictionaryWriter.write(mFilename); + mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); } } } @@ -674,47 +661,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } } - /** - * Dynamically adds a word unigram to the dictionary for testing with blocking-lock. - */ - @UsedForTesting - protected void addWordDynamicallyForTests(final String word, final String shortcutTarget, - final int frequency, final boolean isNotAWord) { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - addWordDynamically(word, shortcutTarget, frequency, isNotAWord); - } - }); - } - - /** - * Dynamically adds a word bigram in the dictionary for testing with blocking-lock. - */ - @UsedForTesting - protected void addBigramDynamicallyForTests(final String word0, final String word1, - final int frequency, final boolean isValid) { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - addBigramDynamically(word0, word1, frequency, isValid); - } - }); - } - - /** - * Dynamically remove a word bigram in the dictionary for testing with blocking-lock. - */ - @UsedForTesting - protected void removeBigramDynamicallyForTests(final String word0, final String word1) { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - removeBigramDynamically(word0, word1); - } - }); - } - // TODO: Implement native binary methods once the dynamic dictionary implementation is done. @UsedForTesting public boolean isInDictionaryForTests(final String word) { diff --git a/java/src/com/android/inputmethod/latin/InputAttributes.java b/java/src/com/android/inputmethod/latin/InputAttributes.java index 21b103e5a..8caf6f17f 100644 --- a/java/src/com/android/inputmethod/latin/InputAttributes.java +++ b/java/src/com/android/inputmethod/latin/InputAttributes.java @@ -103,6 +103,10 @@ public final class InputAttributes { } } + public boolean isTypeNull() { + return InputType.TYPE_NULL == mInputType; + } + public boolean isSameInputType(final EditorInfo editorInfo) { return editorInfo.inputType == mInputType; } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 270dc4c06..96e16de0d 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -81,7 +81,7 @@ import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationDictionarySessionRegister; import com.android.inputmethod.latin.personalization.PersonalizationHelper; import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; -import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary; +import com.android.inputmethod.latin.personalization.UserHistoryDictionary; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.settings.SettingsActivity; import com.android.inputmethod.latin.settings.SettingsValues; @@ -179,7 +179,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private boolean mIsMainDictionaryAvailable; private UserBinaryDictionary mUserDictionary; - private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary; + private UserHistoryDictionary mUserHistoryDictionary; private PersonalizationPredictionDictionary mPersonalizationPredictionDictionary; private PersonalizationDictionary mPersonalizationDictionary; private boolean mIsUserDictionaryAvailable; @@ -623,9 +623,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); - mUserHistoryPredictionDictionary = PersonalizationHelper - .getUserHistoryPredictionDictionary(this, localeStr, prefs); - newSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary); + mUserHistoryDictionary = PersonalizationHelper.getUserHistoryDictionary( + this, localeStr, prefs); + newSuggest.setUserHistoryDictionary(mUserHistoryDictionary); mPersonalizationDictionary = PersonalizationHelper .getPersonalizationDictionary(this, localeStr, prefs); newSuggest.setPersonalizationDictionary(mPersonalizationDictionary); @@ -1521,7 +1521,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mSubtypeState.switchSubtype(token, mRichImm); } - private void sendDownUpKeyEventForBackwardCompatibility(final int code) { + private void sendDownUpKeyEvent(final int code) { final long eventTime = SystemClock.uptimeMillis(); mConnection.sendKeyEvent(new KeyEvent(eventTime, eventTime, KeyEvent.ACTION_DOWN, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, @@ -1538,7 +1538,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // TODO: Remove this special handling of digit letters. // For backward compatibility. See {@link InputMethodService#sendKeyChar(char)}. if (code >= '0' && code <= '9') { - sendDownUpKeyEventForBackwardCompatibility(code - '0' + KeyEvent.KEYCODE_0); + sendDownUpKeyEvent(code - '0' + KeyEvent.KEYCODE_0); return; } @@ -1547,7 +1547,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // a hardware keyboard event on pressing enter or delete. This is bad for many // reasons (there are race conditions with commits) but some applications are // relying on this behavior so we continue to support it for older apps. - sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_ENTER); + sendDownUpKeyEvent(KeyEvent.KEYCODE_ENTER); } else { final String text = new String(new int[] { code }, 0, 1); mConnection.commitText(text, text.length()); @@ -2104,12 +2104,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } final int lengthToDelete = Character.isSupplementaryCodePoint( mConnection.getCodePointBeforeCursor()) ? 2 : 1; - if (mAppWorkAroundsUtils.isBeforeJellyBean()) { - // Backward compatibility mode. Before Jelly bean, the keyboard would simulate - // a hardware keyboard event on pressing enter or delete. This is bad for many - // reasons (there are race conditions with commits) but some applications are - // relying on this behavior so we continue to support it for older apps. - sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_DEL); + if (mAppWorkAroundsUtils.isBeforeJellyBean() || + currentSettings.mInputAttributes.isTypeNull()) { + // There are two possible reasons to send a key event: either the field has + // type TYPE_NULL, in which case the keyboard should send events, or we are + // running in backward compatibility mode. Before Jelly bean, the keyboard + // would simulate a hardware keyboard event on pressing enter or delete. This + // is bad for many reasons (there are race conditions with commits) but some + // applications are relying on this behavior so we continue to support it for + // older apps, so we retain this behavior if the app has target SDK < JellyBean. + sendDownUpKeyEvent(KeyEvent.KEYCODE_DEL); } else { mConnection.deleteSurroundingText(lengthToDelete, 0); } @@ -2553,6 +2557,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private void showSuggestionStripWithTypedWord(final SuggestedWords suggestedWords, final String typedWord) { if (suggestedWords.isEmpty()) { + // No auto-correction is available, clear the cached values. + AccessibilityUtils.getInstance().setAutoCorrection(null, null); clearSuggestionStrip(); return; } @@ -2561,6 +2567,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen setSuggestedWords(suggestedWords, isAutoCorrection); setAutoCorrectionIndicator(isAutoCorrection); setSuggestionStripShown(isSuggestionsStripVisible()); + // An auto-correction is available, cache it in accessibility code so + // we can be speak it if the user touches a key that will insert it. + AccessibilityUtils.getInstance().setAutoCorrection(suggestedWords, typedWord); } private void showSuggestionStrip(final SuggestedWords suggestedWords) { @@ -2746,9 +2755,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final SettingsValues currentSettings = mSettings.getCurrent(); if (!currentSettings.mCorrectionEnabled) return null; - final UserHistoryPredictionDictionary userHistoryPredictionDictionary = - mUserHistoryPredictionDictionary; - if (userHistoryPredictionDictionary == null) return null; + final UserHistoryDictionary userHistoryDictionary = mUserHistoryDictionary; + if (userHistoryDictionary == null) return null; final String prevWord = mConnection.getNthPreviousWord(currentSettings.mWordSeparators, 2); final String secondWord; @@ -2762,8 +2770,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen final int maxFreq = AutoCorrectionUtils.getMaxFrequency( suggest.getUnigramDictionaries(), suggestion); if (maxFreq == 0) return null; - userHistoryPredictionDictionary - .addToPersonalizationPredictionDictionary(prevWord, secondWord, maxFreq > 0); + userHistoryDictionary.addToDictionary(prevWord, secondWord, maxFreq > 0); return prevWord; } @@ -2949,7 +2956,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } mConnection.deleteSurroundingText(deleteLength, 0); if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) { - mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord); + mUserHistoryDictionary.cancelAddingUserHistory(previousWord, committedWord); } final String stringToCommit = originallyTypedWord + mLastComposedWord.mSeparatorString; if (mSettings.getCurrent().mCurrentLanguageHasSpaces) { diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 925381b50..8580a6e54 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -30,6 +30,7 @@ import com.android.inputmethod.latin.define.ProductionFlag; import com.android.inputmethod.latin.settings.SettingsValues; import com.android.inputmethod.latin.utils.CapsModeUtils; import com.android.inputmethod.latin.utils.DebugLogUtils; +import com.android.inputmethod.latin.utils.SpannableStringUtils; import com.android.inputmethod.latin.utils.StringUtils; import com.android.inputmethod.latin.utils.TextRange; import com.android.inputmethod.research.ResearchLogger; @@ -610,8 +611,9 @@ public final class RichInputConnection { // We don't use TextUtils#concat because it copies all spans without respect to their // nature. If the text includes a PARAGRAPH span and it has been split, then // TextUtils#concat will crash when it tries to concat both sides of it. - return new TextRange(StringUtils.concatWithNonParagraphSuggestionSpansOnly(before, after), - startIndexInBefore, before.length() + endIndexInAfter, before.length()); + return new TextRange( + SpannableStringUtils.concatWithNonParagraphSuggestionSpansOnly(before, after), + startIndexInBefore, before.length() + endIndexInAfter, before.length()); } public boolean isCursorTouchingWord(final SettingsValues settingsValues) { diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 6c18c948f..9fd1f53a2 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -26,7 +26,7 @@ import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.PersonalizationDictionary; import com.android.inputmethod.latin.personalization.PersonalizationPredictionDictionary; -import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary; +import com.android.inputmethod.latin.personalization.UserHistoryDictionary; import com.android.inputmethod.latin.settings.Settings; import com.android.inputmethod.latin.utils.AutoCorrectionUtils; import com.android.inputmethod.latin.utils.BoundedTreeSet; @@ -190,10 +190,8 @@ public final class Suggest { addOrReplaceDictionaryInternal(Dictionary.TYPE_CONTACTS, contactsDictionary); } - public void setUserHistoryPredictionDictionary( - final UserHistoryPredictionDictionary userHistoryPredictionDictionary) { - addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, - userHistoryPredictionDictionary); + public void setUserHistoryDictionary(final UserHistoryDictionary userHistoryDictionary) { + addOrReplaceDictionaryInternal(Dictionary.TYPE_USER_HISTORY, userHistoryDictionary); } public void setPersonalizationPredictionDictionary( diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 4dba8e5cf..af61f2979 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -225,6 +225,26 @@ public class BinaryDictEncoderUtils { return position; } + static void writeUIntToStream(final OutputStream stream, final int value, final int size) + throws IOException { + switch(size) { + case 4: + stream.write((value >> 24) & 0xFF); + /* fall through */ + case 3: + stream.write((value >> 16) & 0xFF); + /* fall through */ + case 2: + stream.write((value >> 8) & 0xFF); + /* fall through */ + case 1: + stream.write(value & 0xFF); + break; + default: + /* nop */ + } + } + // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time @@ -365,12 +385,14 @@ public class BinaryDictEncoderUtils { nodeSize + size, ptNode.mChildren)); } nodeSize += getShortcutListSize(ptNode.mShortcutTargets); - if (null != ptNode.mBigrams) { - for (WeightedString bigram : ptNode.mBigrams) { - final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, - nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE, - FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); - nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + if (formatOptions.mVersion < FormatSpec.FIRST_VERSION_WITH_TERMINAL_ID) { + if (null != ptNode.mBigrams) { + for (WeightedString bigram : ptNode.mBigrams) { + final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, + nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE, + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); + nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + } } } ptNode.mCachedSize = nodeSize; @@ -882,8 +904,9 @@ public class BinaryDictEncoderUtils { * @param destination the stream to write the file header to. * @param dict the dictionary to write. * @param formatOptions file format options. + * @return the size of the header. */ - /* package */ static void writeDictionaryHeader(final OutputStream destination, + /* package */ static int writeDictionaryHeader(final OutputStream destination, final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { final int version = formatOptions.mVersion; @@ -932,5 +955,6 @@ public class BinaryDictEncoderUtils { destination.write(bytes); headerBuffer.close(); + return size; } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java index bf3d19101..411e265b3 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -77,7 +77,7 @@ public final class DynamicBinaryDictIOUtils { * @param newParentAddress the absolute address of the parent. * @param formatOptions file format options. */ - public static void updateParentAddress(final DictBuffer dictBuffer, + private static void updateParentAddress(final DictBuffer dictBuffer, final int ptNodeOriginAddress, final int newParentAddress, final FormatOptions formatOptions) { final int originalPosition = dictBuffer.position(); @@ -109,7 +109,7 @@ public final class DynamicBinaryDictIOUtils { * @param newParentAddress the address to be written. * @param formatOptions file format options. */ - public static void updateParentAddresses(final DictBuffer dictBuffer, + private static void updateParentAddresses(final DictBuffer dictBuffer, final int ptNodeOriginAddress, final int newParentAddress, final FormatOptions formatOptions) { final int originalPosition = dictBuffer.position(); @@ -136,7 +136,7 @@ public final class DynamicBinaryDictIOUtils { * @param newChildrenAddress the absolute address of the child. * @param formatOptions file format options. */ - public static void updateChildrenAddress(final DictBuffer dictBuffer, + private static void updateChildrenAddress(final DictBuffer dictBuffer, final int ptNodeOriginAddress, final int newChildrenAddress, final FormatOptions formatOptions) { final int originalPosition = dictBuffer.position(); diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 51b89a02a..9481a8c14 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -263,7 +263,14 @@ public final class FormatSpec { // These values are used only by version 4 or later. static final String TRIE_FILE_EXTENSION = ".trie"; static final String FREQ_FILE_EXTENSION = ".freq"; + // tat = Terminal Address Table + static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; + static final String BIGRAM_FILE_EXTENSION = ".bigram"; + static final String BIGRAM_LOOKUP_TABLE_FILE_EXTENSION = ".bigram_lookup"; + static final String BIGRAM_ADDRESS_TABLE_FILE_EXTENSION = ".bigram_index"; static final int FREQUENCY_AND_FLAGS_SIZE = 2; + static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3; + static final int BIGRAM_ADDRESS_TABLE_BLOCK_SIZE = 4; static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; @@ -322,9 +329,15 @@ public final class FormatSpec { public final int mHeaderSize; public final DictionaryOptions mDictionaryOptions; public final FormatOptions mFormatOptions; - private static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; - private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale"; - private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary"; + // Note that these are corresponding definitions in native code in latinime::HeaderPolicy + // and latinime::HeaderReadWriteUtils. + public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; + public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; + public static final String ATTRIBUTE_VALUE_TRUE = "1"; + + public static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; + public static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale"; + public static final String DICTIONARY_ID_ATTRIBUTE = "dictionary"; private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description"; public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions, final FormatOptions formatOptions) { diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java new file mode 100644 index 000000000..96d057a44 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; + +/** + * SparseTable is an extensible map from integer to integer. + * This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index + * memory. + */ +@UsedForTesting +public class SparseTable { + + /** + * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize + * terminals. + * It contains at index i = j / mBlockSize the index in mContentsTable where the values for + * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array. + */ + private final ArrayList<Integer> mLookupTable; + private final ArrayList<Integer> mContentTable; + + private final int mBlockSize; + public static final int NOT_EXIST = -1; + + @UsedForTesting + public SparseTable(final int initialCapacity, final int blockSize) { + mBlockSize = blockSize; + final int lookupTableSize = initialCapacity / mBlockSize + + (initialCapacity % mBlockSize > 0 ? 1 : 0); + mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST)); + mContentTable = new ArrayList<Integer>(); + } + + @UsedForTesting + public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) { + mBlockSize = blockSize; + mLookupTable = new ArrayList<Integer>(lookupTable.length); + for (int i = 0; i < lookupTable.length; ++i) { + mLookupTable.add(lookupTable[i]); + } + mContentTable = new ArrayList<Integer>(contentTable.length); + for (int i = 0; i < contentTable.length; ++i) { + mContentTable.add(contentTable[i]); + } + } + + /** + * Converts an byte array to an int array considering each set of 4 bytes is an int stored in + * big-endian. + * The length of byteArray must be a multiple of four. + * Otherwise, IndexOutOfBoundsException will be raised. + */ + @UsedForTesting + private static void convertByteArrayToIntegerArray(final byte[] byteArray, + final ArrayList<Integer> integerArray) { + for (int i = 0; i < byteArray.length; i += 4) { + int value = 0; + for (int j = i; j < i + 4; ++j) { + value <<= 8; + value |= byteArray[j] & 0xFF; + } + integerArray.add(value); + } + } + + @UsedForTesting + public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) { + mBlockSize = blockSize; + mLookupTable = new ArrayList<Integer>(lookupTable.length / 4); + mContentTable = new ArrayList<Integer>(contentTable.length / 4); + convertByteArrayToIntegerArray(lookupTable, mLookupTable); + convertByteArrayToIntegerArray(contentTable, mContentTable); + } + + @UsedForTesting + public int get(final int index) { + if (index < 0 || index / mBlockSize >= mLookupTable.size() + || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + return NOT_EXIST; + } + return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); + } + + @UsedForTesting + public void set(final int index, final int value) { + if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + mLookupTable.set(index / mBlockSize, mContentTable.size()); + for (int i = 0; i < mBlockSize; ++i) { + mContentTable.add(NOT_EXIST); + } + } + mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); + } + + public void remove(final int index) { + set(index, NOT_EXIST); + } + + @UsedForTesting + public int size() { + return mLookupTable.size() * mBlockSize; + } + + @UsedForTesting + /* package */ int getContentTableSize() { + return mContentTable.size(); + } + + @UsedForTesting + /* package */ int getLookupTableSize() { + return mLookupTable.size(); + } + + public boolean contains(final int index) { + return get(index) != NOT_EXIST; + } + + @UsedForTesting + public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream) + throws IOException { + for (final int index : mLookupTable) { + BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4); + } + + for (final int index : mContentTable) { + BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4); + } + } + + @UsedForTesting + public void writeToFiles(final File lookupTableFile, final File contentFile) + throws IOException { + FileOutputStream lookupTableOutStream = null; + FileOutputStream contentOutStream = null; + try { + lookupTableOutStream = new FileOutputStream(lookupTableFile); + contentOutStream = new FileOutputStream(contentFile); + write(lookupTableOutStream, contentOutStream); + } finally { + if (lookupTableOutStream != null) { + lookupTableOutStream.close(); + } + if (contentOutStream != null) { + contentOutStream.close(); + } + } + } + + private static byte[] readFileToByteArray(final File file) throws IOException { + final byte[] contents = new byte[(int) file.length()]; + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + inStream.read(contents); + } finally { + if (inStream != null) { + inStream.close(); + } + } + return contents; + } + + @UsedForTesting + public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile, + final int blockSize) throws IOException { + final byte[] lookupTable = readFileToByteArray(lookupTableFile); + final byte[] content = readFileToByteArray(contentFile); + return new SparseTable(lookupTable, content, blockSize); + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 36c5a2720..0aa431966 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -41,11 +41,16 @@ public class Ver4DictDecoder extends DictDecoder { private static final int FILETYPE_TRIE = 1; private static final int FILETYPE_FREQUENCY = 2; + private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; + private static final int FILETYPE_BIGRAM = 4; private final File mDictDirectory; private final DictionaryBufferFactory mBufferFactory; private DictBuffer mDictBuffer; private DictBuffer mFrequencyBuffer; + private DictBuffer mTerminalAddressTableBuffer; + private DictBuffer mBigramBuffer; + private SparseTable mBigramAddressTable; @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { @@ -77,6 +82,12 @@ public class Ver4DictDecoder extends DictDecoder { } else if (fileType == FILETYPE_FREQUENCY) { return new File(mDictDirectory, mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION); + } else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) { + return new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + } else if (fileType == FILETYPE_BIGRAM) { + return new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.BIGRAM_FILE_EXTENSION); } else { throw new RuntimeException("Unsupported kind of file : " + fileType); } @@ -87,6 +98,10 @@ public class Ver4DictDecoder extends DictDecoder { final String filename = mDictDirectory.getName(); mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE)); mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); + mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( + getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); + mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM)); + loadBigramAddressSparseTable(); } @Override @@ -111,6 +126,15 @@ public class Ver4DictDecoder extends DictDecoder { return header; } + private void loadBigramAddressSparseTable() throws IOException { + final File lookupIndexFile = new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION); + final File contentFile = new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION); + mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); + } + protected static class PtNodeReader extends DictDecoder.PtNodeReader { protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) { frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1); @@ -184,8 +208,21 @@ public class Ver4DictDecoder extends DictDecoder { final ArrayList<PendingAttribute> bigrams; if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { bigrams = new ArrayList<PendingAttribute>(); - addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, - addressPointer); + final int posOfBigrams = mBigramAddressTable.get(terminalId); + mBigramBuffer.position(posOfBigrams); + while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, + // remaining bigram entries are ignored. + final int bigramFlags = mBigramBuffer.readUnsignedByte(); + final int targetTerminalId = mBigramBuffer.readUnsignedInt24(); + mTerminalAddressTableBuffer.position( + targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + final int targetAddress = mTerminalAddressTableBuffer.readUnsignedInt24(); + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + targetAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { MakedictLog.d("too many bigrams in a node."); } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 75b75ae2e..4c25faf88 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -26,6 +26,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -41,10 +42,15 @@ import java.util.Iterator; public class Ver4DictEncoder implements DictEncoder { private final File mDictPlacedDir; private byte[] mTrieBuf; - private byte[] mFreqBuf; private int mTriePos; + private int mHeaderSize; + private SparseTable mBigramAddressTable; private OutputStream mTrieOutStream; private OutputStream mFreqOutStream; + private OutputStream mTerminalAddressTableOutStream; + private OutputStream mBigramOutStream; + private File mDictDir; + private String mBaseFilename; @UsedForTesting public Ver4DictEncoder(final File dictPlacedDir) { @@ -54,18 +60,25 @@ public class Ver4DictEncoder implements DictEncoder { private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions) throws FileNotFoundException, IOException { final FileHeader header = new FileHeader(0, dictOptions, formatOptions); - final String filename = header.getId() + "." + header.getVersion(); - final File mDictDir = new File(mDictPlacedDir, filename); - final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION); - final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION); + mBaseFilename = header.getId() + "." + header.getVersion(); + mDictDir = new File(mDictPlacedDir, mBaseFilename); + final File trieFile = new File(mDictDir, mBaseFilename + FormatSpec.TRIE_FILE_EXTENSION); + final File freqFile = new File(mDictDir, mBaseFilename + FormatSpec.FREQ_FILE_EXTENSION); + final File terminalAddressTableFile = new File(mDictDir, + mBaseFilename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + final File bigramFile = new File(mDictDir, + mBaseFilename + FormatSpec.BIGRAM_FILE_EXTENSION); if (!mDictDir.isDirectory()) { if (mDictDir.exists()) mDictDir.delete(); mDictDir.mkdirs(); } if (!trieFile.exists()) trieFile.createNewFile(); if (!freqFile.exists()) freqFile.createNewFile(); + if (!terminalAddressTableFile.exists()) terminalAddressTableFile.createNewFile(); mTrieOutStream = new FileOutputStream(trieFile); mFreqOutStream = new FileOutputStream(freqFile); + mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile); + mBigramOutStream = new FileOutputStream(bigramFile); } private void close() throws IOException { @@ -76,9 +89,17 @@ public class Ver4DictEncoder implements DictEncoder { if (mFreqOutStream != null) { mFreqOutStream.close(); } + if (mTerminalAddressTableOutStream != null) { + mTerminalAddressTableOutStream.close(); + } + if (mBigramOutStream != null) { + mBigramOutStream.close(); + } } finally { mTrieOutStream = null; mFreqOutStream = null; + mTerminalAddressTableOutStream = null; + mBigramOutStream = null; } } @@ -97,7 +118,8 @@ public class Ver4DictEncoder implements DictEncoder { openStreams(formatOptions, dict.mOptions); } - BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, formatOptions); + mHeaderSize = BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, + formatOptions); MakedictLog.i("Flattening the tree..."); ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); @@ -112,10 +134,15 @@ public class Ver4DictEncoder implements DictEncoder { BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + writeTerminalData(flatNodes, terminalCount); + mBigramAddressTable = new SparseTable(terminalCount, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); + writeBigrams(flatNodes, dict); + writeBigramAddressSparseTable(); + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; mTrieBuf = new byte[bufferSize]; - mFreqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE]; MakedictLog.i("Writing file..."); for (PtNodeArray nodeArray : flatNodes) { @@ -126,7 +153,6 @@ public class Ver4DictEncoder implements DictEncoder { MakedictLog.i("has " + terminalCount + " terminals."); } mTrieOutStream.write(mTrieBuf); - mFreqOutStream.write(mFreqBuf); MakedictLog.i("Done"); close(); @@ -185,12 +211,6 @@ public class Ver4DictEncoder implements DictEncoder { FormatSpec.PTNODE_TERMINAL_ID_SIZE); } - private void writeFrequency(final int frequency, final int terminalId) { - final int freqPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE; - BinaryDictEncoderUtils.writeUIntToBuffer(mFreqBuf, freqPos, frequency, - FormatSpec.FREQUENCY_AND_FLAGS_SIZE); - } - private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); if (formatOptions.mSupportsDynamicUpdate) { @@ -226,24 +246,41 @@ public class Ver4DictEncoder implements DictEncoder { shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); } - private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) { - if (bigrams == null) return; - - final Iterator<WeightedString> bigramIterator = bigrams.iterator(); - while (bigramIterator.hasNext()) { - final WeightedString bigram = bigramIterator.next(); - final PtNode target = - FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); - final int addressOfBigram = target.mCachedAddressAfterUpdate; - final int unigramFrequencyForThisWord = target.mFrequency; - final int offset = addressOfBigram - - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), - offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); - mTrieBuf[mTriePos++] = (byte) bigramFlags; - mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, - mTriePos, Math.abs(offset)); + private void writeBigrams(final ArrayList<PtNodeArray> flatNodes, final FusionDictionary dict) + throws IOException { + final ByteArrayOutputStream bigramBuffer = new ByteArrayOutputStream(); + + for (final PtNodeArray nodeArray : flatNodes) { + for (final PtNode ptNode : nodeArray.mData) { + if (ptNode.mBigrams != null) { + final int startPos = bigramBuffer.size(); + mBigramAddressTable.set(ptNode.mTerminalId, startPos); + final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int unigramFrequencyForThisWord = target.mFrequency; + final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags( + bigramIterator.hasNext(), 0, bigram.mFrequency, + unigramFrequencyForThisWord, bigram.mWord); + BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, bigramFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + BinaryDictEncoderUtils.writeUIntToStream(bigramBuffer, target.mTerminalId, + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE); + } + } + } } + bigramBuffer.writeTo(mBigramOutStream); + } + + private void writeBigramAddressSparseTable() throws IOException { + final File lookupIndexFile = + new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION); + final File contentFile = + new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION); + mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile); } @Override @@ -260,10 +297,30 @@ public class Ver4DictEncoder implements DictEncoder { writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); if (ptNode.isTerminal()) { writeTerminalId(ptNode.mTerminalId); - writeFrequency(ptNode.mFrequency, ptNode.mTerminalId); } writeChildrenPosition(ptNode, formatOptions); writeShortcuts(ptNode.mShortcutTargets); - writeBigrams(ptNode.mBigrams, dict); + } + + private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes, + final int terminalCount) throws IOException { + final byte[] freqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE]; + final byte[] terminalAddressTableBuf = + new byte[terminalCount * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE]; + for (final PtNodeArray nodeArray : flatNodes) { + for (final PtNode ptNode : nodeArray.mData) { + if (ptNode.isTerminal()) { + BinaryDictEncoderUtils.writeUIntToBuffer(freqBuf, + ptNode.mTerminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE, + ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf, + ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, + ptNode.mCachedAddressAfterUpdate + mHeaderSize, + FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + } + } + } + mFreqOutStream.write(freqBuf); + mTerminalAddressTableOutStream.write(terminalAddressTableBuf); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 075d7e3c3..7cf4f0c88 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -22,6 +22,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.Dictionary; import com.android.inputmethod.latin.ExpandableBinaryDictionary; import com.android.inputmethod.latin.LatinImeLogger; import com.android.inputmethod.latin.makedict.DictDecoder; @@ -34,12 +35,15 @@ import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListe import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; /** - * This class is a base class of a dictionary for the personalized prediction language model. + * This class is a base class of a dictionary that supports decaying for the personalized language + * model. */ -public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary { - private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName(); +public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary { + private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName(); public static final boolean DBG_SAVE_RESTORE = false; private static final boolean DBG_STRESS_TEST = false; private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; @@ -47,6 +51,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi /** Any pair being typed or picked */ public static final int FREQUENCY_FOR_TYPED = 2; + public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED; + public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY; + /** Locale for which this user history dictionary is storing words */ private final String mLocale; @@ -60,8 +67,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi // Should always be false except when we use this class for test @UsedForTesting boolean mIsTest = false; - /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale, - final SharedPreferences sp, final String dictionaryType, final String fileName) { + /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, + final String locale, final SharedPreferences sp, final String dictionaryType, + final String fileName) { super(context, fileName, dictionaryType, true); mLocale = locale; mFileName = fileName; @@ -84,6 +92,18 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi } @Override + protected Map<String, String> getHeaderAttributeMap() { + HashMap<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName); + attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale); + return attributeMap; + } + + @Override protected boolean hasContentChanged() { return false; } @@ -103,27 +123,29 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi } /** - * Pair will be added to the personalization prediction dictionary. + * Pair will be added to the decaying dictionary. * * The first word may be null. That means we don't know the context, in other words, * it's only a unigram. The first word may also be an empty string : this means start * context, as in beginning of a sentence for example. * The second word may not be null (a NullPointerException would be thrown). */ - public void addToPersonalizationPredictionDictionary( - final String word0, final String word1, final boolean isValid) { + public void addToDictionary(final String word0, final String word1, final boolean isValid) { if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH || (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) { return; } - addWordDynamically(word1, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED, + final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) : + FREQUENCY_FOR_TYPED; + addWordDynamically(word1, null /* the "shortcut" parameter is null */, frequency, false /* isNotAWord */); // Do not insert a word as a bigram of itself if (word1.equals(word0)) { return; } if (null != word0) { - addBigramDynamically(word0, word1, FREQUENCY_FOR_TYPED, isValid); + addBigramDynamically(word0, word1, frequency, isValid); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java index 305088536..039b25337 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -36,6 +36,7 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.Forge import java.io.IOException; import java.util.ArrayList; +import java.util.Map; // Currently this class is used to implement dynamic prodiction dictionary. // TODO: Move to native code. @@ -113,8 +114,8 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr } @Override - protected void writeDictionary(final DictEncoder dictEncoder) - throws IOException, UnsupportedFormatException { + protected void writeDictionary(final DictEncoder dictEncoder, + final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException { UserHistoryDictIOUtils.writeDictionary(dictEncoder, new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams), mBigramList, FORMAT_OPTIONS); diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index ab3de801c..a86f6e584 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -46,7 +46,7 @@ public abstract class PersonalizationDictionaryUpdateSession { // TODO: Use a dynamic binary dictionary instead public WeakReference<PersonalizationDictionary> mDictionary; - public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary; + public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary; public final String mSystemLocale; public PersonalizationDictionaryUpdateSession(String locale) { mSystemLocale = locale; @@ -60,15 +60,16 @@ public abstract class PersonalizationDictionaryUpdateSession { mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); } - public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) { - mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); + public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) { + mPredictionDictionary = + new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary); } protected PersonalizationDictionary getDictionary() { return mDictionary == null ? null : mDictionary.get(); } - protected DynamicPredictionDictionaryBase getPredictionDictionary() { + protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() { return mPredictionDictionary == null ? null : mPredictionDictionary.get(); } @@ -81,7 +82,7 @@ public abstract class PersonalizationDictionaryUpdateSession { } private void unsetPredictionDictionary() { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } @@ -89,7 +90,7 @@ public abstract class PersonalizationDictionaryUpdateSession { } public void clearAndFlushPredictionDictionary(Context context) { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } @@ -105,24 +106,23 @@ public abstract class PersonalizationDictionaryUpdateSession { // TODO: Support multi locale to add bigram public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, int frequency) { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } - dictionary.addToPersonalizationPredictionDictionary(word0, word1, isValid); + dictionary.addToDictionary(word0, word1, isValid); } // Bulk import // TODO: Support multi locale to add bigram public void addBigramsToPersonalizationDictionary( final ArrayList<PersonalizationLanguageModelParam> lmParams) { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } for (final PersonalizationLanguageModelParam lmParam : lmParams) { - dictionary.addToPersonalizationPredictionDictionary( - lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid); + dictionary.addToDictionary(lmParam.mWord0, lmParam.mWord1, lmParam.mIsValid); } } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index 5f702ee3f..8c9484b12 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -30,7 +30,7 @@ public class PersonalizationHelper { private static final String TAG = PersonalizationHelper.class.getSimpleName(); private static final boolean DEBUG = false; - private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>> + private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>> sLangUserHistoryDictCache = CollectionUtils.newConcurrentHashMap(); private static final ConcurrentHashMap<String, SoftReference<PersonalizationDictionary>> @@ -41,25 +41,23 @@ public class PersonalizationHelper { sLangPersonalizationPredictionDictCache = CollectionUtils.newConcurrentHashMap(); - public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary( + public static UserHistoryDictionary getUserHistoryDictionary( final Context context, final String locale, final SharedPreferences sp) { synchronized (sLangUserHistoryDictCache) { if (sLangUserHistoryDictCache.containsKey(locale)) { - final SoftReference<UserHistoryPredictionDictionary> ref = + final SoftReference<UserHistoryDictionary> ref = sLangUserHistoryDictCache.get(locale); - final UserHistoryPredictionDictionary dict = ref == null ? null : ref.get(); + final UserHistoryDictionary dict = ref == null ? null : ref.get(); if (dict != null) { if (DEBUG) { - Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale); + Log.w(TAG, "Use cached UserHistoryDictionary for " + locale); } dict.reloadDictionaryIfRequired(); return dict; } } - final UserHistoryPredictionDictionary dict = - new UserHistoryPredictionDictionary(context, locale, sp); - sLangUserHistoryDictCache.put( - locale, new SoftReference<UserHistoryPredictionDictionary>(dict)); + final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp); + sLangUserHistoryDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict)); return dict; } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java index e80953c05..432954453 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -22,7 +22,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; import android.content.SharedPreferences; -public class PersonalizationPredictionDictionary extends DynamicPredictionDictionaryBase { +public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index b140c919b..a60226d7e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -26,10 +26,10 @@ import android.content.SharedPreferences; * Locally gathers stats about the words user types and various other signals like auto-correction * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. */ -public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase { +public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBase { /* package for tests */ static final String NAME = - UserHistoryPredictionDictionary.class.getSimpleName(); - /* package */ UserHistoryPredictionDictionary(final Context context, final String locale, + UserHistoryDictionary.class.getSimpleName(); + /* package */ UserHistoryDictionary(final Context context, final String locale, final SharedPreferences sp) { super(context, locale, sp, Dictionary.TYPE_USER_HISTORY, getDictionaryFileName(locale)); } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java index 4c1803bdf..55a90ee51 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java @@ -54,7 +54,7 @@ public final class UserHistoryDictionaryBigramList { * Called when loaded from the SQL DB. */ public void addBigram(String word1, String word2, byte fcValue) { - if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { + if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; @@ -74,7 +74,7 @@ public final class UserHistoryDictionaryBigramList { * Called when inserted to the SQL DB. */ public void updateBigram(String word1, String word2, byte fcValue) { - if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { + if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; diff --git a/java/src/com/android/inputmethod/latin/utils/SpannableStringUtils.java b/java/src/com/android/inputmethod/latin/utils/SpannableStringUtils.java new file mode 100644 index 000000000..b51fd9377 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/SpannableStringUtils.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.text.Spannable; +import android.text.SpannableString; +import android.text.Spanned; +import android.text.SpannedString; +import android.text.TextUtils; +import android.text.style.SuggestionSpan; + +public final class SpannableStringUtils { + /** + * Copies the spans from the region <code>start...end</code> in + * <code>source</code> to the region + * <code>destoff...destoff+end-start</code> in <code>dest</code>. + * Spans in <code>source</code> that begin before <code>start</code> + * or end after <code>end</code> but overlap this range are trimmed + * as if they began at <code>start</code> or ended at <code>end</code>. + * Only SuggestionSpans that don't have the SPAN_PARAGRAPH span are copied. + * + * This code is almost entirely taken from {@link TextUtils#copySpansFrom}, except for the + * kind of span that is copied. + * + * @throws IndexOutOfBoundsException if any of the copied spans + * are out of range in <code>dest</code>. + */ + public static void copyNonParagraphSuggestionSpansFrom(Spanned source, int start, int end, + Spannable dest, int destoff) { + Object[] spans = source.getSpans(start, end, SuggestionSpan.class); + + for (int i = 0; i < spans.length; i++) { + int fl = source.getSpanFlags(spans[i]); + if (0 != (fl & Spannable.SPAN_PARAGRAPH)) continue; + + int st = source.getSpanStart(spans[i]); + int en = source.getSpanEnd(spans[i]); + + if (st < start) + st = start; + if (en > end) + en = end; + + dest.setSpan(spans[i], st - start + destoff, en - start + destoff, + fl); + } + } + + /** + * Returns a CharSequence concatenating the specified CharSequences, retaining their + * SuggestionSpans that don't have the PARAGRAPH flag, but not other spans. + * + * This code is almost entirely taken from {@link TextUtils#concat(CharSequence...)}, except + * it calls copyNonParagraphSuggestionSpansFrom instead of {@link TextUtils#copySpansFrom}. + */ + public static CharSequence concatWithNonParagraphSuggestionSpansOnly(CharSequence... text) { + if (text.length == 0) { + return ""; + } + + if (text.length == 1) { + return text[0]; + } + + boolean spanned = false; + for (int i = 0; i < text.length; i++) { + if (text[i] instanceof Spanned) { + spanned = true; + break; + } + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < text.length; i++) { + sb.append(text[i]); + } + + if (!spanned) { + return sb.toString(); + } + + SpannableString ss = new SpannableString(sb); + int off = 0; + for (int i = 0; i < text.length; i++) { + int len = text[i].length(); + + if (text[i] instanceof Spanned) { + copyNonParagraphSuggestionSpansFrom((Spanned) text[i], 0, len, ss, off); + } + + off += len; + } + + return new SpannedString(ss); + } +} diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index 327780ad0..121aecf0f 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -20,12 +20,7 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.settings.SettingsValues; -import android.text.Spannable; -import android.text.SpannableString; -import android.text.Spanned; -import android.text.SpannedString; import android.text.TextUtils; -import android.text.style.SuggestionSpan; import android.util.JsonReader; import android.util.JsonWriter; import android.util.Log; @@ -467,88 +462,4 @@ public final class StringUtils { } return ""; } - - /** - * Copies the spans from the region <code>start...end</code> in - * <code>source</code> to the region - * <code>destoff...destoff+end-start</code> in <code>dest</code>. - * Spans in <code>source</code> that begin before <code>start</code> - * or end after <code>end</code> but overlap this range are trimmed - * as if they began at <code>start</code> or ended at <code>end</code>. - * Only SuggestionSpans that don't have the SPAN_PARAGRAPH span are copied. - * - * This code is almost entirely taken from {@link TextUtils#copySpansFrom}, except for the - * kind of span that is copied. - * - * @throws IndexOutOfBoundsException if any of the copied spans - * are out of range in <code>dest</code>. - */ - public static void copyNonParagraphSuggestionSpansFrom(Spanned source, int start, int end, - Spannable dest, int destoff) { - Object[] spans = source.getSpans(start, end, SuggestionSpan.class); - - for (int i = 0; i < spans.length; i++) { - int fl = source.getSpanFlags(spans[i]); - if (0 != (fl & Spannable.SPAN_PARAGRAPH)) continue; - - int st = source.getSpanStart(spans[i]); - int en = source.getSpanEnd(spans[i]); - - if (st < start) - st = start; - if (en > end) - en = end; - - dest.setSpan(spans[i], st - start + destoff, en - start + destoff, - fl); - } - } - - /** - * Returns a CharSequence concatenating the specified CharSequences, retaining their - * SuggestionSpans that don't have the PARAGRAPH flag, but not other spans. - * - * This code is almost entirely taken from {@link TextUtils#concat(CharSequence...)}, except - * it calls copyNonParagraphSuggestionSpansFrom instead of {@link TextUtils#copySpansFrom}. - */ - public static CharSequence concatWithNonParagraphSuggestionSpansOnly(CharSequence... text) { - if (text.length == 0) { - return ""; - } - - if (text.length == 1) { - return text[0]; - } - - boolean spanned = false; - for (int i = 0; i < text.length; i++) { - if (text[i] instanceof Spanned) { - spanned = true; - break; - } - } - - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < text.length; i++) { - sb.append(text[i]); - } - - if (!spanned) { - return sb.toString(); - } - - SpannableString ss = new SpannableString(sb); - int off = 0; - for (int i = 0; i < text.length; i++) { - int len = text[i].length(); - - if (text[i] instanceof Spanned) { - copyNonParagraphSuggestionSpansFrom((Spanned) text[i], 0, len, ss, off); - } - - off += len; - } - - return new SpannedString(ss); - } } diff --git a/native/jni/Android.mk b/native/jni/Android.mk index c2070327e..36afea54b 100644 --- a/native/jni/Android.mk +++ b/native/jni/Android.mk @@ -85,6 +85,8 @@ LATIN_IME_CORE_SRC_FILES := \ $(addprefix suggest/policyimpl/dictionary/utils/, \ buffer_with_extendable_buffer.cpp \ byte_array_utils.cpp \ + decaying_utils.cpp \ + dict_file_writing_utils.cpp \ format_utils.cpp) \ suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \ $(addprefix suggest/policyimpl/typing/, \ diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 7f47493b2..c5ef264fc 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -26,12 +26,55 @@ #include "suggest/core/dictionary/dictionary.h" #include "suggest/core/suggest_options.h" #include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h" +#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "utils/autocorrection_threshold_utils.h" namespace latinime { class ProximityInfo; +// TODO: Move to makedict. +static jboolean latinime_BinaryDictionary_createEmptyDictFile(JNIEnv *env, jclass clazz, + jstring filePath, jlong dictVersion, jobjectArray attributeKeyStringArray, + jobjectArray attributeValueStringArray) { + const jsize filePathUtf8Length = env->GetStringUTFLength(filePath); + char filePathChars[filePathUtf8Length + 1]; + env->GetStringUTFRegion(filePath, 0, env->GetStringLength(filePath), filePathChars); + filePathChars[filePathUtf8Length] = '\0'; + + const int keyCount = env->GetArrayLength(attributeKeyStringArray); + const int valueCount = env->GetArrayLength(attributeValueStringArray); + if (keyCount != valueCount) { + return false; + } + + HeaderReadWriteUtils::AttributeMap attributeMap; + for (int i = 0; i < keyCount; i++) { + jstring keyString = static_cast<jstring>( + env->GetObjectArrayElement(attributeKeyStringArray, i)); + const jsize keyUtf8Length = env->GetStringUTFLength(keyString); + char keyChars[keyUtf8Length + 1]; + env->GetStringUTFRegion(keyString, 0, env->GetStringLength(keyString), keyChars); + keyChars[keyUtf8Length] = '\0'; + HeaderReadWriteUtils::AttributeMap::key_type key; + HeaderReadWriteUtils::insertCharactersIntoVector(keyChars, &key); + + jstring valueString = static_cast<jstring>( + env->GetObjectArrayElement(attributeValueStringArray, i)); + const jsize valueUtf8Length = env->GetStringUTFLength(valueString); + char valueChars[valueUtf8Length + 1]; + env->GetStringUTFRegion(valueString, 0, env->GetStringLength(valueString), valueChars); + valueChars[valueUtf8Length] = '\0'; + HeaderReadWriteUtils::AttributeMap::mapped_type value; + HeaderReadWriteUtils::insertCharactersIntoVector(valueChars, &value); + + attributeMap[key] = value; + } + + return DictFileWritingUtils::createEmptyDictFile(filePathChars, static_cast<int>(dictVersion), + &attributeMap); +} + static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir, jlong dictOffset, jlong dictSize, jboolean isUpdatable) { PROF_OPEN; @@ -70,10 +113,10 @@ static void latinime_BinaryDictionary_flush(JNIEnv *env, jclass clazz, jlong dic } static bool latinime_BinaryDictionary_needsToRunGC(JNIEnv *env, jclass clazz, - jlong dict) { + jlong dict, jboolean mindsBlockByGC) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return false; - return dictionary->needsToRunGC(); + return dictionary->needsToRunGC(mindsBlockByGC == JNI_TRUE); } static void latinime_BinaryDictionary_flushWithGC(JNIEnv *env, jclass clazz, jlong dict, @@ -280,8 +323,31 @@ static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jcl bigramProbability); } +static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict, + jstring query) { + Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); + if (!dictionary) { + return env->NewStringUTF(""); + } + const jsize queryUtf8Length = env->GetStringUTFLength(query); + char queryChars[queryUtf8Length + 1]; + env->GetStringUTFRegion(query, 0, env->GetStringLength(query), queryChars); + queryChars[queryUtf8Length] = '\0'; + static const int GET_PROPERTY_RESULT_LENGTH = 100; + char resultChars[GET_PROPERTY_RESULT_LENGTH]; + resultChars[0] = '\0'; + dictionary->getDictionaryStructurePolicy()->getProperty(queryChars, resultChars, + GET_PROPERTY_RESULT_LENGTH); + return env->NewStringUTF(resultChars); +} + static const JNINativeMethod sMethods[] = { { + const_cast<char *>("createEmptyDictFileNative"), + const_cast<char *>("(Ljava/lang/String;J[Ljava/lang/String;[Ljava/lang/String;)Z"), + reinterpret_cast<void *>(latinime_BinaryDictionary_createEmptyDictFile) + }, + { const_cast<char *>("openNative"), const_cast<char *>("(Ljava/lang/String;JJZ)J"), reinterpret_cast<void *>(latinime_BinaryDictionary_open) @@ -298,7 +364,7 @@ static const JNINativeMethod sMethods[] = { }, { const_cast<char *>("needsToRunGCNative"), - const_cast<char *>("(J)Z"), + const_cast<char *>("(JZ)Z"), reinterpret_cast<void *>(latinime_BinaryDictionary_needsToRunGC) }, { @@ -350,6 +416,11 @@ static const JNINativeMethod sMethods[] = { const_cast<char *>("calculateProbabilityNative"), const_cast<char *>("(JII)I"), reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative) + }, + { + const_cast<char *>("getPropertyNative"), + const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"), + reinterpret_cast<void *>(latinime_BinaryDictionary_getProperty) } }; diff --git a/native/jni/src/defines.h b/native/jni/src/defines.h index 89dfa39b3..c2aa8ba0e 100644 --- a/native/jni/src/defines.h +++ b/native/jni/src/defines.h @@ -375,7 +375,7 @@ typedef enum { CT_TERMINAL, CT_TERMINAL_INSERTION, // Create new word with space omission - CT_NEW_WORD_SPACE_OMITTION, + CT_NEW_WORD_SPACE_OMISSION, // Create new word with space substitution CT_NEW_WORD_SPACE_SUBSTITUTION, } CorrectionType; diff --git a/native/jni/src/suggest/core/dicnode/dic_node.h b/native/jni/src/suggest/core/dicnode/dic_node.h index 41ef9d2b2..9099e8285 100644 --- a/native/jni/src/suggest/core/dicnode/dic_node.h +++ b/native/jni/src/suggest/core/dicnode/dic_node.h @@ -38,10 +38,10 @@ INTS_TO_CHARS(mDicNodeState.mDicNodeStatePrevWord.mPrevWord, \ mDicNodeState.mDicNodeStatePrevWord.getPrevWordLength(), prevWordCharBuf, \ NELEMS(prevWordCharBuf)); \ - AKLOGI("#%8s, %5f, %5f, %5f, %5f, %s, %s, %d,,", header, \ + AKLOGI("#%8s, %5f, %5f, %5f, %5f, %s, %s, %d, %5f,", header, \ getSpatialDistanceForScoring(), getLanguageDistanceForScoring(), \ getNormalizedCompoundDistance(), getRawLength(), prevWordCharBuf, charBuf, \ - getInputIndex(0)); \ + getInputIndex(0), getNormalizedCompoundDistanceAfterFirstWord()); \ } while (0) #else #define LOGI_SHOW_ADD_COST_PROP @@ -434,6 +434,13 @@ class DicNode { return mDicNodeState.mDicNodeStateScoring.getLanguageDistance(); } + // For space-aware gestures, we store the normalized distance at the char index + // that ends the first word of the suggestion. We call this the distance after + // first word. + float getNormalizedCompoundDistanceAfterFirstWord() const { + return mDicNodeState.mDicNodeStateScoring.getNormalizedCompoundDistanceAfterFirstWord(); + } + float getLanguageDistanceRatePerWordForScoring() const { const float langDist = getLanguageDistanceForScoring(); const float totalWordCount = @@ -565,6 +572,12 @@ class DicNode { inputSize, getTotalInputIndex(), errorType); } + // Saves the current normalized compound distance for space-aware gestures. + // See getNormalizedCompoundDistanceAfterFirstWord for details. + AK_FORCE_INLINE void saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet() { + mDicNodeState.mDicNodeStateScoring.saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet(); + } + // Caveat: Must not be called outside Weighting // This restriction is guaranteed by "friend" AK_FORCE_INLINE void forwardInputIndex(const int pointerId, const int count, diff --git a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h index 4c884225a..3c85d0e9d 100644 --- a/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h +++ b/native/jni/src/suggest/core/dicnode/internal/dic_node_state_scoring.h @@ -31,7 +31,8 @@ class DicNodeStateScoring { mDigraphIndex(DigraphUtils::NOT_A_DIGRAPH_INDEX), mEditCorrectionCount(0), mProximityCorrectionCount(0), mNormalizedCompoundDistance(0.0f), mSpatialDistance(0.0f), mLanguageDistance(0.0f), - mRawLength(0.0f), mExactMatch(true) { + mRawLength(0.0f), mExactMatch(true), + mNormalizedCompoundDistanceAfterFirstWord(MAX_VALUE_FOR_WEIGHTING) { } virtual ~DicNodeStateScoring() {} @@ -45,6 +46,7 @@ class DicNodeStateScoring { mRawLength = 0.0f; mDoubleLetterLevel = NOT_A_DOUBLE_LETTER; mDigraphIndex = DigraphUtils::NOT_A_DIGRAPH_INDEX; + mNormalizedCompoundDistanceAfterFirstWord = MAX_VALUE_FOR_WEIGHTING; mExactMatch = true; } @@ -58,6 +60,8 @@ class DicNodeStateScoring { mDoubleLetterLevel = scoring->mDoubleLetterLevel; mDigraphIndex = scoring->mDigraphIndex; mExactMatch = scoring->mExactMatch; + mNormalizedCompoundDistanceAfterFirstWord = + scoring->mNormalizedCompoundDistanceAfterFirstWord; } void addCost(const float spatialCost, const float languageCost, const bool doNormalization, @@ -86,6 +90,17 @@ class DicNodeStateScoring { } } + // Saves the current normalized distance for space-aware gestures. + // See getNormalizedCompoundDistanceAfterFirstWord for details. + void saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet() { + // We get called here after each word. We only want to store the distance after + // the first word, so if we already have a distance we skip saving -- hence "IfNoneYet" + // in the method name. + if (mNormalizedCompoundDistanceAfterFirstWord >= MAX_VALUE_FOR_WEIGHTING) { + mNormalizedCompoundDistanceAfterFirstWord = getNormalizedCompoundDistance(); + } + } + void addRawLength(const float rawLength) { mRawLength += rawLength; } @@ -102,6 +117,13 @@ class DicNodeStateScoring { return mNormalizedCompoundDistance; } + // For space-aware gestures, we store the normalized distance at the char index + // that ends the first word of the suggestion. We call this the distance after + // first word. + float getNormalizedCompoundDistanceAfterFirstWord() const { + return mNormalizedCompoundDistanceAfterFirstWord; + } + float getSpatialDistance() const { return mSpatialDistance; } @@ -178,6 +200,7 @@ class DicNodeStateScoring { float mLanguageDistance; float mRawLength; bool mExactMatch; + float mNormalizedCompoundDistanceAfterFirstWord; AK_FORCE_INLINE void addDistance(float spatialDistance, float languageDistance, bool doNormalization, int inputSize, int totalInputIndex) { diff --git a/native/jni/src/suggest/core/dictionary/dictionary.cpp b/native/jni/src/suggest/core/dictionary/dictionary.cpp index ec1b63a12..b1d01ed86 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.cpp +++ b/native/jni/src/suggest/core/dictionary/dictionary.cpp @@ -33,6 +33,8 @@ namespace latinime { +const int Dictionary::HEADER_ATTRIBUTE_BUFFER_SIZE = 32; + Dictionary::Dictionary(JNIEnv *env, DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy) : mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy), @@ -121,32 +123,37 @@ void Dictionary::flushWithGC(const char *const filePath) { mDictionaryStructureWithBufferPolicy->flushWithGC(filePath); } -bool Dictionary::needsToRunGC() { - return mDictionaryStructureWithBufferPolicy->needsToRunGC(); +bool Dictionary::needsToRunGC(const bool mindsBlockByGC) { + return mDictionaryStructureWithBufferPolicy->needsToRunGC(mindsBlockByGC); +} + +void Dictionary::getProperty(const char *const query, char *const outResult, + const int maxResultLength) const { + return mDictionaryStructureWithBufferPolicy->getProperty(query, outResult, maxResultLength); } void Dictionary::logDictionaryInfo(JNIEnv *const env) const { - const int BUFFER_SIZE = 16; - int dictionaryIdCodePointBuffer[BUFFER_SIZE]; - int versionStringCodePointBuffer[BUFFER_SIZE]; - int dateStringCodePointBuffer[BUFFER_SIZE]; + int dictionaryIdCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE]; + int versionStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE]; + int dateStringCodePointBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE]; const DictionaryHeaderStructurePolicy *const headerPolicy = getDictionaryStructurePolicy()->getHeaderStructurePolicy(); headerPolicy->readHeaderValueOrQuestionMark("dictionary", dictionaryIdCodePointBuffer, - BUFFER_SIZE); + HEADER_ATTRIBUTE_BUFFER_SIZE); headerPolicy->readHeaderValueOrQuestionMark("version", versionStringCodePointBuffer, - BUFFER_SIZE); - headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer, BUFFER_SIZE); - - char dictionaryIdCharBuffer[BUFFER_SIZE]; - char versionStringCharBuffer[BUFFER_SIZE]; - char dateStringCharBuffer[BUFFER_SIZE]; - intArrayToCharArray(dictionaryIdCodePointBuffer, BUFFER_SIZE, - dictionaryIdCharBuffer, BUFFER_SIZE); - intArrayToCharArray(versionStringCodePointBuffer, BUFFER_SIZE, - versionStringCharBuffer, BUFFER_SIZE); - intArrayToCharArray(dateStringCodePointBuffer, BUFFER_SIZE, - dateStringCharBuffer, BUFFER_SIZE); + HEADER_ATTRIBUTE_BUFFER_SIZE); + headerPolicy->readHeaderValueOrQuestionMark("date", dateStringCodePointBuffer, + HEADER_ATTRIBUTE_BUFFER_SIZE); + + char dictionaryIdCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE]; + char versionStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE]; + char dateStringCharBuffer[HEADER_ATTRIBUTE_BUFFER_SIZE]; + intArrayToCharArray(dictionaryIdCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE, + dictionaryIdCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE); + intArrayToCharArray(versionStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE, + versionStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE); + intArrayToCharArray(dateStringCodePointBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE, + dateStringCharBuffer, HEADER_ATTRIBUTE_BUFFER_SIZE); LogUtils::logToJava(env, "Dictionary info: dictionary = %s ; version = %s ; date = %s", diff --git a/native/jni/src/suggest/core/dictionary/dictionary.h b/native/jni/src/suggest/core/dictionary/dictionary.h index 974447468..d8a0f3e58 100644 --- a/native/jni/src/suggest/core/dictionary/dictionary.h +++ b/native/jni/src/suggest/core/dictionary/dictionary.h @@ -81,7 +81,10 @@ class Dictionary { void flushWithGC(const char *const filePath); - bool needsToRunGC(); + bool needsToRunGC(const bool mindsBlockByGC); + + void getProperty(const char *const query, char *const outResult, + const int maxResultLength) const; const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const { return mDictionaryStructureWithBufferPolicy; @@ -92,6 +95,8 @@ class Dictionary { private: DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary); + static const int HEADER_ATTRIBUTE_BUFFER_SIZE; + DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy; const BigramDictionary *const mBigramDictionary; const SuggestInterface *const mGestureSuggest; diff --git a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h index b95488ebd..c7ffef0d5 100644 --- a/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h +++ b/native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h @@ -78,7 +78,10 @@ class DictionaryStructureWithBufferPolicy { virtual void flushWithGC(const char *const filePath) = 0; - virtual bool needsToRunGC() const = 0; + virtual bool needsToRunGC(const bool mindsBlockByGC) const = 0; + + virtual void getProperty(const char *const query, char *const outResult, + const int maxResultLength) const = 0; protected: DictionaryStructureWithBufferPolicy() {} diff --git a/native/jni/src/suggest/core/policy/weighting.cpp b/native/jni/src/suggest/core/policy/weighting.cpp index f9b777df2..0c4016893 100644 --- a/native/jni/src/suggest/core/policy/weighting.cpp +++ b/native/jni/src/suggest/core/policy/weighting.cpp @@ -38,7 +38,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n case CT_SUBSTITUTION: PROF_SUBSTITUTION(node->mProfiler); return; - case CT_NEW_WORD_SPACE_OMITTION: + case CT_NEW_WORD_SPACE_OMISSION: PROF_NEW_WORD(node->mProfiler); return; case CT_MATCH: @@ -93,6 +93,11 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n } dicNode->addCost(spatialCost, languageCost, weighting->needsToNormalizeCompoundDistance(), inputSize, errorType); + if (CT_NEW_WORD_SPACE_OMISSION == correctionType) { + // When we are on a terminal, we save the current distance for evaluating + // when to auto-commit partial suggestions. + dicNode->saveNormalizedCompoundDistanceAfterFirstWordIfNoneYet(); + } } /* static */ float Weighting::getSpatialCost(const Weighting *const weighting, @@ -108,7 +113,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n case CT_SUBSTITUTION: // only used for typing return weighting->getSubstitutionCost(); - case CT_NEW_WORD_SPACE_OMITTION: + case CT_NEW_WORD_SPACE_OMISSION: return weighting->getNewWordSpatialCost(traverseSession, dicNode, inputStateG); case CT_MATCH: return weighting->getMatchedCost(traverseSession, dicNode, inputStateG); @@ -138,7 +143,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n return 0.0f; case CT_SUBSTITUTION: return 0.0f; - case CT_NEW_WORD_SPACE_OMITTION: + case CT_NEW_WORD_SPACE_OMISSION: return weighting->getNewWordBigramLanguageCost( traverseSession, parentDicNode, multiBigramMap); case CT_MATCH: @@ -173,7 +178,7 @@ static inline void profile(const CorrectionType correctionType, DicNode *const n return 0; /* 0 because CT_MATCH will be called */ case CT_SUBSTITUTION: return 0; /* 0 because CT_MATCH will be called */ - case CT_NEW_WORD_SPACE_OMITTION: + case CT_NEW_WORD_SPACE_OMISSION: return 0; case CT_MATCH: return 1; diff --git a/native/jni/src/suggest/core/suggest.cpp b/native/jni/src/suggest/core/suggest.cpp index b1340e12f..e20bc497a 100644 --- a/native/jni/src/suggest/core/suggest.cpp +++ b/native/jni/src/suggest/core/suggest.cpp @@ -574,7 +574,7 @@ void Suggest::createNextWordDicNode(DicTraverseSession *traverseSession, DicNode DicNodeUtils::initAsRootWithPreviousWord( traverseSession->getDictionaryStructurePolicy(), dicNode, &newDicNode); const CorrectionType correctionType = spaceSubstitution ? - CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMITTION; + CT_NEW_WORD_SPACE_SUBSTITUTION : CT_NEW_WORD_SPACE_OMISSION; Weighting::addCostAndForwardInputIndex(WEIGHTING, correctionType, traverseSession, dicNode, &newDicNode, traverseSession->getMultiBigramMap()); if (newDicNode.getCompoundDistance() < static_cast<float>(MAX_VALUE_FOR_WEIGHTING)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index bc2f5ee58..67a085de3 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -17,10 +17,10 @@ #include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" #include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" -#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" +#include "suggest/policyimpl/dictionary/utils/decaying_utils.h" namespace latinime { @@ -41,9 +41,14 @@ void DynamicBigramListPolicy::getNextBigram(int *const outBigramPos, int *const if (usesAdditionalBuffer && originalBigramPos != NOT_A_DICT_POS) { originalBigramPos += mBuffer->getOriginalBufferSize(); } - *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags); *outHasNext = BigramListReadWriteUtils::hasNext(bigramFlags); + if (mIsDecayingDict && !DecayingUtils::isValidBigram(*outProbability)) { + // This bigram is too weak to output. + *outBigramPos = NOT_A_DICT_POS; + } else { + *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); + } if (usesAdditionalBuffer) { *bigramEntryPos += mBuffer->getOriginalBufferSize(); } @@ -119,7 +124,7 @@ bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const b // Finding useless bigram entries and remove them. Bigram entry is useless when the target PtNode // has been deleted or is not a valid terminal. bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( - int *const bigramListPos) { + int *const bigramListPos, int *const outValidBigramEntryCount) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); if (usesAdditionalBuffer) { *bigramListPos -= mBuffer->getOriginalBufferSize(); @@ -153,14 +158,22 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( const int bigramTargetNodePos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); nodeReader.fetchNodeInfoInBufferFromPtNodePos(bigramTargetNodePos); - // TODO: Update probability for supporting probability decaying. if (nodeReader.isDeleted() || !nodeReader.isTerminal() || bigramTargetNodePos == NOT_A_DICT_POS) { // The target is no longer valid terminal. Invalidate the current bigram entry. if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags, - NOT_A_DICT_POS /* targetOffset */, &bigramEntryPos)) { + NOT_A_DICT_POS /* targetPtNodePos */, &bigramEntryPos)) { return false; } + continue; + } + bool isRemoved = false; + if (!updateProbabilityForDecay(bigramFlags, bigramTargetNodePos, &bigramEntryPos, + &isRemoved)) { + return false; + } + if (!isRemoved) { + (*outValidBigramEntryCount) += 1; } } while(BigramListReadWriteUtils::hasNext(bigramFlags)); return true; @@ -169,7 +182,7 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( // Updates bigram target PtNode positions in the list after the placing step in GC. bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos, const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const - ptNodePositionRelocationMap) { + ptNodePositionRelocationMap, int *const outBigramEntryCount) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); if (usesAdditionalBuffer) { *bigramListPos -= mBuffer->getOriginalBufferSize(); @@ -211,11 +224,12 @@ bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bi return false; } } while(BigramListReadWriteUtils::hasNext(bigramFlags)); + (*outBigramEntryCount) = bigramEntryCount; return true; } bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos, - const int probability, int *const bigramListPos) { + const int probability, int *const bigramListPos, bool *const outAddedNewBigram) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); if (usesAdditionalBuffer) { *bigramListPos -= mBuffer->getOriginalBufferSize(); @@ -243,8 +257,15 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg } if (followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos) == bigramTargetPos) { // Update this bigram entry. + *outAddedNewBigram = false; + const int originalProbability = BigramListReadWriteUtils::getProbabilityFromFlags( + bigramFlags); + const int probabilityToWrite = mIsDecayingDict ? + DecayingUtils::getUpdatedBigramProbabilityDelta( + originalProbability, probability) : probability; const BigramListReadWriteUtils::BigramFlags updatedFlags = - BigramListReadWriteUtils::setProbabilityInFlags(bigramFlags, probability); + BigramListReadWriteUtils::setProbabilityInFlags(bigramFlags, + probabilityToWrite); return BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedFlags, originalBigramPos, &entryPos); } @@ -254,12 +275,14 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg // The current last entry is found. // First, update the flags of the last entry. if (!BigramListReadWriteUtils::setHasNextFlag(mBuffer, true /* hasNext */, entryPos)) { + *outAddedNewBigram = false; return false; } if (usesAdditionalBuffer) { *bigramListPos += mBuffer->getOriginalBufferSize(); } // Then, add a new entry after the last entry. + *outAddedNewBigram = true; return writeNewBigramEntry(bigramTargetPos, probability, bigramListPos); } while(BigramListReadWriteUtils::hasNext(bigramFlags)); // We return directly from the while loop. @@ -270,8 +293,11 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg bool DynamicBigramListPolicy::writeNewBigramEntry(const int bigramTargetPos, const int probability, int *const writingPos) { // hasNext is false because we are adding a new bigram entry at the end of the bigram list. + const int probabilityToWrite = mIsDecayingDict ? + DecayingUtils::getUpdatedBigramProbabilityDelta(NOT_A_PROBABILITY, probability) : + probability; return BigramListReadWriteUtils::createAndWriteBigramEntry(mBuffer, bigramTargetPos, - probability, false /* hasNext */, writingPos); + probabilityToWrite, false /* hasNext */, writingPos); } bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int bigramTargetPos) { @@ -325,7 +351,7 @@ int DynamicBigramListPolicy::followBigramLinkAndGetCurrentBigramPtNodePos( nodeReader.fetchNodeInfoInBufferFromPtNodePos(currentPos); bigramLinkCount++; if (bigramLinkCount > CONTINUING_BIGRAM_LINK_COUNT_LIMIT) { - AKLOGE("Bigram link is invalid. start position: %d", bigramPos); + AKLOGE("Bigram link is invalid. start position: %d", originalBigramPos); ASSERT(false); return NOT_A_DICT_POS; } @@ -333,4 +359,33 @@ int DynamicBigramListPolicy::followBigramLinkAndGetCurrentBigramPtNodePos( return currentPos; } +bool DynamicBigramListPolicy::updateProbabilityForDecay( + BigramListReadWriteUtils::BigramFlags bigramFlags, const int targetPtNodePos, + int *const bigramEntryPos, bool *const outRemoved) const { + *outRemoved = false; + if (mIsDecayingDict) { + // Update bigram probability for decaying. + const int newProbability = DecayingUtils::getBigramProbabilityDeltaToSave( + BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags)); + if (DecayingUtils::isValidBigram(newProbability)) { + // Write new probability. + const BigramListReadWriteUtils::BigramFlags updatedBigramFlags = + BigramListReadWriteUtils::setProbabilityInFlags( + bigramFlags, newProbability); + if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedBigramFlags, + targetPtNodePos, bigramEntryPos)) { + return false; + } + } else { + // Remove current bigram entry. + *outRemoved = true; + if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags, + NOT_A_DICT_POS /* targetPtNodePos */, bigramEntryPos)) { + return false; + } + } + } + return true; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index 8ea318a41..b358b4ed5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -21,6 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" +#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" namespace latinime { @@ -34,8 +35,9 @@ class DictionaryShortcutsStructurePolicy; class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { public: DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer, - const DictionaryShortcutsStructurePolicy *const shortcutPolicy) - : mBuffer(buffer), mShortcutPolicy(shortcutPolicy) {} + const DictionaryShortcutsStructurePolicy *const shortcutPolicy, + const bool isDecayingDict) + : mBuffer(buffer), mShortcutPolicy(shortcutPolicy), mIsDecayingDict(isDecayingDict) {} ~DynamicBigramListPolicy() {} @@ -50,19 +52,20 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos, int *const toPos, int *const outBigramsCount) const; - bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos); + bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos, + int *const outBigramEntryCount); bool updateAllBigramTargetPtNodePositions(int *const bigramListPos, const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const - ptNodePositionRelocationMap); + ptNodePositionRelocationMap, int *const outValidBigramEntryCount); bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability, - int *const bigramListPos); + int *const bigramListPos, bool *const outAddedNewBigram); bool writeNewBigramEntry(const int bigramTargetPos, const int probability, int *const writingPos); - // Return if targetBigramPos is found or not. + // Return whether or not targetBigramPos is found. bool removeBigram(const int bigramListPos, const int bigramTargetPos); private: @@ -73,9 +76,13 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { BufferWithExtendableBuffer *const mBuffer; const DictionaryShortcutsStructurePolicy *const mShortcutPolicy; + const bool mIsDecayingDict; // Follow bigram link and return the position of bigram target PtNode that is currently valid. int followBigramLinkAndGetCurrentBigramPtNodePos(const int originalBigramPos) const; + + bool updateProbabilityForDecay(BigramListReadWriteUtils::BigramFlags bigramFlags, + const int targetPtNodePos, int *const bigramEntryPos, bool *const outRemoved) const; }; } // namespace latinime #endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp index c60e45819..081163a4d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp @@ -16,6 +16,8 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h" +#include "suggest/policyimpl/dictionary/utils/decaying_utils.h" + namespace latinime { bool DynamicPatriciaTrieGcEventListeners @@ -25,6 +27,19 @@ bool DynamicPatriciaTrieGcEventListeners // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless // children. bool isUselessPtNode = !node->isTerminal(); + if (node->isTerminal() && mIsDecayingDict) { + const int newProbability = + DecayingUtils::getUnigramProbabilityToSave(node->getProbability()); + int writingPos = node->getProbabilityFieldPos(); + // Update probability. + if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition( + mBuffer, newProbability, &writingPos)) { + return false; + } + if (!DecayingUtils::isValidUnigram(newProbability)) { + isUselessPtNode = false; + } + } if (mChildrenValue > 0) { isUselessPtNode = false; } else if (node->isTerminal()) { @@ -41,7 +56,27 @@ bool DynamicPatriciaTrieGcEventListeners return false; } } else { - valueStack.back() += 1; + mValueStack.back() += 1; + if (node->isTerminal()) { + mValidUnigramCount += 1; + } + } + return true; +} + +bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability + ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, + const int *const nodeCodePoints) { + if (!node->isDeleted()) { + int pos = node->getBigramsPos(); + if (pos != NOT_A_DICT_POS) { + int bigramEntryCount = 0; + if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos, + &bigramEntryCount)) { + return false; + } + mValidBigramEntryCount += bigramEntryCount; + } } return true; } @@ -137,10 +172,15 @@ bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionField // Updates bigram target PtNode positions in the bigram list. int bigramsPos = node->getBigramsPos(); if (bigramsPos != NOT_A_DICT_POS) { + int bigramEntryCount; if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos, - &mDictPositionRelocationMap->mPtNodePositionRelocationMap)) { + &mDictPositionRelocationMap->mPtNodePositionRelocationMap, &bigramEntryCount)) { return false; } + mBigramCount += bigramEntryCount; + } + if (node->isTerminal()) { + mUnigramCount++; } return true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h index 4256f22fb..463715af5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h @@ -39,23 +39,23 @@ class DynamicPatriciaTrieGcEventListeners { public: TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( DynamicPatriciaTrieWritingHelper *const writingHelper, - BufferWithExtendableBuffer *const buffer) - : mWritingHelper(writingHelper), mBuffer(buffer), valueStack(), - mChildrenValue(0) {} + BufferWithExtendableBuffer *const buffer, const bool isDecayingDict) + : mWritingHelper(writingHelper), mBuffer(buffer), mIsDecayingDict(isDecayingDict), + mValueStack(), mChildrenValue(0), mValidUnigramCount(0) {} ~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {}; bool onAscend() { - if (valueStack.empty()) { + if (mValueStack.empty()) { return false; } - mChildrenValue = valueStack.back(); - valueStack.pop_back(); + mChildrenValue = mValueStack.back(); + mValueStack.pop_back(); return true; } bool onDescend(const int ptNodeArrayPos) { - valueStack.push_back(0); + mValueStack.push_back(0); return true; } @@ -64,14 +64,20 @@ class DynamicPatriciaTrieGcEventListeners { bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, const int *const nodeCodePoints); + int getValidUnigramCount() const { + return mValidUnigramCount; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS( TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted); DynamicPatriciaTrieWritingHelper *const mWritingHelper; BufferWithExtendableBuffer *const mBuffer; - std::vector<int> valueStack; + const int mIsDecayingDict; + std::vector<int> mValueStack; int mChildrenValue; + int mValidUnigramCount; }; // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram @@ -80,7 +86,7 @@ class DynamicPatriciaTrieGcEventListeners { : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { public: TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy) - : mBigramPolicy(bigramPolicy) {} + : mBigramPolicy(bigramPolicy), mValidBigramEntryCount(0) {} bool onAscend() { return true; } @@ -89,22 +95,17 @@ class DynamicPatriciaTrieGcEventListeners { bool onReadingPtNodeArrayTail() { return true; } bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, - const int *const nodeCodePoints) { - if (!node->isDeleted()) { - int pos = node->getBigramsPos(); - if (pos != NOT_A_DICT_POS) { - if (!mBigramPolicy->updateAllBigramEntriesAndDeleteUselessEntries(&pos)) { - return false; - } - } - } - return true; + const int *const nodeCodePoints); + + int getValidBigramEntryCount() const { + return mValidBigramEntryCount; } private: DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability); DynamicBigramListPolicy *const mBigramPolicy; + int mValidBigramEntryCount; }; class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer @@ -150,7 +151,8 @@ class DynamicPatriciaTrieGcEventListeners { dictPositionRelocationMap) : mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy), mBufferToWrite(bufferToWrite), - mDictPositionRelocationMap(dictPositionRelocationMap) {}; + mDictPositionRelocationMap(dictPositionRelocationMap), mUnigramCount(0), + mBigramCount(0) {}; bool onAscend() { return true; } @@ -161,6 +163,14 @@ class DynamicPatriciaTrieGcEventListeners { bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, const int *const nodeCodePoints); + int getUnigramCount() const { + return mUnigramCount; + } + + int getBigramCount() const { + return mBigramCount; + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields); @@ -169,6 +179,8 @@ class DynamicPatriciaTrieGcEventListeners { BufferWithExtendableBuffer *const mBufferToWrite; const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const mDictPositionRelocationMap; + int mUnigramCount; + int mBigramCount; }; private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index 42397c19e..0d8c92768 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -16,6 +16,10 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h" +#include <cstdio> +#include <cstring> +#include <ctime> + #include "defines.h" #include "suggest/core/dicnode/dic_node.h" #include "suggest/core/dicnode/dic_node_vector.h" @@ -24,10 +28,18 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" +#include "suggest/policyimpl/dictionary/utils/decaying_utils.h" #include "suggest/policyimpl/dictionary/utils/probability_utils.h" namespace latinime { +const char *const DynamicPatriciaTriePolicy::UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; +const char *const DynamicPatriciaTriePolicy::BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; +const int DynamicPatriciaTriePolicy::MAX_DICT_EXTENDED_REGION_SIZE = 1024 * 1024; +const int DynamicPatriciaTriePolicy::MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS = + DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE - 1024; +const int DynamicPatriciaTriePolicy::MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING = 2 * 60 * 60; + void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, DicNodeVector *const childDicNodes) const { if (!dicNode->hasChildren()) { @@ -137,14 +149,17 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in int DynamicPatriciaTriePolicy::getProbability(const int unigramProbability, const int bigramProbability) const { - // TODO: check mHeaderPolicy.usesForgettingCurve(); - if (unigramProbability == NOT_A_PROBABILITY) { - return NOT_A_PROBABILITY; - } else if (bigramProbability == NOT_A_PROBABILITY) { - return ProbabilityUtils::backoff(unigramProbability); + if (mHeaderPolicy.isDecayingDict()) { + return DecayingUtils::getProbability(unigramProbability, bigramProbability); } else { - return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, - bigramProbability); + if (unigramProbability == NOT_A_PROBABILITY) { + return NOT_A_PROBABILITY; + } else if (bigramProbability == NOT_A_PROBABILITY) { + return ProbabilityUtils::backoff(unigramProbability); + } else { + return ProbabilityUtils::computeProbabilityForBigram(unigramProbability, + bigramProbability); + } } } @@ -193,12 +208,26 @@ bool DynamicPatriciaTriePolicy::addUnigramWord(const int *const word, const int AKLOGI("Warning: addUnigramWord() is called for non-updatable dictionary."); return false; } + if (mBufferWithExtendableBuffer.getTailPosition() + >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { + AKLOGE("The dictionary is too large to dynamically update."); + return false; + } DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy()); readingHelper.initWithPtNodeArrayPos(getRootPosition()); DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy); - return writingHelper.addUnigramWord(&readingHelper, word, length, probability); + &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); + bool addedNewUnigram = false; + if (writingHelper.addUnigramWord(&readingHelper, word, length, probability, + &addedNewUnigram)) { + if (addedNewUnigram) { + mUnigramCount++; + } + return true; + } else { + return false; + } } bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int length0, @@ -207,6 +236,11 @@ bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int AKLOGI("Warning: addBigramWords() is called for non-updatable dictionary."); return false; } + if (mBufferWithExtendableBuffer.getTailPosition() + >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { + AKLOGE("The dictionary is too large to dynamically update."); + return false; + } const int word0Pos = getTerminalNodePositionOfWord(word0, length0, false /* forceLowerCaseSearch */); if (word0Pos == NOT_A_DICT_POS) { @@ -218,8 +252,16 @@ bool DynamicPatriciaTriePolicy::addBigramWords(const int *const word0, const int return false; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy); - return writingHelper.addBigramWords(word0Pos, word1Pos, probability); + &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); + bool addedNewBigram = false; + if (writingHelper.addBigramWords(word0Pos, word1Pos, probability, &addedNewBigram)) { + if (addedNewBigram) { + mBigramCount++; + } + return true; + } else { + return false; + } } bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const int length0, @@ -228,6 +270,11 @@ bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const AKLOGI("Warning: removeBigramWords() is called for non-updatable dictionary."); return false; } + if (mBufferWithExtendableBuffer.getTailPosition() + >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS) { + AKLOGE("The dictionary is too large to dynamically update."); + return false; + } const int word0Pos = getTerminalNodePositionOfWord(word0, length0, false /* forceLowerCaseSearch */); if (word0Pos == NOT_A_DICT_POS) { @@ -239,8 +286,13 @@ bool DynamicPatriciaTriePolicy::removeBigramWords(const int *const word0, const return false; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy); - return writingHelper.removeBigramWords(word0Pos, word1Pos); + &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); + if (writingHelper.removeBigramWords(word0Pos, word1Pos)) { + mBigramCount--; + return true; + } else { + return false; + } } void DynamicPatriciaTriePolicy::flush(const char *const filePath) { @@ -249,8 +301,8 @@ void DynamicPatriciaTriePolicy::flush(const char *const filePath) { return; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy); - writingHelper.writeToDictFile(filePath, &mHeaderPolicy); + &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); + writingHelper.writeToDictFile(filePath, &mHeaderPolicy, mUnigramCount, mBigramCount); } void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) { @@ -259,17 +311,51 @@ void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) { return; } DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, - &mBigramListPolicy, &mShortcutListPolicy); + &mBigramListPolicy, &mShortcutListPolicy, mHeaderPolicy.isDecayingDict()); writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy); } -bool DynamicPatriciaTriePolicy::needsToRunGC() const { +bool DynamicPatriciaTriePolicy::needsToRunGC(const bool mindsBlockByGC) const { if (!mBuffer->isUpdatable()) { AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); return false; } - // TODO: Implement more properly. - return mBufferWithExtendableBuffer.isNearSizeLimit(); + if (mBufferWithExtendableBuffer.isNearSizeLimit()) { + // Additional buffer size is near the limit. + return true; + } else if (mHeaderPolicy.getExtendedRegionSize() + + mBufferWithExtendableBuffer.getUsedAdditionalBufferSize() + > MAX_DICT_EXTENDED_REGION_SIZE) { + // Total extended region size exceeds the limit. + return true; + } else if (mBufferWithExtendableBuffer.getTailPosition() + >= MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS + && mBufferWithExtendableBuffer.getUsedAdditionalBufferSize() > 0) { + // Needs to reduce dictionary size. + return true; + } else if (mHeaderPolicy.isDecayingDict()) { + if (mUnigramCount >= DecayingUtils::MAX_UNIGRAM_COUNT) { + // Unigram count exceeds the limit. + return true; + } else if (mBigramCount >= DecayingUtils::MAX_BIGRAM_COUNT) { + // Bigram count exceeds the limit. + return true; + } else if (mindsBlockByGC && mHeaderPolicy.getLastUpdatedTime() + + MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING < time(0)) { + // Time to update probabilities for decaying. + return true; + } + } + return false; +} + +void DynamicPatriciaTriePolicy::getProperty(const char *const query, char *const outResult, + const int maxResultLength) const { + if (strncmp(query, UNIGRAM_COUNT_QUERY, maxResultLength) == 0) { + snprintf(outResult, maxResultLength, "%d", mUnigramCount); + } else if (strncmp(query, BIGRAM_COUNT_QUERY, maxResultLength) == 0) { + snprintf(outResult, maxResultLength, "%d", mBigramCount); + } } } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h index 06d8095d8..d3150c6fc 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h @@ -37,7 +37,10 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(), mBuffer->getBufferSize() - mHeaderPolicy.getSize()), mShortcutListPolicy(&mBufferWithExtendableBuffer), - mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy) {} + mBigramListPolicy(&mBufferWithExtendableBuffer, &mShortcutListPolicy, + mHeaderPolicy.isDecayingDict()), + mUnigramCount(mHeaderPolicy.getUnigramCount()), + mBigramCount(mHeaderPolicy.getBigramCount()) {} ~DynamicPatriciaTriePolicy() { delete mBuffer; @@ -89,16 +92,27 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { void flushWithGC(const char *const filePath); - bool needsToRunGC() const; + bool needsToRunGC(const bool mindsBlockByGC) const; + + void getProperty(const char *const query, char *const outResult, + const int maxResultLength) const; private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy); + static const char*const UNIGRAM_COUNT_QUERY; + static const char*const BIGRAM_COUNT_QUERY; + static const int MAX_DICT_EXTENDED_REGION_SIZE; + static const int MIN_DICT_SIZE_TO_REFUSE_DYNAMIC_OPERATIONS; + static const int MIN_SECONDS_TO_REQUIRE_GC_WHEN_WRITING; + const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; BufferWithExtendableBuffer mBufferWithExtendableBuffer; DynamicShortcutListPolicy mShortcutListPolicy; DynamicBigramListPolicy mBigramListPolicy; + int mUnigramCount; + int mBigramCount; }; } // namespace latinime #endif // LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h index 154590fbd..512a4d818 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h @@ -264,7 +264,7 @@ class DynamicPatriciaTrieReadingHelper { AK_FORCE_INLINE void pushReadingStateToStack() { if (mReadingStateStack.size() > MAX_READING_STATE_STACK_SIZE) { - AKLOGI("Reading state stack overflow. Max size: %d", MAX_READING_STATE_STACK_SIZE); + AKLOGI("Reading state stack overflow. Max size: %zd", MAX_READING_STATE_STACK_SIZE); ASSERT(false); mIsError = true; mReadingState.mPos = NOT_A_DICT_POS; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index a51ae5e1d..28124d251 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -16,9 +16,6 @@ #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" -#include <cstdio> -#include <cstring> - #include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h" @@ -28,19 +25,20 @@ #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h" +#include "suggest/policyimpl/dictionary/utils/decaying_utils.h" +#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" #include "utils/hash_map_compat.h" namespace latinime { const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; -const char *const DynamicPatriciaTrieWritingHelper::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = - ".tmp"; // TODO: Make MAX_DICTIONARY_SIZE 8MB. const size_t DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024; bool DynamicPatriciaTrieWritingHelper::addUnigramWord( DynamicPatriciaTrieReadingHelper *const readingHelper, - const int *const wordCodePoints, const int codePointCount, const int probability) { + const int *const wordCodePoints, const int codePointCount, const int probability, + bool *const outAddedNewUnigram) { int parentPos = NOT_A_DICT_POS; while (!readingHelper->isEnd()) { const int matchedCodePointCount = readingHelper->getPrevTotalCodePointCount(); @@ -58,8 +56,11 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( const int nextIndex = matchedCodePointCount + j; if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(j, wordCodePoints[matchedCodePointCount + j])) { + *outAddedNewUnigram = true; return reallocatePtNodeAndAddNewPtNodes(nodeReader, - readingHelper->getMergedNodeCodePoints(), j, probability, + readingHelper->getMergedNodeCodePoints(), j, + getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, + probability), wordCodePoints + matchedCodePointCount, codePointCount - matchedCodePointCount); } @@ -67,10 +68,12 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( // All characters are matched. if (codePointCount == readingHelper->getTotalCodePointCount()) { return setPtNodeProbability(nodeReader, probability, - readingHelper->getMergedNodeCodePoints()); + readingHelper->getMergedNodeCodePoints(), outAddedNewUnigram); } if (!nodeReader->hasChildren()) { - return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability, + *outAddedNewUnigram = true; + return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, + getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, probability), wordCodePoints + readingHelper->getTotalCodePointCount(), codePointCount - readingHelper->getTotalCodePointCount()); } @@ -83,14 +86,15 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( return false; } int pos = readingHelper->getPosOfLastForwardLinkField(); + *outAddedNewUnigram = true; return createAndInsertNodeIntoPtNodeArray(parentPos, wordCodePoints + readingHelper->getPrevTotalCodePointCount(), codePointCount - readingHelper->getPrevTotalCodePointCount(), - probability, &pos); + getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, probability), &pos); } bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos, - const int probability) { + const int probability, bool *const outAddedNewBigram) { int mMergedNodeCodePoints[MAX_WORD_LENGTH]; DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); nodeReader.fetchNodeInfoInBufferFromPtNodePosAndGetNodeCodePoints(word0Pos, MAX_WORD_LENGTH, @@ -111,9 +115,11 @@ bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const if (nodeReader.getBigramsPos() != NOT_A_DICT_POS) { // Insert a new bigram entry into the existing bigram list. int bigramListPos = nodeReader.getBigramsPos(); - return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos); + return mBigramPolicy->addNewBigramEntryToBigramList(word1Pos, probability, &bigramListPos, + outAddedNewBigram); } else { // The PtNode doesn't have a bigram list. + *outAddedNewBigram = true; // First, Write a bigram entry at the tail position of the PtNode. if (!mBigramPolicy->writeNewBigramEntry(word1Pos, probability, &writingPos)) { return false; @@ -142,26 +148,32 @@ bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, con } void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileName, - const HeaderPolicy *const headerPolicy) { + const HeaderPolicy *const headerPolicy, const int unigramCount, const int bigramCount) { BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); - if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) { + const int extendedRegionSize = headerPolicy->getExtendedRegionSize() + + mBuffer->getUsedAdditionalBufferSize(); + if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */, + unigramCount, bigramCount, extendedRegionSize)) { return; } - flushAllToFile(fileName, &headerBuffer, mBuffer); + DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, mBuffer); } void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName, const HeaderPolicy *const headerPolicy) { - BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); - if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) { - return; - } BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */, MAX_DICTIONARY_SIZE); - if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) { + int unigramCount = 0; + int bigramCount = 0; + if (!runGC(rootPtNodeArrayPos, &newDictBuffer, &unigramCount, &bigramCount)) { + return; + } + BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); + if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */, + unigramCount, bigramCount, 0 /* extendedRegionSize */)) { return; } - flushAllToFile(fileName, &headerBuffer, &newDictBuffer); + DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, &newDictBuffer); } bool DynamicPatriciaTrieWritingHelper::markNodeAsDeleted( @@ -339,23 +351,28 @@ bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( const DynamicPatriciaTrieNodeReader *const originalPtNode, const int probability, - const int *const codePoints) { + const int *const codePoints, bool *const outAddedNewUnigram) { if (originalPtNode->isTerminal()) { // Overwrites the probability. + *outAddedNewUnigram = false; + const int probabilityToWrite = getUpdatedProbability(originalPtNode->getProbability(), + probability); int probabilityFieldPos = originalPtNode->getProbabilityFieldPos(); if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer, - probability, &probabilityFieldPos)) { + probabilityToWrite, &probabilityFieldPos)) { return false; } } else { // Make the node terminal and write the probability. + *outAddedNewUnigram = true; int movedPos = mBuffer->getTailPosition(); if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { return false; } if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, originalPtNode, originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(), - probability, &movedPos)) { + getUpdatedProbability(NOT_A_PROBABILITY /* originalProbability */, probability), + &movedPos)) { return false; } } @@ -463,72 +480,23 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( return true; } -// TODO: Create a struct which contains header, body and etc... and use here as an argument. -void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName, - BufferWithExtendableBuffer *const dictHeader, - BufferWithExtendableBuffer *const dictBody) const { - const int tmpFileNameBufSize = strlen(fileName) - + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */; - // Name of a temporary file used for writing that is a connected string of original name and - // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE. - char tmpFileName[tmpFileNameBufSize]; - snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, - TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); - FILE *const file = fopen(tmpFileName, "wb"); - if (!file) { - AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName); - ASSERT(false); - return; - } - // Write the dictionary header. - if (!writeBufferToFilePointer(file, dictHeader)) { - remove(tmpFileName); - AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition()); - ASSERT(false); - return; - } - // Write the dictionary body. - if (!writeBufferToFilePointer(file, dictBody)) { - remove(tmpFileName); - AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition()); - ASSERT(false); - return; - } - fclose(file); - rename(tmpFileName, fileName); -} - -// This closes file pointer when an error is caused and returns whether the writing was succeeded -// or not. -bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file, - const BufferWithExtendableBuffer *const buffer) const { - const int originalBufSize = buffer->getOriginalBufferSize(); - if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */), - originalBufSize, 1, file) < 1) { - fclose(file); - return false; - } - const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize(); - if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */), - additionalBufSize, 1, file) < 1) { - fclose(file); - return false; - } - return true; -} - bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, - BufferWithExtendableBuffer *const bufferToWrite) { + BufferWithExtendableBuffer *const bufferToWrite, int *const outUnigramCount, + int *const outBigramCount) { DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); DynamicPatriciaTrieGcEventListeners ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( - this, mBuffer); + this, mBuffer, mIsDecayingDict); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( &traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) { return false; } + if (mIsDecayingDict && traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted + .getValidUnigramCount() > DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC) { + // TODO: Remove more unigrams. + } readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability @@ -538,6 +506,11 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, return false; } + if (mIsDecayingDict && traversePolicyToUpdateBigramProbability.getValidBigramEntryCount() + > DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC) { + // TODO: Remove more bigrams. + } + // Mapping from positions in mBuffer to positions in bufferToWrite. DictPositionRelocationMap dictPositionRelocationMap; readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); @@ -551,7 +524,8 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, // Create policy instance for the GCed dictionary. DynamicShortcutListPolicy newDictShortcutPolicy(bufferToWrite); - DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy); + DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy, + mIsDecayingDict); // Create reading helper for the GCed dictionary. DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictBigramPolicy, &newDictShortcutPolicy); @@ -563,7 +537,18 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, &traversePolicyToUpdateAllPositionFields)) { return false; } + *outUnigramCount = traversePolicyToUpdateAllPositionFields.getUnigramCount(); + *outBigramCount = traversePolicyToUpdateAllPositionFields.getBigramCount(); return true; } +int DynamicPatriciaTrieWritingHelper::getUpdatedProbability(const int originalProbability, + const int newProbability) { + if (mIsDecayingDict) { + return DecayingUtils::getUpdatedUnigramProbability(originalProbability, newProbability); + } else { + return newProbability; + } +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index 028fa6075..ecee2cdbf 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -17,7 +17,6 @@ #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H -#include <cstdio> #include <stdint.h> #include "defines.h" @@ -48,24 +47,30 @@ class DynamicPatriciaTrieWritingHelper { DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap); }; + static const size_t MAX_DICTIONARY_SIZE; + DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer, DynamicBigramListPolicy *const bigramPolicy, - DynamicShortcutListPolicy *const shortcutPolicy) - : mBuffer(buffer), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy) {} + DynamicShortcutListPolicy *const shortcutPolicy, const bool isDecayingDict) + : mBuffer(buffer), mBigramPolicy(bigramPolicy), mShortcutPolicy(shortcutPolicy), + mIsDecayingDict(isDecayingDict) {} ~DynamicPatriciaTrieWritingHelper() {} // Add a word to the dictionary. If the word already exists, update the probability. bool addUnigramWord(DynamicPatriciaTrieReadingHelper *const readingHelper, - const int *const wordCodePoints, const int codePointCount, const int probability); + const int *const wordCodePoints, const int codePointCount, const int probability, + bool *const outAddedNewUnigram); // Add a bigram relation from word0Pos to word1Pos. - bool addBigramWords(const int word0Pos, const int word1Pos, const int probability); + bool addBigramWords(const int word0Pos, const int word1Pos, const int probability, + bool *const outAddedNewBigram); // Remove a bigram relation from word0Pos to word1Pos. bool removeBigramWords(const int word0Pos, const int word1Pos); - void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy); + void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy, + const int unigramCount, const int bigramCount); void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName, const HeaderPolicy *const headerPolicy); @@ -85,12 +90,11 @@ class DynamicPatriciaTrieWritingHelper { DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); static const int CHILDREN_POSITION_FIELD_SIZE; - static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE; - static const size_t MAX_DICTIONARY_SIZE; BufferWithExtendableBuffer *const mBuffer; DynamicBigramListPolicy *const mBigramPolicy; DynamicShortcutListPolicy *const mShortcutPolicy; + const bool mIsDecayingDict; bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos, const int bigramLinkedNodePos); @@ -109,7 +113,7 @@ class DynamicPatriciaTrieWritingHelper { const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos); bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode, - const int probability, const int *const codePoints); + const int probability, const int *const codePoints, bool *const outAddedNewUnigram); bool createChildrenPtNodeArrayAndAChildPtNode( const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, @@ -124,14 +128,10 @@ class DynamicPatriciaTrieWritingHelper { const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int newNodeCodePointCount); - void flushAllToFile(const char *const fileName, - BufferWithExtendableBuffer *const dictHeader, - BufferWithExtendableBuffer *const dictBody) const; - - bool writeBufferToFilePointer(FILE *const file, - const BufferWithExtendableBuffer *const buffer) const; + bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite, + int *const outUnigramCount, int *const outBigramCount); - bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite); + int getUpdatedProbability(const int originalProbability, const int newProbability); }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp index 5a3983776..30ff10cd6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp @@ -36,6 +36,16 @@ const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000; const int DynamicPatriciaTrieWritingUtils::PROBABILITY_FIELD_SIZE = 1; const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; +/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary( + BufferWithExtendableBuffer *const buffer, const int rootPos) { + int writingPos = rootPos; + if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) { + return false; + } + return writeForwardLinkPositionAndAdvancePosition(buffer, NOT_A_DICT_POS /* forwardLinkPos */, + &writingPos); +} + /* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h index a37e9fb3d..af76bc6b5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h @@ -30,6 +30,8 @@ class DynamicPatriciaTrieWritingUtils { public: static const int NODE_FLAG_FIELD_SIZE; + static bool writeEmptyDictionary(BufferWithExtendableBuffer *const buffer, const int rootPos); + static bool writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 47ace23a1..9ce9994dd 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -16,16 +16,17 @@ #include "suggest/policyimpl/dictionary/header/header_policy.h" -#include <cstddef> -#include <cstdio> -#include <ctime> - namespace latinime { +// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE"; -const char *const HeaderPolicy::USES_FORGETTING_CURVE_KEY = "USES_FORGETTING_CURVE"; +// TODO: Change attribute string to "IS_DECAYING_DICT". +const char *const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE"; const char *const HeaderPolicy::LAST_UPDATED_TIME_KEY = "date"; -const float HeaderPolicy::DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER = 1.0f; +const char *const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT"; +const char *const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT"; +const char *const HeaderPolicy::EXTENDED_REGION_SIZE_KEY = "EXTENDED_REGION_SIZE"; +const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100; const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f; // Used for logging. Question mark is used to indicate that the key is not found. @@ -37,7 +38,7 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out return; } std::vector<int> keyCodePointVector; - insertCharactersIntoVector(key, &keyCodePointVector); + HeaderReadWriteUtils::insertCharactersIntoVector(key, &keyCodePointVector); HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector); if (it == mAttributeMap.end()) { // The key was not found. @@ -53,51 +54,17 @@ void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *out } float HeaderPolicy::readMultipleWordCostMultiplier() const { - int attributeValue = 0; - if (getAttributeValueAsInt(MULTIPLE_WORDS_DEMOTION_RATE_KEY, &attributeValue)) { - if (attributeValue <= 0) { - return static_cast<float>(MAX_VALUE_FOR_WEIGHTING); - } - return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(attributeValue); - } else { - return DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER; - } -} - -bool HeaderPolicy::readUsesForgettingCurveFlag() const { - int attributeValue = 0; - if (getAttributeValueAsInt(USES_FORGETTING_CURVE_KEY, &attributeValue)) { - return attributeValue != 0; - } else { - return false; - } -} - -// Returns S_INT_MIN when the key is not found or the value is invalid. -int HeaderPolicy::readLastUpdatedTime() const { - int attributeValue = 0; - if (getAttributeValueAsInt(LAST_UPDATED_TIME_KEY, &attributeValue)) { - return attributeValue; - } else { - return S_INT_MIN; - } -} - -// Returns whether the key is found or not and stores the found value into outValue. -bool HeaderPolicy::getAttributeValueAsInt(const char *const key, int *const outValue) const { - std::vector<int> keyVector; - insertCharactersIntoVector(key, &keyVector); - HeaderReadWriteUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyVector); - if (it == mAttributeMap.end()) { - // The key was not found. - return false; + const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + MULTIPLE_WORDS_DEMOTION_RATE_KEY, DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE); + if (demotionRate <= 0) { + return static_cast<float>(MAX_VALUE_FOR_WEIGHTING); } - *outValue = parseIntAttributeValue(&(it->second)); - return true; + return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate); } bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, - const bool updatesLastUpdatedTime) const { + const bool updatesLastUpdatedTime, const int unigramCount, const int bigramCount, + const int extendedRegionSize) const { int writingPos = 0; if (!HeaderReadWriteUtils::writeDictionaryVersion(bufferToWrite, mDictFormatVersion, &writingPos)) { @@ -113,26 +80,19 @@ bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferT &writingPos)) { return false; } + HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap); + HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, UNIGRAM_COUNT_KEY, unigramCount); + HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, BIGRAM_COUNT_KEY, bigramCount); + HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, EXTENDED_REGION_SIZE_KEY, + extendedRegionSize); if (updatesLastUpdatedTime) { // Set current time as a last updated time. - HeaderReadWriteUtils::AttributeMap attributeMapTowrite(mAttributeMap); - std::vector<int> updatedTimekey; - insertCharactersIntoVector(LAST_UPDATED_TIME_KEY, &updatedTimekey); - const time_t currentTime = time(NULL); - std::vector<int> updatedTimeValue; - char charBuf[LARGEST_INT_DIGIT_COUNT + 1]; - snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%ld", currentTime); - insertCharactersIntoVector(charBuf, &updatedTimeValue); - attributeMapTowrite[updatedTimekey] = updatedTimeValue; - if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite, - &writingPos)) { - return false; - } - } else { - if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &mAttributeMap, - &writingPos)) { - return false; - } + HeaderReadWriteUtils::setIntAttribute(&attributeMapTowrite, LAST_UPDATED_TIME_KEY, + time(0)); + } + if (!HeaderReadWriteUtils::writeHeaderAttributes(bufferToWrite, &attributeMapTowrite, + &writingPos)) { + return false; } // Writes an actual header size. if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(bufferToWrite, writingPos, @@ -149,30 +109,4 @@ bool HeaderPolicy::writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferT return attributeMap; } -/* static */ int HeaderPolicy::parseIntAttributeValue( - const std::vector<int> *const attributeValue) { - int value = 0; - bool isNegative = false; - for (size_t i = 0; i < attributeValue->size(); ++i) { - if (i == 0 && attributeValue->at(i) == '-') { - isNegative = true; - } else { - if (!isdigit(attributeValue->at(i))) { - // If not a number, return S_INT_MIN - return S_INT_MIN; - } - value *= 10; - value += attributeValue->at(i) - '0'; - } - } - return isNegative ? -value : value; -} - -/* static */ void HeaderPolicy::insertCharactersIntoVector(const char *const characters, - std::vector<int> *const vector) { - for (int i = 0; characters[i]; ++i) { - vector->push_back(characters[i]); - } -} - } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 6b396f3f2..4261667fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -17,7 +17,7 @@ #ifndef LATINIME_HEADER_POLICY_H #define LATINIME_HEADER_POLICY_H -#include <cctype> +#include <ctime> #include <stdint.h> #include "defines.h" @@ -29,15 +29,36 @@ namespace latinime { class HeaderPolicy : public DictionaryHeaderStructurePolicy { public: - explicit HeaderPolicy(const uint8_t *const dictBuf, const int dictSize) - : mDictBuf(dictBuf), - mDictFormatVersion(FormatUtils::detectFormatVersion(dictBuf, dictSize)), + // Reads information from existing dictionary buffer. + HeaderPolicy(const uint8_t *const dictBuf, const int dictSize) + : mDictFormatVersion(FormatUtils::detectFormatVersion(dictBuf, dictSize)), mDictionaryFlags(HeaderReadWriteUtils::getFlags(dictBuf)), mSize(HeaderReadWriteUtils::getHeaderSize(dictBuf)), - mAttributeMap(createAttributeMapAndReadAllAttributes(mDictBuf)), + mAttributeMap(createAttributeMapAndReadAllAttributes(dictBuf)), mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), - mUsesForgettingCurve(readUsesForgettingCurveFlag()), - mLastUpdatedTime(readLastUpdatedTime()) {} + mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, + IS_DECAYING_DICT_KEY, false /* defaultValue */)), + mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), + mUnigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + UNIGRAM_COUNT_KEY, 0 /* defaultValue */)), + mBigramCount(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + BIGRAM_COUNT_KEY, 0 /* defaultValue */)), + mExtendedRegionSize(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + EXTENDED_REGION_SIZE_KEY, 0 /* defaultValue */)) {} + + // Constructs header information using an attribute map. + HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion, + const HeaderReadWriteUtils::AttributeMap *const attributeMap) + : mDictFormatVersion(dictFormatVersion), + mDictionaryFlags(HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( + attributeMap)), mSize(0), mAttributeMap(*attributeMap), + mMultiWordCostMultiplier(readMultipleWordCostMultiplier()), + mIsDecayingDict(HeaderReadWriteUtils::readBoolAttributeValue(&mAttributeMap, + IS_DECAYING_DICT_KEY, false /* defaultValue */)), + mLastUpdatedTime(HeaderReadWriteUtils::readIntAttributeValue(&mAttributeMap, + LAST_UPDATED_TIME_KEY, time(0) /* defaultValue */)), + mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0) {} ~HeaderPolicy() {} @@ -61,53 +82,60 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mMultiWordCostMultiplier; } - AK_FORCE_INLINE bool usesForgettingCurve() const { - return mUsesForgettingCurve; + AK_FORCE_INLINE bool isDecayingDict() const { + return mIsDecayingDict; } AK_FORCE_INLINE int getLastUpdatedTime() const { return mLastUpdatedTime; } + AK_FORCE_INLINE int getUnigramCount() const { + return mUnigramCount; + } + + AK_FORCE_INLINE int getBigramCount() const { + return mBigramCount; + } + + AK_FORCE_INLINE int getExtendedRegionSize() const { + return mExtendedRegionSize; + } + void readHeaderValueOrQuestionMark(const char *const key, int *outValue, int outValueSize) const; bool writeHeaderToBuffer(BufferWithExtendableBuffer *const bufferToWrite, - const bool updatesLastUpdatedTime) const; + const bool updatesLastUpdatedTime, const int unigramCount, + const int bigramCount, const int extendedRegionSize) const; private: DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy); static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY; - static const char *const USES_FORGETTING_CURVE_KEY; + static const char *const IS_DECAYING_DICT_KEY; static const char *const LAST_UPDATED_TIME_KEY; - static const float DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER; + static const char *const UNIGRAM_COUNT_KEY; + static const char *const BIGRAM_COUNT_KEY; + static const char *const EXTENDED_REGION_SIZE_KEY; + static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE; static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE; - const uint8_t *const mDictBuf; const FormatUtils::FORMAT_VERSION mDictFormatVersion; const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags; const int mSize; HeaderReadWriteUtils::AttributeMap mAttributeMap; const float mMultiWordCostMultiplier; - const bool mUsesForgettingCurve; + const bool mIsDecayingDict; const int mLastUpdatedTime; + const int mUnigramCount; + const int mBigramCount; + const int mExtendedRegionSize; float readMultipleWordCostMultiplier() const; - bool readUsesForgettingCurveFlag() const; - - int readLastUpdatedTime() const; - - bool getAttributeValueAsInt(const char *const key, int *const outValue) const; - static HeaderReadWriteUtils::AttributeMap createAttributeMapAndReadAllAttributes( const uint8_t *const dictBuf); - - static int parseIntAttributeValue(const std::vector<int> *const attributeValue); - - static void insertCharactersIntoVector( - const char *const characters, std::vector<int> *const vector); }; } // namespace latinime #endif /* LATINIME_HEADER_POLICY_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp index 80fe88671..2694ce8d5 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.cpp @@ -16,6 +16,8 @@ #include "suggest/policyimpl/dictionary/header/header_read_write_utils.h" +#include <cctype> +#include <cstdio> #include <vector> #include "defines.h" @@ -43,6 +45,13 @@ const HeaderReadWriteUtils::DictionaryFlags const HeaderReadWriteUtils::DictionaryFlags HeaderReadWriteUtils::FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; +// Note that these are corresponding definitions in Java side in FormatSpec.FileHeader. +const char *const HeaderReadWriteUtils::SUPPORTS_DYNAMIC_UPDATE_KEY = "SUPPORTS_DYNAMIC_UPDATE"; +const char *const HeaderReadWriteUtils::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY = + "REQUIRES_GERMAN_UMLAUT_PROCESSING"; +const char *const HeaderReadWriteUtils::REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY = + "REQUIRES_FRENCH_LIGATURE_PROCESSING"; + /* static */ int HeaderReadWriteUtils::getHeaderSize(const uint8_t *const dictBuf) { // See the format of the header in the comment in // BinaryDictionaryFormatUtils::detectFormatVersion() @@ -56,6 +65,22 @@ const HeaderReadWriteUtils::DictionaryFlags HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE); } +/* static */ HeaderReadWriteUtils::DictionaryFlags + HeaderReadWriteUtils::createAndGetDictionaryFlagsUsingAttributeMap( + const HeaderReadWriteUtils::AttributeMap *const attributeMap) { + const bool requiresGermanUmlautProcessing = readBoolAttributeValue(attributeMap, + REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false /* defaultValue */); + const bool requiresFrenchLigatureProcessing = readBoolAttributeValue(attributeMap, + REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY, false /* defaultValue */); + const bool supportsDynamicUpdate = readBoolAttributeValue(attributeMap, + SUPPORTS_DYNAMIC_UPDATE_KEY, false /* defaultValue */); + DictionaryFlags dictflags = NO_FLAGS; + dictflags |= requiresGermanUmlautProcessing ? GERMAN_UMLAUT_PROCESSING_FLAG : 0; + dictflags |= requiresFrenchLigatureProcessing ? FRENCH_LIGATURE_PROCESSING_FLAG : 0; + dictflags |= supportsDynamicUpdate ? SUPPORTS_DYNAMIC_UPDATE_FLAG : 0; + return dictflags; +} + /* static */ void HeaderReadWriteUtils::fetchAllHeaderAttributes(const uint8_t *const dictBuf, AttributeMap *const headerAttributes) { const int headerSize = getHeaderSize(dictBuf); @@ -128,4 +153,72 @@ const HeaderReadWriteUtils::DictionaryFlags return true; } +/* static */ void HeaderReadWriteUtils::setBoolAttribute(AttributeMap *const headerAttributes, + const char *const key, const bool value) { + setIntAttribute(headerAttributes, key, value ? 1 : 0); +} + +/* static */ void HeaderReadWriteUtils::setIntAttribute(AttributeMap *const headerAttributes, + const char *const key, const int value) { + AttributeMap::key_type keyVector; + insertCharactersIntoVector(key, &keyVector); + setIntAttributeInner(headerAttributes, &keyVector, value); +} + +/* static */ void HeaderReadWriteUtils::setIntAttributeInner(AttributeMap *const headerAttributes, + const AttributeMap::key_type *const key, const int value) { + AttributeMap::mapped_type valueVector; + char charBuf[LARGEST_INT_DIGIT_COUNT + 1]; + snprintf(charBuf, LARGEST_INT_DIGIT_COUNT + 1, "%d", value); + insertCharactersIntoVector(charBuf, &valueVector); + (*headerAttributes)[*key] = valueVector; +} + +/* static */ bool HeaderReadWriteUtils::readBoolAttributeValue( + const AttributeMap *const headerAttributes, const char *const key, + const bool defaultValue) { + const int intDefaultValue = defaultValue ? 1 : 0; + const int intValue = readIntAttributeValue(headerAttributes, key, intDefaultValue); + return intValue != 0; +} + +/* static */ int HeaderReadWriteUtils::readIntAttributeValue( + const AttributeMap *const headerAttributes, const char *const key, + const int defaultValue) { + AttributeMap::key_type keyVector; + insertCharactersIntoVector(key, &keyVector); + return readIntAttributeValueInner(headerAttributes, &keyVector, defaultValue); +} + +/* static */ int HeaderReadWriteUtils::readIntAttributeValueInner( + const AttributeMap *const headerAttributes, const AttributeMap::key_type *const key, + const int defaultValue) { + AttributeMap::const_iterator it = headerAttributes->find(*key); + if (it != headerAttributes->end()) { + int value = 0; + bool isNegative = false; + for (size_t i = 0; i < it->second.size(); ++i) { + if (i == 0 && it->second.at(i) == '-') { + isNegative = true; + } else { + if (!isdigit(it->second.at(i))) { + // If not a number. + return defaultValue; + } + value *= 10; + value += it->second.at(i) - '0'; + } + } + return isNegative ? -value : value; + } + return defaultValue; +} + +/* static */ void HeaderReadWriteUtils::insertCharactersIntoVector(const char *const characters, + std::vector<int> *const vector) { + for (int i = 0; characters[i]; ++i) { + vector->push_back(characters[i]); + } +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h index 6cce73375..225968323 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_read_write_utils.h @@ -54,6 +54,9 @@ class HeaderReadWriteUtils { + HEADER_SIZE_FIELD_SIZE; } + static DictionaryFlags createAndGetDictionaryFlagsUsingAttributeMap( + const HeaderReadWriteUtils::AttributeMap *const attributeMap); + static void fetchAllHeaderAttributes(const uint8_t *const dictBuf, AttributeMap *const headerAttributes); @@ -69,6 +72,24 @@ class HeaderReadWriteUtils { static bool writeHeaderAttributes(BufferWithExtendableBuffer *const buffer, const AttributeMap *const headerAttributes, int *const writingPos); + /** + * Methods for header attributes. + */ + static void setBoolAttribute(AttributeMap *const headerAttributes, + const char *const key, const bool value); + + static void setIntAttribute(AttributeMap *const headerAttributes, + const char *const key, const int value); + + static bool readBoolAttributeValue(const AttributeMap *const headerAttributes, + const char *const key, const bool defaultValue); + + static int readIntAttributeValue(const AttributeMap *const headerAttributes, + const char *const key, const int defaultValue); + + static void insertCharactersIntoVector(const char *const characters, + AttributeMap::key_type *const key); + private: DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadWriteUtils); @@ -87,7 +108,16 @@ class HeaderReadWriteUtils { static const DictionaryFlags GERMAN_UMLAUT_PROCESSING_FLAG; static const DictionaryFlags SUPPORTS_DYNAMIC_UPDATE_FLAG; static const DictionaryFlags FRENCH_LIGATURE_PROCESSING_FLAG; - static const DictionaryFlags CONTAINS_BIGRAMS_FLAG; + + static const char *const SUPPORTS_DYNAMIC_UPDATE_KEY; + static const char *const REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY; + static const char *const REQUIRES_FRENCH_LIGATURE_PROCESSING_KEY; + + static void setIntAttributeInner(AttributeMap *const headerAttributes, + const AttributeMap::key_type *const key, const int value); + + static int readIntAttributeValueInner(const AttributeMap *const headerAttributes, + const AttributeMap::key_type *const key, const int defaultValue); }; } #endif /* LATINIME_HEADER_READ_WRITE_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp index 5269795a4..8a84bd261 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.cpp @@ -31,9 +31,21 @@ void PatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const dicNode, return; } int nextPos = dicNode->getChildrenPos(); + if (nextPos < 0 || nextPos >= mDictBufferSize) { + AKLOGE("Children PtNode array position is invalid. pos: %d, dict size: %d", + nextPos, mDictBufferSize); + ASSERT(false); + return; + } const int childCount = PatriciaTrieReadingUtils::getPtNodeArraySizeAndAdvancePosition( mDictRoot, &nextPos); for (int i = 0; i < childCount; i++) { + if (nextPos < 0 || nextPos >= mDictBufferSize) { + AKLOGE("Child PtNode position is invalid. pos: %d, dict size: %d, childCount: %d / %d", + nextPos, mDictBufferSize, i, childCount); + ASSERT(false); + return; + } nextPos = createAndGetLeavingChildNode(dicNode, nextPos, childDicNodes); } } @@ -404,6 +416,11 @@ int PatriciaTriePolicy::createAndGetLeavingChildNode(const DicNode *const dicNod if (PatriciaTrieReadingUtils::hasBigrams(flags)) { getBigramsStructurePolicy()->skipAllBigrams(&pos); } + if (mergedNodeCodePointCount <= 0) { + AKLOGE("Empty PtNode is not allowed. Code point count: %d", mergedNodeCodePointCount); + ASSERT(false); + return pos; + } childDicNodes->pushLeavingChild(dicNode, ptNodePos, childrenPos, probability, PatriciaTrieReadingUtils::isTerminal(flags), PatriciaTrieReadingUtils::hasChildrenInFlags(flags), diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h index 19155f938..8d88c68e8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_policy.h @@ -36,6 +36,7 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { PatriciaTriePolicy(const MmappedBuffer *const buffer) : mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer(), buffer->getBufferSize()), mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()), + mDictBufferSize(mBuffer->getBufferSize() - mHeaderPolicy.getSize()), mBigramListPolicy(mDictRoot), mShortcutListPolicy(mDictRoot) {} ~PatriciaTriePolicy() { @@ -106,18 +107,27 @@ class PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); } - bool needsToRunGC() const { + bool needsToRunGC(const bool mindsBlockByGC) const { // This method should not be called for non-updatable dictionary. AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); return false; } + void getProperty(const char *const query, char *const outResult, + const int maxResultLength) const { + // getProperty is not supported for this class. + if (maxResultLength > 0) { + outResult[0] = '\0'; + } + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(PatriciaTriePolicy); const MmappedBuffer *const mBuffer; const HeaderPolicy mHeaderPolicy; const uint8_t *const mDictRoot; + const int mDictBufferSize; const BigramListPolicy mBigramListPolicy; const ShortcutListPolicy mShortcutListPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp index 1316b425f..7df55815f 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp @@ -71,8 +71,17 @@ const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_BLACKLISTED = 0x01; length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer, pos); } else { - if (maxLength > 0) { - outBuffer[0] = getCodePointAndAdvancePosition(buffer, pos); + const int codePoint = getCodePointAndAdvancePosition(buffer, pos); + if (codePoint == NOT_A_CODE_POINT) { + // CAVEAT: codePoint == NOT_A_CODE_POINT means the code point is + // CHARACTER_ARRAY_TERMINATOR. The code point must not be CHARACTER_ARRAY_TERMINATOR + // when the PtNode has a single code point. + length = 0; + AKLOGE("codePoint is NOT_A_CODE_POINT. pos: %d, codePoint: 0x%x, buffer[pos - 1]: 0x%x", + *pos - 1, codePoint, buffer[*pos - 1]); + ASSERT(false); + } else if (maxLength > 0) { + outBuffer[0] = codePoint; length = 1; } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h index 17d2e39c2..9dc34823c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h @@ -42,6 +42,10 @@ class BufferWithExtendableBuffer { return mOriginalBufferSize + mUsedAdditionalBufferSize; } + AK_FORCE_INLINE int getUsedAdditionalBufferSize() const { + return mUsedAdditionalBufferSize; + } + /** * For reading. */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp new file mode 100644 index 000000000..942a74238 --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.cpp @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/utils/decaying_utils.h" + +#include "suggest/policyimpl/dictionary/utils/probability_utils.h" + +namespace latinime { + +const int DecayingUtils::MAX_UNIGRAM_COUNT = 12000; +const int DecayingUtils::MAX_UNIGRAM_COUNT_AFTER_GC = 10000; +const int DecayingUtils::MAX_BIGRAM_COUNT = 12000; +const int DecayingUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000; + +const int DecayingUtils::MAX_COMPUTED_PROBABILITY = 127; +const int DecayingUtils::MAX_UNIGRAM_PROBABILITY = 120; +const int DecayingUtils::MIN_VALID_UNIGRAM_PROBABILITY = 24; +const int DecayingUtils::UNIGRAM_PROBABILITY_STEP = 8; +const int DecayingUtils::MAX_BIGRAM_PROBABILITY_DELTA = 15; +const int DecayingUtils::MIN_VALID_BIGRAM_PROBABILITY_DELTA = 3; +const int DecayingUtils::BIGRAM_PROBABILITY_DELTA_STEP = 1; + +/* static */ int DecayingUtils::getProbability(const int encodedUnigramProbability, + const int encodedBigramProbabilityDelta) { + if (encodedUnigramProbability == NOT_A_PROBABILITY) { + return NOT_A_PROBABILITY; + } else if (encodedBigramProbabilityDelta == NOT_A_PROBABILITY) { + const int rawProbability = ProbabilityUtils::backoff(decodeUnigramProbability( + encodedUnigramProbability)); + return min(getDecayedProbability(rawProbability), MAX_COMPUTED_PROBABILITY); + } else { + const int rawProbability = ProbabilityUtils::computeProbabilityForBigram( + decodeUnigramProbability(encodedUnigramProbability), + decodeBigramProbabilityDelta(encodedBigramProbabilityDelta)); + return min(getDecayedProbability(rawProbability), MAX_COMPUTED_PROBABILITY); + } +} + +/* static */ int DecayingUtils::getUpdatedUnigramProbability(const int originalEncodedProbability, + const int newProbability) { + if (originalEncodedProbability == NOT_A_PROBABILITY) { + // The unigram is not in this dictionary. + if (newProbability == NOT_A_PROBABILITY) { + // The unigram is not in other dictionaries. + return 0; + } else { + return MIN_VALID_UNIGRAM_PROBABILITY; + } + } else { + if (newProbability != NOT_A_PROBABILITY + && originalEncodedProbability < MIN_VALID_UNIGRAM_PROBABILITY) { + return MIN_VALID_UNIGRAM_PROBABILITY; + } + return min(originalEncodedProbability + UNIGRAM_PROBABILITY_STEP, MAX_UNIGRAM_PROBABILITY); + } +} + +/* static */ int DecayingUtils::getUnigramProbabilityToSave(const int encodedProbability) { + return max(encodedProbability - UNIGRAM_PROBABILITY_STEP, 0); +} + +/* static */ int DecayingUtils::getBigramProbabilityDeltaToSave(const int encodedProbabilityDelta) { + return max(encodedProbabilityDelta - BIGRAM_PROBABILITY_DELTA_STEP, 0); +} + +/* static */ int DecayingUtils::getUpdatedBigramProbabilityDelta( + const int originalEncodedProbabilityDelta, const int newProbability) { + if (originalEncodedProbabilityDelta == NOT_A_PROBABILITY) { + // The bigram relation is not in this dictionary. + if (newProbability == NOT_A_PROBABILITY) { + // The bigram target is not in other dictionaries. + return 0; + } else { + return MIN_VALID_BIGRAM_PROBABILITY_DELTA; + } + } else { + if (newProbability != NOT_A_PROBABILITY + && originalEncodedProbabilityDelta < MIN_VALID_BIGRAM_PROBABILITY_DELTA) { + return MIN_VALID_BIGRAM_PROBABILITY_DELTA; + } + return min(originalEncodedProbabilityDelta + BIGRAM_PROBABILITY_DELTA_STEP, + MAX_BIGRAM_PROBABILITY_DELTA); + } +} + +/* static */ int DecayingUtils::isValidUnigram(const int encodedUnigramProbability) { + return encodedUnigramProbability >= MIN_VALID_UNIGRAM_PROBABILITY; +} + +/* static */ int DecayingUtils::isValidBigram(const int encodedBigramProbabilityDelta) { + return encodedBigramProbabilityDelta >= MIN_VALID_BIGRAM_PROBABILITY_DELTA; +} + +/* static */ int DecayingUtils::decodeUnigramProbability(const int encodedProbability) { + const int probability = encodedProbability - MIN_VALID_UNIGRAM_PROBABILITY; + if (probability < 0) { + return NOT_A_PROBABILITY; + } else { + return min(probability, MAX_UNIGRAM_PROBABILITY); + } +} + +/* static */ int DecayingUtils::decodeBigramProbabilityDelta(const int encodedProbabilityDelta) { + const int probabilityDelta = encodedProbabilityDelta - MIN_VALID_BIGRAM_PROBABILITY_DELTA; + if (probabilityDelta < 0) { + return NOT_A_PROBABILITY; + } else { + return min(probabilityDelta, MAX_BIGRAM_PROBABILITY_DELTA); + } +} + +/* static */ int DecayingUtils::getDecayedProbability(const int rawProbability) { + return rawProbability; +} + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h new file mode 100644 index 000000000..1ca03918f --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/decaying_utils.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DECAYING_UTILS_H +#define LATINIME_DECAYING_UTILS_H + +#include "defines.h" + +namespace latinime { + +// TODO: Check the elapsed time and decrease the probability depending on the time. Time field is +// required to introduced to each terminal PtNode and bigram entry. +// TODO: Quit using bigram probability to indicate the delta. +// TODO: Quit using bigram probability delta. +class DecayingUtils { + public: + static const int MAX_UNIGRAM_COUNT; + static const int MAX_UNIGRAM_COUNT_AFTER_GC; + static const int MAX_BIGRAM_COUNT; + static const int MAX_BIGRAM_COUNT_AFTER_GC; + + static int getProbability(const int encodedUnigramProbability, + const int encodedBigramProbabilityDelta); + + static int getUpdatedUnigramProbability(const int originalEncodedProbability, + const int newProbability); + + static int getUpdatedBigramProbabilityDelta(const int originalEncodedProbabilityDelta, + const int newProbability); + + static int isValidUnigram(const int encodedUnigramProbability); + + static int isValidBigram(const int encodedProbabilityDelta); + + static int getUnigramProbabilityToSave(const int encodedProbability); + + static int getBigramProbabilityDeltaToSave(const int encodedProbabilityDelta); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(DecayingUtils); + + static const int MAX_COMPUTED_PROBABILITY; + static const int MAX_UNIGRAM_PROBABILITY; + static const int MIN_VALID_UNIGRAM_PROBABILITY; + static const int UNIGRAM_PROBABILITY_STEP; + static const int MAX_BIGRAM_PROBABILITY_DELTA; + static const int MIN_VALID_BIGRAM_PROBABILITY_DELTA; + static const int BIGRAM_PROBABILITY_DELTA_STEP; + + static int decodeUnigramProbability(const int encodedProbability); + + static int decodeBigramProbabilityDelta(const int encodedProbability); + + static int getDecayedProbability(const int rawProbability); +}; +} // namespace latinime +#endif /* LATINIME_DECAYING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp new file mode 100644 index 000000000..f22e94c6a --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h" + +#include <cstdio> +#include <cstring> + +#include "suggest/policyimpl/dictionary/header/header_policy.h" +#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h" +#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" +#include "suggest/policyimpl/dictionary/utils/format_utils.h" + +namespace latinime { + +const char *const DictFileWritingUtils::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp"; + +/* static */ bool DictFileWritingUtils::createEmptyDictFile(const char *const filePath, + const int dictVersion, const HeaderReadWriteUtils::AttributeMap *const attributeMap) { + switch (dictVersion) { + case 3: + return createEmptyV3DictFile(filePath, attributeMap); + default: + // Only version 3 dictionary is supported for now. + return false; + } +} + +/* static */ bool DictFileWritingUtils::createEmptyV3DictFile(const char *const filePath, + const HeaderReadWriteUtils::AttributeMap *const attributeMap) { + BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); + HeaderPolicy headerPolicy(FormatUtils::VERSION_3, attributeMap); + headerPolicy.writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */, + 0 /* unigramCount */, 0 /* bigramCount */, 0 /* extendedRegionSize */); + BufferWithExtendableBuffer bodyBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); + if (!DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(&bodyBuffer, 0 /* rootPos */)) { + return false; + } + return flushAllHeaderAndBodyToFile(filePath, &headerBuffer, &bodyBuffer); +} + +/* static */ bool DictFileWritingUtils::flushAllHeaderAndBodyToFile(const char *const filePath, + BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) { + const int tmpFileNameBufSize = strlen(filePath) + + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */; + // Name of a temporary file used for writing that is a connected string of original name and + // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE. + char tmpFileName[tmpFileNameBufSize]; + snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", filePath, + TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); + FILE *const file = fopen(tmpFileName, "wb"); + if (!file) { + AKLOGE("Dictionary file %s cannnot be opened.", tmpFileName); + ASSERT(false); + return false; + } + // Write the dictionary header. + if (!writeBufferToFile(file, dictHeader)) { + remove(tmpFileName); + AKLOGE("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition()); + ASSERT(false); + return false; + } + // Write the dictionary body. + if (!writeBufferToFile(file, dictBody)) { + remove(tmpFileName); + AKLOGE("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition()); + ASSERT(false); + return false; + } + fclose(file); + rename(tmpFileName, filePath); + return true; +} + +// This closes file pointer when an error is caused and returns whether the writing was succeeded +// or not. +/* static */ bool DictFileWritingUtils::writeBufferToFile(FILE *const file, + const BufferWithExtendableBuffer *const buffer) { + const int originalBufSize = buffer->getOriginalBufferSize(); + if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */), + originalBufSize, 1, file) < 1) { + fclose(file); + return false; + } + const int additionalBufSize = buffer->getUsedAdditionalBufferSize(); + if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */), + additionalBufSize, 1, file) < 1) { + fclose(file); + return false; + } + return true; +} + +} // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h new file mode 100644 index 000000000..bd4ac66fd --- /dev/null +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LATINIME_DICT_FILE_WRITING_UTILS_H +#define LATINIME_DICT_FILE_WRITING_UTILS_H + +#include <cstdio> + +#include "defines.h" +#include "suggest/policyimpl/dictionary/header/header_read_write_utils.h" + +namespace latinime { + +class BufferWithExtendableBuffer; + +class DictFileWritingUtils { + public: + static bool createEmptyDictFile(const char *const filePath, const int dictVersion, + const HeaderReadWriteUtils::AttributeMap *const attributeMap); + + static bool flushAllHeaderAndBodyToFile(const char *const filePath, + BufferWithExtendableBuffer *const dictHeader, + BufferWithExtendableBuffer *const dictBody); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(DictFileWritingUtils); + + static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE; + + static bool createEmptyV3DictFile(const char *const filePath, + const HeaderReadWriteUtils::AttributeMap *const attributeMap); + + static bool writeBufferToFile(FILE *const file, + const BufferWithExtendableBuffer *const buffer); +}; +} // namespace latinime +#endif /* LATINIME_DICT_FILE_WRITING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp b/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp index 408b12ae9..5b6b5e874 100644 --- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp +++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.cpp @@ -47,7 +47,7 @@ ErrorType TypingWeighting::getErrorType(const CorrectionType correctionType, case CT_TERMINAL_INSERTION: case CT_TRANSPOSITION: return ET_EDIT_CORRECTION; - case CT_NEW_WORD_SPACE_OMITTION: + case CT_NEW_WORD_SPACE_OMISSION: case CT_NEW_WORD_SPACE_SUBSTITUTION: return ET_NEW_WORD; case CT_TERMINAL: diff --git a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h index b6aa85896..9f0a331e3 100644 --- a/native/jni/src/suggest/policyimpl/typing/typing_weighting.h +++ b/native/jni/src/suggest/policyimpl/typing/typing_weighting.h @@ -74,7 +74,8 @@ class TypingWeighting : public Weighting { // Note: min() required since length can be MAX_POINT_TO_KEY_LENGTH for characters not on // the keyboard (like accented letters) const float normalizedSquaredLength = traverseSession->getProximityInfoState(0) - ->getPointToKeyLength(pointIndex, dicNode->getNodeCodePoint()); + ->getPointToKeyLength(pointIndex, + CharUtils::toBaseLowerCase(dicNode->getNodeCodePoint())); const float normalizedDistance = TouchPositionCorrectionUtils::getSweetSpotFactor( traverseSession->isTouchPositionCorrectionEnabled(), normalizedSquaredLength); const float weightedDistance = ScoringParams::DISTANCE_WEIGHT_LENGTH * normalizedDistance; @@ -113,10 +114,10 @@ class TypingWeighting : public Weighting { const int16_t parentPointIndex = parentDicNode->getInputIndex(0); const int prevCodePoint = parentDicNode->getNodeCodePoint(); const float distance1 = traverseSession->getProximityInfoState(0)->getPointToKeyLength( - parentPointIndex + 1, prevCodePoint); + parentPointIndex + 1, CharUtils::toBaseLowerCase(prevCodePoint)); const int codePoint = dicNode->getNodeCodePoint(); const float distance2 = traverseSession->getProximityInfoState(0)->getPointToKeyLength( - parentPointIndex, codePoint); + parentPointIndex, CharUtils::toBaseLowerCase(codePoint)); const float distance = distance1 + distance2; const float weightedLengthDistance = distance * ScoringParams::DISTANCE_WEIGHT_LENGTH; @@ -133,7 +134,7 @@ class TypingWeighting : public Weighting { const bool existsAdjacentProximityChars = traverseSession->getProximityInfoState(0) ->existsAdjacentProximityChars(insertedPointIndex); const float dist = traverseSession->getProximityInfoState(0)->getPointToKeyLength( - insertedPointIndex + 1, dicNode->getNodeCodePoint()); + insertedPointIndex + 1, CharUtils::toBaseLowerCase(dicNode->getNodeCodePoint())); const float weightedDistance = dist * ScoringParams::DISTANCE_WEIGHT_LENGTH; const bool singleChar = dicNode->getNodeCodePointCount() == 1; float cost = (singleChar ? ScoringParams::INSERTION_COST_FIRST_CHAR : 0.0f); diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java new file mode 100644 index 000000000..cf85d97a0 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; + +import com.android.inputmethod.latin.makedict.FormatSpec; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +@LargeTest +public class BinaryDictionaryDecayingTests extends AndroidTestCase { + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final String TEST_LOCALE = "test"; + + private static final int DUMMY_PROBABILITY = 0; + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { + binaryDictionary.flushWithGC(); + } + + private void forcePassingLongTime(final BinaryDictionary binaryDictionary) { + // Currently, probabilities are decayed when GC is run. All entries that have never been + // typed in 32 GCs are removed. + final int count = 32; + for (int i = 0; i < count; i++) { + binaryDictionary.flushWithGC(); + } + } + + private File createEmptyDictionaryAndGetFile(final String filename) throws IOException { + final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); + Map<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + 3 /* dictVersion */, attributeMap)) { + return file; + } else { + throw new IOException("Empty dictionary cannot be created."); + } + } + + public void testAddValidAndInvalidWords() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidWord("a")); + binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidWord("a")); + binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidWord("a")); + binaryDictionary.addUnigramWord("a", Dictionary.NOT_A_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("a")); + + binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("b")); + + final int unigramProbability = binaryDictionary.getFrequency("a"); + binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidBigram("a", "b")); + binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidBigram("a", "b")); + binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidBigram("a", "b")); + binaryDictionary.addBigramWords("a", "b", Dictionary.NOT_A_PROBABILITY); + assertTrue(binaryDictionary.isValidBigram("a", "b")); + + binaryDictionary.addUnigramWord("c", DUMMY_PROBABILITY); + binaryDictionary.addBigramWords("a", "c", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidBigram("a", "c")); + + binaryDictionary.close(); + dictFile.delete(); + } + + // TODO: Add large tests. + public void testDecayingProbability() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("a")); + forcePassingShortTime(binaryDictionary); + assertFalse(binaryDictionary.isValidWord("a")); + + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + forcePassingShortTime(binaryDictionary); + assertTrue(binaryDictionary.isValidWord("a")); + forcePassingLongTime(binaryDictionary); + assertFalse(binaryDictionary.isValidWord("a")); + + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidBigram("a", "b")); + forcePassingShortTime(binaryDictionary); + assertFalse(binaryDictionary.isValidBigram("a", "b")); + + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("a", DUMMY_PROBABILITY); + binaryDictionary.addUnigramWord("b", DUMMY_PROBABILITY); + binaryDictionary.addBigramWords("a", "b", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidBigram("a", "b")); + forcePassingShortTime(binaryDictionary); + assertTrue(binaryDictionary.isValidBigram("a", "b")); + forcePassingLongTime(binaryDictionary); + assertFalse(binaryDictionary.isValidBigram("a", "b")); + + binaryDictionary.close(); + dictFile.delete(); + } +} diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 96a2217a3..6a21522f9 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -21,24 +21,19 @@ import android.test.suitebuilder.annotation.LargeTest; import android.util.Pair; import com.android.inputmethod.latin.makedict.CodePointUtils; -import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Random; @LargeTest public class BinaryDictionaryTests extends AndroidTestCase { - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */); private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; @@ -52,15 +47,18 @@ public class BinaryDictionaryTests extends AndroidTestCase { super.tearDown(); } - private File createEmptyDictionaryAndGetFile(final String filename) throws IOException, - UnsupportedFormatException { - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + private File createEmptyDictionaryAndGetFile(final String filename) throws IOException { final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - return file; + Map<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + if (BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + 3 /* dictVersion */, attributeMap)) { + return file; + } else { + throw new IOException("Empty dictionary cannot be created."); + } } public void testIsValidDictionary() { @@ -69,8 +67,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -95,8 +91,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -139,8 +133,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -169,8 +161,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -240,8 +230,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -294,8 +282,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -342,8 +328,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -392,8 +376,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, @@ -445,8 +427,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), @@ -516,8 +496,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), @@ -617,8 +595,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); - } catch (UnsupportedFormatException e) { - fail("UnsupportedFormatException while writing an initial dictionary : " + e); } final ArrayList<String> words = new ArrayList<String>(); @@ -630,7 +606,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - while(!binaryDictionary.needsToRunGC()) { + while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); @@ -650,4 +626,57 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile.delete(); } + + public void testUnigramAndBigramCount() { + final int flashWithGCIterationCount = 10; + final int codePointSetSize = 50; + final int unigramCountPerIteration = 1000; + final int bigramCountPerIteration = 2000; + final int seed = 1123581321; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + + final ArrayList<String> words = new ArrayList<String>(); + final HashSet<Pair<String, String>> bigrams = new HashSet<Pair<String, String>>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + BinaryDictionary binaryDictionary; + for (int i = 0; i < flashWithGCIterationCount; i++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + for (int j = 0; j < unigramCountPerIteration; j++) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + for (int j = 0; j < bigramCountPerIteration; j++) { + final String word0 = words.get(random.nextInt(words.size())); + final String word1 = words.get(random.nextInt(words.size())); + bigrams.add(new Pair<String, String>(word0, word1)); + final int bigramProbability = random.nextInt(0xF); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + assertEquals(new HashSet<String>(words).size(), Integer.parseInt( + binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt( + binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY))); + binaryDictionary.flushWithGC(); + assertEquals(new HashSet<String>(words).size(), Integer.parseInt( + binaryDictionary.getPropertyForTests(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + assertEquals(new HashSet<Pair<String, String>>(bigrams).size(), Integer.parseInt( + binaryDictionary.getPropertyForTests(BinaryDictionary.BIGRAM_COUNT_QUERY))); + binaryDictionary.close(); + } + + dictFile.delete(); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java new file mode 100644 index 000000000..132483d5e --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; +import android.util.Log; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Random; + +/** + * Unit tests for SparseTable. + */ +@LargeTest +public class SparseTableTests extends AndroidTestCase { + private static final String TAG = SparseTableTests.class.getSimpleName(); + + private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 }; + private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7}; + + private final Random mRandom; + private final ArrayList<Integer> mRandomIndex; + + private static final int DEFAULT_SIZE = 10000; + private static final int BLOCK_SIZE = 8; + + public SparseTableTests() { + this(System.currentTimeMillis(), DEFAULT_SIZE); + } + + public SparseTableTests(final long seed, final int tableSize) { + super(); + Log.d(TAG, "Seed for test is " + seed + ", size is " + tableSize); + mRandom = new Random(seed); + mRandomIndex = new ArrayList<Integer>(tableSize); + for (int i = 0; i < tableSize; ++i) { + mRandomIndex.add(SparseTable.NOT_EXIST); + } + } + + public void testInitializeWithArray() { + final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE); + for (int i = 0; i < 8; ++i) { + assertEquals(SparseTable.NOT_EXIST, table.get(i)); + } + assertEquals(SparseTable.NOT_EXIST, table.get(8)); + for (int i = 9; i < 16; ++i) { + assertEquals(i - 8, table.get(i)); + } + } + + public void testSet() { + final SparseTable table = new SparseTable(16, BLOCK_SIZE); + table.set(3, 6); + table.set(8, 16); + for (int i = 0; i < 16; ++i) { + if (i == 3 || i == 8) { + assertEquals(i * 2, table.get(i)); + } else { + assertEquals(SparseTable.NOT_EXIST, table.get(i)); + } + } + } + + private void generateRandomIndex(final int size, final int prop) { + for (int i = 0; i < DEFAULT_SIZE; ++i) { + if (mRandom.nextInt(100) < prop) { + mRandomIndex.set(i, mRandom.nextInt()); + } else { + mRandomIndex.set(i, SparseTable.NOT_EXIST); + } + } + } + + private void runTestRandomSet() { + final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE); + int elementCount = 0; + for (int i = 0; i < DEFAULT_SIZE; ++i) { + if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { + table.set(i, mRandomIndex.get(i)); + elementCount++; + } + } + + Log.d(TAG, "table size = " + table.getLookupTableSize() + " + " + + table.getContentTableSize()); + Log.d(TAG, "the table has " + elementCount + " elements"); + for (int i = 0; i < DEFAULT_SIZE; ++i) { + assertEquals(table.get(i), (int)mRandomIndex.get(i)); + } + + // flush and reload + OutputStream lookupOutStream = null; + OutputStream contentOutStream = null; + InputStream lookupInStream = null; + InputStream contentInStream = null; + try { + final File lookupIndexFile = File.createTempFile("testRandomSet", ".small"); + final File contentFile = File.createTempFile("testRandomSet", ".big"); + lookupOutStream = new FileOutputStream(lookupIndexFile); + contentOutStream = new FileOutputStream(contentFile); + table.write(lookupOutStream, contentOutStream); + lookupInStream = new FileInputStream(lookupIndexFile); + contentInStream = new FileInputStream(contentFile); + final byte[] lookupArray = new byte[(int) lookupIndexFile.length()]; + final byte[] contentArray = new byte[(int) contentFile.length()]; + lookupInStream.read(lookupArray); + contentInStream.read(contentArray); + final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE); + for (int i = 0; i < DEFAULT_SIZE; ++i) { + assertEquals(table.get(i), newTable.get(i)); + } + } catch (IOException e) { + Log.d(TAG, "IOException while flushing and realoding", e); + } finally { + if (lookupOutStream != null) { + try { + lookupOutStream.close(); + } catch (IOException e) { + Log.d(TAG, "IOException while closing the stream", e); + } + } + if (contentOutStream != null) { + try { + contentOutStream.close(); + } catch (IOException e) { + Log.d(TAG, "IOException while closing contentStream.", e); + } + } + } + } + + public void testRandomSet() { + for (int i = 0; i <= 100; i += 10) { + generateRandomIndex(DEFAULT_SIZE, i); + runTestRandomSet(); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 06c427193..ddc9546c5 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -75,10 +75,10 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { return new ArrayList<String>(wordSet); } - private void addToDict(final UserHistoryPredictionDictionary dict, final List<String> words) { + private void addToDict(final UserHistoryDictionary dict, final List<String> words) { String prevWord = null; for (String word : words) { - dict.addToPersonalizationPredictionDictionary(prevWord, word, true); + dict.addToDictionary(prevWord, word, true); prevWord = word; } } @@ -90,8 +90,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private void addAndWriteRandomWords(final String testFilenameSuffix, final int numberOfWords, final Random random, final boolean checkContents) { final List<String> words = generateWords(numberOfWords, random); - final UserHistoryPredictionDictionary dict = - PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), testFilenameSuffix /* locale */, mPrefs); // Add random words to the user history dictionary. addToDict(dict, words); @@ -122,8 +122,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { true /* checksContents */); } finally { try { - final UserHistoryPredictionDictionary dict = - PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), testFilenameSuffix, mPrefs); Log.d(TAG, "waiting for writing ..."); dict.shutdownExecutorForTests(); @@ -134,7 +134,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Log.d(TAG, "InterruptedException: " + e); } - final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFile = new File(getContext().getFilesDir(), fileName); @@ -159,7 +159,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { // Create filename suffixes for this test. for (int i = 0; i < numberOfLanguages; i++) { testFilenameSuffixes[i] = "testSwitchingLanguages" + i; - final String fileName = UserHistoryPredictionDictionary.NAME + "." + + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFiles[i] = new File(getContext().getFilesDir(), fileName); } @@ -181,8 +181,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { try { Log.d(TAG, "waiting for writing ..."); for (int i = 0; i < numberOfLanguages; i++) { - final UserHistoryPredictionDictionary dict = - PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), testFilenameSuffixes[i], mPrefs); dict.shutdownExecutorForTests(); while (!dict.isTerminatedForTests()) { @@ -210,8 +210,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 10000 : 1000; final Random random = new Random(123456); - UserHistoryPredictionDictionary dict = - PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), testFilenameSuffix, mPrefs); try { addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, @@ -227,7 +227,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } catch (InterruptedException e) { Log.d(TAG, "InterruptedException: ", e); } - final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFile = new File(getContext().getFilesDir(), fileName); if (dictFile != null) { diff --git a/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java new file mode 100644 index 000000000..fa6ad16c1 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/SpannableStringUtilsTests.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.SmallTest; +import android.text.style.SuggestionSpan; +import android.text.style.URLSpan; +import android.text.SpannableStringBuilder; +import android.text.Spannable; +import android.text.Spanned; + +@SmallTest +public class SpannableStringUtilsTests extends AndroidTestCase { + public void testConcatWithSuggestionSpansOnly() { + SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n" + + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n" + + "test string\ntest string\n"); + final int N = 10; + for (int i = 0; i < N; ++i) { + // Put a PARAGRAPH-flagged span that should not be found in the result. + s.setSpan(new SuggestionSpan(getContext(), + new String[] {"" + i}, Spannable.SPAN_PARAGRAPH), + i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH); + // Put a normal suggestion span that should be found in the result. + s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0); + // Put a URL span than should not be found in the result. + s.setSpan(new URLSpan("http://a"), i, i * 2, 0); + } + + final CharSequence a = s.subSequence(0, 15); + final CharSequence b = s.subSequence(15, s.length()); + final Spanned result = + (Spanned)SpannableStringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b); + + Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class); + for (int i = 0; i < spans.length; i++) { + final int flags = result.getSpanFlags(spans[i]); + assertEquals("Should not find a span with PARAGRAPH flag", + flags & Spannable.SPAN_PARAGRAPH, 0); + assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan); + } + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java index eb9fb984e..4e396a1cf 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java @@ -20,11 +20,6 @@ import com.android.inputmethod.latin.settings.SettingsValues; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; -import android.text.style.SuggestionSpan; -import android.text.style.URLSpan; -import android.text.SpannableStringBuilder; -import android.text.Spannable; -import android.text.Spanned; import java.util.Arrays; import java.util.List; @@ -285,34 +280,4 @@ public class StringUtilsTests extends AndroidTestCase { assertEquals(objs[i], newObjArray.get(i)); } } - - public void testConcatWithSuggestionSpansOnly() { - SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n" - + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n" - + "test string\ntest string\n"); - final int N = 10; - for (int i = 0; i < N; ++i) { - // Put a PARAGRAPH-flagged span that should not be found in the result. - s.setSpan(new SuggestionSpan(getContext(), - new String[] {"" + i}, Spannable.SPAN_PARAGRAPH), - i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH); - // Put a normal suggestion span that should be found in the result. - s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0); - // Put a URL span than should not be found in the result. - s.setSpan(new URLSpan("http://a"), i, i * 2, 0); - } - - final CharSequence a = s.subSequence(0, 15); - final CharSequence b = s.subSequence(15, s.length()); - final Spanned result = - (Spanned)StringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b); - - Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class); - for (int i = 0; i < spans.length; i++) { - final int flags = result.getSpanFlags(spans[i]); - assertEquals("Should not find a span with PARAGRAPH flag", - flags & Spannable.SPAN_PARAGRAPH, 0); - assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan); - } - } } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index fa80385fc..bd06e9f3a 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -80,17 +80,17 @@ public final class BinaryDictOffdeviceUtils { } /** - * Returns a decrypted/uncompressed binary dictionary. + * Returns a decrypted/uncompressed dictionary. * - * This will decrypt/uncompress any number of times as necessary until it finds the binary + * This will decrypt/uncompress any number of times as necessary until it finds the * dictionary signature, and copy the decoded file to a temporary place. - * If this is not a binary dictionary, the method returns null. + * If this is not a dictionary, the method returns null. */ - public static DecoderChainSpec getRawBinaryDictionaryOrNull(final File src) { - return getRawBinaryDictionaryOrNullInternal(new DecoderChainSpec(), src, 0); + public static DecoderChainSpec getRawDictionaryOrNull(final File src) { + return getRawDictionaryOrNullInternal(new DecoderChainSpec(), src, 0); } - private static DecoderChainSpec getRawBinaryDictionaryOrNullInternal( + private static DecoderChainSpec getRawDictionaryOrNullInternal( final DecoderChainSpec spec, final File src, final int depth) { // Unfortunately the decoding scheme we use can consider any data to be encrypted // and will product some output, meaning it's not possible to reliably detect encrypted @@ -98,7 +98,8 @@ public final class BinaryDictOffdeviceUtils { // over and over, ending in a stack overflow. Hence we limit the depth at which we try // decoding the file. if (depth > MAX_DECODE_DEPTH) return null; - if (BinaryDictDecoderUtils.isBinaryDictionary(src)) { + if (BinaryDictDecoderUtils.isBinaryDictionary(src) + || CombinedInputOutput.isCombinedDictionary(src.getAbsolutePath())) { spec.mFile = src; return spec; } @@ -106,7 +107,7 @@ public final class BinaryDictOffdeviceUtils { final File uncompressedFile = tryGetUncompressedFile(src); if (null != uncompressedFile) { final DecoderChainSpec newSpec = - getRawBinaryDictionaryOrNullInternal(spec, uncompressedFile, depth + 1); + getRawDictionaryOrNullInternal(spec, uncompressedFile, depth + 1); if (null == newSpec) return null; return newSpec.addStep(COMPRESSION); } @@ -114,7 +115,7 @@ public final class BinaryDictOffdeviceUtils { final File decryptedFile = tryGetDecryptedFile(src); if (null != decryptedFile) { final DecoderChainSpec newSpec = - getRawBinaryDictionaryOrNullInternal(spec, decryptedFile, depth + 1); + getRawDictionaryOrNullInternal(spec, decryptedFile, depth + 1); if (null == newSpec) return null; return newSpec.addStep(ENCRYPTION); } @@ -175,17 +176,18 @@ public final class BinaryDictOffdeviceUtils { return XmlDictInputOutput.readDictionaryXml( new BufferedInputStream(new FileInputStream(file)), null /* shortcuts */, null /* bigrams */); - } else if (CombinedInputOutput.isCombinedDictionary(filename)) { - if (report) System.out.println("Format : Combined format"); - return CombinedInputOutput.readDictionaryCombined( - new BufferedInputStream(new FileInputStream(file))); } else { - final DecoderChainSpec decodedSpec = getRawBinaryDictionaryOrNull(file); + final DecoderChainSpec decodedSpec = getRawDictionaryOrNull(file); if (null == decodedSpec) { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); + } else if (CombinedInputOutput.isCombinedDictionary( + decodedSpec.mFile.getAbsolutePath())){ + if (report) System.out.println("Format : Combined format"); + return CombinedInputOutput.readDictionaryCombined( + new BufferedInputStream(new FileInputStream(decodedSpec.mFile))); } else { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java index 9274dcd2e..dff3387be 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java @@ -79,7 +79,7 @@ public class Package { throw new RuntimeException("Too many/too few arguments for command " + COMMAND); } final BinaryDictOffdeviceUtils.DecoderChainSpec decodedSpec = - BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(new File(mArgs[0])); + BinaryDictOffdeviceUtils.getRawDictionaryOrNull(new File(mArgs[0])); if (null == decodedSpec) { System.out.println(mArgs[0] + " does not seem to be a dictionary"); return; diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 1eff497c1..1baeb7a47 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -64,7 +64,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { // Test for an actually compressed dictionary and its contents final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec = - BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(dst); + BinaryDictOffdeviceUtils.getRawDictionaryOrNull(dst); for (final String step : decodeSpec.mDecoderSpec) { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } @@ -90,7 +90,7 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { // Test that a random data file actually fails assertNull("Wrongly identified data file", - BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(dst)); + BinaryDictOffdeviceUtils.getRawDictionaryOrNull(dst)); final File gzDst = File.createTempFile("testGetRawDict", ".tmp"); gzDst.deleteOnExit(); @@ -103,6 +103,6 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { // Test that a compressed random data file actually fails assertNull("Wrongly identified data file", - BinaryDictOffdeviceUtils.getRawBinaryDictionaryOrNull(gzDst)); + BinaryDictOffdeviceUtils.getRawDictionaryOrNull(gzDst)); } } |
