diff options
46 files changed, 1368 insertions, 197 deletions
diff --git a/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png Binary files differindex de1493151..10f8e97e4 100644 --- a/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png +++ b/java/res/drawable-hdpi/btn_keyboard_key_popup_selected_holo.9.png diff --git a/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png Binary files differindex 5b202e25b..ee0aae28b 100644 --- a/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png +++ b/java/res/drawable-mdpi/btn_keyboard_key_popup_selected_holo.9.png diff --git a/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png Binary files differindex af9187ed2..891d00024 100644 --- a/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png +++ b/java/res/drawable-xhdpi/btn_keyboard_key_popup_selected_holo.9.png diff --git a/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png b/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png Binary files differindex c62051f37..0cbb2ec84 100644 --- a/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png +++ b/java/res/drawable-xxhdpi/btn_keyboard_key_popup_selected_holo.9.png diff --git a/java/res/layout/emoji_keyboard_view.xml b/java/res/layout/emoji_keyboard_view.xml index 6a953a702..4566a5a1f 100644 --- a/java/res/layout/emoji_keyboard_view.xml +++ b/java/res/layout/emoji_keyboard_view.xml @@ -40,7 +40,7 @@ <TabWidget android:id="@android:id/tabs" android:layout_width="match_parent" - android:layout_height="wrap_content" + android:layout_height="match_parent" android:background="@drawable/tab_selected" android:divider="@null" android:tabStripEnabled="true" @@ -61,11 +61,16 @@ android:visibility="gone" /> </FrameLayout> </TabHost> + <View + android:layout_width="2dip" + android:layout_height="match_parent" + android:background="@drawable/suggestions_strip_divider" /> <ImageButton android:id="@+id/emoji_keyboard_delete" android:layout_width="0dip" android:layout_weight="12.5" android:layout_height="match_parent" + android:background="@color/emoji_key_background_color" android:src="@drawable/sym_keyboard_delete_holo_dark" /> </LinearLayout> <android.support.v4.view.ViewPager diff --git a/java/res/values-hy/donottranslate.xml b/java/res/values-hy/donottranslate.xml new file mode 100644 index 000000000..4a6d188fb --- /dev/null +++ b/java/res/values-hy/donottranslate.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ +--> +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- Same list as in English, but add armenian period and comma: --> + <!-- U+055D: "՝" ARMENIAN COMMA --> + <!-- U+0589: "։" ARMENIAN FULL STOP --> + <!-- Symbols that are normally followed by a space (used to add an auto-space after these) --> + <string name="symbols_followed_by_space">.,;:!?)]}&։՝</string> + <!-- Symbols that separate words. Adding armenian period and comma. --> + <!-- Don't remove the enclosing double quotes, they protect whitespace (not just U+0020) --> + <string name="symbols_word_separators">"	 \n"()[]{}*&<>+=|.,;:!?/_\"։՝</string> +</resources> diff --git a/java/res/values/colors.xml b/java/res/values/colors.xml index ea7400d4b..3803cb776 100644 --- a/java/res/values/colors.xml +++ b/java/res/values/colors.xml @@ -62,4 +62,8 @@ <color name="setup_welcome_video_margin_color">#FFCCCCCC</color> <color name="emoji_category_page_id_view_background">#FF000000</color> <color name="emoji_category_page_id_view_foreground">#80FFFFFF</color> + + <!-- TODO: Color which should be included in the theme --> + <color name="emoji_key_background_color">#00000000</color> + <color name="emoji_key_pressed_background_color">#30FFFFFF</color> </resources> diff --git a/java/res/values/keypress-vibration-durations.xml b/java/res/values/keypress-vibration-durations.xml index 53448c3e1..ee0ac003c 100644 --- a/java/res/values/keypress-vibration-durations.xml +++ b/java/res/values/keypress-vibration-durations.xml @@ -55,8 +55,8 @@ <item>MODEL=HTL22:MANUFACTURER=HTC,15</item> <!-- Motorola Razor M --> <item>MODEL=XT907:MANUFACTURER=motorola,30</item> - <!-- Sony Xperia Z --> - <item>MODEL=C6603:MANUFACTURER=Sony,35</item> + <!-- Sony Xperia Z, Z Ultra --> + <item>MODEL=C6603|C6806:MANUFACTURER=Sony,35</item> <!-- Default value for unknown device. The negative value means system default. --> <item>,-1</item> </string-array> diff --git a/java/res/xml/key_styles_currency.xml b/java/res/xml/key_styles_currency.xml index b7677a20d..84c2abc08 100644 --- a/java/res/xml/key_styles_currency.xml +++ b/java/res/xml/key_styles_currency.xml @@ -103,6 +103,8 @@ vi: Vietnamese (Dong) --> <!-- TODO: The currency sign of Turkish Lira was created in 2012 and assigned U+20BA for its unicode, although there is no font glyph for it as of November 2012. --> + <!-- TODO: The currency sign of Armenian Dram was created in 2012 and assigned U+058F for + its unicode, although there is no font glyph for it as of September 2013. --> <case latin:languageCode="fa|hi|iw|lo|mn|ne|th|uk|vi" > diff --git a/java/res/xml/keys_comma_period.xml b/java/res/xml/keys_comma_period.xml index 7e7c7282e..02b46c23a 100644 --- a/java/res/xml/keys_comma_period.xml +++ b/java/res/xml/keys_comma_period.xml @@ -73,6 +73,20 @@ latin:backgroundType="functional" latin:keyStyle="hasShiftedLetterHintStyle" /> </case> + <case + latin:languageCode="hy" + > + <!-- U+0589: "։" ARMENIAN FULL STOP --> + <Key + latin:keyLabel="։" + latin:keyLabelFlags="hasPopupHint" + latin:backgroundType="functional" + latin:moreKeys="!text/more_keys_for_punctuation" /> + <!-- U+055D: "՝" ARMENIAN COMMA --> + <Key + latin:keyLabel="՝" + latin:backgroundType="functional" /> + </case> <default> <Key latin:keyLabel="." diff --git a/java/res/xml/row_qwerty4.xml b/java/res/xml/row_qwerty4.xml index 340beb99b..578bc1234 100644 --- a/java/res/xml/row_qwerty4.xml +++ b/java/res/xml/row_qwerty4.xml @@ -49,6 +49,14 @@ <include latin:keyboardLayout="@xml/key_nepali_traditional_period" /> </case> + <case + latin:languageCode="hy" + > + <!-- U+0589: "։" ARMENIAN FULL STOP --> + <Key + latin:keyLabel="։" + latin:keyStyle="punctuationKeyStyle" /> + </case> <default> <Key latin:keyStyle="punctuationKeyStyle" /> diff --git a/java/src/com/android/inputmethod/compat/ActivityManagerCompatUtils.java b/java/src/com/android/inputmethod/compat/ActivityManagerCompatUtils.java new file mode 100644 index 000000000..385e3e025 --- /dev/null +++ b/java/src/com/android/inputmethod/compat/ActivityManagerCompatUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.compat; + +import android.app.ActivityManager; +import android.content.Context; + +import java.lang.reflect.Method; + +public class ActivityManagerCompatUtils { + private static final Object LOCK = new Object(); + private static volatile Boolean sBoolean = null; + private static final Method METHOD_isLowRamDevice = CompatUtils.getMethod( + ActivityManager.class, "isLowRamDevice"); + + private ActivityManagerCompatUtils() { + // Do not instantiate this class. + } + + public static boolean isLowRamDevice(Context context) { + if (sBoolean == null) { + synchronized(LOCK) { + if (sBoolean == null) { + final ActivityManager am = + (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE); + sBoolean = (Boolean)CompatUtils.invoke(am, false, METHOD_isLowRamDevice); + } + } + } + return sBoolean; + } +} diff --git a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java index db65de2ad..4e61edac2 100644 --- a/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/EmojiKeyboardView.java @@ -28,11 +28,13 @@ import android.os.Build; import android.preference.PreferenceManager; import android.support.v4.view.PagerAdapter; import android.support.v4.view.ViewPager; +import android.text.format.DateUtils; import android.util.AttributeSet; import android.util.Log; import android.util.Pair; import android.util.SparseArray; import android.view.LayoutInflater; +import android.view.MotionEvent; import android.view.View; import android.view.ViewGroup; import android.widget.ImageView; @@ -76,6 +78,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange private final int mEmojiFunctionalKeyBackgroundId; private final KeyboardLayoutSet mLayoutSet; private final ColorStateList mTabLabelColor; + private final DeleteKeyOnTouchListener mDeleteKeyOnTouchListener; private EmojiKeyboardAdapter mEmojiKeyboardAdapter; private TabHost mTabHost; @@ -203,7 +206,14 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange } public int getCategoryPageSize(int categoryId) { - return mShownCategories.get(categoryId).mPageCount; + for (final CategoryProperties prop : mShownCategories) { + if (prop.mCategoryId == categoryId) { + return prop.mPageCount; + } + } + Log.w(TAG, "Invalid category id: " + categoryId); + // Should not reach here. + return 0; } public void setCurrentCategoryId(int categoryId) { @@ -395,6 +405,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange mLayoutSet = builder.build(); mEmojiCategory = new EmojiCategory(PreferenceManager.getDefaultSharedPreferences(context), context.getResources(), builder.build()); + mDeleteKeyOnTouchListener = new DeleteKeyOnTouchListener(context); } @Override @@ -459,11 +470,9 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange final LinearLayout actionBar = (LinearLayout)findViewById(R.id.emoji_action_bar); emojiLp.setActionBarProperties(actionBar); - // TODO: Implement auto repeat, using View.OnTouchListener? final ImageView deleteKey = (ImageView)findViewById(R.id.emoji_keyboard_delete); - deleteKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); deleteKey.setTag(Constants.CODE_DELETE); - deleteKey.setOnClickListener(this); + deleteKey.setOnTouchListener(mDeleteKeyOnTouchListener); final ImageView alphabetKey = (ImageView)findViewById(R.id.emoji_keyboard_alphabet); alphabetKey.setBackgroundResource(mEmojiFunctionalKeyBackgroundId); alphabetKey.setTag(Constants.CODE_SWITCH_ALPHA_SYMBOL); @@ -556,6 +565,7 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange public void setKeyboardActionListener(final KeyboardActionListener listener) { mKeyboardActionListener = listener; + mDeleteKeyOnTouchListener.setKeyboardActionListener(mKeyboardActionListener); } private void updateEmojiCategoryPageIdView() { @@ -665,4 +675,92 @@ public final class EmojiKeyboardView extends LinearLayout implements OnTabChange container.removeView(keyboardView); } } + + // TODO: Do the same things done in PointerTracker + private static class DeleteKeyOnTouchListener implements OnTouchListener { + private static final long MAX_REPEAT_COUNT_TIME = 30 * DateUtils.SECOND_IN_MILLIS; + private final int mDeleteKeyPressedBackgroundColor; + private final long mKeyRepeatStartTimeout; + private final long mKeyRepeatInterval; + + public DeleteKeyOnTouchListener(Context context) { + final Resources res = context.getResources(); + mDeleteKeyPressedBackgroundColor = + res.getColor(R.color.emoji_key_pressed_background_color); + mKeyRepeatStartTimeout = res.getInteger(R.integer.config_key_repeat_start_timeout); + mKeyRepeatInterval = res.getInteger(R.integer.config_key_repeat_interval); + } + + private KeyboardActionListener mKeyboardActionListener = + KeyboardActionListener.EMPTY_LISTENER; + private DummyRepeatKeyRepeatTimer mTimer; + + private synchronized void startRepeat() { + if (mTimer != null) { + abortRepeat(); + } + mTimer = new DummyRepeatKeyRepeatTimer(); + mTimer.start(); + } + + private synchronized void abortRepeat() { + mTimer.abort(); + mTimer = null; + } + + // TODO: Remove + // This function is mimicking the repeat code in PointerTracker. + // Specifically referring to PointerTracker#startRepeatKey and PointerTracker#onKeyRepeat. + private class DummyRepeatKeyRepeatTimer extends Thread { + public boolean mAborted = false; + + @Override + public void run() { + int timeCount = 0; + while (timeCount < MAX_REPEAT_COUNT_TIME && !mAborted) { + if (timeCount > mKeyRepeatStartTimeout) { + pressDelete(); + } + timeCount += mKeyRepeatInterval; + try { + Thread.sleep(mKeyRepeatInterval); + } catch (InterruptedException e) { + } + } + } + + public void abort() { + mAborted = true; + } + } + + public void pressDelete() { + mKeyboardActionListener.onPressKey( + Constants.CODE_DELETE, 0 /* repeatCount */, true /* isSinglePointer */); + mKeyboardActionListener.onCodeInput( + Constants.CODE_DELETE, NOT_A_COORDINATE, NOT_A_COORDINATE); + mKeyboardActionListener.onReleaseKey( + Constants.CODE_DELETE, false /* withSliding */); + } + + public void setKeyboardActionListener(KeyboardActionListener listener) { + mKeyboardActionListener = listener; + } + + @Override + public boolean onTouch(View v, MotionEvent event) { + switch(event.getAction()) { + case MotionEvent.ACTION_DOWN: + v.setBackgroundColor(mDeleteKeyPressedBackgroundColor); + pressDelete(); + startRepeat(); + return true; + case MotionEvent.ACTION_UP: + v.setBackgroundColor(0); + abortRepeat(); + return true; + } + return false; + } + } } diff --git a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java index f8ad43e74..13db47004 100644 --- a/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java +++ b/java/src/com/android/inputmethod/keyboard/MainKeyboardView.java @@ -155,7 +155,6 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack private final SlidingKeyInputPreview mSlidingKeyInputPreview; // Key preview - private static final int PREVIEW_ALPHA = 240; private final int mKeyPreviewLayoutId; private final int mKeyPreviewOffset; private final int mKeyPreviewHeight; @@ -816,7 +815,6 @@ public final class MainKeyboardView extends KeyboardView implements PointerTrack if (background != null) { final int hasMoreKeys = (key.getMoreKeys() != null) ? STATE_HAS_MOREKEYS : STATE_NORMAL; background.setState(KEY_PREVIEW_BACKGROUND_STATE_TABLE[statePosition][hasMoreKeys]); - background.setAlpha(PREVIEW_ALPHA); } ViewLayoutUtils.placeViewAt( previewText, previewX, previewY, previewWidth, previewHeight); diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index 2af2f6995..67553fb75 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -1769,15 +1769,27 @@ public final class KeyboardTextsSet { null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, /* ~52 */ + // U+058A: "֊" ARMENIAN HYPHEN + // U+055C: "՜" ARMENIAN EXCLAMATION MARK + // U+055D: "՝" ARMENIAN COMMA // U+055E: "՞" ARMENIAN QUESTION MARK - /* 53 */ "!fixedColumnOrder!4,\u055E,!,\\,,?,:,;,@", + // U+0559: "ՙ" ARMENIAN MODIFIER LETTER LEFT HALF RING + // U+055A: "՚" ARMENIAN APOSTROPHE + // U+055B: "՛" ARMENIAN EMPHASIS MARK + // U+055F: "՟" ARMENIAN ABBREVIATION MARK + /* 53 */ "!fixedColumnOrder!8,!,?,\\,,.,\u058A,\u055C,\u055D,\u055E,:,;,@,\u0559,\u055A,\u055B,\u055F", /* 54~ */ null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, null, - /* ~107 */ - /* 108 */ "\u055E,?", + null, + /* ~99 */ + // U+055C: "՜" ARMENIAN EXCLAMATION MARK + // U+00A1: "¡" INVERTED EXCLAMATION MARK + /* 100 */ "\u055C,\u00A1", + // U+055E: "՞" ARMENIAN QUESTION MARK + // U+00BF: "¿" INVERTED QUESTION MARK + /* 101 */ "\u055E,\u00BF", }; /* Language is: Icelandic */ diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 632ee0da4..61ccfcfad 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.StringUtils; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; @@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary { return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); } + private void runGCIfRequired() { + if (needsToRunGCNative(mNativeDict)) { + flushWithGC(); + } + } + // Add a unigram entry to binary dictionary in native code. public void addUnigramWord(final String word, final int probability) { if (TextUtils.isEmpty(word)) { return; } + runGCIfRequired(); final int[] codePoints = StringUtils.toCodePointArray(word); addUnigramWordNative(mNativeDict, codePoints, probability); } @@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } + runGCIfRequired(); final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability); @@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary { if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { return; } + runGCIfRequired(); final int[] codePoints0 = StringUtils.toCodePointArray(word0); final int[] codePoints1 = StringUtils.toCodePointArray(word1); removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); } - @UsedForTesting public void flush() { if (!isValidDictionary()) return; flushNative(mNativeDict, mDictFilePath); + closeNative(mNativeDict); + final File dictFile = new File(mDictFilePath); + mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, + dictFile.length(), true /* isUpdatable */); } - @UsedForTesting public void flushWithGC() { if (!isValidDictionary()) return; flushWithGCNative(mNativeDict, mDictFilePath); + closeNative(mNativeDict); + final File dictFile = new File(mDictFilePath); + mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */, + dictFile.length(), true /* isUpdatable */); } - @UsedForTesting public boolean needsToRunGC() { if (!isValidDictionary()) return false; return needsToRunGCNative(mNativeDict); diff --git a/java/src/com/android/inputmethod/latin/Constants.java b/java/src/com/android/inputmethod/latin/Constants.java index 28d9e8652..c4f96016c 100644 --- a/java/src/com/android/inputmethod/latin/Constants.java +++ b/java/src/com/android/inputmethod/latin/Constants.java @@ -138,6 +138,9 @@ public final class Constants { public static final int SPELL_CHECKER_COORDINATE = -3; public static final int EXTERNAL_KEYBOARD_COORDINATE = -4; + // A hint on how many characters to cache from the TextView. A good value of this is given by + // how many characters we need to be able to almost always find the caps mode. + public static final int EDITOR_CONTENTS_CACHE_SIZE = 1024; // Must be equal to MAX_WORD_LENGTH in native/jni/src/defines.h public static final int DICTIONARY_MAX_WORD_LENGTH = 48; @@ -162,6 +165,7 @@ public final class Constants { public static final int CODE_TAB = '\t'; public static final int CODE_SPACE = ' '; public static final int CODE_PERIOD = '.'; + public static final int CODE_ARMENIAN_PERIOD = 0x0589; public static final int CODE_DASH = '-'; public static final int CODE_SINGLE_QUOTE = '\''; public static final int CODE_DOUBLE_QUOTE = '"'; diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index fcd7ede1a..0774ce203 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -22,14 +22,22 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FusionDictionary; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; +import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; +import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; @@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Whether to print debug output to log */ private static boolean DEBUG = false; - // TODO: Remove and enable dynamic update in native code. + // TODO: Remove. /** Whether to call binary dictionary dynamically updating methods. */ - private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false; + public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true; private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; @@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */); + /** * A static map of time recorders, each of which records the time of accesses to a single binary * dictionary file. The key for this map is the filename and the value is the shared dictionary @@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static AbstractDictionaryWriter getDictionaryWriter(final Context context, final String dictType, final boolean isDynamicPersonalizationDictionary) { if (isDynamicPersonalizationDictionary) { - return new DynamicPersonalizationDictionaryWriter(context, dictType); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + return null; + } else { + return new DynamicPersonalizationDictionaryWriter(context, dictType); + } } else { return new DictionaryWriter(context, dictType); } @@ -198,7 +213,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary.close(); mBinaryDictionary = null; } - mDictionaryWriter.close(); + if (mDictionaryWriter != null) { + mDictionaryWriter.close(); + } } }); } @@ -220,7 +237,23 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - mDictionaryWriter.clear(); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { + mBinaryDictionary.close(); + final File file = new File(mContext.getFilesDir(), mFilename); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), + false, false)); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + try { + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + } catch (IOException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } catch (UnsupportedFormatException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } + } else { + mDictionaryWriter.clear(); + } } }); } @@ -257,9 +290,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { mBinaryDictionary.addUnigramWord(word, frequency); + } else { + // TODO: Remove. + mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); } - // TODO: Remove. - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); } }); } @@ -280,10 +314,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { mBinaryDictionary.addBigramWords(word0, word1, frequency); + } else { + // TODO: Remove. + mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, + 0 /* lastTouchedTime */); } - // TODO: Remove. - mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, - 0 /* lastTouchedTime */); } }); } @@ -303,9 +338,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { mBinaryDictionary.removeBigramWords(word0, word1); + } else { + // TODO: Remove. + mDictionaryWriter.removeBigramWords(word0, word1); } - // TODO: Remove. - mDictionaryWriter.removeBigramWords(word0, word1); } }); } @@ -322,26 +358,39 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ? - null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, additionalFeaturesOptions, - sessionId); - // TODO: Remove checking mIsUpdatable and use native suggestion. - if (mBinaryDictionary != null && !mIsUpdatable) { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + if (mBinaryDictionary == null) { + holder.set(null); + return; + } final ArrayList<SuggestedWordInfo> binarySuggestion = mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId); - if (inMemDictSuggestion == null) { - holder.set(binarySuggestion); - } else if (binarySuggestion == null) { - holder.set(inMemDictSuggestion); + holder.set(binarySuggestion); + } else { + final ArrayList<SuggestedWordInfo> inMemDictSuggestion = + composer.isBatchMode() ? null : + mDictionaryWriter.getSuggestionsWithSessionId(composer, + prevWord, proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId); + // TODO: Remove checking mIsUpdatable and use native suggestion. + if (mBinaryDictionary != null && !mIsUpdatable) { + final ArrayList<SuggestedWordInfo> binarySuggestion = + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, + additionalFeaturesOptions, sessionId); + if (inMemDictSuggestion == null) { + holder.set(binarySuggestion); + } else if (binarySuggestion == null) { + holder.set(inMemDictSuggestion); + } else { + binarySuggestion.addAll(inMemDictSuggestion); + holder.set(binarySuggestion); + } } else { - binarySuggestion.addAll(inMemDictSuggestion); - holder.set(binarySuggestion); + holder.set(inMemDictSuggestion); } - } else { - holder.set(inMemDictSuggestion); } } }); @@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - // Ensure all threads accessing the current dictionary have finished before swapping in - // the new one. + // Ensure all threads accessing the current dictionary have finished before + // swapping in the new one. + // TODO: Ensure multi-thread assignment of mBinaryDictionary. final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; getExecutor(mFilename).executePrioritized(new Runnable() { @Override @@ -443,8 +493,33 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); loadDictionaryAsync(); + mDictionaryWriter.write(mFilename); + } else { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { + final File file = new File(mContext.getFilesDir(), mFilename); + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), + false, false)); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + try { + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + } catch (IOException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } catch (UnsupportedFormatException e) { + Log.e(TAG, "Exception in creating new dictionary file.", e); + } + } else { + if (mBinaryDictionary.needsToRunGC()) { + mBinaryDictionary.flushWithGC(); + } else { + mBinaryDictionary.flush(); + } + } + } else { + mDictionaryWriter.write(mFilename); + } } - mDictionaryWriter.write(mFilename); } /** @@ -539,7 +614,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).executePrioritized(new Runnable() { @Override public void run() { - loadDictionaryAsync(); + if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + loadDictionaryAsync(); + } } }); } @@ -547,7 +624,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Generate binary dictionary using DictionaryWriter. */ - protected void asyncWriteBinaryDictionary() { + protected void asyncFlashAllBinaryDictionary() { final Runnable newTask = new Runnable() { @Override public void run() { @@ -620,8 +697,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public void run() { if (mDictType == Dictionary.TYPE_USER_HISTORY) { - holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) - .isInDictionaryForTests(word)); + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + holder.set(mBinaryDictionary.isValidWord(word)); + } else { + holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) + .isInDictionaryForTests(word)); + } } } }); diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index c9311a6d8..270dc4c06 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -233,6 +233,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen private static final int MSG_RESUME_SUGGESTIONS = 4; private static final int MSG_REOPEN_DICTIONARIES = 5; private static final int MSG_ON_END_BATCH_INPUT = 6; + private static final int MSG_RESET_CACHES = 7; private static final int ARG1_NOT_GESTURE_INPUT = 0; private static final int ARG1_DISMISS_GESTURE_FLOATING_PREVIEW_TEXT = 1; @@ -297,6 +298,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen case MSG_ON_END_BATCH_INPUT: latinIme.onEndBatchInputAsyncInternal((SuggestedWords) msg.obj); break; + case MSG_RESET_CACHES: + latinIme.retryResetCaches(msg.arg1 == 1 /* tryResumeSuggestions */, + msg.arg2 /* remainingTries */); + break; } } @@ -313,6 +318,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen sendMessageDelayed(obtainMessage(MSG_RESUME_SUGGESTIONS), mDelayUpdateSuggestions); } + public void postResetCaches(final boolean tryResumeSuggestions, final int remainingTries) { + removeMessages(MSG_RESET_CACHES); + sendMessage(obtainMessage(MSG_RESET_CACHES, tryResumeSuggestions ? 1 : 0, + remainingTries, null)); + } + public void cancelUpdateSuggestionStrip() { removeMessages(MSG_UPDATE_SUGGESTION_STRIP); } @@ -852,7 +863,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // span, so we should reset our state unconditionally, even if restarting is true. mEnteredText = null; resetComposingState(true /* alsoResetLastComposedWord */); - if (isDifferentTextField) mHandler.postResumeSuggestions(); mDeleteCount = 0; mSpaceState = SPACE_STATE_NONE; mRecapitalizeStatus.deactivate(); @@ -871,8 +881,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } mSuggestedWords = SuggestedWords.EMPTY; - mConnection.resetCachesUponCursorMove(editorInfo.initialSelStart, - false /* shouldFinishComposition */); + // Sometimes, while rotating, for some reason the framework tells the app we are not + // connected to it and that means we can't refresh the cache. In this case, schedule a + // refresh later. + if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(editorInfo.initialSelStart, + false /* shouldFinishComposition */)) { + // We try resetting the caches up to 5 times before giving up. + mHandler.postResetCaches(isDifferentTextField, 5 /* remainingTries */); + } else { + if (isDifferentTextField) mHandler.postResumeSuggestions(); + } if (isDifferentTextField) { mainKeyboardView.closing(); @@ -899,6 +917,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mLastSelectionStart = editorInfo.initialSelStart; mLastSelectionEnd = editorInfo.initialSelEnd; + // In some cases (namely, after rotation of the device) editorInfo.initialSelStart is lying + // so we try using some heuristics to find out about these and fix them. + tryFixLyingCursorPosition(); mHandler.cancelUpdateSuggestionStrip(); mHandler.cancelDoubleSpacePeriodTimer(); @@ -918,6 +939,35 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen if (TRACE) Debug.startMethodTracing("/data/trace/latinime"); } + /** + * Try to get the text from the editor to expose lies the framework may have been + * telling us. Concretely, when the device rotates, the frameworks tells us about where the + * cursor used to be initially in the editor at the time it first received the focus; this + * may be completely different from the place it is upon rotation. Since we don't have any + * means to get the real value, try at least to ask the text view for some characters and + * detect the most damaging cases: when the cursor position is declared to be much smaller + * than it really is. + */ + private void tryFixLyingCursorPosition() { + final CharSequence textBeforeCursor = + mConnection.getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0); + if (null == textBeforeCursor) { + mLastSelectionStart = mLastSelectionEnd = NOT_A_CURSOR_POSITION; + } else { + final int textLength = textBeforeCursor.length(); + if (textLength > mLastSelectionStart + || (textLength < Constants.EDITOR_CONTENTS_CACHE_SIZE + && mLastSelectionStart < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { + mLastSelectionStart = textLength; + // We can't figure out the value of mLastSelectionEnd :( + // But at least if it's smaller than mLastSelectionStart something is wrong + if (mLastSelectionStart > mLastSelectionEnd) { + mLastSelectionEnd = mLastSelectionStart; + } + } + } + } + // Initialization of personalization debug settings. This must be called inside // onStartInputView. private void initPersonalizationDebugSettings(SettingsValues currentSettingsValues) { @@ -1072,7 +1122,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // argument as true. But in all cases where we don't reset the entire input state, // we still want to tell the rich input connection about the new cursor position so // that it can update its caches. - mConnection.resetCachesUponCursorMove(newSelStart, + mConnection.resetCachesUponCursorMoveAndReturnSuccess(newSelStart, false /* shouldFinishComposition */); } @@ -1308,7 +1358,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } else { setSuggestedWords(settingsValues.mSuggestPuncList, false); } - mConnection.resetCachesUponCursorMove(newCursorPosition, shouldFinishComposition); + mConnection.resetCachesUponCursorMoveAndReturnSuccess(newCursorPosition, + shouldFinishComposition); } private void resetComposingState(final boolean alsoResetLastComposedWord) { @@ -2853,6 +2904,27 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mHandler.postUpdateSuggestionStrip(); } + /** + * Retry resetting caches in the rich input connection. + * + * When the editor can't be accessed we can't reset the caches, so we schedule a retry. + * This method handles the retry, and re-schedules a new retry if we still can't access. + * We only retry up to 5 times before giving up. + * + * @param tryResumeSuggestions Whether we should resume suggestions or not. + * @param remainingTries How many times we may try again before giving up. + */ + private void retryResetCaches(final boolean tryResumeSuggestions, final int remainingTries) { + if (!mConnection.resetCachesUponCursorMoveAndReturnSuccess(mLastSelectionStart, false)) { + if (0 < remainingTries) { + mHandler.postResetCaches(tryResumeSuggestions, remainingTries - 1); + } + return; + } + tryFixLyingCursorPosition(); + if (tryResumeSuggestions) mHandler.postResumeSuggestions(); + } + private void revertCommit() { final String previousWord = mLastComposedWord.mPrevWord; final String originallyTypedWord = mLastComposedWord.mTypedWord; diff --git a/java/src/com/android/inputmethod/latin/RichInputConnection.java b/java/src/com/android/inputmethod/latin/RichInputConnection.java index 2ecf1463f..925381b50 100644 --- a/java/src/com/android/inputmethod/latin/RichInputConnection.java +++ b/java/src/com/android/inputmethod/latin/RichInputConnection.java @@ -73,9 +73,6 @@ public final class RichInputConnection { * This contains the currently composing text, as LatinIME thinks the TextView is seeing it. */ private final StringBuilder mComposingText = new StringBuilder(); - // A hint on how many characters to cache from the TextView. A good value of this is given by - // how many characters we need to be able to almost always find the caps mode. - private static final int DEFAULT_TEXT_CACHE_SIZE = 100; private final InputMethodService mParent; InputConnection mIC; @@ -93,7 +90,8 @@ public final class RichInputConnection { r.token = 1; r.flags = 0; final ExtractedText et = mIC.getExtractedText(r, 0); - final CharSequence beforeCursor = getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0); + final CharSequence beforeCursor = getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, + 0); final StringBuilder internal = new StringBuilder().append(mCommittedTextBeforeComposingText) .append(mComposingText); if (null == et || null == beforeCursor) return; @@ -142,19 +140,56 @@ public final class RichInputConnection { if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); } - public void resetCachesUponCursorMove(final int newCursorPosition, + /** + * Reset the cached text and retrieve it again from the editor. + * + * This should be called when the cursor moved. It's possible that we can't connect to + * the application when doing this; notably, this happens sometimes during rotation, probably + * because of a race condition in the framework. In this case, we just can't retrieve the + * data, so we empty the cache and note that we don't know the new cursor position, and we + * return false so that the caller knows about this and can retry later. + * + * @param newCursorPosition The new position of the cursor, as received from the system. + * @param shouldFinishComposition Whether we should finish the composition in progress. + * @return true if we were able to connect to the editor successfully, false otherwise. When + * this method returns false, the caches could not be correctly refreshed so they were only + * reset: the caller should try again later to return to normal operation. + */ + public boolean resetCachesUponCursorMoveAndReturnSuccess(final int newCursorPosition, final boolean shouldFinishComposition) { mExpectedCursorPosition = newCursorPosition; mComposingText.setLength(0); mCommittedTextBeforeComposingText.setLength(0); - final CharSequence textBeforeCursor = getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0); - if (null != textBeforeCursor) mCommittedTextBeforeComposingText.append(textBeforeCursor); + mIC = mParent.getCurrentInputConnection(); + // Call upon the inputconnection directly since our own method is using the cache, and + // we want to refresh it. + final CharSequence textBeforeCursor = null == mIC ? null : + mIC.getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0); + if (null == textBeforeCursor) { + // For some reason the app thinks we are not connected to it. This looks like a + // framework bug... Fall back to ground state and return false. + mExpectedCursorPosition = INVALID_CURSOR_POSITION; + Log.e(TAG, "Unable to connect to the editor to retrieve text... will retry later"); + return false; + } + mCommittedTextBeforeComposingText.append(textBeforeCursor); + final int lengthOfTextBeforeCursor = textBeforeCursor.length(); + if (lengthOfTextBeforeCursor > newCursorPosition + || (lengthOfTextBeforeCursor < Constants.EDITOR_CONTENTS_CACHE_SIZE + && newCursorPosition < Constants.EDITOR_CONTENTS_CACHE_SIZE)) { + // newCursorPosition may be lying -- when rotating the device (probably a framework + // bug). If we have less chars than we asked for, then we know how many chars we have, + // and if we got more than newCursorPosition says, then we know it was lying. In both + // cases the length is more reliable + mExpectedCursorPosition = lengthOfTextBeforeCursor; + } if (null != mIC && shouldFinishComposition) { mIC.finishComposingText(); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS) { ResearchLogger.richInputConnection_finishComposingText(); } } + return true; } private void checkBatchEdit() { @@ -233,7 +268,8 @@ public final class RichInputConnection { // getCapsMode should be updated to be able to return a "not enough info" result so that // we can get more context only when needed. if (TextUtils.isEmpty(mCommittedTextBeforeComposingText) && 0 != mExpectedCursorPosition) { - final CharSequence textBeforeCursor = getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0); + final CharSequence textBeforeCursor = getTextBeforeCursor( + Constants.EDITOR_CONTENTS_CACHE_SIZE, 0); if (!TextUtils.isEmpty(textBeforeCursor)) { mCommittedTextBeforeComposingText.append(textBeforeCursor); } @@ -364,7 +400,7 @@ public final class RichInputConnection { if (DEBUG_BATCH_NESTING) checkBatchEdit(); if (DEBUG_PREVIOUS_TEXT) checkConsistencyForDebug(); final CharSequence textBeforeCursor = - getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE + (end - start), 0); + getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE + (end - start), 0); mCommittedTextBeforeComposingText.setLength(0); if (!TextUtils.isEmpty(textBeforeCursor)) { final int indexOfStartOfComposingText = @@ -406,7 +442,8 @@ public final class RichInputConnection { } mExpectedCursorPosition = start; mCommittedTextBeforeComposingText.setLength(0); - mCommittedTextBeforeComposingText.append(getTextBeforeCursor(DEFAULT_TEXT_CACHE_SIZE, 0)); + mCommittedTextBeforeComposingText.append( + getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, 0)); } public void commitCorrection(final CorrectionInfo correctionInfo) { @@ -525,9 +562,9 @@ public final class RichInputConnection { if (mIC == null || sep == null) { return null; } - final CharSequence before = mIC.getTextBeforeCursor(1000, + final CharSequence before = mIC.getTextBeforeCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, InputConnection.GET_TEXT_WITH_STYLES); - final CharSequence after = mIC.getTextAfterCursor(1000, + final CharSequence after = mIC.getTextAfterCursor(Constants.EDITOR_CONTENTS_CACHE_SIZE, InputConnection.GET_TEXT_WITH_STYLES); if (before == null || after == null) { return null; @@ -570,8 +607,11 @@ public final class RichInputConnection { } } - return new TextRange(TextUtils.concat(before, after), startIndexInBefore, - before.length() + endIndexInAfter, before.length()); + // We don't use TextUtils#concat because it copies all spans without respect to their + // nature. If the text includes a PARAGRAPH span and it has been split, then + // TextUtils#concat will crash when it tries to concat both sides of it. + return new TextRange(StringUtils.concatWithNonParagraphSuggestionSpansOnly(before, after), + startIndexInBefore, before.length() + endIndexInAfter, before.length()); } public boolean isCursorTouchingWord(final SettingsValues settingsValues) { diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java index e43e74d87..0af028a9e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -19,6 +19,7 @@ package com.android.inputmethod.latin.personalization; import android.content.Context; import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.compat.ActivityManagerCompatUtils; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.AbstractDictionaryWriter; import com.android.inputmethod.latin.ExpandableDictionary; @@ -41,7 +42,8 @@ import java.util.ArrayList; public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWriter { private static final String TAG = DynamicPersonalizationDictionaryWriter.class.getSimpleName(); /** Maximum number of pairs. Pruning will start when databases goes above this number. */ - public static final int MAX_HISTORY_BIGRAMS = 10000; + public static final int DEFAULT_MAX_HISTORY_BIGRAMS = 10000; + public static final int LOW_MEMORY_MAX_HISTORY_BIGRAMS = 2000; /** Any pair being typed or picked */ private static final int FREQUENCY_FOR_TYPED = 2; @@ -53,10 +55,14 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr private final UserHistoryDictionaryBigramList mBigramList = new UserHistoryDictionaryBigramList(); private final ExpandableDictionary mExpandableDictionary; + private final int mMaxHistoryBigrams; public DynamicPersonalizationDictionaryWriter(final Context context, final String dictType) { super(context, dictType); mExpandableDictionary = new ExpandableDictionary(dictType); + final boolean isLowRamDevice = ActivityManagerCompatUtils.isLowRamDevice(context); + mMaxHistoryBigrams = isLowRamDevice ? + LOW_MEMORY_MAX_HISTORY_BIGRAMS : DEFAULT_MAX_HISTORY_BIGRAMS; } @Override @@ -72,6 +78,10 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr @Override public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, final boolean isNotAWord) { + if (mBigramList.size() > mMaxHistoryBigrams * 2) { + // Too many entries: just stop adding new vocabrary and wait next refresh. + return; + } mExpandableDictionary.addWord(word, shortcutTarget, frequency); mBigramList.addBigram(null, word, (byte)frequency); } @@ -79,6 +89,10 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr @Override public void addBigramWords(final String word0, final String word1, final int frequency, final boolean isValid, final long lastModifiedTime) { + if (mBigramList.size() > mMaxHistoryBigrams * 2) { + // Too many entries: just stop adding new vocabrary and wait next refresh. + return; + } if (lastModifiedTime > 0) { mExpandableDictionary.setBigramAndGetFrequency(word0, word1, new ForgettingCurveParams(frequency, System.currentTimeMillis(), @@ -102,19 +116,22 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr protected void writeDictionary(final DictEncoder dictEncoder) throws IOException, UnsupportedFormatException { UserHistoryDictIOUtils.writeDictionary(dictEncoder, - new FrequencyProvider(mBigramList, mExpandableDictionary), mBigramList, - FORMAT_OPTIONS); + new FrequencyProvider(mBigramList, mExpandableDictionary, mMaxHistoryBigrams), + mBigramList, FORMAT_OPTIONS); } private static class FrequencyProvider implements BigramDictionaryInterface { - final private UserHistoryDictionaryBigramList mBigramList; - final private ExpandableDictionary mExpandableDictionary; + private final UserHistoryDictionaryBigramList mBigramList; + private final ExpandableDictionary mExpandableDictionary; + private final int mMaxHistoryBigrams; public FrequencyProvider(final UserHistoryDictionaryBigramList bigramList, - final ExpandableDictionary expandableDictionary) { + final ExpandableDictionary expandableDictionary, final int maxHistoryBigrams) { mBigramList = bigramList; mExpandableDictionary = expandableDictionary; + mMaxHistoryBigrams = maxHistoryBigrams; } + @Override public int getFrequency(final String word0, final String word1) { final int freq; @@ -130,7 +147,7 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr if (prevFc > 0 && prevFc == fc) { freq = fc & 0xFF; } else if (UserHistoryForgettingCurveUtils. - needsToSave(fc, isValid, mBigramList.size() <= MAX_HISTORY_BIGRAMS)) { + needsToSave(fc, isValid, mBigramList.size() <= mMaxHistoryBigrams)) { freq = fc & 0xFF; } else { // Delete this entry diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index 9364fb034..075d7e3c3 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi @Override public void close() { - // Close only binary dictionary to reuse this dictionary. - // super.close(); - closeBinaryDictionary(); + if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + closeBinaryDictionary(); + } // Flush pending writes. // TODO: Remove after this class become to use a dynamic binary dictionary. - asyncWriteBinaryDictionary(); + asyncFlashAllBinaryDictionary(); Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale); } @@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi // Clear the node structure on memory clear(); // Then flush the cleared state of the dictionary on disk. - asyncWriteBinaryDictionary(); + asyncFlashAllBinaryDictionary(); } } diff --git a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java index 2f91c5743..60b24d5d5 100644 --- a/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/CapsModeUtils.java @@ -60,6 +60,11 @@ public final class CapsModeUtils { || WordComposer.CAPS_MODE_AUTO_SHIFT_LOCKED == mode; } + private static boolean isPeriod(final int codePoint) { + // TODO: make this a resource. + return codePoint == Constants.CODE_PERIOD || codePoint == Constants.CODE_ARMENIAN_PERIOD; + } + /** * Determine what caps mode should be in effect at the current offset in * the text. Only the mode bits set in <var>reqModes</var> will be @@ -190,7 +195,7 @@ public final class CapsModeUtils { if (c == Constants.CODE_QUESTION_MARK || c == Constants.CODE_EXCLAMATION_MARK) { return (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_SENTENCES) & reqModes; } - if (c != Constants.CODE_PERIOD || j <= 0) { + if (!isPeriod(c) || j <= 0) { return (TextUtils.CAP_MODE_CHARACTERS | TextUtils.CAP_MODE_WORDS) & reqModes; } @@ -240,7 +245,7 @@ public final class CapsModeUtils { case WORD: if (Character.isLetter(c)) { state = WORD; - } else if (c == Constants.CODE_PERIOD) { + } else if (isPeriod(c)) { state = PERIOD; } else { return caps; @@ -256,7 +261,7 @@ public final class CapsModeUtils { case LETTER: if (Character.isLetter(c)) { state = LETTER; - } else if (c == Constants.CODE_PERIOD) { + } else if (isPeriod(c)) { state = PERIOD; } else { return noCaps; diff --git a/java/src/com/android/inputmethod/latin/utils/StringUtils.java b/java/src/com/android/inputmethod/latin/utils/StringUtils.java index 121aecf0f..327780ad0 100644 --- a/java/src/com/android/inputmethod/latin/utils/StringUtils.java +++ b/java/src/com/android/inputmethod/latin/utils/StringUtils.java @@ -20,7 +20,12 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.settings.SettingsValues; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.Spanned; +import android.text.SpannedString; import android.text.TextUtils; +import android.text.style.SuggestionSpan; import android.util.JsonReader; import android.util.JsonWriter; import android.util.Log; @@ -462,4 +467,88 @@ public final class StringUtils { } return ""; } + + /** + * Copies the spans from the region <code>start...end</code> in + * <code>source</code> to the region + * <code>destoff...destoff+end-start</code> in <code>dest</code>. + * Spans in <code>source</code> that begin before <code>start</code> + * or end after <code>end</code> but overlap this range are trimmed + * as if they began at <code>start</code> or ended at <code>end</code>. + * Only SuggestionSpans that don't have the SPAN_PARAGRAPH span are copied. + * + * This code is almost entirely taken from {@link TextUtils#copySpansFrom}, except for the + * kind of span that is copied. + * + * @throws IndexOutOfBoundsException if any of the copied spans + * are out of range in <code>dest</code>. + */ + public static void copyNonParagraphSuggestionSpansFrom(Spanned source, int start, int end, + Spannable dest, int destoff) { + Object[] spans = source.getSpans(start, end, SuggestionSpan.class); + + for (int i = 0; i < spans.length; i++) { + int fl = source.getSpanFlags(spans[i]); + if (0 != (fl & Spannable.SPAN_PARAGRAPH)) continue; + + int st = source.getSpanStart(spans[i]); + int en = source.getSpanEnd(spans[i]); + + if (st < start) + st = start; + if (en > end) + en = end; + + dest.setSpan(spans[i], st - start + destoff, en - start + destoff, + fl); + } + } + + /** + * Returns a CharSequence concatenating the specified CharSequences, retaining their + * SuggestionSpans that don't have the PARAGRAPH flag, but not other spans. + * + * This code is almost entirely taken from {@link TextUtils#concat(CharSequence...)}, except + * it calls copyNonParagraphSuggestionSpansFrom instead of {@link TextUtils#copySpansFrom}. + */ + public static CharSequence concatWithNonParagraphSuggestionSpansOnly(CharSequence... text) { + if (text.length == 0) { + return ""; + } + + if (text.length == 1) { + return text[0]; + } + + boolean spanned = false; + for (int i = 0; i < text.length; i++) { + if (text[i] instanceof Spanned) { + spanned = true; + break; + } + } + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < text.length; i++) { + sb.append(text[i]); + } + + if (!spanned) { + return sb.toString(); + } + + SpannableString ss = new SpannableString(sb); + int off = 0; + for (int i = 0; i < text.length; i++) { + int len = text[i].length(); + + if (text[i] instanceof Spanned) { + copyNonParagraphSuggestionSpansFrom((Spanned) text[i], 0, len, ss, off); + } + + off += len; + } + + return new SpannedString(ss); + } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp index f8ed2b9aa..1926b9831 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp @@ -16,6 +16,7 @@ #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" +#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" @@ -78,8 +79,10 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; offset = ByteArrayUtils::readUint24AndAdvancePosition(bigramsBuf, pos); break; } - if (offset == 0) { + if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID) { return NOT_A_DICT_POS; + } else if (offset == DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET) { + return origin; } if (isOffsetNegative(flags)) { return origin - offset; @@ -88,6 +91,24 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; } } +/* static */ bool BigramListReadWriteUtils::setHasNextFlag( + BufferWithExtendableBuffer *const buffer, const bool hasNext, const int entryPos) { + const bool usesAdditionalBuffer = buffer->isInAdditionalBuffer(entryPos); + int readingPos = entryPos; + if (usesAdditionalBuffer) { + readingPos -= buffer->getOriginalBufferSize(); + } + BigramFlags bigramFlags = ByteArrayUtils::readUint8AndAdvancePosition( + buffer->getBuffer(usesAdditionalBuffer), &readingPos); + if (hasNext) { + bigramFlags = bigramFlags | FLAG_ATTRIBUTE_HAS_NEXT; + } else { + bigramFlags = bigramFlags & (~FLAG_ATTRIBUTE_HAS_NEXT); + } + int writingPos = entryPos; + return buffer->writeUintAndAdvancePosition(bigramFlags, 1 /* size */, &writingPos); +} + /* static */ bool BigramListReadWriteUtils::createAndWriteBigramEntry( BufferWithExtendableBuffer *const buffer, const int targetPos, const int probability, const bool hasNext, int *const writingPos) { @@ -101,10 +122,12 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; /* static */ bool BigramListReadWriteUtils::writeBigramEntry( BufferWithExtendableBuffer *const bufferToWrite, const BigramFlags flags, const int targetPtNodePos, int *const writingPos) { - if (!bufferToWrite->writeUintAndAdvancePosition(flags, 1 /* size */, writingPos)) { + const int offset = getBigramTargetOffset(targetPtNodePos, *writingPos); + const BigramFlags flagsToWrite = (offset < 0) ? + (flags | FLAG_ATTRIBUTE_OFFSET_NEGATIVE) : (flags & ~FLAG_ATTRIBUTE_OFFSET_NEGATIVE); + if (!bufferToWrite->writeUintAndAdvancePosition(flagsToWrite, 1 /* size */, writingPos)) { return false; } - const int offset = (targetPtNodePos != NOT_A_DICT_POS) ? targetPtNodePos - *writingPos : 0; const uint32_t absOffest = abs(offset); const int bigramTargetFieldSize = attributeAddressSize(flags); return bufferToWrite->writeUintAndAdvancePosition(absOffest, bigramTargetFieldSize, @@ -113,14 +136,13 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; // Returns true if the bigram entry is valid and put entry flags into out*. /* static */ bool BigramListReadWriteUtils::createAndGetBigramFlags(const int entryPos, - const int targetPos, const int probability, const bool hasNext, + const int targetPtNodePos, const int probability, const bool hasNext, BigramFlags *const outBigramFlags) { BigramFlags flags = probability & MASK_ATTRIBUTE_PROBABILITY; if (hasNext) { flags |= FLAG_ATTRIBUTE_HAS_NEXT; } - const int targetFieldPos = entryPos + 1; - const int offset = (targetPos != NOT_A_DICT_POS) ? targetPos - targetFieldPos : 0; + const int offset = getBigramTargetOffset(targetPtNodePos, entryPos); if (offset < 0) { flags |= FLAG_ATTRIBUTE_OFFSET_NEGATIVE; } @@ -143,4 +165,18 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; return true; } +/* static */ int BigramListReadWriteUtils::getBigramTargetOffset(const int targetPtNodePos, + const int entryPos) { + if (targetPtNodePos == NOT_A_DICT_POS) { + return DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; + } else { + const int offset = targetPtNodePos - (entryPos + 1 /* bigramFlagsField */); + if (offset == 0) { + return DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; + } else { + return offset; + } + } +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h index 234a0ea58..eabe4e099 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h @@ -59,9 +59,8 @@ public: */ } - static AK_FORCE_INLINE BigramFlags setHasNextFlag(const BigramFlags flags) { - return flags | FLAG_ATTRIBUTE_HAS_NEXT; - } + static bool setHasNextFlag(BufferWithExtendableBuffer *const buffer, + const bool hasNext, const int entryPos); static AK_FORCE_INLINE BigramFlags setProbabilityInFlags(const BigramFlags flags, const int probability) { @@ -96,6 +95,8 @@ private: static int getBigramAddressAndAdvancePosition(const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos); + + static int getBigramTargetOffset(const int targetPtNodePos, const int entryPos); }; } // namespace latinime #endif // LATINIME_BIGRAM_LIST_READ_WRITE_UTILS_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index ba43bdb10..bc2f5ee58 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -19,6 +19,7 @@ #include "suggest/core/policy/dictionary_shortcuts_structure_policy.h" #include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h" #include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h" +#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" #include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h" namespace latinime { @@ -69,9 +70,11 @@ bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const b *outBigramsCount = 0; BigramListReadWriteUtils::BigramFlags bigramFlags; int bigramEntryCount = 0; + int lastWrittenEntryPos = NOT_A_DICT_POS; do { if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) { - AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); + AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d", + bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); ASSERT(false); return false; } @@ -88,6 +91,11 @@ bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const b originalBigramPos += mBuffer->getOriginalBufferSize(); } const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos); + if (bigramPos == NOT_A_DICT_POS) { + // Target PtNode has been invalidated. + continue; + } + lastWrittenEntryPos = *toPos; if (!BigramListReadWriteUtils::createAndWriteBigramEntry(bufferToWrite, bigramPos, BigramListReadWriteUtils::getProbabilityFromFlags(bigramFlags), BigramListReadWriteUtils::hasNext(bigramFlags), toPos)) { @@ -95,6 +103,13 @@ bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const b } (*outBigramsCount)++; } while(BigramListReadWriteUtils::hasNext(bigramFlags)); + // Makes the last entry the terminal of the list. Updates the flags. + if (lastWrittenEntryPos != NOT_A_DICT_POS) { + if (!BigramListReadWriteUtils::setHasNextFlag(bufferToWrite, false /* hasNext */, + lastWrittenEntryPos)) { + return false; + } + } if (usesAdditionalBuffer) { *fromPos += mBuffer->getOriginalBufferSize(); } @@ -114,7 +129,8 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( int bigramEntryCount = 0; do { if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) { - AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); + AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d", + bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); ASSERT(false); return false; } @@ -150,6 +166,54 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( return true; } +// Updates bigram target PtNode positions in the list after the placing step in GC. +bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos, + const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const + ptNodePositionRelocationMap) { + const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); + if (usesAdditionalBuffer) { + *bigramListPos -= mBuffer->getOriginalBufferSize(); + } + BigramListReadWriteUtils::BigramFlags bigramFlags; + int bigramEntryCount = 0; + do { + if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) { + AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d", + bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); + ASSERT(false); + return false; + } + int bigramEntryPos = *bigramListPos; + if (usesAdditionalBuffer) { + bigramEntryPos += mBuffer->getOriginalBufferSize(); + } + int bigramTargetPtNodePos; + // The buffer address can be changed after calling buffer writing methods. + BigramListReadWriteUtils::getBigramEntryPropertiesAndAdvancePosition( + mBuffer->getBuffer(usesAdditionalBuffer), &bigramFlags, &bigramTargetPtNodePos, + bigramListPos); + if (bigramTargetPtNodePos == NOT_A_DICT_POS) { + continue; + } + if (usesAdditionalBuffer) { + bigramTargetPtNodePos += mBuffer->getOriginalBufferSize(); + } + + DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it = + ptNodePositionRelocationMap->find(bigramTargetPtNodePos); + if (it != ptNodePositionRelocationMap->end()) { + bigramTargetPtNodePos = it->second; + } else { + bigramTargetPtNodePos = NOT_A_DICT_POS; + } + if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags, + bigramTargetPtNodePos, &bigramEntryPos)) { + return false; + } + } while(BigramListReadWriteUtils::hasNext(bigramFlags)); + return true; +} + bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability, int *const bigramListPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos); @@ -160,7 +224,8 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg int bigramEntryCount = 0; do { if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) { - AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); + AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d", + bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); ASSERT(false); return false; } @@ -188,10 +253,7 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg } // The current last entry is found. // First, update the flags of the last entry. - const BigramListReadWriteUtils::BigramFlags updatedFlags = - BigramListReadWriteUtils::setHasNextFlag(bigramFlags); - if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, updatedFlags, originalBigramPos, - &entryPos)) { + if (!BigramListReadWriteUtils::setHasNextFlag(mBuffer, true /* hasNext */, entryPos)) { return false; } if (usesAdditionalBuffer) { @@ -222,7 +284,8 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int bi int bigramEntryCount = 0; do { if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) { - AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); + AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d", + bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT); ASSERT(false); return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h index 16b080ae5..8ea318a41 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h @@ -21,6 +21,7 @@ #include "defines.h" #include "suggest/core/policy/dictionary_bigrams_structure_policy.h" +#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h" namespace latinime { @@ -51,6 +52,10 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos); + bool updateAllBigramTargetPtNodePositions(int *const bigramListPos, + const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const + ptNodePositionRelocationMap); + bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability, int *const bigramListPos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp index ffa02e3f6..c60e45819 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp @@ -19,7 +19,7 @@ namespace latinime { bool DynamicPatriciaTrieGcEventListeners - ::ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted + ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, const int *const nodeCodePoints) { // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless @@ -47,11 +47,13 @@ bool DynamicPatriciaTrieGcEventListeners } // Writes dummy PtNode array size when the head of PtNode array is read. -bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer +bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onDescend(const int ptNodeArrayPos) { mValidPtNodeCount = 0; int writingPos = mBufferToWrite->getTailPosition(); - mPositionMap->insert(hash_map_compat<int, int>::value_type(ptNodeArrayPos, writingPos)); + mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert( + DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type( + ptNodeArrayPos, writingPos)); // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes. // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count. mPtNodeArraySizeFieldPos = writingPos; @@ -60,7 +62,7 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod } // Write PtNode array terminal and actual PtNode array size. -bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer +bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onReadingPtNodeArrayTail() { int writingPos = mBufferToWrite->getTailPosition(); // Write PtNode array terminal. @@ -77,22 +79,71 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod } // Write valid PtNode to buffer and memorize mapping from the old position to the new position. -bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer +bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, const int *const nodeCodePoints) { if (node->isDeleted()) { // Current PtNode is not written in new buffer because it has been deleted. - mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(), - NOT_A_DICT_POS)); + mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert( + DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type( + node->getHeadPos(), NOT_A_DICT_POS)); return true; } int writingPos = mBufferToWrite->getTailPosition(); - mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(), writingPos)); + mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert( + DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type( + node->getHeadPos(), writingPos)); mValidPtNodeCount++; // Writes current PtNode. return mWritingHelper->writePtNodeToBufferByCopyingPtNodeInfo(mBufferToWrite, node, - node->getParentPos(), nodeCodePoints, node->getCodePointCount(), + node->getParentPos(), nodeCodePoints, node->getCodePointCount(), node->getProbability(), &writingPos); } +bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields + ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, + const int *const nodeCodePoints) { + // Updates parent position. + int parentPos = node->getParentPos(); + if (parentPos != NOT_A_DICT_POS) { + DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it = + mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos); + if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) { + parentPos = it->second; + } + } + int writingPos = node->getHeadPos() + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + // Write updated parent offset. + if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(mBufferToWrite, + parentPos, node->getHeadPos(), &writingPos)) { + return false; + } + + // Updates children position. + int childrenPos = node->getChildrenPos(); + if (childrenPos != NOT_A_DICT_POS) { + DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it = + mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos); + if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) { + childrenPos = it->second; + } + } + writingPos = node->getChildrenPosFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite, + childrenPos, &writingPos)) { + return false; + } + + // Updates bigram target PtNode positions in the bigram list. + int bigramsPos = node->getBigramsPos(); + if (bigramsPos != NOT_A_DICT_POS) { + if (!mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos, + &mDictPositionRelocationMap->mPtNodePositionRelocationMap)) { + return false; + } + } + + return true; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h index 728559330..4256f22fb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h @@ -34,16 +34,16 @@ class DynamicPatriciaTrieGcEventListeners { // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC. // TODO: Concatenate non-terminal PtNodes. - class ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted + class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { public: - ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted( + TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( DynamicPatriciaTrieWritingHelper *const writingHelper, BufferWithExtendableBuffer *const buffer) : mWritingHelper(writingHelper), mBuffer(buffer), valueStack(), mChildrenValue(0) {} - ~ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted() {}; + ~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {}; bool onAscend() { if (valueStack.empty()) { @@ -66,7 +66,7 @@ class DynamicPatriciaTrieGcEventListeners { private: DISALLOW_IMPLICIT_CONSTRUCTORS( - ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted); + TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted); DynamicPatriciaTrieWritingHelper *const mWritingHelper; BufferWithExtendableBuffer *const mBuffer; @@ -76,10 +76,10 @@ class DynamicPatriciaTrieGcEventListeners { // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram // entries. - class ListenerForUpdatingBigramProbability + class TraversePolicyToUpdateBigramProbability : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { public: - ListenerForUpdatingBigramProbability(DynamicBigramListPolicy *const bigramPolicy) + TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy) : mBigramPolicy(bigramPolicy) {} bool onAscend() { return true; } @@ -102,20 +102,21 @@ class DynamicPatriciaTrieGcEventListeners { } private: - DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForUpdatingBigramProbability); + DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability); DynamicBigramListPolicy *const mBigramPolicy; }; - class ListenerForPlacingAndWritingValidPtNodesToBuffer + class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { public: - ListenerForPlacingAndWritingValidPtNodesToBuffer( + TraversePolicyToPlaceAndWriteValidPtNodesToBuffer( DynamicPatriciaTrieWritingHelper *const writingHelper, BufferWithExtendableBuffer *const bufferToWrite, - hash_map_compat<int, int> *const positionMap) + DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const + dictPositionRelocationMap) : mWritingHelper(writingHelper), mBufferToWrite(bufferToWrite), - mPositionMap(positionMap), mValidPtNodeCount(0), + mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0), mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {}; bool onAscend() { return true; } @@ -127,20 +128,49 @@ class DynamicPatriciaTrieGcEventListeners { bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, const int *const nodeCodePoints); - hash_map_compat<int, int> *getPositionMap() const { - return mPositionMap; - } - private: - DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForPlacingAndWritingValidPtNodesToBuffer); + DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToPlaceAndWriteValidPtNodesToBuffer); DynamicPatriciaTrieWritingHelper *const mWritingHelper; BufferWithExtendableBuffer *const mBufferToWrite; - hash_map_compat<int, int> *const mPositionMap; + DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const + mDictPositionRelocationMap; int mValidPtNodeCount; int mPtNodeArraySizeFieldPos; }; + class TraversePolicyToUpdateAllPositionFields + : public DynamicPatriciaTrieReadingHelper::TraversingEventListener { + public: + TraversePolicyToUpdateAllPositionFields( + DynamicPatriciaTrieWritingHelper *const writingHelper, + DynamicBigramListPolicy *const bigramPolicy, + BufferWithExtendableBuffer *const bufferToWrite, + const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const + dictPositionRelocationMap) + : mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy), + mBufferToWrite(bufferToWrite), + mDictPositionRelocationMap(dictPositionRelocationMap) {}; + + bool onAscend() { return true; } + + bool onDescend(const int ptNodeArrayPos) { return true; } + + bool onReadingPtNodeArrayTail() { return true; } + + bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node, + const int *const nodeCodePoints); + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields); + + DynamicPatriciaTrieWritingHelper *const mWritingHelper; + DynamicBigramListPolicy *const mBigramPolicy; + BufferWithExtendableBuffer *const mBufferToWrite; + const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const + mDictPositionRelocationMap; + }; + private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners); }; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index d52de7ada..456352c17 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -40,9 +40,10 @@ void DynamicPatriciaTrieNodeReader::fetchPtNodeInfoFromBufferAndProcessMovedPtNo pos -= mBuffer->getOriginalBufferSize(); } mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); - const int parentPos = - DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictBuf, &pos); - mParentPos = (parentPos != 0) ? ptNodePos + parentPos : NOT_A_DICT_POS; + const int parentPosOffset = + DynamicPatriciaTrieReadingUtils::getParentPtNodePosOffsetAndAdvancePosition(dictBuf, + &pos); + mParentPos = DynamicPatriciaTrieReadingUtils::getParentPtNodePos(parentPosOffset, mHeadPos); if (outCodePoints != 0) { mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( dictBuf, mFlags, maxCodePointCount, outCodePoints, &pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp index 70a09c245..42397c19e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp @@ -258,7 +258,9 @@ void DynamicPatriciaTriePolicy::flushWithGC(const char *const filePath) { AKLOGI("Warning: flushWithGC() is called for non-updatable dictionary."); return; } - // TODO: Implement. + DynamicPatriciaTrieWritingHelper writingHelper(&mBufferWithExtendableBuffer, + &mBigramListPolicy, &mShortcutListPolicy); + writingHelper.writeToDictFileWithGC(getRootPosition(), filePath, &mHeaderPolicy); } bool DynamicPatriciaTriePolicy::needsToRunGC() const { @@ -266,8 +268,8 @@ bool DynamicPatriciaTriePolicy::needsToRunGC() const { AKLOGI("Warning: needsToRunGC() is called for non-updatable dictionary."); return false; } - // TODO: Implement. - return false; + // TODO: Implement more properly. + return mBufferWithExtendableBuffer.isNearSizeLimit(); } } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp index 754b679d5..f4a2ef389 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.cpp @@ -59,6 +59,9 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPostorderDepthFirstMa if (!listener->onReadingPtNodeArrayTail()) { return false; } + if (mReadingStateStack.size() <= 0) { + break; + } if (!listener->onAscend()) { return false; } @@ -101,6 +104,9 @@ bool DynamicPatriciaTrieReadingHelper::traverseAllPtNodesInPtNodeArrayLevelPreor if (!listener->onAscend()) { return false; } + if (mReadingStateStack.size() <= 0) { + break; + } popReadingStateFromStack(); alreadyVisitedChildren = true; alreadyVisitedAllPtNodesInArray = true; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp index 8428c0b15..d68446db6 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp @@ -28,24 +28,42 @@ const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; +// TODO: Make DICT_OFFSET_ZERO_OFFSET = 0. +// Currently, DICT_OFFSET_INVALID is 0 in Java side but offset can be 0 during GC. So, the maximum +// value of offsets, which is 0x7FFFFF is used to represent 0 offset. +const int DptReadingUtils::DICT_OFFSET_INVALID = 0; +const int DptReadingUtils::DICT_OFFSET_ZERO_OFFSET = 0x7FFFFF; + /* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } -/* static */ int DptReadingUtils::getParentPosAndAdvancePosition(const uint8_t *const buffer, - int *const pos) { +/* static */ int DptReadingUtils::getParentPtNodePosOffsetAndAdvancePosition( + const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } +/* static */ int DptReadingUtils::getParentPtNodePos(const int parentOffset, const int ptNodePos) { + if (parentOffset == DICT_OFFSET_INVALID) { + return NOT_A_DICT_POS; + } else if (parentOffset == DICT_OFFSET_ZERO_OFFSET) { + return ptNodePos; + } else { + return parentOffset + ptNodePos; + } +} + /* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const int base = *pos; const int offset = ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); - if (offset == 0) { - // 0 offset means that the node does not have children. + if (offset == DICT_OFFSET_INVALID) { + // The PtNode does not have children. return NOT_A_DICT_POS; + } else if (offset == DICT_OFFSET_ZERO_OFFSET) { + return base; } else { return base + offset; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h index db5f9b1bd..67c3cc57e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h @@ -27,13 +27,19 @@ class DynamicPatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; + static const int DICT_OFFSET_INVALID; + static const int DICT_OFFSET_ZERO_OFFSET; + static int getForwardLinkPosition(const uint8_t *const buffer, const int pos); static AK_FORCE_INLINE bool isValidForwardLinkPosition(const int forwardLinkAddress) { return forwardLinkAddress != 0; } - static int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos); + static int getParentPtNodePosOffsetAndAdvancePosition(const uint8_t *const buffer, + int *const pos); + + static int getParentPtNodePos(const int parentOffset, const int ptNodePos); static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, int *const pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index 3fc9c4ce0..a51ae5e1d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -28,12 +28,15 @@ #include "suggest/policyimpl/dictionary/header/header_policy.h" #include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" #include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h" +#include "utils/hash_map_compat.h" namespace latinime { const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3; const char *const DynamicPatriciaTrieWritingHelper::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE = ".tmp"; +// TODO: Make MAX_DICTIONARY_SIZE 8MB. +const size_t DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024; bool DynamicPatriciaTrieWritingHelper::addUnigramWord( DynamicPatriciaTrieReadingHelper *const readingHelper, @@ -153,7 +156,8 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNod if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) { return; } - BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); + BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */, + MAX_DICTIONARY_SIZE); if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) { return; } @@ -202,9 +206,8 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( return false; } // Update moved position, which is stored in the parent offset field. - const int movedPosOffset = movedPos - originalNode->getHeadPos(); - if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( - mBuffer, movedPosOffset, &writingPos)) { + if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + mBuffer, movedPos, originalNode->getHeadPos(), &writingPos)) { return false; } // Update bigram linked node position, which is stored in the children position field. @@ -219,11 +222,10 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader(); readingHelper.initWithPtNodeArrayPos(originalNode->getChildrenPos()); while (!readingHelper.isEnd()) { - const int childPtNodeWrittenPos = nodeReader->getHeadPos(); - const int parentOffset = movedPos - childPtNodeWrittenPos; - int parentOffsetFieldPos = childPtNodeWrittenPos + 1 /* Flags */; - if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( - mBuffer, parentOffset, &parentOffsetFieldPos)) { + int parentOffsetFieldPos = nodeReader->getHeadPos() + + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE; + if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + mBuffer, movedPos, nodeReader->getHeadPos(), &parentOffsetFieldPos)) { // Parent offset cannot be written because of a bug or a broken dictionary; thus, // we give up to update dictionary. return false; @@ -249,9 +251,8 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer( return false; } // Calculate a parent offset and write the offset. - const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; - if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(bufferToWrite, - parentOffset, writingPos)) { + if (!DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition(bufferToWrite, + parentPos, nodePos, writingPos)) { return false; } // Write code points @@ -521,30 +522,48 @@ bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy); readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); DynamicPatriciaTrieGcEventListeners - ::ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted - listenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted( + ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted + traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted( this, mBuffer); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( - &listenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted)) { + &traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted)) { return false; } readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::ListenerForUpdatingBigramProbability - listenerForupdatingBigramProbability(mBigramPolicy); + DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateBigramProbability + traversePolicyToUpdateBigramProbability(mBigramPolicy); if (!readingHelper.traverseAllPtNodesInPostorderDepthFirstManner( - &listenerForupdatingBigramProbability)) { + &traversePolicyToUpdateBigramProbability)) { return false; } // Mapping from positions in mBuffer to positions in bufferToWrite. - hash_map_compat<int, int> positionMap; + DictPositionRelocationMap dictPositionRelocationMap; readingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); - DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer - listenerForPlacingAndWritingLivingPtNodesToBuffer(this, mBuffer, &positionMap); + DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer + traversePolicyToPlaceAndWriteValidPtNodesToBuffer(this, bufferToWrite, + &dictPositionRelocationMap); + if (!readingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( + &traversePolicyToPlaceAndWriteValidPtNodesToBuffer)) { + return false; + } - // TODO: Implement. - return false; + // Create policy instance for the GCed dictionary. + DynamicShortcutListPolicy newDictShortcutPolicy(bufferToWrite); + DynamicBigramListPolicy newDictBigramPolicy(bufferToWrite, &newDictShortcutPolicy); + // Create reading helper for the GCed dictionary. + DynamicPatriciaTrieReadingHelper newDictReadingHelper(bufferToWrite, &newDictBigramPolicy, + &newDictShortcutPolicy); + newDictReadingHelper.initWithPtNodeArrayPos(rootPtNodeArrayPos); + DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields + traversePolicyToUpdateAllPositionFields(this, &newDictBigramPolicy, bufferToWrite, + &dictPositionRelocationMap); + if (!newDictReadingHelper.traverseAllPtNodesInPtNodeArrayLevelPreorderDepthFirstManner( + &traversePolicyToUpdateAllPositionFields)) { + return false; + } + return true; } } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index e82b80ae5..028fa6075 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -21,6 +21,7 @@ #include <stdint.h> #include "defines.h" +#include "utils/hash_map_compat.h" namespace latinime { @@ -33,6 +34,20 @@ class HeaderPolicy; class DynamicPatriciaTrieWritingHelper { public: + typedef hash_map_compat<int, int> PtNodeArrayPositionRelocationMap; + typedef hash_map_compat<int, int> PtNodePositionRelocationMap; + struct DictPositionRelocationMap { + public: + DictPositionRelocationMap() + : mPtNodeArrayPositionRelocationMap(), mPtNodePositionRelocationMap() {} + + PtNodeArrayPositionRelocationMap mPtNodeArrayPositionRelocationMap; + PtNodePositionRelocationMap mPtNodePositionRelocationMap; + + private: + DISALLOW_COPY_AND_ASSIGN(DictPositionRelocationMap); + }; + DynamicPatriciaTrieWritingHelper(BufferWithExtendableBuffer *const buffer, DynamicBigramListPolicy *const bigramPolicy, DynamicShortcutListPolicy *const shortcutPolicy) @@ -71,6 +86,7 @@ class DynamicPatriciaTrieWritingHelper { static const int CHILDREN_POSITION_FIELD_SIZE; static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE; + static const size_t MAX_DICTIONARY_SIZE; BufferWithExtendableBuffer *const mBuffer; DynamicBigramListPolicy *const mBigramPolicy; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp index b261e594d..5a3983776 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp @@ -39,18 +39,20 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; /* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos) { - const int offset = (forwardLinkPos != NOT_A_DICT_POS) ? - forwardLinkPos - (*forwardLinkFieldPos) : 0; - return writeDictOffset(buffer, offset, forwardLinkFieldPos); + return writeDictOffset(buffer, forwardLinkPos, (*forwardLinkFieldPos), forwardLinkFieldPos); } /* static */ bool DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const size_t arraySize, int *const arraySizeFieldPos) { - if (arraySize <= MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD) { + // Currently, all array size field to be created has LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE to + // simplify updating process. + // TODO: Use SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE for small arrays. + /*if (arraySize <= MAX_PTNODE_ARRAY_SIZE_TO_USE_SMALL_SIZE_FIELD) { return buffer->writeUintAndAdvancePosition(arraySize, SMALL_PTNODE_ARRAY_SIZE_FIELD_SIZE, arraySizeFieldPos); - } else if (arraySize <= MAX_PTNODE_ARRAY_SIZE) { + } else */ + if (arraySize <= MAX_PTNODE_ARRAY_SIZE) { uint32_t data = arraySize | LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE_FLAG; return buffer->writeUintAndAdvancePosition(data, LARGE_PTNODE_ARRAY_SIZE_FIELD_SIZE, arraySizeFieldPos); @@ -69,11 +71,10 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; } // Note that parentOffset is offset from node's head position. -/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( - BufferWithExtendableBuffer *const buffer, const int parentOffset, +/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPosOffsetAndAdvancePosition( + BufferWithExtendableBuffer *const buffer, const int parentPos, const int basePos, int *const parentPosFieldPos) { - int offset = (parentOffset != NOT_A_DICT_POS) ? parentOffset : 0; - return writeDictOffset(buffer, offset, parentPosFieldPos); + return writeDictOffset(buffer, parentPos, basePos, parentPosFieldPos); } /* static */ bool DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition( @@ -106,13 +107,19 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; /* static */ bool DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int childrenPosition, int *const childrenPositionFieldPos) { - int offset = (childrenPosition != NOT_A_DICT_POS) ? - childrenPosition - (*childrenPositionFieldPos) : 0; - return writeDictOffset(buffer, offset, childrenPositionFieldPos); + return writeDictOffset(buffer, childrenPosition, (*childrenPositionFieldPos), + childrenPositionFieldPos); } /* static */ bool DynamicPatriciaTrieWritingUtils::writeDictOffset( - BufferWithExtendableBuffer *const buffer, const int offset, int *const offsetFieldPos) { + BufferWithExtendableBuffer *const buffer, const int targetPos, const int basePos, + int *const offsetFieldPos) { + int offset = targetPos - basePos; + if (targetPos == NOT_A_DICT_POS) { + offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_INVALID; + } else if (offset == 0) { + offset = DynamicPatriciaTrieReadingUtils::DICT_OFFSET_ZERO_OFFSET; + } if (offset > MAX_DICT_OFFSET_VALUE || offset < MIN_DICT_OFFSET_VALUE) { AKLOGI("offset cannot be written because the offset is too large or too small: %d", offset); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h index 183ede444..a37e9fb3d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h @@ -28,6 +28,8 @@ class BufferWithExtendableBuffer; class DynamicPatriciaTrieWritingUtils { public: + static const int NODE_FLAG_FIELD_SIZE; + static bool writeForwardLinkPositionAndAdvancePosition( BufferWithExtendableBuffer *const buffer, const int forwardLinkPos, int *const forwardLinkFieldPos); @@ -39,8 +41,8 @@ class DynamicPatriciaTrieWritingUtils { const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos); - static bool writeParentOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, - const int parentPosition, int *const parentPosFieldPos); + static bool writeParentPosOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, + const int parentPosition, const int basePos, int *const parentPosFieldPos); static bool writeCodePointsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, const int *const codePoints, const int codePointCount, int *const codePointFieldPos); @@ -63,11 +65,10 @@ class DynamicPatriciaTrieWritingUtils { static const int MAX_DICT_OFFSET_VALUE; static const int MIN_DICT_OFFSET_VALUE; static const int DICT_OFFSET_NEGATIVE_FLAG; - static const int NODE_FLAG_FIELD_SIZE; static const int PROBABILITY_FIELD_SIZE; - static bool writeDictOffset(BufferWithExtendableBuffer *const buffer, const int offset, - int *const offsetFieldPos); + static bool writeDictOffset(BufferWithExtendableBuffer *const buffer, const int targetPos, + const int basePos, int *const offsetFieldPos); }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_UTILS_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp index 0fed275e9..f692882f2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp @@ -18,9 +18,10 @@ namespace latinime { -const size_t BufferWithExtendableBuffer::INITIAL_ADDITIONAL_BUFFER_SIZE = 16 * 1024; const size_t BufferWithExtendableBuffer::MAX_ADDITIONAL_BUFFER_SIZE = 1024 * 1024; -const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 16 * 1024; +const int BufferWithExtendableBuffer::NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE = 90; +// TODO: Needs to allocate larger memory corresponding to the current vector size. +const size_t BufferWithExtendableBuffer::EXTEND_ADDITIONAL_BUFFER_SIZE_STEP = 128 * 1024; bool BufferWithExtendableBuffer::writeUintAndAdvancePosition(const uint32_t data, const int size, int *const pos) { @@ -64,6 +65,16 @@ bool BufferWithExtendableBuffer::writeCodePointsAndAdvancePosition(const int *co return true; } +bool BufferWithExtendableBuffer::extendBuffer() { + const size_t sizeAfterExtending = + mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP; + if (sizeAfterExtending > mMaxAdditionalBufferSize) { + return false; + } + mAdditionalBuffer.resize(mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP); + return true; +} + bool BufferWithExtendableBuffer::checkAndPrepareWriting(const int pos, const int size) { if (isInAdditionalBuffer(pos)) { const int tailPosition = getTailPosition(); diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h index c6a484131..17d2e39c2 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h @@ -32,9 +32,11 @@ namespace latinime { // raw pointer but provides several methods that handle boundary checking for writing data. class BufferWithExtendableBuffer { public: - BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize) + BufferWithExtendableBuffer(uint8_t *const originalBuffer, const int originalBufferSize, + const int maxAdditionalBufferSize = MAX_ADDITIONAL_BUFFER_SIZE) : mOriginalBuffer(originalBuffer), mOriginalBufferSize(originalBufferSize), - mAdditionalBuffer(INITIAL_ADDITIONAL_BUFFER_SIZE), mUsedAdditionalBufferSize(0) {} + mAdditionalBuffer(EXTEND_ADDITIONAL_BUFFER_SIZE_STEP), mUsedAdditionalBufferSize(0), + mMaxAdditionalBufferSize(maxAdditionalBufferSize) {} AK_FORCE_INLINE int getTailPosition() const { return mOriginalBufferSize + mUsedAdditionalBufferSize; @@ -61,6 +63,11 @@ class BufferWithExtendableBuffer { return mOriginalBufferSize; } + AK_FORCE_INLINE bool isNearSizeLimit() const { + return mAdditionalBuffer.size() >= ((mMaxAdditionalBufferSize + * NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE) / 100); + } + /** * For writing. * @@ -75,28 +82,22 @@ class BufferWithExtendableBuffer { private: DISALLOW_COPY_AND_ASSIGN(BufferWithExtendableBuffer); - static const size_t INITIAL_ADDITIONAL_BUFFER_SIZE; static const size_t MAX_ADDITIONAL_BUFFER_SIZE; + static const int NEAR_BUFFER_LIMIT_THRESHOLD_PERCENTILE; static const size_t EXTEND_ADDITIONAL_BUFFER_SIZE_STEP; uint8_t *const mOriginalBuffer; const int mOriginalBufferSize; std::vector<uint8_t> mAdditionalBuffer; int mUsedAdditionalBufferSize; + const size_t mMaxAdditionalBufferSize; // Return if the buffer is successfully extended or not. - AK_FORCE_INLINE bool extendBuffer() { - if (mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP - > MAX_ADDITIONAL_BUFFER_SIZE) { - return false; - } - mAdditionalBuffer.resize(mAdditionalBuffer.size() + EXTEND_ADDITIONAL_BUFFER_SIZE_STEP); - return true; - } + bool extendBuffer(); // Returns if it is possible to write size-bytes from pos. When pos is at the tail position of // the additional buffer, try extending the buffer. - AK_FORCE_INLINE bool checkAndPrepareWriting(const int pos, const int size); + bool checkAndPrepareWriting(const int pos, const int size); }; } #endif /* LATINIME_BUFFER_WITH_EXTENDABLE_BUFFER_H */ diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 00d76c990..96a2217a3 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; +import android.util.Pair; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.DictEncoder; @@ -384,4 +385,269 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile.delete(); } + + public void testFlushWithGCDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int bigramProbability = 10; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + dictFile.delete(); + } + + // TODO: Evaluate performance of GC + public void testAddBigramWordsAndFlashWithGC() { + final int wordCount = 100; + final int bigramCount = 1000; + final int codePointSetSize = 30; + // TODO: Use various seeds such as a current timestamp to make this test more random. + final int seed = 314159265; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final int[] unigramProbabilities = new int[wordCount]; + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities[i] = unigramProbability; + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + final int[][] probabilities = new int[wordCount][wordCount]; + + for (int i = 0; i < wordCount; ++i) { + for (int j = 0; j < wordCount; ++j) { + probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; + } + } + + for (int i = 0; i < bigramCount; i++) { + final int word0Index = random.nextInt(wordCount); + final int word1Index = random.nextInt(wordCount); + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( + unigramProbabilities[word1Index], bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + for (int i = 0; i < words.size(); i++) { + for (int j = 0; j < words.size(); j++) { + assertEquals(probabilities[i][j], + binaryDictionary.getBigramProbability(words.get(i), words.get(j))); + } + } + dictFile.delete(); + } + + public void testRandomOperetionsAndFlashWithGC() { + final int flashWithGCIterationCount = 50; + final int operationCountInEachIteration = 200; + final int initialUnigramCount = 100; + final float addUnigramProb = 0.5f; + final float addBigramProb = 0.8f; + final float removeBigramProb = 0.2f; + final int codePointSetSize = 30; + final int seed = 141421356; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = + new HashMap<Pair<String, String>, Integer>(); + for (int i = 0; i < initialUnigramCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) { + // Add unigram. + if (random.nextFloat() < addUnigramProb) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + // Add bigram. + if (random.nextFloat() < addBigramProb && words.size() > 2) { + final int word0Index = random.nextInt(words.size()); + int word1Index = random.nextInt(words.size() - 1); + if (word0Index <= word1Index) { + word1Index++; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + bigramWords.add(bigram); + bigramProbabilities.put(bigram, bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + // Remove bigram. + if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) { + final int bigramIndex = random.nextInt(bigramWords.size()); + final Pair<String, String> bigram = bigramWords.get(bigramIndex); + bigramWords.remove(bigramIndex); + bigramProbabilities.remove(bigram); + binaryDictionary.removeBigramWords(bigram.first, bigram.second); + } + } + + // Test whether the all unigram operations are collectlly handled. + for (int i = 0; i < words.size(); i++) { + final String word = words.get(i); + final int unigramProbability = unigramProbabilities.get(word); + assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); + } + // Test whether the all bigram operations are collectlly handled. + for (int i = 0; i < bigramWords.size(); i++) { + final Pair<String, String> bigram = bigramWords.get(i); + final int unigramProbability = unigramProbabilities.get(bigram.second); + final int probability; + if (bigramProbabilities.containsKey(bigram)) { + final int bigramProbability = bigramProbabilities.get(bigram); + probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + } else { + probability = Dictionary.NOT_A_PROBABILITY; + } + assertEquals(probability, + binaryDictionary.getBigramProbability(bigram.first, bigram.second)); + } + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + } + + dictFile.delete(); + } + + public void testAddManyUnigramsAndFlushWithGC() { + final int flashWithGCIterationCount = 3; + final int codePointSetSize = 50; + final int seed = 22360679; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + + final ArrayList<String> words = new ArrayList<String>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + BinaryDictionary binaryDictionary; + for (int i = 0; i < flashWithGCIterationCount; i++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + while(!binaryDictionary.needsToRunGC()) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + for (int j = 0; j < words.size(); j++) { + final String word = words.get(j); + final int unigramProbability = unigramProbabilities.get(word); + assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); + } + + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + } + + dictFile.delete(); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index cedd0df88..a4d94262f 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -494,7 +494,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { formatOptions, "unigram")); results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, formatOptions, "chain")); - results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, + results.add(runReadUnigramsAndBigramsBinary(sWords, sStarBigrams, bufferType, formatOptions, "star")); } diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index bf44a1424..d605cdb84 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); } catch (InterruptedException e) { } - for (int i = 0; i < 10 && i < numberOfWords; ++i) { + // Limit word count to check when using a Java on memory dictionary. + final int wordCountToCheck = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + numberOfWords : 10; + for (int i = 0; i < wordCountToCheck; ++i) { final String word = words.get(i); // This may fail as long as we use tryLock on inserting the bigram words assertTrue(dict.isInDictionaryForTests(word)); @@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } } } + + public void testAddManyWords() { + File dictFile = null; + final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final int numberOfWords = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + 10000 : 1000; + final Random random = new Random(123456); + + UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); + try { + addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, + true /* checksContents */); + dict.close(); + } finally { + try { + Log.d(TAG, "waiting for writing ..."); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } catch (InterruptedException e) { + Log.d(TAG, "InterruptedException: ", e); + } + final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + dictFile = new File(getContext().getFilesDir(), fileName); + if (dictFile != null) { + assertTrue(dictFile.exists()); + assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); + dictFile.delete(); + } + } + } + } diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java index 4e396a1cf..eb9fb984e 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java @@ -20,6 +20,11 @@ import com.android.inputmethod.latin.settings.SettingsValues; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import android.text.style.SuggestionSpan; +import android.text.style.URLSpan; +import android.text.SpannableStringBuilder; +import android.text.Spannable; +import android.text.Spanned; import java.util.Arrays; import java.util.List; @@ -280,4 +285,34 @@ public class StringUtilsTests extends AndroidTestCase { assertEquals(objs[i], newObjArray.get(i)); } } + + public void testConcatWithSuggestionSpansOnly() { + SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n" + + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n" + + "test string\ntest string\n"); + final int N = 10; + for (int i = 0; i < N; ++i) { + // Put a PARAGRAPH-flagged span that should not be found in the result. + s.setSpan(new SuggestionSpan(getContext(), + new String[] {"" + i}, Spannable.SPAN_PARAGRAPH), + i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH); + // Put a normal suggestion span that should be found in the result. + s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0); + // Put a URL span than should not be found in the result. + s.setSpan(new URLSpan("http://a"), i, i * 2, 0); + } + + final CharSequence a = s.subSequence(0, 15); + final CharSequence b = s.subSequence(15, s.length()); + final Spanned result = + (Spanned)StringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b); + + Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class); + for (int i = 0; i < spans.length; i++) { + final int flags = result.getSpanFlags(spans[i]); + assertEquals("Should not find a span with PARAGRAPH flag", + flags & Spannable.SPAN_PARAGRAPH, 0); + assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan); + } + } } diff --git a/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml b/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml index f6c64285c..2f34128bd 100644 --- a/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml +++ b/tools/make-keyboard-text/res/values-hy/donottranslate-more-keys.xml @@ -18,7 +18,23 @@ */ --> <resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <!-- U+058A: "֊" ARMENIAN HYPHEN --> + <!-- U+055C: "՜" ARMENIAN EXCLAMATION MARK --> + <!-- U+055D: "՝" ARMENIAN COMMA --> <!-- U+055E: "՞" ARMENIAN QUESTION MARK --> - <string name="more_keys_for_punctuation">"!fixedColumnOrder!4,՞,!,\\,,\?,:,;,\@"</string> - <string name="more_keys_for_tablet_period">՞,\?</string> + <!-- U+0559: "ՙ" ARMENIAN MODIFIER LETTER LEFT HALF RING --> + <!-- U+055A: "՚" ARMENIAN APOSTROPHE --> + <!-- U+055B: "՛" ARMENIAN EMPHASIS MARK --> + <!-- U+055F: "՟" ARMENIAN ABBREVIATION MARK --> + <string name="more_keys_for_punctuation">"!fixedColumnOrder!8,!,?,\\,,.,֊,՜,՝,՞,:,;,\@,ՙ,՚,՛,՟"</string> + <!-- U+055E: "՞" ARMENIAN QUESTION MARK --> + <!-- U+00BF: "¿" INVERTED QUESTION MARK --> + <string name="more_keys_for_symbols_question">՞,¿</string> + <!-- U+055C: "՜" ARMENIAN EXCLAMATION MARK --> + <!-- U+00A1: "¡" INVERTED EXCLAMATION MARK --> + <string name="more_keys_for_symbols_exclamation">՜,¡</string> + <!-- U+058F: "֏" ARMENIAN DRAM SIGN --> + <!-- TODO: Enable this when we have glyph for the following letter + <string name="keylabel_for_currency">֏</string> + --> </resources> |
