diff options
25 files changed, 383 insertions, 493 deletions
diff --git a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java b/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java deleted file mode 100644 index 1c6a14efe..000000000 --- a/java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin; - -import android.util.Log; - -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver4DictEncoder; -import com.android.inputmethod.latin.utils.FileUtils; - -import java.io.File; -import java.io.IOException; -import java.util.Map; - -abstract public class AbstractDictionaryWriter { - /** Used for Log actions from this class */ - private static final String TAG = AbstractDictionaryWriter.class.getSimpleName(); - - public AbstractDictionaryWriter() { - } - - abstract public void clear(); - - /** - * Add a unigram with an optional shortcut to the dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - abstract public void addUnigramWord(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq, final boolean isNotAWord); - - // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve. - abstract public void addBigramWords(final String word0, final String word1, - final int frequency, final boolean isValid, final long lastModifiedTime); - - abstract public void removeBigramWords(final String word0, final String word1); - - abstract protected void writeDictionary(final DictEncoder dictEncoder, - final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException; - - public void write(final File file, final Map<String, String> attributeMap) { - try { - FileUtils.deleteRecursively(file); - file.mkdir(); - final DictEncoder dictEncoder = new Ver4DictEncoder(file); - writeDictionary(dictEncoder, attributeMap); - } catch (IOException e) { - Log.e(TAG, "IO exception while writing file", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Unsupported format", e); - } - } -} diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index c450a1d4f..851ecc042 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -28,9 +28,9 @@ import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.makedict.WordProperty; +import com.android.inputmethod.latin.personalization.PersonalizationHelper; import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.StringUtils; @@ -508,7 +508,9 @@ public final class BinaryDictionary extends Dictionary { */ @UsedForTesting public static int setCurrentTimeForTest(final int currentTime) { - return setCurrentTimeForTestNative(currentTime); + final int currentNativeTimestamp = setCurrentTimeForTestNative(currentTime); + PersonalizationHelper.currentTimeChangedForTesting(currentNativeTimestamp); + return currentNativeTimestamp; } @UsedForTesting diff --git a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java index ae9bdf3fc..c2941e424 100644 --- a/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java @@ -78,7 +78,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { public ContactsBinaryDictionary(final Context context, final Locale locale, final File dictFile) { super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_CONTACTS, - false /* isUpdatable */, dictFile); + dictFile); mLocale = locale; mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale); registerObserver(context); @@ -114,14 +114,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } @Override - public void loadDictionaryAsync() { - loadDeviceAccountsEmailAddresses(); - loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI); + public void loadInitialContentsLocked() { + loadDeviceAccountsEmailAddressesLocked(); + loadDictionaryForUriLocked(ContactsContract.Profile.CONTENT_URI); // TODO: Switch this URL to the newer ContactsContract too - loadDictionaryAsyncForUri(Contacts.CONTENT_URI); + loadDictionaryForUriLocked(Contacts.CONTENT_URI); } - private void loadDeviceAccountsEmailAddresses() { + private void loadDeviceAccountsEmailAddressesLocked() { final List<String> accountVocabulary = AccountUtils.getDeviceAccountsEmailAddresses(mContext); if (accountVocabulary == null || accountVocabulary.isEmpty()) { @@ -131,12 +131,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (DEBUG) { Log.d(TAG, "loadAccountVocabulary: " + word); } - super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, 0 /* shortcutFreq */, - false /* isNotAWord */); + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */, + 0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); } } - private void loadDictionaryAsyncForUri(final Uri uri) { + private void loadDictionaryForUriLocked(final Uri uri) { Cursor cursor = null; try { cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null); @@ -145,7 +147,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } if (cursor.moveToFirst()) { sContactCountAtLastRebuild = getContactCount(); - addWords(cursor); + addWordsLocked(cursor); } } catch (final SQLiteException e) { Log.e(TAG, "SQLiteException in the remote Contacts process.", e); @@ -166,12 +168,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { return false; } - private void addWords(final Cursor cursor) { + private void addWordsLocked(final Cursor cursor) { int count = 0; while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) { String name = cursor.getString(INDEX_NAME); if (isValidName(name)) { - addName(name); + addNameLocked(name); ++count; } else { if (DEBUG_DUMP) { @@ -207,7 +209,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { * Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their * bigrams depending on locale. */ - private void addName(final String name) { + private void addNameLocked(final String name) { int len = StringUtils.codePointCount(name); String prevWord = null; // TODO: Better tokenization for non-Latin writing systems @@ -226,13 +228,15 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (DEBUG) { Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord); } - super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, - 0 /* shortcutFreq */, false /* isNotAWord */); - if (!TextUtils.isEmpty(prevWord)) { - if (mUseFirstLastBigrams) { - super.addBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM, - 0 /* lastModifiedTime */); - } + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, + null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) { + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addBigramDynamicallyLocked(prevWord, word, + FREQUENCY_FOR_CONTACTS_BIGRAM, + BinaryDictionary.NOT_A_VALID_TIMESTAMP); } prevWord = word; } @@ -258,12 +262,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { } @Override - protected boolean needsToReloadBeforeWriting() { + protected boolean needsToReloadAfterCreation() { return true; } @Override - protected boolean hasContentChanged() { + protected boolean haveContentsChanged() { final long startTime = SystemClock.uptimeMillis(); final int contactCount = getContactCount(); if (contactCount > MAX_CONTACT_COUNT) { @@ -291,7 +295,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { if (cursor.moveToFirst()) { while (!cursor.isAfterLast()) { String name = cursor.getString(INDEX_NAME); - if (isValidName(name) && !isNameInDictionary(name)) { + if (isValidName(name) && !isNameInDictionaryLocked(name)) { if (DEBUG) { Log.d(TAG, "Contact name missing: " + name + " (runtime = " + (SystemClock.uptimeMillis() - startTime) + " ms)"); @@ -321,7 +325,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { /** * Checks if the words in a name are in the current binary dictionary. */ - private boolean isNameInDictionary(final String name) { + private boolean isNameInDictionaryLocked(final String name) { int len = StringUtils.codePointCount(name); String prevWord = null; for (int i = 0; i < len; i++) { @@ -332,11 +336,11 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { final int wordLen = StringUtils.codePointCount(word); if (wordLen < MAX_WORD_LENGTH && wordLen > 1) { if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) { - if (!super.isValidBigramLocked(prevWord, word)) { + if (!isValidBigramLocked(prevWord, word)) { return false; } } else { - if (!super.isValidWordLocked(word)) { + if (!isValidWordLocked(word)) { return false; } } diff --git a/java/src/com/android/inputmethod/latin/DictionaryWriter.java b/java/src/com/android/inputmethod/latin/DictionaryWriter.java deleted file mode 100644 index b931c66d1..000000000 --- a/java/src/com/android/inputmethod/latin/DictionaryWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin; - -import com.android.inputmethod.latin.makedict.DictEncoder; -import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.makedict.ProbabilityInfo; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.utils.CollectionUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -/** - * An in memory dictionary for memorizing entries and writing a binary dictionary. - */ -public class DictionaryWriter extends AbstractDictionaryWriter { - private static final int BINARY_DICT_VERSION = FormatSpec.VERSION4; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(BINARY_DICT_VERSION, false /* hasTimestamp */); - - private FusionDictionary mFusionDictionary; - - public DictionaryWriter() { - clear(); - } - - @Override - public void clear() { - final HashMap<String, String> attributes = CollectionUtils.newHashMap(); - mFusionDictionary = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(attributes)); - } - - /** - * Adds a word unigram to the fusion dictionary. - */ - // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries, - // considering performance regression. - @Override - public void addUnigramWord(final String word, final String shortcutTarget, - final int probability, final int shortcutProbability, final boolean isNotAWord) { - if (shortcutTarget == null) { - mFusionDictionary.add(word, new ProbabilityInfo(probability), null, isNotAWord); - } else { - // TODO: Do this in the subclass, with this class taking an arraylist. - final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList(); - shortcutTargets.add(new WeightedString(shortcutTarget, shortcutProbability)); - mFusionDictionary.add(word, new ProbabilityInfo(probability), shortcutTargets, - isNotAWord); - } - } - - @Override - public void addBigramWords(final String word0, final String word1, final int probability, - final boolean isValid, final long lastModifiedTime) { - mFusionDictionary.setBigram(word0, word1, new ProbabilityInfo(probability)); - } - - @Override - public void removeBigramWords(final String word0, final String word1) { - // This class don't support removing bigram words. - } - - @Override - protected void writeDictionary(final DictEncoder dictEncoder, - final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException { - for (final Map.Entry<String, String> entry : attributeMap.entrySet()) { - mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue()); - } - dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS); - } -} diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index c3e33c0c1..26545acbd 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -32,7 +32,6 @@ import com.android.inputmethod.latin.utils.CombinedFormatUtils; import com.android.inputmethod.latin.utils.ExecutorUtils; import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; -import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; import java.util.ArrayList; @@ -65,6 +64,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; private static final int TIMEOUT_FOR_READ_OPS_FOR_TESTS_IN_MILLISECONDS = 10000; + private static final int DEFAULT_MAX_UNIGRAM_COUNT = 10000; + private static final int DEFAULT_MAX_BIGRAM_COUNT = 10000; + /** * The maximum length of a word in this dictionary. */ @@ -90,10 +92,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private BinaryDictionary mBinaryDictionary; - // TODO: Remove and handle dictionaries in native code. - /** The in-memory dictionary used to generate the binary dictionary. */ - protected AbstractDictionaryWriter mDictionaryWriter; - /** * The name of this dictionary, used as a part of the filename for storing the binary * dictionary. Multiple dictionary instances with the same name is supported, with access @@ -104,9 +102,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Dictionary locale */ private final Locale mLocale; - /** Whether to support dynamically updating the dictionary */ - private final boolean mIsUpdatable; - /** Dictionary file */ private final File mDictFile; @@ -126,23 +121,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { new AtomicReference<Runnable>(); /** - * Abstract method for loading the unigrams and bigrams of a given dictionary in a background - * thread. + * Abstract method for loading initial contents of a given dictionary. */ - protected abstract void loadDictionaryAsync(); + protected abstract void loadInitialContentsLocked(); /** - * Indicates that the source dictionary content has changed and a rebuild of the binary file is - * required. If it returns false, the next reload will only read the current binary dictionary - * from file. Note that the shared binary dictionary is locked when this is called. + * Indicates that the source dictionary contents have changed and a rebuild of the binary file + * is required. If it returns false, the next reload will only read the current binary + * dictionary from file. Note that the shared binary dictionary is locked when this is called. */ - protected abstract boolean hasContentChanged(); + protected abstract boolean haveContentsChanged(); private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { return formatVersion == FormatSpec.VERSION4; } - public boolean isValidDictionary() { + public boolean isValidDictionaryLocked() { return mBinaryDictionary.isValidDictionary(); } @@ -161,15 +155,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return recorder; } - private static AbstractDictionaryWriter getDictionaryWriter( - final boolean isDynamicPersonalizationDictionary) { - if (isDynamicPersonalizationDictionary) { - return null; - } else { - return new DictionaryWriter(); - } - } - /** * Creates a new expandable binary dictionary. * @@ -178,24 +163,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * name is supported. * @param locale the dictionary locale. * @param dictType the dictionary type, as a human-readable string - * @param isUpdatable whether to support dynamically updating the dictionary. Please note that - * dynamic dictionary has negative effects on memory space and computation time. * @param dictFile dictionary file path. if null, use default dictionary path based on * dictionary type. */ public ExpandableBinaryDictionary(final Context context, final String dictName, - final Locale locale, final String dictType, final boolean isUpdatable, - final File dictFile) { + final Locale locale, final String dictType, final File dictFile) { super(dictType); mDictName = dictName; mContext = context; mLocale = locale; - mIsUpdatable = isUpdatable; mDictFile = getDictFile(context, dictName, dictFile); mBinaryDictionary = null; mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName); - // Currently, only dynamic personalization dictionary is updatable. - mDictionaryWriter = getDictionaryWriter(isUpdatable); } public static File getDictFile(final Context context, final String dictName, @@ -225,25 +204,16 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } - protected void closeBinaryDictionary() { - // Ensure that no other threads are accessing the local binary dictionary. - ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { - @Override - public void run() { - if (mBinaryDictionary != null) { - mBinaryDictionary.close(); - mBinaryDictionary = null; - } - } - }); - } - protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); + attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, + String.valueOf(DEFAULT_MAX_UNIGRAM_COUNT)); + attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, + String.valueOf(DEFAULT_MAX_BIGRAM_COUNT)); return attributeMap; } @@ -257,61 +227,48 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary = null; } + private void createBinaryDictionaryLocked() { + BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); + } + + private void openBinaryDictionaryLocked() { + mBinaryDictionary = new BinaryDictionary( + mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(), + true /* useFullEditDistance */, mLocale, mDictType, true /* isUpdatable */); + } + protected void clear() { ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - if (mDictionaryWriter == null) { - removeBinaryDictionaryLocked(); - BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); - mBinaryDictionary = new BinaryDictionary( - mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(), - true /* useFullEditDistance */, mLocale, mDictType, mIsUpdatable); - } else { - mDictionaryWriter.clear(); - } + removeBinaryDictionaryLocked(); + createBinaryDictionaryLocked(); + openBinaryDictionaryLocked(); } }); } /** - * Adds a word unigram to the dictionary. Used for loading a dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - protected void addWord(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq, final boolean isNotAWord) { - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, isNotAWord); - } - - /** - * Adds a word bigram in the dictionary. Used for loading a dictionary. - */ - protected void addBigram(final String prevWord, final String word, final int frequency, - final long lastModifiedTime) { - mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */, - lastModifiedTime); - } - - /** * Check whether GC is needed and run GC if required. */ protected void runGCIfRequired(final boolean mindsBlockByGC) { ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(mindsBlockByGC); + runGCAfterAllPrioritizedTasksIfRequiredLocked(mindsBlockByGC); } }); } - private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { - // Calls to needsToRunGC() need to be serialized. + protected void runGCIfRequiredLocked(final boolean mindsBlockByGC) { + if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { + mBinaryDictionary.flushWithGC(); + } + } + + private void runGCAfterAllPrioritizedTasksIfRequiredLocked(final boolean mindsBlockByGC) { + // needsToRunGC() have to be called with lock. if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { if (setProcessingLargeTaskIfNot()) { // Run GC after currently existing time sensitive operations. @@ -335,52 +292,50 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected void addWordDynamically(final String word, final int frequency, final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { - if (!mIsUpdatable) { - Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq, isNotAWord, isBlacklisted, timestamp); } }); } + protected void addWordDynamicallyLocked(final String word, final int frequency, + final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, + final boolean isBlacklisted, final int timestamp) { + mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, + isNotAWord, isBlacklisted, timestamp); + } + /** * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. */ protected void addBigramDynamically(final String word0, final String word1, final int frequency, final int timestamp) { - if (!mIsUpdatable) { - Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: " - + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + addBigramDynamicallyLocked(word0, word1, frequency, timestamp); } }); } + protected void addBigramDynamicallyLocked(final String word0, final String word1, + final int frequency, final int timestamp) { + mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + } + /** * Dynamically remove a word bigram in the dictionary. */ protected void removeBigramDynamically(final String word0, final String word1) { - if (!mIsUpdatable) { - Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: " - + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); mBinaryDictionary.removeBigramWords(word0, word1); } }); @@ -396,11 +351,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected void addMultipleDictionaryEntriesDynamically( final ArrayList<LanguageModelParam> languageModelParams, final AddMultipleDictionaryEntriesCallback callback) { - if (!mIsUpdatable) { - Log.w(TAG, "addMultipleDictionaryEntriesDynamically is called for non-updatable " + - "dictionary: " + mDictName); - return; - } ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { @@ -463,10 +413,6 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public boolean isValidWord(final String word) { reloadDictionaryIfRequired(); - return isValidWordInner(word); - } - - protected boolean isValidWordInner(final String word) { if (processingLargeTask()) { return false; } @@ -503,7 +449,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Loads the current binary dictionary from internal storage. Assumes the dictionary file * exists. */ - private void loadBinaryDictionary() { + private void loadBinaryDictionaryLocked() { if (DEBUG) { Log.d(TAG, "Loading binary dictionary: " + mDictName + " request=" + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" @@ -519,65 +465,40 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } catch (InterruptedException e) { } } - - final String filename = mDictFile.getAbsolutePath(); - final long length = mDictFile.length(); - - // Build the new binary dictionary - final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0 /* offset */, - length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - - // Ensure all threads accessing the current dictionary have finished before - // swapping in the new one. - // TODO: Ensure multi-thread assignment of mBinaryDictionary. final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; - ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { - @Override - public void run() { - mBinaryDictionary = newBinaryDictionary; - if (oldBinaryDictionary != null) { - oldBinaryDictionary.close(); - } - } - }); + openBinaryDictionaryLocked(); + if (oldBinaryDictionary != null) { + oldBinaryDictionary.close(); + } } /** * Abstract method for checking if it is required to reload the dictionary before writing * a binary dictionary. */ - abstract protected boolean needsToReloadBeforeWriting(); + abstract protected boolean needsToReloadAfterCreation(); /** - * Writes a new binary dictionary based on the contents of the fusion dictionary. + * Create a new binary dictionary and load initial contents. */ - private void writeBinaryDictionary() { + private void createNewDictionaryLocked() { if (DEBUG) { Log.d(TAG, "Generating binary dictionary: " + mDictName + " request=" + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + mDictNameDictionaryUpdateController.mLastUpdateTime); } - if (needsToReloadBeforeWriting()) { - mDictionaryWriter.clear(); - loadDictionaryAsync(); - mDictionaryWriter.write(mDictFile, getHeaderAttributeMap()); + removeBinaryDictionaryLocked(); + createBinaryDictionaryLocked(); + openBinaryDictionaryLocked(); + loadInitialContentsLocked(); + mBinaryDictionary.flushWithGC(); + } + + private void flushDictionaryLocked() { + if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + mBinaryDictionary.flushWithGC(); } else { - if (mBinaryDictionary == null || !isValidDictionary() - // TODO: remove the check below - || !matchesExpectedBinaryDictFormatVersionForThisType( - mBinaryDictionary.getFormatVersion())) { - if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) { - Log.e(TAG, "Can't remove a file: " + mDictFile.getName()); - } - BinaryDictionary.createEmptyDictFile(mDictFile.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); - } else { - if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { - mBinaryDictionary.flushWithGC(); - } else { - mBinaryDictionary.flush(); - } - } + mBinaryDictionary.flush(); } } @@ -638,52 +559,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void run() { try { final long time = System.currentTimeMillis(); - final boolean dictionaryFileExists = dictionaryFileExists(); - if (mDictNameDictionaryUpdateController.isOutOfDate() - || !dictionaryFileExists) { - // If the shared dictionary file does not exist or is out of date, the - // first instance that acquires the lock will generate a new one. - if (hasContentChanged() || !dictionaryFileExists) { - // If the source content has changed or the dictionary does not exist, - // rebuild the binary dictionary. Empty dictionaries are supported (in - // the case where loadDictionaryAsync() adds nothing) in order to - // provide a uniform framework. - mDictNameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); - } else { - // If not, the reload request was unnecessary so revert - // LastUpdateRequestTime to LastUpdateTime. - mDictNameDictionaryUpdateController.mLastUpdateRequestTime = - mDictNameDictionaryUpdateController.mLastUpdateTime; - } + final boolean openedDictIsOutOfDate = + mDictNameDictionaryUpdateController.isOutOfDate(); + if (!dictionaryFileExists() + || (openedDictIsOutOfDate && haveContentsChanged())) { + // If the shared dictionary file does not exist or is out of date and + // contents have been updated, the first instance that acquires the lock + // will generate a new one + mDictNameDictionaryUpdateController.mLastUpdateTime = time; + createNewDictionaryLocked(); + } else if (openedDictIsOutOfDate) { + // If not, the reload request was unnecessary so revert + // LastUpdateRequestTime to LastUpdateTime. + mDictNameDictionaryUpdateController.mLastUpdateRequestTime = + mDictNameDictionaryUpdateController.mLastUpdateTime; } else if (mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.mLastUpdateTime < mDictNameDictionaryUpdateController.mLastUpdateTime) { // Otherwise, if the local dictionary is older than the shared dictionary, // load the shared dictionary. - loadBinaryDictionary(); + loadBinaryDictionaryLocked(); } - // If we just loaded the binary dictionary, then mBinaryDictionary is not - // up-to-date yet so it's useless to test it right away. Schedule the check - // for right after it's loaded instead. - ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { - @Override - public void run() { - if (mBinaryDictionary != null && !(isValidDictionary() - // TODO: remove the check below - && matchesExpectedBinaryDictFormatVersionForThisType( - mBinaryDictionary.getFormatVersion()))) { - // Binary dictionary or its format version is not valid. Regenerate - // the dictionary file. writeBinaryDictionary will remove the - // existing files if appropriate. - mDictNameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); - } - mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; - } - }); + if (mBinaryDictionary != null && !(isValidDictionaryLocked() + // TODO: remove the check below + && matchesExpectedBinaryDictFormatVersionForThisType( + mBinaryDictionary.getFormatVersion()))) { + // Binary dictionary or its format version is not valid. Regenerate + // the dictionary file. writeBinaryDictionary will remove the + // existing files if appropriate. + mDictNameDictionaryUpdateController.mLastUpdateTime = time; + createNewDictionaryLocked(); + } + mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; } finally { mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); } @@ -697,13 +604,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Generate binary dictionary using DictionaryWriter. + * Flush binary dictionary to dictionary file. */ protected void asyncFlushBinaryDictionary() { final Runnable newTask = new Runnable() { @Override public void run() { - writeBinaryDictionary(); + flushDictionaryLocked(); } }; final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask); diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 8ab1bb6a1..bfc578082 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -172,7 +172,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } public void onCreate() { - final Resources res = getOwnerInstance().getResources(); + final LatinIME latinIme = getOwnerInstance(); + if (latinIme == null) { + return; + } + final Resources res = latinIme.getResources(); mDelayUpdateSuggestions = res.getInteger(R.integer.config_delay_update_suggestions); mDelayUpdateShiftState = res.getInteger(R.integer.config_delay_update_shift_state); mDoubleSpacePeriodTimeout = @@ -182,6 +186,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen @Override public void handleMessage(final Message msg) { final LatinIME latinIme = getOwnerInstance(); + if (latinIme == null) { + return; + } final KeyboardSwitcher switcher = latinIme.mKeyboardSwitcher; switch (msg.what) { case MSG_UPDATE_SUGGESTION_STRIP: @@ -239,7 +246,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } public void postResumeSuggestions() { - if (!getOwnerInstance().mSettings.getCurrent().isSuggestionStripVisible()) { + final LatinIME latinIme = getOwnerInstance(); + if (latinIme == null) { + return; + } + if (!latinIme.mSettings.getCurrent().isSuggestionStripVisible()) { return; } removeMessages(MSG_RESUME_SUGGESTIONS); @@ -326,6 +337,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen resetPendingImsCallback(); mIsOrientationChanging = true; final LatinIME latinIme = getOwnerInstance(); + if (latinIme == null) { + return; + } if (latinIme.isInputViewShown()) { latinIme.mKeyboardSwitcher.saveKeyboardState(); } @@ -362,8 +376,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mPendingSuccessiveImsCallback = true; } final LatinIME latinIme = getOwnerInstance(); - executePendingImsCallback(latinIme, editorInfo, restarting); - latinIme.onStartInputInternal(editorInfo, restarting); + if (latinIme != null) { + executePendingImsCallback(latinIme, editorInfo, restarting); + latinIme.onStartInputInternal(editorInfo, restarting); + } } } @@ -381,9 +397,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen PENDING_IMS_CALLBACK_DURATION); } final LatinIME latinIme = getOwnerInstance(); - executePendingImsCallback(latinIme, editorInfo, restarting); - latinIme.onStartInputViewInternal(editorInfo, restarting); - mAppliedEditorInfo = editorInfo; + if (latinIme != null) { + executePendingImsCallback(latinIme, editorInfo, restarting); + latinIme.onStartInputViewInternal(editorInfo, restarting); + mAppliedEditorInfo = editorInfo; + } } } @@ -393,8 +411,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mHasPendingFinishInputView = true; } else { final LatinIME latinIme = getOwnerInstance(); - latinIme.onFinishInputViewInternal(finishingInput); - mAppliedEditorInfo = null; + if (latinIme != null) { + latinIme.onFinishInputViewInternal(finishingInput); + mAppliedEditorInfo = null; + } } } @@ -404,8 +424,10 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mHasPendingFinishInput = true; } else { final LatinIME latinIme = getOwnerInstance(); - executePendingImsCallback(latinIme, null, false); - latinIme.onFinishInputInternal(); + if (latinIme != null) { + executePendingImsCallback(latinIme, null, false); + latinIme.onFinishInputInternal(); + } } } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index e77cce0d4..f0e7d2f61 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -142,8 +142,8 @@ public final class Suggest { whitelistedWord = firstSuggestion = null; } else { final SuggestedWordInfo firstSuggestedWordInfo = getTransformedSuggestedWordInfo( - suggestionsSet.first(), mLocale, isAllUpperCase, isFirstCharCapitalized, - trailingSingleQuotesCount); + suggestionsSet.first(), mLocale, isAllUpperCase, isFirstCharCapitalized, + trailingSingleQuotesCount); firstSuggestion = firstSuggestedWordInfo.mWord; if (SuggestedWordInfo.KIND_WHITELIST != firstSuggestedWordInfo.mKind) { whitelistedWord = null; @@ -161,10 +161,10 @@ public final class Suggest { // TODO: stop relying on indices to find where is the auto-correction in the suggested // words, and correct this test. final boolean allowsToBeAutoCorrected = (null != whitelistedWord - && !whitelistedWord.equals(consideredWord)) + && !whitelistedWord.equals(typedWord)) || (consideredWord.length() > 1 && !mDictionaryFacilitator.isValidWord( consideredWord, wordComposer.isFirstCharCapitalized()) - && !consideredWord.equals(firstSuggestion)); + && !typedWord.equals(firstSuggestion)); final boolean hasAutoCorrection; // TODO: using isCorrectionEnabled here is not very good. It's probably useless, because diff --git a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java index 3e3cbf063..8078ab541 100644 --- a/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/UserBinaryDictionary.java @@ -86,8 +86,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { public UserBinaryDictionary(final Context context, final Locale locale, final boolean alsoUseMoreRestrictiveLocales, final File dictFile) { - super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER, - false /* isUpdatable */, dictFile); + super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_USER, dictFile); if (null == locale) throw new NullPointerException(); // Catch the error earlier final String localeStr = locale.toString(); if (SubtypeLocaleUtils.NO_LANGUAGE.equals(localeStr)) { @@ -130,7 +129,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } @Override - public void loadDictionaryAsync() { + public void loadInitialContentsLocked() { // Split the locale. For example "en" => ["en"], "de_DE" => ["de", "DE"], // "en_US_foo_bar_qux" => ["en", "US", "foo_bar_qux"] because of the limit of 3. // This is correct for locale processing. @@ -182,7 +181,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { try { cursor = mContext.getContentResolver().query( Words.CONTENT_URI, PROJECTION_QUERY, request.toString(), requestArguments, null); - addWords(cursor); + addWordsLocked(cursor); } catch (final SQLiteException e) { Log.e(TAG, "SQLiteException in the remote User dictionary process.", e); } finally { @@ -236,7 +235,7 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } } - private void addWords(final Cursor cursor) { + private void addWordsLocked(final Cursor cursor) { final boolean hasShortcutColumn = Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN; if (cursor == null) return; if (cursor.moveToFirst()) { @@ -250,12 +249,16 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { final int adjustedFrequency = scaleFrequencyFromDefaultToLatinIme(frequency); // Safeguard against adding really long words. if (word.length() < MAX_WORD_LENGTH) { - super.addWord(word, null, adjustedFrequency, 0 /* shortcutFreq */, - false /* isNotAWord */); - } - if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) { - super.addWord(shortcut, word, adjustedFrequency, USER_DICT_SHORTCUT_FREQUENCY, - true /* isNotAWord */); + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, adjustedFrequency, null /* shortcutTarget */, + 0 /* shortcutFreq */, false /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + if (null != shortcut && shortcut.length() < MAX_WORD_LENGTH) { + runGCIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(shortcut, adjustedFrequency, word, + USER_DICT_SHORTCUT_FREQUENCY, true /* isNotAWord */, + false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); + } } cursor.moveToNext(); } @@ -263,12 +266,12 @@ public class UserBinaryDictionary extends ExpandableBinaryDictionary { } @Override - protected boolean hasContentChanged() { + protected boolean haveContentsChanged() { return true; } @Override - protected boolean needsToReloadBeforeWriting() { + protected boolean needsToReloadAfterCreation() { return true; } } diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java index b09e20591..db96de305 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogicHandler.java @@ -139,7 +139,7 @@ class InputLogicHandler implements Handler.Callback { forEnd /* dismissGestureFloatingPreviewText */); if (forEnd) { mInBatchInput = false; - // The following call schedules onEndBatchInputAsyncInternal + // The following call schedules onEndBatchInputInternal // to be called on the UI thread. mLatinIME.mHandler.onEndBatchInput(suggestedWords); } diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java index a5dc45691..678c5ca6b 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin.makedict; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; @@ -25,6 +26,7 @@ import java.io.IOException; * An interface of binary dictionary encoder. */ public interface DictEncoder { + @UsedForTesting public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java b/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java index ed9c39602..b32eb9195 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictionaryHeader.java @@ -42,6 +42,10 @@ public final class DictionaryHeader { "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP"; public static final String FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY = "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID"; + public static final String FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY = + "FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS"; + public static final String MAX_UNIGRAM_COUNT_KEY = "MAX_UNIGRAM_COUNT"; + public static final String MAX_BIGRAM_COUNT_KEY = "MAX_BIGRAM_COUNT"; public static final String ATTRIBUTE_VALUE_TRUE = "1"; public DictionaryHeader(final int headerSize, final DictionaryOptions dictionaryOptions, diff --git a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 8c358cd50..074ec4074 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -27,10 +27,8 @@ import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; import java.util.ArrayList; -import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; /** * This class is a base class of a dictionary that supports decaying for the personalized language @@ -49,15 +47,13 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB /** The locale for this dictionary. */ public final Locale mLocale; - private final String mDictName; private Map<String, String> mAdditionalAttributeMap = null; protected DecayingExpandableBinaryDictionaryBase(final Context context, final String dictName, final Locale locale, final String dictionaryType, final File dictFile) { - super(context, dictName, locale, dictionaryType, true /* isUpdatable */, dictFile); + super(context, dictName, locale, dictionaryType, dictFile); mLocale = locale; - mDictName = dictName; if (mLocale != null && mLocale.toString().length() > 1) { reloadDictionaryIfRequired(); } @@ -79,7 +75,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB @Override protected Map<String, String> getHeaderAttributeMap() { - final Map<String, String> attributeMap = new HashMap<String, String>(); + final Map<String, String> attributeMap = super.getHeaderAttributeMap(); if (mAdditionalAttributeMap != null) { attributeMap.putAll(mAdditionalAttributeMap); } @@ -87,20 +83,16 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB DictionaryHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, DictionaryHeader.ATTRIBUTE_VALUE_TRUE); - attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); - attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); - attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, - String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); return attributeMap; } @Override - protected boolean hasContentChanged() { + protected boolean haveContentsChanged() { return false; } @Override - protected boolean needsToReloadBeforeWriting() { + protected boolean needsToReloadAfterCreation() { return false; } @@ -144,8 +136,8 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB } @Override - protected void loadDictionaryAsync() { - // Never loaded to memory in Java side. + protected void loadInitialContentsLocked() { + // No initial contents. } @UsedForTesting @@ -163,7 +155,7 @@ public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableB clearAndFlushDictionary(); } - /* package */ void decayIfNeeded() { + /* package */ void runGCIfRequired() { runGCIfRequired(false /* mindsBlockByGC */); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java b/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java index e9ca662e7..de2744f29 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java +++ b/java/src/com/android/inputmethod/latin/personalization/DictionaryDecayBroadcastReciever.java @@ -43,7 +43,7 @@ public class DictionaryDecayBroadcastReciever extends BroadcastReceiver { /** * Interval to update for decaying dictionaries. */ - private static final long DICTIONARY_DECAY_INTERVAL = TimeUnit.MINUTES.toMillis(60); + /* package */ static final long DICTIONARY_DECAY_INTERVAL = TimeUnit.MINUTES.toMillis(60); public static void setUpIntervalAlarmForDictionaryDecaying(Context context) { AlarmManager alarmManager = (AlarmManager)context.getSystemService(Context.ALARM_SERVICE); @@ -60,7 +60,7 @@ public class DictionaryDecayBroadcastReciever extends BroadcastReceiver { public void onReceive(final Context context, final Intent intent) { final String action = intent.getAction(); if (action.equals(DICTIONARY_DECAY_INTENT_ACTION)) { - PersonalizationHelper.tryDecayingAllOpeningUserHistoryDictionary(); + PersonalizationHelper.runGCOnAllOpenedUserHistoryDictionaries(); } } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index 5ae2fb6f8..385b525b6 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin.personalization; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.FileUtils; @@ -27,6 +28,7 @@ import java.io.FilenameFilter; import java.lang.ref.SoftReference; import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; public class PersonalizationHelper { private static final String TAG = PersonalizationHelper.class.getSimpleName(); @@ -59,14 +61,35 @@ public class PersonalizationHelper { } } - public static void tryDecayingAllOpeningUserHistoryDictionary() { - for (final ConcurrentHashMap.Entry<String, SoftReference<UserHistoryDictionary>> entry - : sLangUserHistoryDictCache.entrySet()) { - if (entry.getValue() != null) { - final UserHistoryDictionary dict = entry.getValue().get(); - if (dict != null) { - dict.decayIfNeeded(); - } + private static int sCurrentTimestampForTesting = 0; + public static void currentTimeChangedForTesting(final int currentTimestamp) { + if (TimeUnit.MILLISECONDS.toSeconds( + DictionaryDecayBroadcastReciever.DICTIONARY_DECAY_INTERVAL) + < currentTimestamp - sCurrentTimestampForTesting) { + // TODO: Run GC for both PersonalizationDictionary and UserHistoryDictionary. + runGCOnAllOpenedUserHistoryDictionaries(); + } + } + + public static void runGCOnAllOpenedUserHistoryDictionaries() { + runGCOnAllDictionariesIfRequired(sLangUserHistoryDictCache); + } + + @UsedForTesting + public static void runGCOnAllOpenedPersonalizationDictionaries() { + runGCOnAllDictionariesIfRequired(sLangPersonalizationDictCache); + } + + private static <T extends DecayingExpandableBinaryDictionaryBase> + void runGCOnAllDictionariesIfRequired( + final ConcurrentHashMap<String, SoftReference<T>> dictionaryMap) { + for (final ConcurrentHashMap.Entry<String, SoftReference<T>> entry + : dictionaryMap.entrySet()) { + final DecayingExpandableBinaryDictionaryBase dict = entry.getValue().get(); + if (dict != null) { + dict.runGCIfRequired(); + } else { + dictionaryMap.remove(entry.getKey()); } } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp index 7f916677a..5df2096a4 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp @@ -162,7 +162,7 @@ bool Ver4BigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries(const i } else if (bigramEntry.hasHistoricalInfo()) { const HistoricalInfo historicalInfo = ForgettingCurveUtils::createHistoricalInfoToSave( bigramEntry.getHistoricalInfo(), mHeaderPolicy); - if (ForgettingCurveUtils::needsToKeep(&historicalInfo)) { + if (ForgettingCurveUtils::needsToKeep(&historicalInfo, mHeaderPolicy)) { const BigramEntry updatedBigramEntry = bigramEntry.updateHistoricalInfoAndGetEntry(&historicalInfo); if (!mBigramDictContent->writeBigramEntry(&updatedBigramEntry, entryPos)) { diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp index 2ac417b33..7c7b05ca8 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp @@ -37,11 +37,22 @@ const char *const HeaderPolicy::FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY = "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP"; const char *const HeaderPolicy::FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY = "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID"; +const char *const HeaderPolicy::FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY = + "FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS"; + +const char *const HeaderPolicy::MAX_UNIGRAM_COUNT_KEY = "MAX_UNIGRAM_COUNT"; +const char *const HeaderPolicy::MAX_BIGRAM_COUNT_KEY = "MAX_BIGRAM_COUNT"; const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100; const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f; const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP = 4; const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID = 0; +// 4 days +const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS = + 4 * 24 * 60 * 60; + +const int HeaderPolicy::DEFAULT_MAX_UNIGRAM_COUNT = 10000; +const int HeaderPolicy::DEFAULT_MAX_BIGRAM_COUNT = 10000; // Used for logging. Question mark is used to indicate that the key is not found. void HeaderPolicy::readHeaderValueOrQuestionMark(const char *const key, int *outValue, diff --git a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h index 8fa7e168c..66824245e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h @@ -58,7 +58,14 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)), mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY, - DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)) {} + DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)), + mForgettingCurveDurationToLevelDown(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY, + DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS)), + mMaxUnigramCount(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, MAX_UNIGRAM_COUNT_KEY, DEFAULT_MAX_UNIGRAM_COUNT)), + mMaxBigramCount(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, MAX_BIGRAM_COUNT_KEY, DEFAULT_MAX_BIGRAM_COUNT)) {} // Constructs header information using an attribute map. HeaderPolicy(const FormatUtils::FORMAT_VERSION dictFormatVersion, @@ -83,7 +90,14 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP)), mForgettingCurveProbabilityValuesTableId(HeaderReadWriteUtils::readIntAttributeValue( &mAttributeMap, FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY, - DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)) {} + DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID)), + mForgettingCurveDurationToLevelDown(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY, + DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS)), + mMaxUnigramCount(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, MAX_UNIGRAM_COUNT_KEY, DEFAULT_MAX_UNIGRAM_COUNT)), + mMaxBigramCount(HeaderReadWriteUtils::readIntAttributeValue( + &mAttributeMap, MAX_BIGRAM_COUNT_KEY, DEFAULT_MAX_BIGRAM_COUNT)) {} // Temporary dummy header. HeaderPolicy() @@ -92,7 +106,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { mRequiresGermanUmlautProcessing(false), mIsDecayingDict(false), mDate(0), mLastDecayedTime(0), mUnigramCount(0), mBigramCount(0), mExtendedRegionSize(0), mHasHistoricalInfoOfWords(false), - mForgettingCurveOccurrencesToLevelUp(0), mForgettingCurveProbabilityValuesTableId(0) {} + mForgettingCurveOccurrencesToLevelUp(0), mForgettingCurveProbabilityValuesTableId(0), + mForgettingCurveDurationToLevelDown(0), mMaxUnigramCount(0), mMaxBigramCount(0) {} ~HeaderPolicy() {} @@ -179,6 +194,18 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { return mForgettingCurveProbabilityValuesTableId; } + AK_FORCE_INLINE int getForgettingCurveDurationToLevelDown() const { + return mForgettingCurveDurationToLevelDown; + } + + AK_FORCE_INLINE int getMaxUnigramCount() const { + return mMaxUnigramCount; + } + + AK_FORCE_INLINE int getMaxBigramCount() const { + return mMaxBigramCount; + } + void readHeaderValueOrQuestionMark(const char *const key, int *outValue, int outValueSize) const; @@ -205,10 +232,16 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { static const char *const LOCALE_KEY; static const char *const FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY; static const char *const FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY; + static const char *const FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY; + static const char *const MAX_UNIGRAM_COUNT_KEY; + static const char *const MAX_BIGRAM_COUNT_KEY; static const int DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE; static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE; static const int DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP; static const int DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID; + static const int DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS; + static const int DEFAULT_MAX_UNIGRAM_COUNT; + static const int DEFAULT_MAX_BIGRAM_COUNT; const FormatUtils::FORMAT_VERSION mDictFormatVersion; const HeaderReadWriteUtils::DictionaryFlags mDictionaryFlags; @@ -226,6 +259,9 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy { const bool mHasHistoricalInfoOfWords; const int mForgettingCurveOccurrencesToLevelUp; const int mForgettingCurveProbabilityValuesTableId; + const int mForgettingCurveDurationToLevelDown; + const int mMaxUnigramCount; + const int mMaxBigramCount; const std::vector<int> readLocale() const; float readMultipleWordCostMultiplier() const; diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp index 13ae9d923..f24c2e1af 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_node_writer.cpp @@ -168,7 +168,7 @@ bool Ver4PatriciaTrieNodeWriter::updatePtNodeProbabilityAndGetNeedsToKeepPtNodeA toBeUpdatedPtNodeParams->getTerminalId()); return false; } - const bool isValid = ForgettingCurveUtils::needsToKeep(&historicalInfo); + const bool isValid = ForgettingCurveUtils::needsToKeep(&historicalInfo, mHeaderPolicy); if (!isValid) { if (!markPtNodeAsWillBecomeNonTerminal(toBeUpdatedPtNodeParams)) { AKLOGE("Cannot mark PtNode as willBecomeNonTerminal."); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index 197250ff3..4d1b0dadb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -329,11 +329,15 @@ void Ver4PatriciaTriePolicy::getProperty(const char *const query, const int quer snprintf(outResult, maxResultLength, "%d", mBigramCount); } else if (strncmp(query, MAX_UNIGRAM_COUNT_QUERY, compareLength) == 0) { snprintf(outResult, maxResultLength, "%d", - mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_UNIGRAM_COUNT : + mHeaderPolicy->isDecayingDict() ? + ForgettingCurveUtils::getUnigramCountHardLimit( + mHeaderPolicy->getMaxUnigramCount()) : static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE)); } else if (strncmp(query, MAX_BIGRAM_COUNT_QUERY, compareLength) == 0) { snprintf(outResult, maxResultLength, "%d", - mHeaderPolicy->isDecayingDict() ? ForgettingCurveUtils::MAX_BIGRAM_COUNT : + mHeaderPolicy->isDecayingDict() ? + ForgettingCurveUtils::getBigramCountHardLimit( + mHeaderPolicy->getMaxBigramCount()) : static_cast<int>(Ver4DictConstants::MAX_DICTIONARY_SIZE)); } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp index 2da295054..3907c84a0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_writing_helper.cpp @@ -96,12 +96,11 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, } const int unigramCount = traversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted .getValidUnigramCount(); - if (headerPolicy->isDecayingDict() - && unigramCount > ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC) { - if (!truncateUnigrams(&ptNodeReader, &ptNodeWriter, - ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC)) { + const int maxUnigramCount = headerPolicy->getMaxUnigramCount(); + if (headerPolicy->isDecayingDict() && unigramCount > maxUnigramCount) { + if (!truncateUnigrams(&ptNodeReader, &ptNodeWriter, maxUnigramCount)) { AKLOGE("Cannot remove unigrams. current: %d, max: %d", unigramCount, - ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC); + maxUnigramCount); return false; } } @@ -114,11 +113,10 @@ bool Ver4PatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, return false; } const int bigramCount = traversePolicyToUpdateBigramProbability.getValidBigramEntryCount(); - if (headerPolicy->isDecayingDict() - && bigramCount > ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC) { - if (!truncateBigrams(ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC)) { - AKLOGE("Cannot remove bigrams. current: %d, max: %d", bigramCount, - ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC); + const int maxBigramCount = headerPolicy->getMaxBigramCount(); + if (headerPolicy->isDecayingDict() && bigramCount > maxBigramCount) { + if (!truncateBigrams(maxBigramCount)) { + AKLOGE("Cannot remove bigrams. current: %d, max: %d", bigramCount, maxBigramCount); return false; } } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp index 51b0eb23f..35e05d77a 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.cpp @@ -25,20 +25,17 @@ namespace latinime { -const int ForgettingCurveUtils::MAX_UNIGRAM_COUNT = 12000; -const int ForgettingCurveUtils::MAX_UNIGRAM_COUNT_AFTER_GC = 10000; -const int ForgettingCurveUtils::MAX_BIGRAM_COUNT = 12000; -const int ForgettingCurveUtils::MAX_BIGRAM_COUNT_AFTER_GC = 10000; - const int ForgettingCurveUtils::MULTIPLIER_TWO_IN_PROBABILITY_SCALE = 8; const int ForgettingCurveUtils::DECAY_INTERVAL_SECONDS = 2 * 60 * 60; const int ForgettingCurveUtils::MAX_LEVEL = 3; const int ForgettingCurveUtils::MIN_VALID_LEVEL = 1; -const int ForgettingCurveUtils::TIME_STEP_DURATION_IN_SECONDS = 6 * 60 * 60; const int ForgettingCurveUtils::MAX_ELAPSED_TIME_STEP_COUNT = 15; const int ForgettingCurveUtils::DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD = 14; +const float ForgettingCurveUtils::UNIGRAM_COUNT_HARD_LIMIT_WEIGHT = 1.2; +const float ForgettingCurveUtils::BIGRAM_COUNT_HARD_LIMIT_WEIGHT = 1.2; + const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityTable; // TODO: Revise the logic to decide the initial probability depending on the given probability. @@ -71,7 +68,8 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT /* static */ int ForgettingCurveUtils::decodeProbability( const HistoricalInfo *const historicalInfo, const HeaderPolicy *const headerPolicy) { - const int elapsedTimeStepCount = getElapsedTimeStepCount(historicalInfo->getTimeStamp()); + const int elapsedTimeStepCount = getElapsedTimeStepCount(historicalInfo->getTimeStamp(), + headerPolicy->getForgettingCurveDurationToLevelDown()); return sProbabilityTable.getProbability( headerPolicy->getForgettingCurveProbabilityValuesTableId(), historicalInfo->getLevel(), min(max(elapsedTimeStepCount, 0), MAX_ELAPSED_TIME_STEP_COUNT)); @@ -90,10 +88,12 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT } } -/* static */ bool ForgettingCurveUtils::needsToKeep(const HistoricalInfo *const historicalInfo) { +/* static */ bool ForgettingCurveUtils::needsToKeep(const HistoricalInfo *const historicalInfo, + const HeaderPolicy *const headerPolicy) { return historicalInfo->getLevel() > 0 - || getElapsedTimeStepCount(historicalInfo->getTimeStamp()) - < DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD; + || getElapsedTimeStepCount(historicalInfo->getTimeStamp(), + headerPolicy->getForgettingCurveDurationToLevelDown()) + < DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD; } /* static */ const HistoricalInfo ForgettingCurveUtils::createHistoricalInfoToSave( @@ -102,7 +102,9 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT if (originalHistoricalInfo->getTimeStamp() == NOT_A_TIMESTAMP) { return HistoricalInfo(); } - const int elapsedTimeStep = getElapsedTimeStepCount(originalHistoricalInfo->getTimeStamp()); + const int durationToLevelDownInSeconds = headerPolicy->getForgettingCurveDurationToLevelDown(); + const int elapsedTimeStep = getElapsedTimeStepCount( + originalHistoricalInfo->getTimeStamp(), durationToLevelDownInSeconds); if (elapsedTimeStep <= MAX_ELAPSED_TIME_STEP_COUNT) { // No need to update historical info. return *originalHistoricalInfo; @@ -111,18 +113,18 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT const int maxLevelDownAmonut = elapsedTimeStep / (MAX_ELAPSED_TIME_STEP_COUNT + 1); const int levelDownAmount = (maxLevelDownAmonut >= originalHistoricalInfo->getLevel()) ? originalHistoricalInfo->getLevel() : maxLevelDownAmonut; - const int adjustedTimestamp = originalHistoricalInfo->getTimeStamp() + - levelDownAmount * (MAX_ELAPSED_TIME_STEP_COUNT + 1) * TIME_STEP_DURATION_IN_SECONDS; - return HistoricalInfo(adjustedTimestamp, + const int adjustedTimestampInSeconds = originalHistoricalInfo->getTimeStamp() + + levelDownAmount * durationToLevelDownInSeconds; + return HistoricalInfo(adjustedTimestampInSeconds, originalHistoricalInfo->getLevel() - levelDownAmount, 0 /* count */); } /* static */ bool ForgettingCurveUtils::needsToDecay(const bool mindsBlockByDecay, const int unigramCount, const int bigramCount, const HeaderPolicy *const headerPolicy) { - if (unigramCount >= ForgettingCurveUtils::MAX_UNIGRAM_COUNT) { + if (unigramCount >= getUnigramCountHardLimit(headerPolicy->getMaxUnigramCount())) { // Unigram count exceeds the limit. return true; - } else if (bigramCount >= ForgettingCurveUtils::MAX_BIGRAM_COUNT) { + } else if (bigramCount >= getBigramCountHardLimit(headerPolicy->getMaxBigramCount())) { // Bigram count exceeds the limit. return true; } @@ -143,8 +145,12 @@ const ForgettingCurveUtils::ProbabilityTable ForgettingCurveUtils::sProbabilityT return unigramProbability; } -/* static */ int ForgettingCurveUtils::getElapsedTimeStepCount(const int timestamp) { - return (TimeKeeper::peekCurrentTime() - timestamp) / TIME_STEP_DURATION_IN_SECONDS; +/* static */ int ForgettingCurveUtils::getElapsedTimeStepCount(const int timestamp, + const int durationToLevelDownInSeconds) { + const int elapsedTimeInSeconds = TimeKeeper::peekCurrentTime() - timestamp; + const int timeStepDurationInSeconds = + durationToLevelDownInSeconds / (MAX_ELAPSED_TIME_STEP_COUNT + 1); + return elapsedTimeInSeconds / timeStepDurationInSeconds; } const int ForgettingCurveUtils::ProbabilityTable::PROBABILITY_TABLE_COUNT = 4; @@ -172,12 +178,10 @@ ForgettingCurveUtils::ProbabilityTable::ProbabilityTable() : mTables() { mTables[tableId][level][timeStepCount] = NOT_A_PROBABILITY; continue; } - const int elapsedTime = timeStepCount * TIME_STEP_DURATION_IN_SECONDS; const float probability = initialProbability * powf(initialProbability / endProbability, - -1.0f * static_cast<float>(elapsedTime) - / static_cast<float>(TIME_STEP_DURATION_IN_SECONDS - * (MAX_ELAPSED_TIME_STEP_COUNT + 1))); + -1.0f * static_cast<float>(timeStepCount) + / static_cast<float>(MAX_ELAPSED_TIME_STEP_COUNT + 1)); mTables[tableId][level][timeStepCount] = min(max(static_cast<int>(probability), 1), MAX_PROBABILITY); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h index 1a285e573..bb8690939 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/forgetting_curve_utils.h @@ -28,11 +28,6 @@ class HeaderPolicy; class ForgettingCurveUtils { public: - static const int MAX_UNIGRAM_COUNT; - static const int MAX_UNIGRAM_COUNT_AFTER_GC; - static const int MAX_BIGRAM_COUNT; - static const int MAX_BIGRAM_COUNT_AFTER_GC; - static const HistoricalInfo createUpdatedHistoricalInfo( const HistoricalInfo *const originalHistoricalInfo, const int newProbability, const int timestamp, const HeaderPolicy *const headerPolicy); @@ -47,11 +42,22 @@ class ForgettingCurveUtils { static int getProbability(const int encodedUnigramProbability, const int encodedBigramProbability); - static bool needsToKeep(const HistoricalInfo *const historicalInfo); + static bool needsToKeep(const HistoricalInfo *const historicalInfo, + const HeaderPolicy *const headerPolicy); static bool needsToDecay(const bool mindsBlockByDecay, const int unigramCount, const int bigramCount, const HeaderPolicy *const headerPolicy); + AK_FORCE_INLINE static int getUnigramCountHardLimit(const int maxUnigramCount) { + return static_cast<int>(static_cast<float>(maxUnigramCount) + * UNIGRAM_COUNT_HARD_LIMIT_WEIGHT); + } + + AK_FORCE_INLINE static int getBigramCountHardLimit(const int maxBigramCount) { + return static_cast<int>(static_cast<float>(maxBigramCount) + * BIGRAM_COUNT_HARD_LIMIT_WEIGHT); + } + private: DISALLOW_IMPLICIT_CONSTRUCTORS(ForgettingCurveUtils); @@ -88,16 +94,17 @@ class ForgettingCurveUtils { static const int MAX_LEVEL; static const int MIN_VALID_LEVEL; - static const int TIME_STEP_DURATION_IN_SECONDS; static const int MAX_ELAPSED_TIME_STEP_COUNT; static const int DISCARD_LEVEL_ZERO_ENTRY_TIME_STEP_COUNT_THRESHOLD; - static const int HALF_LIFE_TIME_IN_SECONDS; + + static const float UNIGRAM_COUNT_HARD_LIMIT_WEIGHT; + static const float BIGRAM_COUNT_HARD_LIMIT_WEIGHT; static const ProbabilityTable sProbabilityTable; static int backoff(const int unigramProbability); - static int getElapsedTimeStepCount(const int timestamp); + static int getElapsedTimeStepCount(const int timestamp, const int durationToLevelDown); }; } // namespace latinime #endif /* LATINIME_FORGETTING_CURVE_UTILS_H */ diff --git a/tests/src/com/android/inputmethod/keyboard/KeyboardLayoutSetTestsBase.java b/tests/src/com/android/inputmethod/keyboard/KeyboardLayoutSetTestsBase.java index 6dcf51370..9939a4335 100644 --- a/tests/src/com/android/inputmethod/keyboard/KeyboardLayoutSetTestsBase.java +++ b/tests/src/com/android/inputmethod/keyboard/KeyboardLayoutSetTestsBase.java @@ -80,12 +80,25 @@ public class KeyboardLayoutSetTestsBase extends AndroidTestCase { || mScreenMetrics == Constants.SCREEN_METRICS_LARGE_PHONE; } + private static String toString(final ArrayList<InputMethodSubtype> subtypeList) { + final StringBuilder sb = new StringBuilder(); + for (int index = 0; index < subtypeList.size(); index++) { + final InputMethodSubtype subtype = subtypeList.get(index); + sb.append((index + 1) + ": "); + sb.append(SubtypeLocaleUtils.getSubtypeNameForLogging(subtype)); + sb.append("\n"); + } + return sb.toString(); + } + public final void testAllSubtypesCount() { - assertEquals(NUMBER_OF_SUBTYPES, mAllSubtypesList.size()); + assertEquals(toString(mAllSubtypesList), + NUMBER_OF_SUBTYPES, mAllSubtypesList.size()); } public final void testAsciiCapableSubtypesCount() { - assertEquals(NUMBER_OF_ASCII_CAPABLE_SUBTYPES, mAsciiCapableSubtypesList.size()); + assertEquals(toString(mAsciiCapableSubtypesList), + NUMBER_OF_ASCII_CAPABLE_SUBTYPES, mAsciiCapableSubtypesList.size()); } protected final InputMethodSubtype getSubtype(final Locale locale, diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index b1239f0af..6ace2de4f 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -267,13 +267,13 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word)); } forcePassingShortTime(); - dict.decayIfNeeded(); + dict.runGCIfRequired(); dict.waitAllTasksForTests(); for (final String word : words) { assertTrue(dict.isInUnderlyingBinaryDictionaryForTests(word)); } forcePassingLongTime(); - dict.decayIfNeeded(); + dict.runGCIfRequired(); dict.waitAllTasksForTests(); for (final String word : words) { assertFalse(dict.isInUnderlyingBinaryDictionaryForTests(word)); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/tools/dicttool/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java new file mode 100644 index 000000000..a4ad6b514 --- /dev/null +++ b/tools/dicttool/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.personalization; + +public class PersonalizationHelper { + public static void currentTimeChangedForTesting(final int currentTimestamp) { + } +} |