diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java')
| -rw-r--r-- | java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java | 709 |
1 files changed, 347 insertions, 362 deletions
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index eb8650e6f..64e9d2b51 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -17,23 +17,31 @@ package com.android.inputmethod.latin; import android.content.Context; -import android.os.SystemClock; import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; +import com.android.inputmethod.latin.makedict.DictionaryHeader; import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; +import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.CollectionUtils; -import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; +import com.android.inputmethod.latin.utils.CombinedFormatUtils; +import com.android.inputmethod.latin.utils.ExecutorUtils; +import com.android.inputmethod.latin.utils.FileUtils; +import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; import java.util.ArrayList; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -52,34 +60,29 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Whether to print debug output to log */ private static boolean DEBUG = false; - - // TODO: Remove. - /** Whether to call binary dictionary dynamically updating methods. */ - public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true; + private static final boolean DBG_STRESS_TEST = false; private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; + private static final int TIMEOUT_FOR_READ_OPS_FOR_TESTS_IN_MILLISECONDS = 10000; + + private static final int DEFAULT_MAX_UNIGRAM_COUNT = 10000; + private static final int DEFAULT_MAX_BIGRAM_COUNT = 10000; /** * The maximum length of a word in this dictionary. */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; - private static final int DICTIONARY_FORMAT_VERSION = 3; - - private static final String SUPPORTS_DYNAMIC_UPDATE = - FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE; + private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4; /** * A static map of update controllers, each of which records the time of accesses to a single * binary dictionary file and tracks whether the file is regenerating. The key for this map is - * the filename and the value is the shared dictionary time recorder associated with that - * filename. + * the dictionary name and the value is the shared dictionary time recorder associated with + * that dictionary name. */ private static final ConcurrentHashMap<String, DictionaryUpdateController> - sFilenameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap(); - - private static final ConcurrentHashMap<String, PrioritizedSerialExecutor> - sFilenameExecutorMap = CollectionUtils.newConcurrentHashMap(); + sDictNameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap(); /** The application context. */ protected final Context mContext; @@ -90,23 +93,22 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private BinaryDictionary mBinaryDictionary; - // TODO: Remove and handle dictionaries in native code. - /** The in-memory dictionary used to generate the binary dictionary. */ - protected AbstractDictionaryWriter mDictionaryWriter; - /** - * The name of this dictionary, used as the filename for storing the binary dictionary. Multiple - * dictionary instances with the same filename is supported, with access controlled by - * DictionaryTimeRecorder. + * The name of this dictionary, used as a part of the filename for storing the binary + * dictionary. Multiple dictionary instances with the same name is supported, with access + * controlled by DictionaryUpdateController. */ - private final String mFilename; + private final String mDictName; - /** Whether to support dynamically updating the dictionary */ - private final boolean mIsUpdatable; + /** Dictionary locale */ + private final Locale mLocale; + + /** Dictionary file */ + private final File mDictFile; // TODO: remove, once dynamic operations is serialized /** Controls updating the shared binary dictionary file across multiple instances. */ - private final DictionaryUpdateController mFilenameDictionaryUpdateController; + private final DictionaryUpdateController mDictNameDictionaryUpdateController; // TODO: remove, once dynamic operations is serialized /** Controls updating the local binary dictionary for this instance. */ @@ -114,90 +116,82 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { new DictionaryUpdateController(); /* A extension for a binary dictionary file. */ - public static final String DICT_FILE_EXTENSION = ".dict"; + protected static final String DICT_FILE_EXTENSION = ".dict"; private final AtomicReference<Runnable> mUnfinishedFlushingTask = new AtomicReference<Runnable>(); /** - * Abstract method for loading the unigrams and bigrams of a given dictionary in a background - * thread. + * Abstract method for loading initial contents of a given dictionary. */ - protected abstract void loadDictionaryAsync(); + protected abstract void loadInitialContentsLocked(); /** - * Indicates that the source dictionary content has changed and a rebuild of the binary file is - * required. If it returns false, the next reload will only read the current binary dictionary - * from file. Note that the shared binary dictionary is locked when this is called. + * Indicates that the source dictionary contents have changed and a rebuild of the binary file + * is required. If it returns false, the next reload will only read the current binary + * dictionary from file. Note that the shared binary dictionary is locked when this is called. */ - protected abstract boolean hasContentChanged(); + protected abstract boolean haveContentsChanged(); + + private boolean matchesExpectedBinaryDictFormatVersionForThisType(final int formatVersion) { + return formatVersion == FormatSpec.VERSION4; + } + + private boolean needsToMigrateDictionary(final int formatVersion) { + // TODO: Check version. + return false; + } + + public boolean isValidDictionaryLocked() { + return mBinaryDictionary.isValidDictionary(); + } /** - * Gets the dictionary update controller for the given filename. + * Gets the dictionary update controller for the given dictionary name. */ private static DictionaryUpdateController getDictionaryUpdateController( - String filename) { - DictionaryUpdateController recorder = sFilenameDictionaryUpdateControllerMap.get(filename); + final String dictName) { + DictionaryUpdateController recorder = sDictNameDictionaryUpdateControllerMap.get(dictName); if (recorder == null) { - synchronized(sFilenameDictionaryUpdateControllerMap) { + synchronized(sDictNameDictionaryUpdateControllerMap) { recorder = new DictionaryUpdateController(); - sFilenameDictionaryUpdateControllerMap.put(filename, recorder); + sDictNameDictionaryUpdateControllerMap.put(dictName, recorder); } } return recorder; } /** - * Gets the executor for the given filename. - */ - private static PrioritizedSerialExecutor getExecutor(final String filename) { - PrioritizedSerialExecutor executor = sFilenameExecutorMap.get(filename); - if (executor == null) { - synchronized(sFilenameExecutorMap) { - executor = new PrioritizedSerialExecutor(); - sFilenameExecutorMap.put(filename, executor); - } - } - return executor; - } - - private static AbstractDictionaryWriter getDictionaryWriter(final Context context, - final String dictType, final boolean isDynamicPersonalizationDictionary) { - if (isDynamicPersonalizationDictionary) { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - return null; - } else { - return new DynamicPersonalizationDictionaryWriter(context, dictType); - } - } else { - return new DictionaryWriter(context, dictType); - } - } - - /** * Creates a new expandable binary dictionary. * * @param context The application context of the parent. - * @param filename The filename for this binary dictionary. Multiple dictionaries with the same - * filename is supported. + * @param dictName The name of the dictionary. Multiple instances with the same + * name is supported. + * @param locale the dictionary locale. * @param dictType the dictionary type, as a human-readable string - * @param isUpdatable whether to support dynamically updating the dictionary. Please note that - * dynamic dictionary has negative effects on memory space and computation time. + * @param dictFile dictionary file path. if null, use default dictionary path based on + * dictionary type. */ - public ExpandableBinaryDictionary(final Context context, final String filename, - final String dictType, final boolean isUpdatable) { + public ExpandableBinaryDictionary(final Context context, final String dictName, + final Locale locale, final String dictType, final File dictFile) { super(dictType); - mFilename = filename; + mDictName = dictName; mContext = context; - mIsUpdatable = isUpdatable; + mLocale = locale; + mDictFile = getDictFile(context, dictName, dictFile); mBinaryDictionary = null; - mFilenameDictionaryUpdateController = getDictionaryUpdateController(filename); - // Currently, only dynamic personalization dictionary is updatable. - mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); + mDictNameDictionaryUpdateController = getDictionaryUpdateController(dictName); } - protected static String getFilenameWithLocale(final String name, final String localeStr) { - return name + "." + localeStr + DICT_FILE_EXTENSION; + public static File getDictFile(final Context context, final String dictName, + final File dictFile) { + return (dictFile != null) ? dictFile + : new File(context.getFilesDir(), dictName + DICT_FILE_EXTENSION); + } + + public static String getDictName(final String name, final Locale locale, + final File dictFile) { + return dictFile != null ? dictFile.getName() : name + "." + locale.toString(); } /** @@ -205,23 +199,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ @Override public void close() { - getExecutor(mFilename).execute(new Runnable() { - @Override - public void run() { - if (mBinaryDictionary!= null) { - mBinaryDictionary.close(); - mBinaryDictionary = null; - } - if (mDictionaryWriter != null) { - mDictionaryWriter.close(); - } - } - }); - } - - protected void closeBinaryDictionary() { - // Ensure that no other threads are accessing the local binary dictionary. - getExecutor(mFilename).execute(new Runnable() { + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { if (mBinaryDictionary != null) { @@ -234,80 +212,82 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected Map<String, String> getHeaderAttributeMap() { HashMap<String, String> attributeMap = new HashMap<String, String>(); - attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, - SUPPORTS_DYNAMIC_UPDATE); - attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFilename); + attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, mDictName); + attributeMap.put(DictionaryHeader.DICTIONARY_LOCALE_KEY, mLocale.toString()); + attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, + String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); + attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, + String.valueOf(DEFAULT_MAX_UNIGRAM_COUNT)); + attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, + String.valueOf(DEFAULT_MAX_BIGRAM_COUNT)); return attributeMap; } + private void removeBinaryDictionaryLocked() { + if (mBinaryDictionary != null) { + mBinaryDictionary.close(); + } + if (mDictFile.exists() && !FileUtils.deleteRecursively(mDictFile)) { + Log.e(TAG, "Can't remove a file: " + mDictFile.getName()); + } + mBinaryDictionary = null; + } + + private void createBinaryDictionaryLocked() { + BinaryDictionaryUtils.createEmptyDictFile(mDictFile.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, mLocale, getHeaderAttributeMap()); + } + + private void openBinaryDictionaryLocked() { + mBinaryDictionary = new BinaryDictionary( + mDictFile.getAbsolutePath(), 0 /* offset */, mDictFile.length(), + true /* useFullEditDistance */, mLocale, mDictType, true /* isUpdatable */); + } + protected void clear() { - getExecutor(mFilename).execute(new Runnable() { + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { - mBinaryDictionary.close(); - final File file = new File(mContext.getFilesDir(), mFilename); - BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); - mBinaryDictionary = new BinaryDictionary( - file.getAbsolutePath(), 0 /* offset */, file.length(), - true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - } else { - mDictionaryWriter.clear(); - } + removeBinaryDictionaryLocked(); + createBinaryDictionaryLocked(); + openBinaryDictionaryLocked(); } }); } /** - * Adds a word unigram to the dictionary. Used for loading a dictionary. - * @param word The word to add. - * @param shortcutTarget A shortcut target for this word, or null if none. - * @param frequency The frequency for this unigram. - * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored - * if shortcutTarget is null. - * @param isNotAWord true if this is not a word, i.e. shortcut only. - */ - protected void addWord(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq, final boolean isNotAWord) { - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, isNotAWord); - } - - /** - * Adds a word bigram in the dictionary. Used for loading a dictionary. - */ - protected void addBigram(final String prevWord, final String word, final int frequency, - final long lastModifiedTime) { - mDictionaryWriter.addBigramWords(prevWord, word, frequency, true /* isValid */, - lastModifiedTime); - } - - /** * Check whether GC is needed and run GC if required. */ protected void runGCIfRequired(final boolean mindsBlockByGC) { - if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; - getExecutor(mFilename).execute(new Runnable() { + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - runGCIfRequiredInternalLocked(mindsBlockByGC); + if (mBinaryDictionary == null) { + return; + } + runGCAfterAllPrioritizedTasksIfRequiredLocked(mindsBlockByGC); } }); } - private void runGCIfRequiredInternalLocked(final boolean mindsBlockByGC) { - if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) return; - // Calls to needsToRunGC() need to be serialized. + protected void runGCIfRequiredLocked(final boolean mindsBlockByGC) { + if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { + mBinaryDictionary.flushWithGC(); + } + } + + private void runGCAfterAllPrioritizedTasksIfRequiredLocked(final boolean mindsBlockByGC) { + // needsToRunGC() have to be called with lock. if (mBinaryDictionary.needsToRunGC(mindsBlockByGC)) { - if (setIsRegeneratingIfNotRegenerating()) { + if (setProcessingLargeTaskIfNot()) { // Run GC after currently existing time sensitive operations. - getExecutor(mFilename).executePrioritized(new Runnable() { + ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { @Override public void run() { try { mBinaryDictionary.flushWithGC(); } finally { - mFilenameDictionaryUpdateController.mIsRegenerating.set(false); + mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); } } }); @@ -318,70 +298,99 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** * Dynamically adds a word unigram to the dictionary. May overwrite an existing entry. */ - protected void addWordDynamically(final String word, final String shortcutTarget, - final int frequency, final int shortcutFreq, final boolean isNotAWord) { - if (!mIsUpdatable) { - Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename); - return; - } - getExecutor(mFilename).execute(new Runnable() { + protected void addWordDynamically(final String word, final int frequency, + final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, + final boolean isBlacklisted, final int timestamp) { + reloadDictionaryIfRequired(); + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addUnigramWord(word, frequency); - } else { - // TODO: Remove. - mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, shortcutFreq, - isNotAWord); + if (mBinaryDictionary == null) { + return; } + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + addWordDynamicallyLocked(word, frequency, shortcutTarget, shortcutFreq, + isNotAWord, isBlacklisted, timestamp); } }); } + protected void addWordDynamicallyLocked(final String word, final int frequency, + final String shortcutTarget, final int shortcutFreq, final boolean isNotAWord, + final boolean isBlacklisted, final int timestamp) { + mBinaryDictionary.addUnigramWord(word, frequency, shortcutTarget, shortcutFreq, + isNotAWord, isBlacklisted, timestamp); + } + /** * Dynamically adds a word bigram in the dictionary. May overwrite an existing entry. */ protected void addBigramDynamically(final String word0, final String word1, - final int frequency, final boolean isValid) { - if (!mIsUpdatable) { - Log.w(TAG, "addBigramDynamically is called for non-updatable dictionary: " - + mFilename); - return; - } - getExecutor(mFilename).execute(new Runnable() { + final int frequency, final int timestamp) { + reloadDictionaryIfRequired(); + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.addBigramWords(word0, word1, frequency); - } else { - // TODO: Remove. - mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, - 0 /* lastTouchedTime */); + if (mBinaryDictionary == null) { + return; } + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + addBigramDynamicallyLocked(word0, word1, frequency, timestamp); } }); } + protected void addBigramDynamicallyLocked(final String word0, final String word1, + final int frequency, final int timestamp) { + mBinaryDictionary.addBigramWords(word0, word1, frequency, timestamp); + } + /** * Dynamically remove a word bigram in the dictionary. */ protected void removeBigramDynamically(final String word0, final String word1) { - if (!mIsUpdatable) { - Log.w(TAG, "removeBigramDynamically is called for non-updatable dictionary: " - + mFilename); - return; - } - getExecutor(mFilename).execute(new Runnable() { + reloadDictionaryIfRequired(); + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - runGCIfRequiredInternalLocked(true /* mindsBlockByGC */); - mBinaryDictionary.removeBigramWords(word0, word1); - } else { - // TODO: Remove. - mDictionaryWriter.removeBigramWords(word0, word1); + if (mBinaryDictionary == null) { + return; + } + runGCAfterAllPrioritizedTasksIfRequiredLocked(true /* mindsBlockByGC */); + mBinaryDictionary.removeBigramWords(word0, word1); + } + }); + } + + public interface AddMultipleDictionaryEntriesCallback { + public void onFinished(); + } + + /** + * Dynamically add multiple entries to the dictionary. + */ + protected void addMultipleDictionaryEntriesDynamically( + final ArrayList<LanguageModelParam> languageModelParams, + final AddMultipleDictionaryEntriesCallback callback) { + reloadDictionaryIfRequired(); + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { + @Override + public void run() { + if (mBinaryDictionary == null) { + return; + } + final boolean locked = setProcessingLargeTaskIfNot(); + try { + mBinaryDictionary.addMultipleDictionaryEntries( + languageModelParams.toArray( + new LanguageModelParam[languageModelParams.size()])); + } finally { + if (callback != null) { + callback.onFinished(); + } + if (locked) { + mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); + } } } }); @@ -391,50 +400,27 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, - final int sessionId) { + final int sessionId, final float[] inOutLanguageWeight) { reloadDictionaryIfRequired(); - if (isRegenerating()) { + if (processingLargeTask()) { return null; } - final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = new AsyncResultHolder<ArrayList<SuggestedWordInfo>>(); - getExecutor(mFilename).executePrioritized(new Runnable() { + ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { @Override public void run() { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - if (mBinaryDictionary == null) { - holder.set(null); - return; - } - final ArrayList<SuggestedWordInfo> binarySuggestion = - mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, additionalFeaturesOptions, - sessionId); - holder.set(binarySuggestion); - } else { - final ArrayList<SuggestedWordInfo> inMemDictSuggestion = - composer.isBatchMode() ? null : - mDictionaryWriter.getSuggestionsWithSessionId(composer, - prevWord, proximityInfo, blockOffensiveWords, - additionalFeaturesOptions, sessionId); - // TODO: Remove checking mIsUpdatable and use native suggestion. - if (mBinaryDictionary != null && !mIsUpdatable) { - final ArrayList<SuggestedWordInfo> binarySuggestion = - mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, - additionalFeaturesOptions, sessionId); - if (inMemDictSuggestion == null) { - holder.set(binarySuggestion); - } else if (binarySuggestion == null) { - holder.set(inMemDictSuggestion); - } else { - binarySuggestion.addAll(inMemDictSuggestion); - holder.set(binarySuggestion); - } - } else { - holder.set(inMemDictSuggestion); - } + if (mBinaryDictionary == null) { + holder.set(null); + return; + } + final ArrayList<SuggestedWordInfo> binarySuggestion = + mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord, + proximityInfo, blockOffensiveWords, additionalFeaturesOptions, + sessionId, inOutLanguageWeight); + holder.set(binarySuggestion); + if (mBinaryDictionary.isCorrupted()) { + removeBinaryDictionaryLocked(); } } }); @@ -444,23 +430,20 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { + final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, + final float[] inOutLanguageWeight) { return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, - additionalFeaturesOptions, 0 /* sessionId */); + additionalFeaturesOptions, 0 /* sessionId */, inOutLanguageWeight); } @Override public boolean isValidWord(final String word) { reloadDictionaryIfRequired(); - return isValidWordInner(word); - } - - protected boolean isValidWordInner(final String word) { - if (isRegenerating()) { + if (processingLargeTask()) { return false; } final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); - getExecutor(mFilename).executePrioritized(new Runnable() { + ExecutorUtils.getExecutor(mDictName).executePrioritized(new Runnable() { @Override public void run() { holder.set(isValidWordLocked(word)); @@ -484,7 +467,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * dictionary exists, this method will generate one. */ protected void loadDictionary() { - mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = SystemClock.uptimeMillis(); + mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = System.currentTimeMillis(); reloadDictionaryIfRequired(); } @@ -492,71 +475,57 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * Loads the current binary dictionary from internal storage. Assumes the dictionary file * exists. */ - private void loadBinaryDictionary() { + private void loadBinaryDictionaryLocked() { if (DEBUG) { - Log.d(TAG, "Loading binary dictionary: " + mFilename + " request=" - + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update=" - + mFilenameDictionaryUpdateController.mLastUpdateTime); + Log.d(TAG, "Loading binary dictionary: " + mDictName + " request=" + + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + + mDictNameDictionaryUpdateController.mLastUpdateTime); } - - final File file = new File(mContext.getFilesDir(), mFilename); - final String filename = file.getAbsolutePath(); - final long length = file.length(); - - // Build the new binary dictionary - final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0 /* offset */, - length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - - // Ensure all threads accessing the current dictionary have finished before - // swapping in the new one. - // TODO: Ensure multi-thread assignment of mBinaryDictionary. - final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - mBinaryDictionary = newBinaryDictionary; - if (oldBinaryDictionary != null) { - oldBinaryDictionary.close(); - } + if (DBG_STRESS_TEST) { + // Test if this class does not cause problems when it takes long time to load binary + // dictionary. + try { + Log.w(TAG, "Start stress in loading: " + mDictName); + Thread.sleep(15000); + Log.w(TAG, "End stress in loading"); + } catch (InterruptedException e) { } - }); + } + final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; + openBinaryDictionaryLocked(); + if (oldBinaryDictionary != null) { + oldBinaryDictionary.close(); + } + if (mBinaryDictionary.isValidDictionary() + && needsToMigrateDictionary(mBinaryDictionary.getFormatVersion())) { + mBinaryDictionary.migrateTo(DICTIONARY_FORMAT_VERSION); + } } /** - * Abstract method for checking if it is required to reload the dictionary before writing - * a binary dictionary. + * Create a new binary dictionary and load initial contents. */ - abstract protected boolean needsToReloadBeforeWriting(); - - /** - * Writes a new binary dictionary based on the contents of the fusion dictionary. - */ - private void writeBinaryDictionary() { + private void createNewDictionaryLocked() { if (DEBUG) { - Log.d(TAG, "Generating binary dictionary: " + mFilename + " request=" - + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update=" - + mFilenameDictionaryUpdateController.mLastUpdateTime); + Log.d(TAG, "Generating binary dictionary: " + mDictName + " request=" + + mDictNameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + + mDictNameDictionaryUpdateController.mLastUpdateTime); } - if (needsToReloadBeforeWriting()) { - mDictionaryWriter.clear(); - loadDictionaryAsync(); - mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); + removeBinaryDictionaryLocked(); + createBinaryDictionaryLocked(); + openBinaryDictionaryLocked(); + loadInitialContentsLocked(); + mBinaryDictionary.flushWithGC(); + } + + private void flushDictionaryLocked() { + if (mBinaryDictionary == null) { + return; + } + if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { + mBinaryDictionary.flushWithGC(); } else { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { - final File file = new File(mContext.getFilesDir(), mFilename); - BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), - DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); - } else { - if (mBinaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { - mBinaryDictionary.flushWithGC(); - } else { - mBinaryDictionary.flush(); - } - } - } else { - mDictionaryWriter.write(mFilename, getHeaderAttributeMap()); - } + mBinaryDictionary.flush(); } } @@ -568,12 +537,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * the current binary dictionary from file. */ protected void setRequiresReload(final boolean requiresRebuild) { - final long time = SystemClock.uptimeMillis(); + final long time = System.currentTimeMillis(); mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = time; - mFilenameDictionaryUpdateController.mLastUpdateRequestTime = time; + mDictNameDictionaryUpdateController.mLastUpdateRequestTime = time; if (DEBUG) { - Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update=" - + mFilenameDictionaryUpdateController.mLastUpdateTime); + Log.d(TAG, "Reload request: " + mDictName + ": request=" + time + " update=" + + mDictNameDictionaryUpdateController.mLastUpdateTime); } } @@ -582,7 +551,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ public final void reloadDictionaryIfRequired() { if (!isReloadRequired()) return; - if (setIsRegeneratingIfNotRegenerating()) { + if (setProcessingLargeTaskIfNot()) { reloadDictionary(); } } @@ -594,13 +563,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { return mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.isOutOfDate(); } - private boolean isRegenerating() { - return mFilenameDictionaryUpdateController.mIsRegenerating.get(); + private boolean processingLargeTask() { + return mDictNameDictionaryUpdateController.mProcessingLargeTask.get(); } - // Returns whether the dictionary can be regenerated. - private boolean setIsRegeneratingIfNotRegenerating() { - return mFilenameDictionaryUpdateController.mIsRegenerating.compareAndSet( + // Returns whether the dictionary is being used for a large task. If true, we should not use + // this dictionary for latency sensitive operations. + private boolean setProcessingLargeTaskIfNot() { + return mDictNameDictionaryUpdateController.mProcessingLargeTask.compareAndSet( false /* expect */ , true /* update */); } @@ -611,46 +581,45 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private final void reloadDictionary() { // Ensure that only one thread attempts to read or write to the shared binary dictionary // file at the same time. - getExecutor(mFilename).execute(new Runnable() { + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { try { - final long time = SystemClock.uptimeMillis(); - final boolean dictionaryFileExists = dictionaryFileExists(); - if (mFilenameDictionaryUpdateController.isOutOfDate() - || !dictionaryFileExists) { - // If the shared dictionary file does not exist or is out of date, the - // first instance that acquires the lock will generate a new one. - if (hasContentChanged() || !dictionaryFileExists) { - // If the source content has changed or the dictionary does not exist, - // rebuild the binary dictionary. Empty dictionaries are supported (in - // the case where loadDictionaryAsync() adds nothing) in order to - // provide a uniform framework. - mFilenameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); - } else { - // If not, the reload request was unnecessary so revert - // LastUpdateRequestTime to LastUpdateTime. - mFilenameDictionaryUpdateController.mLastUpdateRequestTime = - mFilenameDictionaryUpdateController.mLastUpdateTime; - } + final long time = System.currentTimeMillis(); + final boolean openedDictIsOutOfDate = + mDictNameDictionaryUpdateController.isOutOfDate(); + if (!dictionaryFileExists() + || (openedDictIsOutOfDate && haveContentsChanged())) { + // If the shared dictionary file does not exist or is out of date and + // contents have been updated, the first instance that acquires the lock + // will generate a new one + mDictNameDictionaryUpdateController.mLastUpdateTime = time; + createNewDictionaryLocked(); + } else if (openedDictIsOutOfDate) { + // If not, the reload request was unnecessary so revert + // LastUpdateRequestTime to LastUpdateTime. + mDictNameDictionaryUpdateController.mLastUpdateRequestTime = + mDictNameDictionaryUpdateController.mLastUpdateTime; } else if (mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.mLastUpdateTime - < mFilenameDictionaryUpdateController.mLastUpdateTime) { + < mDictNameDictionaryUpdateController.mLastUpdateTime) { // Otherwise, if the local dictionary is older than the shared dictionary, // load the shared dictionary. - loadBinaryDictionary(); + loadBinaryDictionaryLocked(); } - if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { - // Binary dictionary is not valid. Regenerate the dictionary file. - mFilenameDictionaryUpdateController.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); + if (mBinaryDictionary != null && !(isValidDictionaryLocked() + // TODO: remove the check below + && matchesExpectedBinaryDictFormatVersionForThisType( + mBinaryDictionary.getFormatVersion()))) { + // Binary dictionary or its format version is not valid. Regenerate + // the dictionary file. writeBinaryDictionary will remove the + // existing files if appropriate. + mDictNameDictionaryUpdateController.mLastUpdateTime = time; + createNewDictionaryLocked(); } mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; } finally { - mFilenameDictionaryUpdateController.mIsRegenerating.set(false); + mDictNameDictionaryUpdateController.mProcessingLargeTask.set(false); } } }); @@ -658,79 +627,95 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // TODO: cache the file's existence so that we avoid doing a disk access each time. private boolean dictionaryFileExists() { - final File file = new File(mContext.getFilesDir(), mFilename); - return file.exists(); + return mDictFile.exists(); } /** - * Load the dictionary to memory. + * Flush binary dictionary to dictionary file. */ - protected void asyncLoadDictionaryToMemory() { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - loadDictionaryAsync(); - } - } - }); - } - - /** - * Generate binary dictionary using DictionaryWriter. - */ - protected void asyncFlashAllBinaryDictionary() { + protected void asyncFlushBinaryDictionary() { final Runnable newTask = new Runnable() { @Override public void run() { - writeBinaryDictionary(); + flushDictionaryLocked(); } }; final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask); - getExecutor(mFilename).replaceAndExecute(oldTask, newTask); + ExecutorUtils.getExecutor(mDictName).replaceAndExecute(oldTask, newTask); } /** - * For tracking whether the dictionary is out of date and the dictionary is regenerating. - * Can be shared across multiple dictionary instances that access the same filename. + * For tracking whether the dictionary is out of date and the dictionary is used in a large + * task. Can be shared across multiple dictionary instances that access the same filename. */ private static class DictionaryUpdateController { public volatile long mLastUpdateTime = 0; public volatile long mLastUpdateRequestTime = 0; - public volatile AtomicBoolean mIsRegenerating = new AtomicBoolean(); + public volatile AtomicBoolean mProcessingLargeTask = new AtomicBoolean(); public boolean isOutOfDate() { return (mLastUpdateRequestTime > mLastUpdateTime); } } - // TODO: Implement native binary methods once the dynamic dictionary implementation is done. + // TODO: Implement BinaryDictionary.isInDictionary(). @UsedForTesting - public boolean isInDictionaryForTests(final String word) { + public boolean isInUnderlyingBinaryDictionaryForTests(final String word) { final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); - getExecutor(mFilename).executePrioritized(new Runnable() { + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { @Override public void run() { if (mDictType == Dictionary.TYPE_USER_HISTORY) { - if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { - holder.set(mBinaryDictionary.isValidWord(word)); - } else { - holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) - .isInBigramListForTests(word)); - } + holder.set(mBinaryDictionary.isValidWord(word)); } } }); - return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); + return holder.get(false, TIMEOUT_FOR_READ_OPS_FOR_TESTS_IN_MILLISECONDS); } @UsedForTesting - public void shutdownExecutorForTests() { - getExecutor(mFilename).shutdown(); + public void waitAllTasksForTests() { + final CountDownLatch countDownLatch = new CountDownLatch(1); + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { + @Override + public void run() { + countDownLatch.countDown(); + } + }); + try { + countDownLatch.await(); + } catch (InterruptedException e) { + Log.e(TAG, "Interrupted while waiting for finishing dictionary operations.", e); + } } @UsedForTesting - public boolean isTerminatedForTests() { - return getExecutor(mFilename).isTerminated(); + public void dumpAllWordsForDebug() { + reloadDictionaryIfRequired(); + ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { + @Override + public void run() { + Log.d(TAG, "Dump dictionary: " + mDictName); + try { + final DictionaryHeader header = mBinaryDictionary.getHeader(); + Log.d(TAG, CombinedFormatUtils.formatAttributeMap( + header.mDictionaryOptions.mAttributes)); + } catch (final UnsupportedFormatException e) { + Log.d(TAG, "Cannot fetch header information.", e); + } + int token = 0; + do { + final BinaryDictionary.GetNextWordPropertyResult result = + mBinaryDictionary.getNextWordProperty(token); + final WordProperty wordProperty = result.mWordProperty; + if (wordProperty == null) { + Log.d(TAG, " dictionary is empty."); + break; + } + Log.d(TAG, wordProperty.toString()); + token = result.mNextToken; + } while (token != 0); + } + }); } } |