diff options
25 files changed, 940 insertions, 446 deletions
diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 740b86d86..c884e7b1f 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -24,14 +24,14 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; -import com.android.inputmethod.latin.personalization.DynamicPredictionDictionaryBase; +import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; import java.util.ArrayList; -import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Abstract base class for an expandable dictionary that can be created and updated dynamically @@ -49,19 +49,27 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { /** Whether to print debug output to log */ private static boolean DEBUG = false; + // TODO: Remove and enable dynamic update in native code. + /** Whether to call binary dictionary dynamically updating methods. */ + private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false; + + private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100; + /** * The maximum length of a word in this dictionary. */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; /** - * A static map of locks, each of which controls access to a single binary dictionary file. They - * ensure that only one instance can update the same dictionary at the same time. The key for - * this map is the filename and the value is the shared dictionary controller associated with - * that filename. + * A static map of time recorders, each of which records the time of accesses to a single binary + * dictionary file. The key for this map is the filename and the value is the shared dictionary + * time recorder associated with that filename. */ - private static final HashMap<String, DictionaryController> sSharedDictionaryControllers = - CollectionUtils.newHashMap(); + private static volatile ConcurrentHashMap<String, DictionaryTimeRecorder> + sFilenameDictionaryTimeRecorderMap = CollectionUtils.newConcurrentHashMap(); + + private static volatile ConcurrentHashMap<String, PrioritizedSerialExecutor> + sFilenameExecutorMap = CollectionUtils.newConcurrentHashMap(); /** The application context. */ protected final Context mContext; @@ -72,13 +80,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ private BinaryDictionary mBinaryDictionary; + // TODO: Remove and handle dictionaries in native code. /** The in-memory dictionary used to generate the binary dictionary. */ protected AbstractDictionaryWriter mDictionaryWriter; /** * The name of this dictionary, used as the filename for storing the binary dictionary. Multiple * dictionary instances with the same filename is supported, with access controlled by - * DictionaryController. + * DictionaryTimeRecorder. */ private final String mFilename; @@ -86,18 +95,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private final boolean mIsUpdatable; // TODO: remove, once dynamic operations is serialized - /** Controls access to the shared binary dictionary file across multiple instances. */ - private final DictionaryController mSharedDictionaryController; + /** Records access to the shared binary dictionary file across multiple instances. */ + private final DictionaryTimeRecorder mFilenameDictionaryTimeRecorder; // TODO: remove, once dynamic operations is serialized - /** Controls access to the local binary dictionary for this instance. */ - private final DictionaryController mLocalDictionaryController = new DictionaryController(); + /** Records access to the local binary dictionary for this instance. */ + private final DictionaryTimeRecorder mPerInstanceDictionaryTimeRecorder = + new DictionaryTimeRecorder(); /* A extension for a binary dictionary file. */ public static final String DICT_FILE_EXTENSION = ".dict"; - private final AtomicReference<AsyncWriteBinaryDictionaryTask> mWaitingTask = - new AtomicReference<AsyncWriteBinaryDictionaryTask>(); + private final AtomicReference<Runnable> mUnfinishedFlushingTask = + new AtomicReference<Runnable>(); /** * Abstract method for loading the unigrams and bigrams of a given dictionary in a background @@ -113,16 +123,32 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected abstract boolean hasContentChanged(); /** - * Gets the shared dictionary controller for the given filename. + * Gets the dictionary time recorder for the given filename. */ - private static synchronized DictionaryController getSharedDictionaryController( + private static DictionaryTimeRecorder getDictionaryTimeRecorder( String filename) { - DictionaryController controller = sSharedDictionaryControllers.get(filename); - if (controller == null) { - controller = new DictionaryController(); - sSharedDictionaryControllers.put(filename, controller); + DictionaryTimeRecorder recorder = sFilenameDictionaryTimeRecorderMap.get(filename); + if (recorder == null) { + synchronized(sFilenameDictionaryTimeRecorderMap) { + recorder = new DictionaryTimeRecorder(); + sFilenameDictionaryTimeRecorderMap.put(filename, recorder); + } + } + return recorder; + } + + /** + * Gets the executor for the given filename. + */ + private static PrioritizedSerialExecutor getExecutor(final String filename) { + PrioritizedSerialExecutor executor = sFilenameExecutorMap.get(filename); + if (executor == null) { + synchronized(sFilenameExecutorMap) { + executor = new PrioritizedSerialExecutor(); + sFilenameExecutorMap.put(filename, executor); + } } - return controller; + return executor; } private static AbstractDictionaryWriter getDictionaryWriter(final Context context, @@ -151,7 +177,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mContext = context; mIsUpdatable = isUpdatable; mBinaryDictionary = null; - mSharedDictionaryController = getSharedDictionaryController(filename); + mFilenameDictionaryTimeRecorder = getDictionaryTimeRecorder(filename); // Currently, only dynamic personalization dictionary is updatable. mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); } @@ -165,35 +191,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ @Override public void close() { - closeBinaryDictionary(); - mLocalDictionaryController.writeLock().lock(); - try { - mDictionaryWriter.close(); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + if (mBinaryDictionary!= null) { + mBinaryDictionary.close(); + mBinaryDictionary = null; + } + mDictionaryWriter.close(); + } + }); } protected void closeBinaryDictionary() { // Ensure that no other threads are accessing the local binary dictionary. - mLocalDictionaryController.writeLock().lock(); - try { - if (mBinaryDictionary != null) { - mBinaryDictionary.close(); - mBinaryDictionary = null; + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + if (mBinaryDictionary != null) { + mBinaryDictionary.close(); + mBinaryDictionary = null; + } } - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + }); } protected void clear() { - mLocalDictionaryController.writeLock().lock(); - try { - mDictionaryWriter.clear(); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + mDictionaryWriter.clear(); + } + }); } /** @@ -222,14 +251,17 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { Log.w(TAG, "addWordDynamically is called for non-updatable dictionary: " + mFilename); return; } - // TODO: Use a queue to reflect what needs to be reflected. - if (mLocalDictionaryController.writeLock().tryLock()) { - try { + + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + mBinaryDictionary.addUnigramWord(word, frequency); + } + // TODO: Remove. mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord); - } finally { - mLocalDictionaryController.writeLock().unlock(); } - } + }); } /** @@ -242,15 +274,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { + mFilename); return; } - // TODO: Use a queue to reflect what needs to be reflected. - if (mLocalDictionaryController.writeLock().tryLock()) { - try { + + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + mBinaryDictionary.addBigramWords(word0, word1, frequency); + } + // TODO: Remove. mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid, 0 /* lastTouchedTime */); - } finally { - mLocalDictionaryController.writeLock().unlock(); } - } + }); } /** @@ -262,24 +297,30 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { + mFilename); return; } - // TODO: Use a queue to reflect what needs to be reflected. - if (mLocalDictionaryController.writeLock().tryLock()) { - try { + + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { + if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { + mBinaryDictionary.removeBigramWords(word0, word1); + } + // TODO: Remove. mDictionaryWriter.removeBigramWords(word0, word1); - } finally { - mLocalDictionaryController.writeLock().unlock(); } - } + }); } @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { - asyncReloadDictionaryIfRequired(); - // Write lock because getSuggestions in native updates session status. - if (mLocalDictionaryController.writeLock().tryLock()) { - try { + reloadDictionaryIfRequired(); + final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); + final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = + new AsyncResultHolder<ArrayList<SuggestedWordInfo>>(); + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { final ArrayList<SuggestedWordInfo> inMemDictSuggestion = mDictionaryWriter.getSuggestions(composer, prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions); @@ -289,38 +330,37 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mBinaryDictionary.getSuggestions(composer, prevWord, proximityInfo, blockOffensiveWords, additionalFeaturesOptions); if (inMemDictSuggestion == null) { - return binarySuggestion; + holder.set(binarySuggestion); } else if (binarySuggestion == null) { - return inMemDictSuggestion; + holder.set(inMemDictSuggestion); } else { binarySuggestion.addAll(inMemDictSuggestion); - return binarySuggestion; + holder.set(binarySuggestion); } } else { - return inMemDictSuggestion; + holder.set(inMemDictSuggestion); } - } finally { - mLocalDictionaryController.writeLock().unlock(); } - } - return null; + }); + + return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } @Override public boolean isValidWord(final String word) { - asyncReloadDictionaryIfRequired(); + reloadDictionaryIfRequired(); return isValidWordInner(word); } protected boolean isValidWordInner(final String word) { - if (mLocalDictionaryController.readLock().tryLock()) { - try { - return isValidWordLocked(word); - } finally { - mLocalDictionaryController.readLock().unlock(); + final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + holder.set(isValidWordLocked(word)); } - } - return false; + }); + return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } protected boolean isValidWordLocked(final String word) { @@ -338,8 +378,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * dictionary exists, this method will generate one. */ protected void loadDictionary() { - mLocalDictionaryController.mLastUpdateRequestTime = SystemClock.uptimeMillis(); - asyncReloadDictionaryIfRequired(); + mPerInstanceDictionaryTimeRecorder.mLastUpdateRequestTime = SystemClock.uptimeMillis(); + reloadDictionaryIfRequired(); } /** @@ -349,8 +389,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private void loadBinaryDictionary() { if (DEBUG) { Log.d(TAG, "Loading binary dictionary: " + mFilename + " request=" - + mSharedDictionaryController.mLastUpdateRequestTime + " update=" - + mSharedDictionaryController.mLastUpdateTime); + + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime + " update=" + + mFilenameDictionaryTimeRecorder.mLastUpdateTime); } final File file = new File(mContext.getFilesDir(), mFilename); @@ -361,20 +401,18 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length, true /* useFullEditDistance */, null, mDictType, mIsUpdatable); - if (mBinaryDictionary != null) { - // Ensure all threads accessing the current dictionary have finished before swapping in - // the new one. - final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; - mLocalDictionaryController.writeLock().lock(); - try { + // Ensure all threads accessing the current dictionary have finished before swapping in + // the new one. + final BinaryDictionary oldBinaryDictionary = mBinaryDictionary; + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { mBinaryDictionary = newBinaryDictionary; - } finally { - mLocalDictionaryController.writeLock().unlock(); + if (oldBinaryDictionary != null) { + oldBinaryDictionary.close(); + } } - oldBinaryDictionary.close(); - } else { - mBinaryDictionary = newBinaryDictionary; - } + }); } /** @@ -389,8 +427,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private void writeBinaryDictionary() { if (DEBUG) { Log.d(TAG, "Generating binary dictionary: " + mFilename + " request=" - + mSharedDictionaryController.mLastUpdateRequestTime + " update=" - + mSharedDictionaryController.mLastUpdateTime); + + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime + " update=" + + mFilenameDictionaryTimeRecorder.mLastUpdateTime); } if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); @@ -408,54 +446,42 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected void setRequiresReload(final boolean requiresRebuild) { final long time = SystemClock.uptimeMillis(); - mLocalDictionaryController.mLastUpdateRequestTime = time; - mSharedDictionaryController.mLastUpdateRequestTime = time; + mPerInstanceDictionaryTimeRecorder.mLastUpdateRequestTime = time; + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime = time; if (DEBUG) { Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update=" - + mSharedDictionaryController.mLastUpdateTime); + + mFilenameDictionaryTimeRecorder.mLastUpdateTime); } } /** - * Reloads the dictionary if required. Reload will occur asynchronously in a separate thread. - */ - public void asyncReloadDictionaryIfRequired() { - if (!isReloadRequired()) return; - if (DEBUG) { - Log.d(TAG, "Starting AsyncReloadDictionaryTask: " + mFilename); - } - new AsyncReloadDictionaryTask().start(); - } - - /** * Reloads the dictionary if required. */ - public final void syncReloadDictionaryIfRequired() { + public final void reloadDictionaryIfRequired() { if (!isReloadRequired()) return; - syncReloadDictionaryInternal(); + reloadDictionary(); } /** * Returns whether a dictionary reload is required. */ private boolean isReloadRequired() { - return mBinaryDictionary == null || mLocalDictionaryController.isOutOfDate(); + return mBinaryDictionary == null || mPerInstanceDictionaryTimeRecorder.isOutOfDate(); } /** * Reloads the dictionary. Access is controlled on a per dictionary file basis and supports * concurrent calls from multiple instances that share the same dictionary file. */ - private final void syncReloadDictionaryInternal() { + private final void reloadDictionary() { // Ensure that only one thread attempts to read or write to the shared binary dictionary // file at the same time. - mSharedDictionaryController.writeLock().lock(); - try { - mLocalDictionaryController.writeLock().lock(); - try { + getExecutor(mFilename).execute(new Runnable() { + @Override + public void run() { final long time = SystemClock.uptimeMillis(); final boolean dictionaryFileExists = dictionaryFileExists(); - if (mSharedDictionaryController.isOutOfDate() || !dictionaryFileExists) { + if (mFilenameDictionaryTimeRecorder.isOutOfDate() || !dictionaryFileExists) { // If the shared dictionary file does not exist or is out of date, the first // instance that acquires the lock will generate a new one. if (hasContentChanged() || !dictionaryFileExists) { @@ -463,34 +489,31 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // rebuild the binary dictionary. Empty dictionaries are supported (in the // case where loadDictionaryAsync() adds nothing) in order to provide a // uniform framework. - mSharedDictionaryController.mLastUpdateTime = time; + mFilenameDictionaryTimeRecorder.mLastUpdateTime = time; writeBinaryDictionary(); loadBinaryDictionary(); } else { // If not, the reload request was unnecessary so revert // LastUpdateRequestTime to LastUpdateTime. - mSharedDictionaryController.mLastUpdateRequestTime = - mSharedDictionaryController.mLastUpdateTime; + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime = + mFilenameDictionaryTimeRecorder.mLastUpdateTime; } - } else if (mBinaryDictionary == null || mLocalDictionaryController.mLastUpdateTime - < mSharedDictionaryController.mLastUpdateTime) { + } else if (mBinaryDictionary == null || + mPerInstanceDictionaryTimeRecorder.mLastUpdateTime + < mFilenameDictionaryTimeRecorder.mLastUpdateTime) { // Otherwise, if the local dictionary is older than the shared dictionary, load // the shared dictionary. loadBinaryDictionary(); } if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { // Binary dictionary is not valid. Regenerate the dictionary file. - mSharedDictionaryController.mLastUpdateTime = time; + mFilenameDictionaryTimeRecorder.mLastUpdateTime = time; writeBinaryDictionary(); loadBinaryDictionary(); } - mLocalDictionaryController.mLastUpdateTime = time; - } finally { - mLocalDictionaryController.writeLock().unlock(); + mPerInstanceDictionaryTimeRecorder.mLastUpdateTime = time; } - } finally { - mSharedDictionaryController.writeLock().unlock(); - } + }); } // TODO: cache the file's existence so that we avoid doing a disk access each time. @@ -500,83 +523,36 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Thread class for asynchronously reloading and rewriting the binary dictionary. - */ - private class AsyncReloadDictionaryTask extends Thread { - @Override - public void run() { - syncReloadDictionaryInternal(); - } - } - - /** * Load the dictionary to memory. */ protected void asyncLoadDictionaryToMemory() { - new AsyncLoadDictionaryToMemoryTask().start(); - } - - /** - * Thread class for asynchronously loading dictionary to memory. - */ - private class AsyncLoadDictionaryToMemoryTask extends Thread { - @Override - public void run() { - mSharedDictionaryController.readLock().lock(); - try { - mLocalDictionaryController.writeLock().lock(); - try { - loadDictionaryAsync(); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } - } finally { - mSharedDictionaryController.readLock().unlock(); + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + loadDictionaryAsync(); } - } + }); } /** * Generate binary dictionary using DictionaryWriter. */ protected void asyncWriteBinaryDictionary() { - final AsyncWriteBinaryDictionaryTask newTask = new AsyncWriteBinaryDictionaryTask(); - newTask.start(); - final AsyncWriteBinaryDictionaryTask oldTask = mWaitingTask.getAndSet(newTask); - if (oldTask != null) { - oldTask.interrupt(); - } - } - - /** - * Thread class for asynchronously writing the binary dictionary. - */ - private class AsyncWriteBinaryDictionaryTask extends Thread { - @Override - public void run() { - mSharedDictionaryController.writeLock().lock(); - try { - mLocalDictionaryController.writeLock().lock(); - try { - if (isInterrupted()) { - return; - } - writeBinaryDictionary(); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } - } finally { - mSharedDictionaryController.writeLock().unlock(); + final Runnable newTask = new Runnable() { + @Override + public void run() { + writeBinaryDictionary(); } - } + }; + final Runnable oldTask = mUnfinishedFlushingTask.getAndSet(newTask); + getExecutor(mFilename).replaceAndExecute(oldTask, newTask); } /** - * Lock for controlling access to a given binary dictionary and for tracking whether the - * dictionary is out of date. Can be shared across multiple dictionary instances that access the - * same filename. + * Time recorder for tracking whether the dictionary is out of date. + * Can be shared across multiple dictionary instances that access the same filename. */ - private static class DictionaryController extends ReentrantReadWriteLock { + private static class DictionaryTimeRecorder { private volatile long mLastUpdateTime = 0; private volatile long mLastUpdateRequestTime = 0; @@ -591,12 +567,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @UsedForTesting protected void addWordDynamicallyForTests(final String word, final String shortcutTarget, final int frequency, final boolean isNotAWord) { - mLocalDictionaryController.writeLock().lock(); - try { - addWordDynamically(word, shortcutTarget, frequency, isNotAWord); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + addWordDynamically(word, shortcutTarget, frequency, isNotAWord); + } + }); } /** @@ -605,12 +581,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { @UsedForTesting protected void addBigramDynamicallyForTests(final String word0, final String word1, final int frequency, final boolean isValid) { - mLocalDictionaryController.writeLock().lock(); - try { - addBigramDynamically(word0, word1, frequency, isValid); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + addBigramDynamically(word0, word1, frequency, isValid); + } + }); } /** @@ -618,42 +594,27 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ @UsedForTesting protected void removeBigramDynamicallyForTests(final String word0, final String word1) { - mLocalDictionaryController.writeLock().lock(); - try { - removeBigramDynamically(word0, word1); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + removeBigramDynamically(word0, word1); + } + }); } // TODO: Implement native binary methods once the dynamic dictionary implementation is done. @UsedForTesting public boolean isInDictionaryForTests(final String word) { - mLocalDictionaryController.writeLock().lock(); - try { - if (mDictType == Dictionary.TYPE_USER_HISTORY) { - return ((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) - .isInDictionaryForTests(word); + final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); + getExecutor(mFilename).executePrioritized(new Runnable() { + @Override + public void run() { + if (mDictType == Dictionary.TYPE_USER_HISTORY) { + holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) + .isInDictionaryForTests(word)); + } } - } finally { - mLocalDictionaryController.writeLock().unlock(); - } - return false; - } - - // TODO: Remove and use addToPersonalizationPredictionDictionary instead!!!!!!!!!!!!!!!! - @UsedForTesting - public void forceAddWordForTest( - final String word0, final String word1, final boolean isValid) { - mLocalDictionaryController.writeLock().lock(); - try { - mDictionaryWriter.addUnigramWord(word1, null /* the "shortcut" parameter is null */, - DynamicPredictionDictionaryBase.FREQUENCY_FOR_TYPED, false /* isNotAWord */); - mDictionaryWriter.addBigramWords(word0, word1, - DynamicPredictionDictionaryBase.FREQUENCY_FOR_TYPED, isValid, - 0 /* lastTouchedTime */); - } finally { - mLocalDictionaryController.writeLock().unlock(); - } + }); + return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 3d29c5a0b..6c83ac7ed 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -86,6 +86,7 @@ import com.android.inputmethod.latin.settings.SettingsActivity; import com.android.inputmethod.latin.settings.SettingsValues; import com.android.inputmethod.latin.suggestions.SuggestionStripView; import com.android.inputmethod.latin.utils.ApplicationUtils; +import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.AutoCorrectionUtils; import com.android.inputmethod.latin.utils.CapsModeUtils; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -107,8 +108,6 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Locale; import java.util.TreeSet; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; /** * Input method implementation for Qwerty'ish keyboard. @@ -1668,8 +1667,15 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return didAutoCorrect; } - // Called from the end of onTextInput - private void completeOnTextInput(final String rawText) { + // Called from PointerTracker through the KeyboardActionListener interface + @Override + public void onTextInput(final String rawText) { + mConnection.beginBatchEdit(); + if (mWordComposer.isComposingWord()) { + commitCurrentAutoCorrection(rawText); + } else { + resetComposingState(true /* alsoResetLastComposedWord */); + } mHandler.postUpdateSuggestionStrip(); if (ProductionFlag.USES_DEVELOPMENT_ONLY_DIAGNOSTICS && ResearchLogger.RESEARCH_KEY_OUTPUT_TEXT.equals(rawText)) { @@ -1692,44 +1698,12 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mEnteredText = text; } - // Called from PointerTracker through the KeyboardActionListener interface - @Override - public void onTextInput(final String rawText) { - mConnection.beginBatchEdit(); - boolean isReturningAsynchronously = false; - if (mWordComposer.isComposingWord()) { - commitCurrentAutoCorrection(rawText, new Runnable() { - @Override - public void run() { - completeOnTextInput(rawText); - } - }); - isReturningAsynchronously = true; - } else { - resetComposingState(true /* alsoResetLastComposedWord */); - } - if (!isReturningAsynchronously) { - completeOnTextInput(rawText); - } - } - - private void completeOnStartBatchInput(final SettingsValues settingsValues) { - final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); - if (Character.isLetterOrDigit(codePointBeforeCursor) - || settingsValues.isUsuallyFollowedBySpace(codePointBeforeCursor)) { - mSpaceState = SPACE_STATE_PHANTOM; - } - mConnection.endBatchEdit(); - mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); - } - @Override public void onStartBatchInput() { mInputUpdater.onStartBatchInput(); mHandler.cancelUpdateSuggestionStrip(); mConnection.beginBatchEdit(); final SettingsValues settingsValues = mSettings.getCurrent(); - boolean isReturningAsynchronously = false; if (mWordComposer.isComposingWord()) { if (settingsValues.mIsInternal) { if (mWordComposer.isBatchMode()) { @@ -1751,21 +1725,19 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // tapping probably is that the word you intend to type is not in the dictionary, // so we do not attempt to correct, on the assumption that if that was a dictionary // word, the user would probably have gestured instead. - commitCurrentAutoCorrection(LastComposedWord.NOT_A_SEPARATOR, new Runnable() { - @Override - public void run() { - completeOnStartBatchInput(settingsValues); - } - }); - isReturningAsynchronously = true; + commitCurrentAutoCorrection(LastComposedWord.NOT_A_SEPARATOR); } else { commitTyped(LastComposedWord.NOT_A_SEPARATOR); } mExpectingUpdateSelection = true; } - if (!isReturningAsynchronously) { - completeOnStartBatchInput(settingsValues); + final int codePointBeforeCursor = mConnection.getCodePointBeforeCursor(); + if (Character.isLetterOrDigit(codePointBeforeCursor) + || settingsValues.isUsuallyFollowedBySpace(codePointBeforeCursor)) { + mSpaceState = SPACE_STATE_PHANTOM; } + mConnection.endBatchEdit(); + mWordComposer.setCapitalizedModeAtStartComposingTime(getActualCapsMode()); } private static final class InputUpdater implements Handler.Callback { @@ -2245,9 +2217,30 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mKeyboardSwitcher.updateShiftState(); } - private void completeHandleSeparator(final int primaryCode, final int x, final int y, - final int spaceState, final SettingsValues currentSettings, - final boolean shouldAvoidSendingCode) { + // Returns true if we do an autocorrection, false otherwise. + private boolean handleSeparator(final int primaryCode, final int x, final int y, + final int spaceState) { + boolean didAutoCorrect = false; + final SettingsValues currentSettings = mSettings.getCurrent(); + // We avoid sending spaces in languages without spaces if we were composing. + final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == primaryCode + && !currentSettings.mCurrentLanguageHasSpaces && mWordComposer.isComposingWord(); + if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { + // If we are in the middle of a recorrection, we need to commit the recorrection + // first so that we can insert the separator at the current cursor position. + resetEntireInputState(mLastSelectionStart); + } + if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing + if (currentSettings.mCorrectionEnabled) { + final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR + : new String(new int[] { primaryCode }, 0, 1); + commitCurrentAutoCorrection(separator); + didAutoCorrect = true; + } else { + commitTyped(new String(new int[]{primaryCode}, 0, 1)); + } + } + final boolean swapWeakSpace = maybeStripSpace(primaryCode, spaceState, Constants.SUGGESTION_STRIP_COORDINATE == x); @@ -2302,44 +2295,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } mKeyboardSwitcher.updateShiftState(); - } - - // Returns true if we do an autocorrection, false otherwise. - private boolean handleSeparator(final int primaryCode, final int x, final int y, - final int spaceState) { - boolean doesAutoCorrect = false; - final SettingsValues currentSettings = mSettings.getCurrent(); - // We avoid sending spaces in languages without spaces if we were composing. - final boolean shouldAvoidSendingCode = Constants.CODE_SPACE == primaryCode - && !currentSettings.mCurrentLanguageHasSpaces && mWordComposer.isComposingWord(); - if (mWordComposer.isCursorFrontOrMiddleOfComposingWord()) { - // If we are in the middle of a recorrection, we need to commit the recorrection - // first so that we can insert the separator at the current cursor position. - resetEntireInputState(mLastSelectionStart); - } - boolean isReturningAsynchronously = false; - if (mWordComposer.isComposingWord()) { // May have changed since we stored wasComposing - if (currentSettings.mCorrectionEnabled) { - final String separator = shouldAvoidSendingCode ? LastComposedWord.NOT_A_SEPARATOR - : new String(new int[] { primaryCode }, 0, 1); - commitCurrentAutoCorrection(separator, new Runnable() { - @Override - public void run() { - completeHandleSeparator(primaryCode, x, y, spaceState, currentSettings, - shouldAvoidSendingCode); - } - }); - doesAutoCorrect = true; - isReturningAsynchronously = true; - } else { - commitTyped(new String(new int[]{primaryCode}, 0, 1)); - } - } - if (!isReturningAsynchronously) { - completeHandleSeparator(primaryCode, x, y, spaceState, currentSettings, - shouldAvoidSendingCode); - } - return doesAutoCorrect; + return didAutoCorrect; } private CharSequence getTextWithUnderline(final String text) { @@ -2428,31 +2384,20 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen return; } - final CountDownLatch latch = new CountDownLatch(1); - final SuggestedWords[] suggestedWordsArray = new SuggestedWords[1]; + final AsyncResultHolder<SuggestedWords> holder = new AsyncResultHolder<SuggestedWords>(); getSuggestedWordsOrOlderSuggestionsAsync(Suggest.SESSION_TYPING, new OnGetSuggestedWordsCallback() { @Override public void onGetSuggestedWords(final SuggestedWords suggestedWords) { - suggestedWordsArray[0] = suggestedWords; - latch.countDown(); + holder.set(suggestedWords); } } ); - // TODO: Quit blocking the main thread. - try { - // Wait for the result of getSuggestedWords - // We set the time out to avoid ANR. - latch.await(GET_SUGGESTED_WORDS_TIMEOUT, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - // TODO: Cancel all pending "getSuggestedWords" tasks when it failed. We may want to add - // "onGetSuggestionFailed" to "OnGetSuggestedWordsCallback". - Log.e(TAG, "InterruptedException while waiting for getSuggestedWords.", e); - return; - } - if (suggestedWordsArray[0] != null) { - showSuggestionStrip(suggestedWordsArray[0]); + // This line may cause the current thread to wait. + final SuggestedWords suggestedWords = holder.get(null, GET_SUGGESTED_WORDS_TIMEOUT); + if (suggestedWords != null) { + showSuggestionStrip(suggestedWords); } } @@ -2555,7 +2500,11 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen setSuggestionStripShown(isSuggestionsStripVisible()); } - private void completeCommitCurrentAutoCorrection(final String separator) { + private void commitCurrentAutoCorrection(final String separator) { + // Complete any pending suggestions query first + if (mHandler.hasPendingUpdateSuggestions()) { + updateSuggestionStrip(); + } final String typedAutoCorrection = mWordComposer.getAutoCorrectionOrNull(); final String typedWord = mWordComposer.getTypedWord(); final String autoCorrection = (typedAutoCorrection != null) @@ -2591,22 +2540,6 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } } - private void commitCurrentAutoCorrection(final String separator, final Runnable callback) { - getSuggestedWordsOrOlderSuggestionsAsync(Suggest.SESSION_TYPING, - new OnGetSuggestedWordsCallback() { - @Override - public void onGetSuggestedWords(final SuggestedWords suggestedWords) { - if (suggestedWords != null) { - setAutoCorrection(suggestedWords); - } - completeCommitCurrentAutoCorrection(separator); - if (callback != null) { - callback.run(); - } - } - }); - } - // Called from {@link SuggestionStripView} through the {@link SuggestionStripView#Listener} // interface @Override diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java index 67ef538ac..3213c92c7 100644 --- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedContactsBinaryDictionary.java @@ -35,14 +35,14 @@ public final class SynchronouslyLoadedContactsBinaryDictionary extends ContactsB public synchronized ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer codes, final String prevWordForBigrams, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { - syncReloadDictionaryIfRequired(); + reloadDictionaryIfRequired(); return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, blockOffensiveWords, additionalFeaturesOptions); } @Override public synchronized boolean isValidWord(final String word) { - syncReloadDictionaryIfRequired(); + reloadDictionaryIfRequired(); return isValidWordInner(word); } diff --git a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java index bea522320..6405b5e46 100644 --- a/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/SynchronouslyLoadedUserBinaryDictionary.java @@ -38,14 +38,14 @@ public final class SynchronouslyLoadedUserBinaryDictionary extends UserBinaryDic public synchronized ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer codes, final String prevWordForBigrams, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions) { - syncReloadDictionaryIfRequired(); + reloadDictionaryIfRequired(); return super.getSuggestions(codes, prevWordForBigrams, proximityInfo, blockOffensiveWords, additionalFeaturesOptions); } @Override public synchronized boolean isValidWord(final String word) { - syncReloadDictionaryIfRequired(); + reloadDictionaryIfRequired(); return isValidWordInner(word); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 9385ba3a0..3f26ff378 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -68,6 +68,12 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion > 3) { + throw new UnsupportedFormatException( + "The given format options has wrong version number : " + + formatOptions.mVersion); + } + if (mOutStream == null) { openStream(); } diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java index be4c7c42d..5b1d0647b 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java @@ -69,7 +69,7 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi mPrefs = sp; if (mLocale != null && mLocale.length() > 1) { asyncLoadDictionaryToMemory(); - asyncReloadDictionaryIfRequired(); + reloadDictionaryIfRequired(); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java index c8deaf90d..5f702ee3f 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java @@ -52,7 +52,7 @@ public class PersonalizationHelper { if (DEBUG) { Log.w(TAG, "Use cached UserHistoryPredictionDictionary for " + locale); } - dict.asyncReloadDictionaryIfRequired(); + dict.reloadDictionaryIfRequired(); return dict; } } diff --git a/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java b/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java new file mode 100644 index 000000000..c2e97a36f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/AsyncResultHolder.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * This class is a holder of a result of asynchronous computation. + * + * @param <E> the type of the result. + */ +public class AsyncResultHolder<E> { + + private final Object mLock = new Object(); + + private E mResult; + private final CountDownLatch mLatch; + + public AsyncResultHolder() { + mLatch = new CountDownLatch(1); + } + + /** + * Sets the result value to this holder. + * + * @param result the value which is set. + */ + public void set(final E result) { + synchronized(mLock) { + if (mLatch.getCount() > 0) { + mResult = result; + mLatch.countDown(); + } + } + } + + /** + * Gets the result value held in this holder. + * Causes the current thread to wait unless the value is set or the specified time is elapsed. + * + * @param defaultValue the default value. + * @param timeOut the time to wait. + * @return if the result is set until the time limit then the result, otherwise defaultValue. + */ + public E get(final E defaultValue, final long timeOut) { + try { + if(mLatch.await(timeOut, TimeUnit.MILLISECONDS)) { + return mResult; + } else { + return defaultValue; + } + } catch (InterruptedException e) { + return defaultValue; + } + } +} diff --git a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java new file mode 100644 index 000000000..3c1db6529 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java @@ -0,0 +1,126 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import java.util.ArrayDeque; +import java.util.Queue; + +/** + * An object that executes submitted tasks using a thread. + */ +public class PrioritizedSerialExecutor { + public static final String TAG = PrioritizedSerialExecutor.class.getSimpleName(); + + private final Object mLock = new Object(); + + // The default value of capacities of task queues. + private static final int TASK_QUEUE_CAPACITY = 1000; + private final Queue<Runnable> mTasks; + private final Queue<Runnable> mPrioritizedTasks; + + // The task which is running now. + private Runnable mActive; + + public PrioritizedSerialExecutor() { + mTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); + mPrioritizedTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); + } + + /** + * Clears all queued tasks. + */ + public void clearAllTasks() { + synchronized(mLock) { + mTasks.clear(); + mPrioritizedTasks.clear(); + } + } + + /** + * Enqueues the given task into the task queue. + * @param r the enqueued task + */ + public void execute(final Runnable r) { + synchronized(mLock) { + mTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } + } + } + + /** + * Enqueues the given task into the prioritized task queue. + * @param r the enqueued task + */ + public void executePrioritized(final Runnable r) { + synchronized(mLock) { + mPrioritizedTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } + } + } + + private boolean fetchNextTasks() { + synchronized(mLock) { + mActive = mPrioritizedTasks.poll(); + if (mActive == null) { + mActive = mTasks.poll(); + } + return mActive != null; + } + } + + private void scheduleNext() { + synchronized(mLock) { + if (!fetchNextTasks()) { + return; + } + new Thread(new Runnable() { + @Override + public void run() { + try { + do { + synchronized(mLock) { + if (mActive != null) { + mActive.run(); + } + } + } while (fetchNextTasks()); + } finally { + scheduleNext(); + } + } + }).start(); + } + } + + public void remove(final Runnable r) { + synchronized(mLock) { + mTasks.remove(r); + mPrioritizedTasks.remove(r); + } + } + + public void replaceAndExecute(final Runnable oldTask, final Runnable newTask) { + synchronized(mLock) { + if (oldTask != null) remove(oldTask); + execute(newTask); + } + } +} diff --git a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp index 86c2394d1..8da1859c4 100644 --- a/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +++ b/native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp @@ -204,6 +204,7 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, } jsize wordLength = env->GetArrayLength(word); int codePoints[wordLength]; + env->GetIntArrayRegion(word, 0, wordLength, codePoints); dictionary->addUnigramWord(codePoints, wordLength, probability); } @@ -215,8 +216,10 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz, } jsize word0Length = env->GetArrayLength(word0); int word0CodePoints[word0Length]; + env->GetIntArrayRegion(word0, 0, word0Length, word0CodePoints); jsize word1Length = env->GetArrayLength(word1); int word1CodePoints[word1Length]; + env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length, probability); } @@ -229,8 +232,10 @@ static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass claz } jsize word0Length = env->GetArrayLength(word0); int word0CodePoints[word0Length]; + env->GetIntArrayRegion(word0, 0, word0Length, word0CodePoints); jsize word1Length = env->GetArrayLength(word1); int word1CodePoints[word1Length]; + env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints); dictionary->removeBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp index e31a91069..936dc9c5d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp @@ -20,12 +20,13 @@ namespace latinime { bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); } BigramListReadWriteUtils::BigramFlags flags; do { + // The buffer address can be changed after calling buffer writing methods. + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, fromPos); int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer( buffer, flags, fromPos); @@ -63,7 +64,6 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int probability, int *const pos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { *pos -= mBuffer->getOriginalBufferSize(); } @@ -73,6 +73,8 @@ bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int prob if (usesAdditionalBuffer) { entryPos += mBuffer->getOriginalBufferSize(); } + // The buffer address can be changed after calling buffer writing methods. + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos); BigramListReadWriteUtils::getBigramAddressAndForwardPointer(buffer, flags, pos); if (BigramListReadWriteUtils::hasNext(flags)) { @@ -118,13 +120,14 @@ bool DynamicBigramListPolicy::addBigramEntry(const int bigramPos, const int prob bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int targetBigramPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(bigramListPos); - const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); int pos = bigramListPos; if (usesAdditionalBuffer) { pos -= mBuffer->getOriginalBufferSize(); } BigramListReadWriteUtils::BigramFlags flags; do { + // The buffer address can be changed after calling buffer writing methods. + const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, &pos); int bigramOffsetFieldPos = pos; if (usesAdditionalBuffer) { @@ -139,8 +142,7 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta continue; } // Target entry is found. Write 0 into the bigram pos field to mark the bigram invalid. - const int bigramOffsetFieldSize = - BigramListReadWriteUtils::attributeAddressSize(flags); + const int bigramOffsetFieldSize = BigramListReadWriteUtils::attributeAddressSize(flags); if (!mBuffer->writeUintAndAdvancePosition(0 /* data */, bigramOffsetFieldSize, &bigramOffsetFieldPos)) { return false; diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp index 405628b30..5674cb48e 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp @@ -34,7 +34,7 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos); const int parentPos = DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictBuf, &pos); - mParentPos = (parentPos != 0) ? mNodePos + parentPos : NOT_A_DICT_POS; + mParentPos = (parentPos != 0) ? nodePos + parentPos : NOT_A_DICT_POS; if (outCodePoints != 0) { mCodePointCount = PatriciaTrieReadingUtils::getCharsAndAdvancePosition( dictBuf, mFlags, maxCodePointCount, outCodePoints, &pos); diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h index 62d73bb02..2e604a202 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h @@ -59,9 +59,9 @@ class DynamicPatriciaTrieReadingUtils { static AK_FORCE_INLINE NodeFlags updateAndGetFlags(const NodeFlags originalFlags, const bool isMoved, const bool isDeleted) { NodeFlags flags = originalFlags; - flags = isMoved ? ((flags & (!MASK_MOVED)) | FLAG_IS_MOVED) : flags; - flags = isDeleted ? ((flags & (!MASK_MOVED)) | FLAG_IS_DELETED) : flags; - flags = (!isMoved && !isDeleted) ? ((flags & (!MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags; + flags = isMoved ? ((flags & (~MASK_MOVED)) | FLAG_IS_MOVED) : flags; + flags = isDeleted ? ((flags & (~MASK_MOVED)) | FLAG_IS_DELETED) : flags; + flags = (!isMoved && !isDeleted) ? ((flags & (~MASK_MOVED)) | FLAG_IS_NOT_MOVED) : flags; return flags; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp index e24421219..7c0b6286c 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp @@ -26,13 +26,10 @@ namespace latinime { -// TODO: Enable dynamic update and remove this flag. -const bool DynamicPatriciaTrieWritingHelper::ENABLE_DYNAMIC_UPDATE = false; - bool DynamicPatriciaTrieWritingHelper::addUnigramWord( DynamicPatriciaTrieReadingHelper *const readingHelper, const int *const wordCodePoints, const int codePointCount, const int probability) { - int parentPos = NOT_A_VALID_WORD_POS; + int parentPos = NOT_A_DICT_POS; while (!readingHelper->isEnd()) { const int matchedCodePointCount = readingHelper->getPrevTotalCodePointCount(); if (!readingHelper->isMatchedCodePoint(0 /* index */, @@ -47,33 +44,23 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( const int nodeCodePointCount = nodeReader->getCodePointCount(); for (int j = 1; j < nodeCodePointCount; ++j) { const int nextIndex = matchedCodePointCount + j; - if (nextIndex >= codePointCount) { - // TODO: split current node after j - 1, create child and make this terminal. - return false; - } - if (!readingHelper->isMatchedCodePoint(j, + if (nextIndex >= codePointCount || !readingHelper->isMatchedCodePoint(j, wordCodePoints[matchedCodePointCount + j])) { - // TODO: split current node after j - 1 and create two children. - return false; + return reallocatePtNodeAndAddNewPtNodes(nodeReader, + readingHelper->getMergedNodeCodePoints(), j, probability, + wordCodePoints + matchedCodePointCount, + codePointCount - matchedCodePointCount); } } // All characters are matched. if (codePointCount == readingHelper->getTotalCodePointCount()) { - if (ENABLE_DYNAMIC_UPDATE) { - return setPtNodeProbability(nodeReader, probability, - readingHelper->getMergedNodeCodePoints()); - } else { - return false; - } + return setPtNodeProbability(nodeReader, probability, + readingHelper->getMergedNodeCodePoints()); } if (!nodeReader->hasChildren()) { - if (ENABLE_DYNAMIC_UPDATE) { - return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability, - wordCodePoints + readingHelper->getTotalCodePointCount(), - codePointCount - readingHelper->getTotalCodePointCount()); - } else { - return false; - } + return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability, + wordCodePoints + readingHelper->getTotalCodePointCount(), + codePointCount - readingHelper->getTotalCodePointCount()); } // Advance to the children nodes. parentPos = nodeReader->getNodePos(); @@ -84,14 +71,10 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( return false; } int pos = readingHelper->getPosOfLastForwardLinkField(); - if (ENABLE_DYNAMIC_UPDATE) { - return createAndInsertNodeIntoPtNodeArray(parentPos, - wordCodePoints + readingHelper->getPrevTotalCodePointCount(), - codePointCount - readingHelper->getPrevTotalCodePointCount(), - probability, &pos); - } else { - return false; - } + return createAndInsertNodeIntoPtNodeArray(parentPos, + wordCodePoints + readingHelper->getPrevTotalCodePointCount(), + codePointCount - readingHelper->getPrevTotalCodePointCount(), + probability, &pos); } bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const int word1Pos, @@ -136,20 +119,22 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( &writingPos)) { return false; } - // Update moved position, which is stored in the parent position field. - if (!DynamicPatriciaTrieWritingUtils::writeParentPositionAndAdvancePosition( - mBuffer, movedPos, &writingPos)) { + // Update moved position, which is stored in the parent offset field. + const int movedPosOffset = movedPos - originalNode->getNodePos(); + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( + mBuffer, movedPosOffset, &writingPos)) { return false; } return true; } -// Write new node at writingPos. -bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlacklisted, +// Write new PtNode at writingPos. +bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos) { + const int nodePos = *writingPos; // Create node flags and write them. const PatriciaTrieReadingUtils::NodeFlags nodeFlags = PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord, @@ -160,9 +145,10 @@ bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlackliste writingPos)) { return false; } - // Write parent position - if (!DynamicPatriciaTrieWritingUtils::writeParentPositionAndAdvancePosition(mBuffer, parentPos, - writingPos)) { + // Calculate a parent offset and write the offset. + const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; + if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBuffer, + parentOffset, writingPos)) { return false; } // Write code points @@ -186,7 +172,9 @@ bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlackliste // Copy shortcut list when the originalShortcutListPos is valid dictionary position. if (originalShortcutListPos != NOT_A_DICT_POS) { int fromPos = originalShortcutListPos; - mShortcutPolicy->copyAllShortcuts(&fromPos, writingPos); + if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(&fromPos, writingPos)) { + return false; + } } // Copy bigram list when the originalBigramListPos is valid dictionary position. if (originalBigramListPos != NOT_A_DICT_POS) { @@ -198,6 +186,25 @@ bool DynamicPatriciaTrieWritingHelper::writeNodeToBuffer(const bool isBlackliste return true; } +bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(const int parentPos, + const int *const codePoints, const int codePointCount, const int probability, + int *const writingPos) { + return writePtNodeWithFullInfoToBuffer(false /* isBlacklisted */, false /* isNotAWord */, + parentPos, codePoints, codePointCount, probability, + NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, + NOT_A_DICT_POS /* originalShortcutPos */, writingPos); +} + +bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( + const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, + const int *const codePoints, const int codePointCount, const int probability, + int *const writingPos) { + return writePtNodeWithFullInfoToBuffer(originalNode->isBlacklisted(), + originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, + originalNode->getChildrenPos(), originalNode->getBigramsPos(), + originalNode->getShortcutPos(), writingPos); +} + bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos) { @@ -226,10 +233,8 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos)) { return false; } - if (!writeNodeToBuffer(originalPtNode->isBlacklisted(), originalPtNode->isNotAWord(), - originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(), - probability, originalPtNode->getChildrenPos(), originalPtNode->getBigramsPos(), - originalPtNode->getShortcutPos(), &movedPos)) { + if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), + codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { return false; } } @@ -257,9 +262,7 @@ bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( 1 /* arraySize */, &writingPos)) { return false; } - if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPtNodePos, - nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */, - NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, + if (!writePtNodeToBuffer(parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, &writingPos)) { return false; } @@ -270,4 +273,69 @@ bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( return true; } +// Returns whether the dictionary updating was succeeded or not. +bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( + const DynamicPatriciaTrieNodeReader *const reallocatingPtNode, + const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, + const int probabilityOfNewPtNode, const int *const newNodeCodePoints, + const int newNodeCodePointCount) { + // When addsExtraChild is true, split the reallocating PtNode and add new child. + // Reallocating PtNode: abcde, newNode: abcxy. + // abc (1st, not terminal) __ de (2nd) + // \_ xy (extra child, terminal) + // Otherwise, this method makes 1st part terminal and write probabilityOfNewPtNode. + // Reallocating PtNode: abcde, newNode: abc. + // abc (1st, terminal) __ de (2nd) + const bool addsExtraChild = newNodeCodePointCount > overlappingCodePointCount; + const int firstPtNodePos = mBuffer->getTailPosition(); + if (!markNodeAsMovedAndSetPosition(reallocatingPtNode, firstPtNodePos)) { + return false; + } + int writingPos = firstPtNodePos; + // Write the 1st part of the reallocating node. The children position will be updated later + // with actual children position. + const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; + if (!writePtNodeToBuffer(reallocatingPtNode->getParentPos(), reallocatingPtNodeCodePoints, + overlappingCodePointCount, newProbability, &writingPos)) { + return false; + } + const int actualChildrenPos = writingPos; + // Create new children PtNode array. + const size_t newPtNodeCount = addsExtraChild ? 2 : 1; + if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, + newPtNodeCount, &writingPos)) { + return false; + } + // Write the 2nd part of the reallocating node. + if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, + reallocatingPtNode->getNodePos(), + reallocatingPtNodeCodePoints + overlappingCodePointCount, + reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, + reallocatingPtNode->getProbability(), &writingPos)) { + return false; + } + if (addsExtraChild) { + if (!writePtNodeToBuffer(reallocatingPtNode->getNodePos(), + newNodeCodePoints + overlappingCodePointCount, + newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, + &writingPos)) { + return false; + } + } + if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, + NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { + return false; + } + // Load node info. Information of the 1st part will be fetched. + DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy); + nodeReader.fetchNodeInfoFromBuffer(firstPtNodePos); + // Update children position. + int childrenPosFieldPos = nodeReader.getChildrenPosFieldPos(); + if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, + actualChildrenPos, &childrenPosFieldPos)) { + return false; + } + return true; +} + } // namespace latinime diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h index 16b84bac3..ada634a54 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h @@ -49,7 +49,6 @@ class DynamicPatriciaTrieWritingHelper { private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); - static const bool ENABLE_DYNAMIC_UPDATE; BufferWithExtendableBuffer *const mBuffer; DynamicBigramListPolicy *const mBigramPolicy; DynamicShortcutListPolicy *const mShortcutPolicy; @@ -57,11 +56,19 @@ class DynamicPatriciaTrieWritingHelper { bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos); - bool writeNodeToBuffer(const bool isBlacklisted, const bool isNotAWord, const int parentPos, - const int *const codePoints, const int codePointCount, const int probability, - const int childrenPos, const int originalBigramListPos, + bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, + const int parentPos, const int *const codePoints, const int codePointCount, + const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos); + bool writePtNodeToBuffer(const int parentPos, const int *const codePoints, + const int codePointCount, const int probability, int *const writingPos); + + bool writePtNodeToBufferByCopyingPtNodeInfo( + const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, + const int *const codePoints, const int codePointCount, const int probability, + int *const writingPos); + bool createAndInsertNodeIntoPtNodeArray(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability, int *const forwardLinkFieldPos); @@ -74,6 +81,12 @@ class DynamicPatriciaTrieWritingHelper { bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability); + + bool reallocatePtNodeAndAddNewPtNodes( + const DynamicPatriciaTrieNodeReader *const reallocatingPtNode, + const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, + const int probabilityOfNewPtNode, const int *const newNodeCodePoints, + const int newNodeCodePointCount); }; } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp index 4187504b4..b261e594d 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp @@ -68,11 +68,11 @@ const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1; return buffer->writeUintAndAdvancePosition(nodeFlags, NODE_FLAG_FIELD_SIZE, nodeFlagsFieldPos); } -/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentPositionAndAdvancePosition( - BufferWithExtendableBuffer *const buffer, const int parentPosition, +// Note that parentOffset is offset from node's head position. +/* static */ bool DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition( + BufferWithExtendableBuffer *const buffer, const int parentOffset, int *const parentPosFieldPos) { - // Note that parentPosition is offset from node's head position. - int offset = (parentPosition != NOT_A_DICT_POS) ? parentPosition : 0; + int offset = (parentOffset != NOT_A_DICT_POS) ? parentOffset : 0; return writeDictOffset(buffer, offset, parentPosFieldPos); } diff --git a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h index 801042ddf..183ede444 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h +++ b/native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h @@ -39,7 +39,7 @@ class DynamicPatriciaTrieWritingUtils { const DynamicPatriciaTrieReadingUtils::NodeFlags nodeFlags, int *const nodeFlagsFieldPos); - static bool writeParentPositionAndAdvancePosition(BufferWithExtendableBuffer *const buffer, + static bool writeParentOffsetAndAdvancePosition(BufferWithExtendableBuffer *const buffer, const int parentPosition, int *const parentPosFieldPos); static bool writeCodePointsAndAdvancePosition(BufferWithExtendableBuffer *const buffer, diff --git a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h index 5e9c52950..1803c09cb 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h +++ b/native/jni/src/suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h @@ -83,8 +83,8 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { } // Copy shortcuts from the shortcut list that starts at fromPos to toPos and advance these - // positions after the shortcut lists. - void copyAllShortcuts(int *const fromPos, int *const toPos) { + // positions after the shortcut lists. This returns whether the copy was succeeded or not. + bool copyAllShortcutsAndReturnIfSucceededOrNot(int *const fromPos, int *const toPos) { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer); if (usesAdditionalBuffer) { @@ -93,16 +93,23 @@ class DynamicShortcutListPolicy : public DictionaryShortcutsStructurePolicy { const int shortcutListSize = ShortcutListReadingUtils ::getShortcutListSizeAndForwardPointer(buffer, fromPos); // Copy shortcut list size. - mBuffer->writeUintAndAdvancePosition( + if (!mBuffer->writeUintAndAdvancePosition( shortcutListSize + ShortcutListReadingUtils::getShortcutListSizeFieldSize(), - ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos); + ShortcutListReadingUtils::getShortcutListSizeFieldSize(), toPos)) { + return false; + } + // Copy shortcut list. for (int i = 0; i < shortcutListSize; ++i) { - const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition(buffer, fromPos); - mBuffer->writeUintAndAdvancePosition(data, 1 /* size */, toPos); + const uint8_t data = ByteArrayUtils::readUint8AndAdvancePosition( + mBuffer->getBuffer(usesAdditionalBuffer), fromPos); + if (!mBuffer->writeUintAndAdvancePosition(data, 1 /* size */, toPos)) { + return false; + } } if (usesAdditionalBuffer) { *fromPos += mBuffer->getOriginalBufferSize(); } + return true; } private: diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp index 6326754c2..0fed275e9 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.cpp @@ -66,16 +66,17 @@ bool BufferWithExtendableBuffer::writeCodePointsAndAdvancePosition(const int *co bool BufferWithExtendableBuffer::checkAndPrepareWriting(const int pos, const int size) { if (isInAdditionalBuffer(pos)) { - if (pos == mUsedAdditionalBufferSize) { + const int tailPosition = getTailPosition(); + if (pos == tailPosition) { // Append data to the tail. - if (pos + size > static_cast<int>(mAdditionalBuffer.size())) { + if (pos + size > static_cast<int>(mAdditionalBuffer.size()) + mOriginalBufferSize) { // Need to extend buffer. if (!extendBuffer()) { return false; } } mUsedAdditionalBufferSize += size; - } else if (pos + size >= mUsedAdditionalBufferSize) { + } else if (pos + size > tailPosition) { // The access will beyond the tail of used region. return false; } diff --git a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h index b35b47d7a..c6a484131 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h +++ b/native/jni/src/suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h @@ -47,6 +47,7 @@ class BufferWithExtendableBuffer { return position >= mOriginalBufferSize; } + // TODO: Resolve the issue that the address can be changed when the vector is resized. // CAVEAT!: Be careful about array out of bound access with buffers AK_FORCE_INLINE const uint8_t *getBuffer(const bool usesAdditionalBuffer) const { if (usesAdditionalBuffer) { diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java new file mode 100644 index 000000000..bf4954b34 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.LargeTest; + +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.FusionDictionary; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; + +@LargeTest +public class BinaryDictionaryTests extends AndroidTestCase { + private static final FormatSpec.FormatOptions FORMAT_OPTIONS = + new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */); + private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final String TEST_LOCALE = "test"; + + @Override + protected void setUp() throws Exception { + super.setUp(); + } + + @Override + protected void tearDown() throws Exception { + super.tearDown(); + } + + private File createEmptyDictionaryAndGetFile(final String filename) throws IOException, + UnsupportedFormatException { + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), + new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + final File file = File.createTempFile(filename, TEST_DICT_FILE_EXTENSION, + getContext().getCacheDir()); + final DictEncoder dictEncoder = new Ver3DictEncoder(file); + dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + return file; + } + + public void testIsValidDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + assertTrue("binaryDictionary must be valid for existing valid dictionary file.", + binaryDictionary.isValidDictionary()); + binaryDictionary.close(); + assertFalse("binaryDictionary must be invalid after closing.", + binaryDictionary.isValidDictionary()); + dictFile.delete(); + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, + dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), + TEST_LOCALE, true /* isUpdatable */); + assertFalse("binaryDictionary must be invalid for not existing dictionary file.", + binaryDictionary.isValidDictionary()); + binaryDictionary.close(); + } + + public void testAddUnigramWord() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int probability = 100; + binaryDictionary.addUnigramWord("aaa", probability); + // Reallocate and create. + binaryDictionary.addUnigramWord("aab", probability); + // Insert into children. + binaryDictionary.addUnigramWord("aac", probability); + // Make terminal. + binaryDictionary.addUnigramWord("aa", probability); + // Create children. + binaryDictionary.addUnigramWord("aaaa", probability); + // Reallocate and make termianl. + binaryDictionary.addUnigramWord("a", probability); + + final int updatedProbability = 200; + // Update. + binaryDictionary.addUnigramWord("aaa", updatedProbability); + + assertEquals(probability, binaryDictionary.getFrequency("aab")); + assertEquals(probability, binaryDictionary.getFrequency("aac")); + assertEquals(probability, binaryDictionary.getFrequency("aac")); + assertEquals(probability, binaryDictionary.getFrequency("aaaa")); + assertEquals(probability, binaryDictionary.getFrequency("a")); + assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa")); + } +} diff --git a/tests/src/com/android/inputmethod/latin/InputTestsBase.java b/tests/src/com/android/inputmethod/latin/InputTestsBase.java index da1fb6f0d..2603b35f5 100644 --- a/tests/src/com/android/inputmethod/latin/InputTestsBase.java +++ b/tests/src/com/android/inputmethod/latin/InputTestsBase.java @@ -48,8 +48,6 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { protected static final int DELAY_TO_WAIT_FOR_UNDERLINE = 200; // The message that sets predictions is posted with a 200 ms delay protected static final int DELAY_TO_WAIT_FOR_PREDICTIONS = 200; - // The message that sets auto-corrections is posted within a 100 ms delay. - protected static final int DELAY_TO_WAIT_FOR_AUTOCORRECTION = 100; protected LatinIME mLatinIME; protected Keyboard mKeyboard; @@ -223,7 +221,6 @@ public class InputTestsBase extends ServiceTestCase<LatinIMEForTests> { protected void type(final String stringToType) { for (int i = 0; i < stringToType.length(); i = stringToType.offsetByCodePoints(i, 1)) { type(stringToType.codePointAt(i)); - sleep(DELAY_TO_WAIT_FOR_AUTOCORRECTION); } } diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index 1a20ec52d..d15e88bdb 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -77,7 +77,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { private void addToDict(final UserHistoryPredictionDictionary dict, final List<String> words) { String prevWord = null; for (String word : words) { - dict.forceAddWordForTest(prevWord, word, true); + dict.addToPersonalizationPredictionDictionary(prevWord, word, true); prevWord = word; } } diff --git a/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java b/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java new file mode 100644 index 000000000..7fd167977 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/AsyncResultHolderTests.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.MediumTest; +import android.util.Log; + +@MediumTest +public class AsyncResultHolderTests extends AndroidTestCase { + private static final String TAG = AsyncResultHolderTests.class.getSimpleName(); + + private static final int TIMEOUT_IN_MILLISECONDS = 500; + private static final int MARGIN_IN_MILLISECONDS = 250; + private static final int DEFAULT_VALUE = 2; + private static final int SET_VALUE = 1; + + private <T> void setAfterGivenTime(final AsyncResultHolder<T> holder, final T value, + final long time) { + new Thread(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(time); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping", e); + } + holder.set(value); + } + }).start(); + } + + public void testGetWithoutSet() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(DEFAULT_VALUE, resultValue); + } + + public void testGetBeforeSet() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + setAfterGivenTime(holder, SET_VALUE, TIMEOUT_IN_MILLISECONDS + MARGIN_IN_MILLISECONDS); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(DEFAULT_VALUE, resultValue); + } + + public void testGetAfterSet() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + holder.set(SET_VALUE); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(SET_VALUE, resultValue); + } + + public void testGetBeforeTimeout() { + final AsyncResultHolder<Integer> holder = new AsyncResultHolder<Integer>(); + setAfterGivenTime(holder, SET_VALUE, TIMEOUT_IN_MILLISECONDS - MARGIN_IN_MILLISECONDS); + final int resultValue = holder.get(DEFAULT_VALUE, TIMEOUT_IN_MILLISECONDS); + assertEquals(SET_VALUE, resultValue); + } +} diff --git a/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java b/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java new file mode 100644 index 000000000..e0755483c --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutorTests.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.utils; + +import android.test.AndroidTestCase; +import android.test.suitebuilder.annotation.MediumTest; +import android.util.Log; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Unit tests for PrioritizedSerialExecutor. + * TODO: Add more detailed tests to make use of priorities, etc. + */ +@MediumTest +public class PrioritizedSerialExecutorTests extends AndroidTestCase { + private static final String TAG = PrioritizedSerialExecutorTests.class.getSimpleName(); + + private static final int NUM_OF_TASKS = 10; + private static final int DELAY_FOR_WAITING_TASKS_MILLISECONDS = 500; + + public void testExecute() { + final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.execute(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + try { + Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(NUM_OF_TASKS, v.get()); + } + + public void testExecutePrioritized() { + final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.executePrioritized(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + try { + Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(NUM_OF_TASKS, v.get()); + } + + public void testExecuteCombined() { + final PrioritizedSerialExecutor executor = new PrioritizedSerialExecutor(); + final AtomicInteger v = new AtomicInteger(0); + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.execute(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + + for (int i = 0; i < NUM_OF_TASKS; ++i) { + executor.executePrioritized(new Runnable() { + @Override + public void run() { + v.incrementAndGet(); + } + }); + } + + try { + Thread.sleep(DELAY_FOR_WAITING_TASKS_MILLISECONDS); + } catch (InterruptedException e) { + Log.d(TAG, "Exception while sleeping.", e); + } + + assertEquals(2 * NUM_OF_TASKS, v.get()); + } +} |