diff options
Diffstat (limited to 'java/src/com')
14 files changed, 400 insertions, 197 deletions
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index fdde98da1..6ec7aeec3 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -31,6 +31,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Locale; +import java.util.Map; /** * Implements a static, compacted, binary dictionary of standard words. @@ -47,6 +48,11 @@ public final class BinaryDictionary extends Dictionary { // TODO: Remove this heuristic. private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3; + @UsedForTesting + public static final String UNIGRAM_COUNT_QUERY = "UNIGRAM_COUNT"; + @UsedForTesting + public static final String BIGRAM_COUNT_QUERY = "BIGRAM_COUNT"; + private long mNativeDict; private final Locale mLocale; private final long mDictSize; @@ -104,6 +110,8 @@ public final class BinaryDictionary extends Dictionary { JniUtils.loadNativeLibrary(); } + private static native boolean createEmptyDictFileNative(String filePath, long dictVersion, + String[] attributeKeyStringArray, String[] attributeValueStringArray); private static native long openNative(String sourceDir, long dictOffset, long dictSize, boolean isUpdatable); private static native void flushNative(long dict, String filePath); @@ -126,6 +134,21 @@ public final class BinaryDictionary extends Dictionary { private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); private static native int calculateProbabilityNative(long dict, int unigramProbability, int bigramProbability); + private static native String getPropertyNative(long dict, String query); + + @UsedForTesting + public static boolean createEmptyDictFile(final String filePath, final long dictVersion, + final Map<String, String> attributeMap) { + final String[] keyArray = new String[attributeMap.size()]; + final String[] valueArray = new String[attributeMap.size()]; + int index = 0; + for (final String key : attributeMap.keySet()) { + keyArray[index] = key; + valueArray[index] = attributeMap.get(key); + index++; + } + return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray); + } // TODO: Move native dict into session private final void loadDictionary(final String path, final long startOffset, @@ -314,6 +337,12 @@ public final class BinaryDictionary extends Dictionary { return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability); } + @UsedForTesting + public String getPropertyForTests(String query) { + if (!isValidDictionary()) return ""; + return getPropertyNative(mNativeDict, query); + } + @Override public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { // TODO: actually use the confidence rather than use this completely broken heuristic diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 0774ce203..d3da068bd 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -22,12 +22,7 @@ import android.util.Log; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; -import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; -import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; -import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; @@ -35,10 +30,11 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** @@ -68,18 +64,21 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */); + private static final int DICTIONARY_FORMAT_VERSION = 3; + + private static final String SUPPORTS_DYNAMIC_UPDATE = + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE; /** - * A static map of time recorders, each of which records the time of accesses to a single binary - * dictionary file. The key for this map is the filename and the value is the shared dictionary - * time recorder associated with that filename. + * A static map of update controllers, each of which records the time of accesses to a single + * binary dictionary file and tracks whether the file is regenerating. The key for this map is + * the filename and the value is the shared dictionary time recorder associated with that + * filename. */ - private static volatile ConcurrentHashMap<String, DictionaryTimeRecorder> - sFilenameDictionaryTimeRecorderMap = CollectionUtils.newConcurrentHashMap(); + private static final ConcurrentHashMap<String, DictionaryUpdateController> + sFilenameDictionaryUpdateControllerMap = CollectionUtils.newConcurrentHashMap(); - private static volatile ConcurrentHashMap<String, PrioritizedSerialExecutor> + private static final ConcurrentHashMap<String, PrioritizedSerialExecutor> sFilenameExecutorMap = CollectionUtils.newConcurrentHashMap(); /** The application context. */ @@ -106,13 +105,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private final boolean mIsUpdatable; // TODO: remove, once dynamic operations is serialized - /** Records access to the shared binary dictionary file across multiple instances. */ - private final DictionaryTimeRecorder mFilenameDictionaryTimeRecorder; + /** Controls updating the shared binary dictionary file across multiple instances. */ + private final DictionaryUpdateController mFilenameDictionaryUpdateController; // TODO: remove, once dynamic operations is serialized - /** Records access to the local binary dictionary for this instance. */ - private final DictionaryTimeRecorder mPerInstanceDictionaryTimeRecorder = - new DictionaryTimeRecorder(); + /** Controls updating the local binary dictionary for this instance. */ + private final DictionaryUpdateController mPerInstanceDictionaryUpdateController = + new DictionaryUpdateController(); /* A extension for a binary dictionary file. */ public static final String DICT_FILE_EXTENSION = ".dict"; @@ -134,15 +133,15 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { protected abstract boolean hasContentChanged(); /** - * Gets the dictionary time recorder for the given filename. + * Gets the dictionary update controller for the given filename. */ - private static DictionaryTimeRecorder getDictionaryTimeRecorder( + private static DictionaryUpdateController getDictionaryUpdateController( String filename) { - DictionaryTimeRecorder recorder = sFilenameDictionaryTimeRecorderMap.get(filename); + DictionaryUpdateController recorder = sFilenameDictionaryUpdateControllerMap.get(filename); if (recorder == null) { - synchronized(sFilenameDictionaryTimeRecorderMap) { - recorder = new DictionaryTimeRecorder(); - sFilenameDictionaryTimeRecorderMap.put(filename, recorder); + synchronized(sFilenameDictionaryUpdateControllerMap) { + recorder = new DictionaryUpdateController(); + sFilenameDictionaryUpdateControllerMap.put(filename, recorder); } } return recorder; @@ -192,7 +191,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { mContext = context; mIsUpdatable = isUpdatable; mBinaryDictionary = null; - mFilenameDictionaryTimeRecorder = getDictionaryTimeRecorder(filename); + mFilenameDictionaryUpdateController = getDictionaryUpdateController(filename); // Currently, only dynamic personalization dictionary is updatable. mDictionaryWriter = getDictionaryWriter(context, dictType, isUpdatable); } @@ -233,6 +232,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); } + protected Map<String, String> getHeaderAttributeMap() { + HashMap<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + SUPPORTS_DYNAMIC_UPDATE); + return attributeMap; + } + protected void clear() { getExecutor(mFilename).execute(new Runnable() { @Override @@ -240,17 +246,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) { mBinaryDictionary.close(); final File file = new File(mContext.getFilesDir(), mFilename); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), - false, false)); - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - try { - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } + BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); } else { mDictionaryWriter.clear(); } @@ -352,6 +349,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId) { reloadDictionaryIfRequired(); + if (isRegenerating()) { + return null; + } final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); final AsyncResultHolder<ArrayList<SuggestedWordInfo>> holder = new AsyncResultHolder<ArrayList<SuggestedWordInfo>>(); @@ -412,6 +412,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } protected boolean isValidWordInner(final String word) { + if (isRegenerating()) { + return false; + } final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); getExecutor(mFilename).executePrioritized(new Runnable() { @Override @@ -437,7 +440,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { * dictionary exists, this method will generate one. */ protected void loadDictionary() { - mPerInstanceDictionaryTimeRecorder.mLastUpdateRequestTime = SystemClock.uptimeMillis(); + mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = SystemClock.uptimeMillis(); reloadDictionaryIfRequired(); } @@ -448,8 +451,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private void loadBinaryDictionary() { if (DEBUG) { Log.d(TAG, "Loading binary dictionary: " + mFilename + " request=" - + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime + " update=" - + mFilenameDictionaryTimeRecorder.mLastUpdateTime); + + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + + mFilenameDictionaryUpdateController.mLastUpdateTime); } final File file = new File(mContext.getFilesDir(), mFilename); @@ -487,8 +490,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { private void writeBinaryDictionary() { if (DEBUG) { Log.d(TAG, "Generating binary dictionary: " + mFilename + " request=" - + mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime + " update=" - + mFilenameDictionaryTimeRecorder.mLastUpdateTime); + + mFilenameDictionaryUpdateController.mLastUpdateRequestTime + " update=" + + mFilenameDictionaryUpdateController.mLastUpdateTime); } if (needsToReloadBeforeWriting()) { mDictionaryWriter.clear(); @@ -498,17 +501,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) { if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) { final File file = new File(mContext.getFilesDir(), mFilename); - final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), - false, false)); - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - try { - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); - } catch (IOException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } catch (UnsupportedFormatException e) { - Log.e(TAG, "Exception in creating new dictionary file.", e); - } + BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(), + DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap()); } else { if (mBinaryDictionary.needsToRunGC()) { mBinaryDictionary.flushWithGC(); @@ -531,11 +525,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ protected void setRequiresReload(final boolean requiresRebuild) { final long time = SystemClock.uptimeMillis(); - mPerInstanceDictionaryTimeRecorder.mLastUpdateRequestTime = time; - mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime = time; + mPerInstanceDictionaryUpdateController.mLastUpdateRequestTime = time; + mFilenameDictionaryUpdateController.mLastUpdateRequestTime = time; if (DEBUG) { Log.d(TAG, "Reload request: " + mFilename + ": request=" + time + " update=" - + mFilenameDictionaryTimeRecorder.mLastUpdateTime); + + mFilenameDictionaryUpdateController.mLastUpdateTime); } } @@ -544,14 +538,26 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { */ public final void reloadDictionaryIfRequired() { if (!isReloadRequired()) return; - reloadDictionary(); + if (setIsRegeneratingIfNotRegenerating()) { + reloadDictionary(); + } } /** * Returns whether a dictionary reload is required. */ private boolean isReloadRequired() { - return mBinaryDictionary == null || mPerInstanceDictionaryTimeRecorder.isOutOfDate(); + return mBinaryDictionary == null || mPerInstanceDictionaryUpdateController.isOutOfDate(); + } + + private boolean isRegenerating() { + return mFilenameDictionaryUpdateController.mIsRegenerating.get(); + } + + // Returns whether the dictionary can be regenerated. + private boolean setIsRegeneratingIfNotRegenerating() { + return mFilenameDictionaryUpdateController.mIsRegenerating.compareAndSet( + false /* expect */ , true /* update */); } /** @@ -564,39 +570,44 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mFilename).execute(new Runnable() { @Override public void run() { - final long time = SystemClock.uptimeMillis(); - final boolean dictionaryFileExists = dictionaryFileExists(); - if (mFilenameDictionaryTimeRecorder.isOutOfDate() || !dictionaryFileExists) { - // If the shared dictionary file does not exist or is out of date, the first - // instance that acquires the lock will generate a new one. - if (hasContentChanged() || !dictionaryFileExists) { - // If the source content has changed or the dictionary does not exist, - // rebuild the binary dictionary. Empty dictionaries are supported (in the - // case where loadDictionaryAsync() adds nothing) in order to provide a - // uniform framework. - mFilenameDictionaryTimeRecorder.mLastUpdateTime = time; + try { + final long time = SystemClock.uptimeMillis(); + final boolean dictionaryFileExists = dictionaryFileExists(); + if (mFilenameDictionaryUpdateController.isOutOfDate() + || !dictionaryFileExists) { + // If the shared dictionary file does not exist or is out of date, the + // first instance that acquires the lock will generate a new one. + if (hasContentChanged() || !dictionaryFileExists) { + // If the source content has changed or the dictionary does not exist, + // rebuild the binary dictionary. Empty dictionaries are supported (in + // the case where loadDictionaryAsync() adds nothing) in order to + // provide a uniform framework. + mFilenameDictionaryUpdateController.mLastUpdateTime = time; + writeBinaryDictionary(); + loadBinaryDictionary(); + } else { + // If not, the reload request was unnecessary so revert + // LastUpdateRequestTime to LastUpdateTime. + mFilenameDictionaryUpdateController.mLastUpdateRequestTime = + mFilenameDictionaryUpdateController.mLastUpdateTime; + } + } else if (mBinaryDictionary == null || + mPerInstanceDictionaryUpdateController.mLastUpdateTime + < mFilenameDictionaryUpdateController.mLastUpdateTime) { + // Otherwise, if the local dictionary is older than the shared dictionary, + // load the shared dictionary. + loadBinaryDictionary(); + } + if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { + // Binary dictionary is not valid. Regenerate the dictionary file. + mFilenameDictionaryUpdateController.mLastUpdateTime = time; writeBinaryDictionary(); loadBinaryDictionary(); - } else { - // If not, the reload request was unnecessary so revert - // LastUpdateRequestTime to LastUpdateTime. - mFilenameDictionaryTimeRecorder.mLastUpdateRequestTime = - mFilenameDictionaryTimeRecorder.mLastUpdateTime; } - } else if (mBinaryDictionary == null || - mPerInstanceDictionaryTimeRecorder.mLastUpdateTime - < mFilenameDictionaryTimeRecorder.mLastUpdateTime) { - // Otherwise, if the local dictionary is older than the shared dictionary, load - // the shared dictionary. - loadBinaryDictionary(); - } - if (mBinaryDictionary != null && !mBinaryDictionary.isValidDictionary()) { - // Binary dictionary is not valid. Regenerate the dictionary file. - mFilenameDictionaryTimeRecorder.mLastUpdateTime = time; - writeBinaryDictionary(); - loadBinaryDictionary(); + mPerInstanceDictionaryUpdateController.mLastUpdateTime = time; + } finally { + mFilenameDictionaryUpdateController.mIsRegenerating.set(false); } - mPerInstanceDictionaryTimeRecorder.mLastUpdateTime = time; } }); } @@ -636,59 +647,19 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } /** - * Time recorder for tracking whether the dictionary is out of date. + * For tracking whether the dictionary is out of date and the dictionary is regenerating. * Can be shared across multiple dictionary instances that access the same filename. */ - private static class DictionaryTimeRecorder { - private volatile long mLastUpdateTime = 0; - private volatile long mLastUpdateRequestTime = 0; + private static class DictionaryUpdateController { + public volatile long mLastUpdateTime = 0; + public volatile long mLastUpdateRequestTime = 0; + public volatile AtomicBoolean mIsRegenerating = new AtomicBoolean(); - private boolean isOutOfDate() { + public boolean isOutOfDate() { return (mLastUpdateRequestTime > mLastUpdateTime); } } - /** - * Dynamically adds a word unigram to the dictionary for testing with blocking-lock. - */ - @UsedForTesting - protected void addWordDynamicallyForTests(final String word, final String shortcutTarget, - final int frequency, final boolean isNotAWord) { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - addWordDynamically(word, shortcutTarget, frequency, isNotAWord); - } - }); - } - - /** - * Dynamically adds a word bigram in the dictionary for testing with blocking-lock. - */ - @UsedForTesting - protected void addBigramDynamicallyForTests(final String word0, final String word1, - final int frequency, final boolean isValid) { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - addBigramDynamically(word0, word1, frequency, isValid); - } - }); - } - - /** - * Dynamically remove a word bigram in the dictionary for testing with blocking-lock. - */ - @UsedForTesting - protected void removeBigramDynamicallyForTests(final String word0, final String word1) { - getExecutor(mFilename).executePrioritized(new Runnable() { - @Override - public void run() { - removeBigramDynamically(word0, word1); - } - }); - } - // TODO: Implement native binary methods once the dynamic dictionary implementation is done. @UsedForTesting public boolean isInDictionaryForTests(final String word) { @@ -701,7 +672,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { holder.set(mBinaryDictionary.isValidWord(word)); } else { holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter) - .isInDictionaryForTests(word)); + .isInBigramListForTests(word)); } } } diff --git a/java/src/com/android/inputmethod/latin/InputAttributes.java b/java/src/com/android/inputmethod/latin/InputAttributes.java index 21b103e5a..8caf6f17f 100644 --- a/java/src/com/android/inputmethod/latin/InputAttributes.java +++ b/java/src/com/android/inputmethod/latin/InputAttributes.java @@ -103,6 +103,10 @@ public final class InputAttributes { } } + public boolean isTypeNull() { + return InputType.TYPE_NULL == mInputType; + } + public boolean isSameInputType(final EditorInfo editorInfo) { return editorInfo.inputType == mInputType; } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 270dc4c06..5fbbc1764 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1521,7 +1521,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mSubtypeState.switchSubtype(token, mRichImm); } - private void sendDownUpKeyEventForBackwardCompatibility(final int code) { + private void sendDownUpKeyEvent(final int code) { final long eventTime = SystemClock.uptimeMillis(); mConnection.sendKeyEvent(new KeyEvent(eventTime, eventTime, KeyEvent.ACTION_DOWN, code, 0, 0, KeyCharacterMap.VIRTUAL_KEYBOARD, 0, @@ -1538,7 +1538,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // TODO: Remove this special handling of digit letters. // For backward compatibility. See {@link InputMethodService#sendKeyChar(char)}. if (code >= '0' && code <= '9') { - sendDownUpKeyEventForBackwardCompatibility(code - '0' + KeyEvent.KEYCODE_0); + sendDownUpKeyEvent(code - '0' + KeyEvent.KEYCODE_0); return; } @@ -1547,7 +1547,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen // a hardware keyboard event on pressing enter or delete. This is bad for many // reasons (there are race conditions with commits) but some applications are // relying on this behavior so we continue to support it for older apps. - sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_ENTER); + sendDownUpKeyEvent(KeyEvent.KEYCODE_ENTER); } else { final String text = new String(new int[] { code }, 0, 1); mConnection.commitText(text, text.length()); @@ -2104,12 +2104,16 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen } final int lengthToDelete = Character.isSupplementaryCodePoint( mConnection.getCodePointBeforeCursor()) ? 2 : 1; - if (mAppWorkAroundsUtils.isBeforeJellyBean()) { - // Backward compatibility mode. Before Jelly bean, the keyboard would simulate - // a hardware keyboard event on pressing enter or delete. This is bad for many - // reasons (there are race conditions with commits) but some applications are - // relying on this behavior so we continue to support it for older apps. - sendDownUpKeyEventForBackwardCompatibility(KeyEvent.KEYCODE_DEL); + if (mAppWorkAroundsUtils.isBeforeJellyBean() || + currentSettings.mInputAttributes.isTypeNull()) { + // There are two possible reasons to send a key event: either the field has + // type TYPE_NULL, in which case the keyboard should send events, or we are + // running in backward compatibility mode. Before Jelly bean, the keyboard + // would simulate a hardware keyboard event on pressing enter or delete. This + // is bad for many reasons (there are race conditions with commits) but some + // applications are relying on this behavior so we continue to support it for + // older apps, so we retain this behavior if the app has target SDK < JellyBean. + sendDownUpKeyEvent(KeyEvent.KEYCODE_DEL); } else { mConnection.deleteSurroundingText(lengthToDelete, 0); } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 3b1d2427b..6cc0bfb76 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -225,6 +225,26 @@ public class BinaryDictEncoderUtils { return position; } + static void writeUIntToStream(final OutputStream stream, final int value, final int size) + throws IOException { + switch(size) { + case 4: + stream.write((value >> 24) & 0xFF); + /* fall through */ + case 3: + stream.write((value >> 16) & 0xFF); + /* fall through */ + case 2: + stream.write((value >> 8) & 0xFF); + /* fall through */ + case 1: + stream.write(value & 0xFF); + break; + default: + /* nop */ + } + } + // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index aa5129ccb..849bff050 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -325,6 +325,12 @@ public final class FormatSpec { public final int mHeaderSize; public final DictionaryOptions mDictionaryOptions; public final FormatOptions mFormatOptions; + // Note that these are corresponding definitions in native code in latinime::HeaderPolicy + // and latinime::HeaderReadWriteUtils. + public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; + public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; + public static final String ATTRIBUTE_VALUE_TRUE = "1"; + private static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale"; private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary"; diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java new file mode 100644 index 000000000..0b9cf91d2 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; + +/** + * SparseTable is an extensible map from integer to integer. + * This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index + * memory. + */ +@UsedForTesting +public class SparseTable { + + /** + * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize + * terminals. + * It contains at index i = j / mBlockSize the index in mContentsTable where the values for + * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array. + */ + private final ArrayList<Integer> mLookupTable; + private final ArrayList<Integer> mContentTable; + + private final int mBlockSize; + public static final int NOT_EXIST = -1; + + @UsedForTesting + public SparseTable(final int initialCapacity, final int blockSize) { + mBlockSize = blockSize; + final int lookupTableSize = initialCapacity / mBlockSize + + (initialCapacity % mBlockSize > 0 ? 1 : 0); + mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST)); + mContentTable = new ArrayList<Integer>(); + } + + @UsedForTesting + public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) { + mBlockSize = blockSize; + mLookupTable = new ArrayList<Integer>(lookupTable.length); + for (int i = 0; i < lookupTable.length; ++i) { + mLookupTable.add(lookupTable[i]); + } + mContentTable = new ArrayList<Integer>(contentTable.length); + for (int i = 0; i < contentTable.length; ++i) { + mContentTable.add(contentTable[i]); + } + } + + /** + * Converts an byte array to an int array considering each set of 4 bytes is an int stored in + * big-endian. + * The length of byteArray must be a multiple of four. + * Otherwise, IndexOutOfBoundsException will be raised. + */ + @UsedForTesting + private static void convertByteArrayToIntegerArray(final byte[] byteArray, + final ArrayList<Integer> integerArray) { + for (int i = 0; i < byteArray.length; i += 4) { + int value = 0; + for (int j = i; j < i + 4; ++j) { + value <<= 8; + value |= byteArray[j] & 0xFF; + } + integerArray.add(value); + } + } + + @UsedForTesting + public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) { + mBlockSize = blockSize; + mLookupTable = new ArrayList<Integer>(lookupTable.length / 4); + mContentTable = new ArrayList<Integer>(contentTable.length / 4); + convertByteArrayToIntegerArray(lookupTable, mLookupTable); + convertByteArrayToIntegerArray(contentTable, mContentTable); + } + + @UsedForTesting + public int get(final int index) { + if (index < 0 || index / mBlockSize >= mLookupTable.size() + || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + return NOT_EXIST; + } + return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); + } + + @UsedForTesting + public void set(final int index, final int value) { + if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + mLookupTable.set(index / mBlockSize, mContentTable.size()); + for (int i = 0; i < mBlockSize; ++i) { + mContentTable.add(NOT_EXIST); + } + } + mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); + } + + public void remove(final int index) { + set(index, NOT_EXIST); + } + + @UsedForTesting + public int size() { + return mLookupTable.size() * mBlockSize; + } + + @UsedForTesting + /* package */ int getContentTableSize() { + return mContentTable.size(); + } + + @UsedForTesting + /* package */ int getLookupTableSize() { + return mLookupTable.size(); + } + + public boolean contains(final int index) { + return get(index) != NOT_EXIST; + } + + @UsedForTesting + public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream) + throws IOException { + for (final int index : mLookupTable) { + BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4); + } + + for (final int index : mContentTable) { + BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4); + } + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java index 075d7e3c3..66517a800 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java +++ b/java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java @@ -34,12 +34,15 @@ import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListe import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; /** - * This class is a base class of a dictionary for the personalized prediction language model. + * This class is a base class of a dictionary that supports decaying for the personalized language + * model. */ -public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary { - private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName(); +public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary { + private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName(); public static final boolean DBG_SAVE_RESTORE = false; private static final boolean DBG_STRESS_TEST = false; private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG; @@ -60,8 +63,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi // Should always be false except when we use this class for test @UsedForTesting boolean mIsTest = false; - /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale, - final SharedPreferences sp, final String dictionaryType, final String fileName) { + /* package */ DecayingExpandableBinaryDictionaryBase(final Context context, + final String locale, final SharedPreferences sp, final String dictionaryType, + final String fileName) { super(context, fileName, dictionaryType, true); mLocale = locale; mFileName = fileName; @@ -84,6 +88,16 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi } @Override + protected Map<String, String> getHeaderAttributeMap() { + HashMap<String, String> attributeMap = new HashMap<String, String>(); + attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE, + FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE); + return attributeMap; + } + + @Override protected boolean hasContentChanged() { return false; } diff --git a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java index 0af028a9e..305088536 100644 --- a/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java +++ b/java/src/com/android/inputmethod/latin/personalization/DynamicPersonalizationDictionaryWriter.java @@ -79,7 +79,7 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr public void addUnigramWord(final String word, final String shortcutTarget, final int frequency, final boolean isNotAWord) { if (mBigramList.size() > mMaxHistoryBigrams * 2) { - // Too many entries: just stop adding new vocabrary and wait next refresh. + // Too many entries: just stop adding new vocabulary and wait next refresh. return; } mExpandableDictionary.addWord(word, shortcutTarget, frequency); @@ -90,7 +90,7 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr public void addBigramWords(final String word0, final String word1, final int frequency, final boolean isValid, final long lastModifiedTime) { if (mBigramList.size() > mMaxHistoryBigrams * 2) { - // Too many entries: just stop adding new vocabrary and wait next refresh. + // Too many entries: just stop adding new vocabulary and wait next refresh. return; } if (lastModifiedTime > 0) { @@ -176,8 +176,8 @@ public class DynamicPersonalizationDictionaryWriter extends AbstractDictionaryWr } @UsedForTesting - public boolean isInDictionaryForTests(final String word) { + public boolean isInBigramListForTests(final String word) { // TODO: Use native method to determine whether the word is in dictionary or not - return mBigramList.containsKey(word); + return mBigramList.containsKey(word) || mBigramList.getBigrams(null).containsKey(word); } } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java index ab3de801c..c616a296c 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java @@ -46,7 +46,7 @@ public abstract class PersonalizationDictionaryUpdateSession { // TODO: Use a dynamic binary dictionary instead public WeakReference<PersonalizationDictionary> mDictionary; - public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary; + public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary; public final String mSystemLocale; public PersonalizationDictionaryUpdateSession(String locale) { mSystemLocale = locale; @@ -60,15 +60,16 @@ public abstract class PersonalizationDictionaryUpdateSession { mDictionary = new WeakReference<PersonalizationDictionary>(dictionary); } - public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) { - mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary); + public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) { + mPredictionDictionary = + new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary); } protected PersonalizationDictionary getDictionary() { return mDictionary == null ? null : mDictionary.get(); } - protected DynamicPredictionDictionaryBase getPredictionDictionary() { + protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() { return mPredictionDictionary == null ? null : mPredictionDictionary.get(); } @@ -81,7 +82,7 @@ public abstract class PersonalizationDictionaryUpdateSession { } private void unsetPredictionDictionary() { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } @@ -89,7 +90,7 @@ public abstract class PersonalizationDictionaryUpdateSession { } public void clearAndFlushPredictionDictionary(Context context) { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } @@ -105,7 +106,7 @@ public abstract class PersonalizationDictionaryUpdateSession { // TODO: Support multi locale to add bigram public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid, int frequency) { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } @@ -116,7 +117,7 @@ public abstract class PersonalizationDictionaryUpdateSession { // TODO: Support multi locale to add bigram public void addBigramsToPersonalizationDictionary( final ArrayList<PersonalizationLanguageModelParam> lmParams) { - final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary(); + final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary(); if (dictionary == null) { return; } diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java index e80953c05..432954453 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java @@ -22,7 +22,7 @@ import com.android.inputmethod.latin.ExpandableBinaryDictionary; import android.content.Context; import android.content.SharedPreferences; -public class PersonalizationPredictionDictionary extends DynamicPredictionDictionaryBase { +public class PersonalizationPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName(); /* package */ PersonalizationPredictionDictionary(final Context context, final String locale, diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java index 4c1803bdf..55a90ee51 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryBigramList.java @@ -54,7 +54,7 @@ public final class UserHistoryDictionaryBigramList { * Called when loaded from the SQL DB. */ public void addBigram(String word1, String word2, byte fcValue) { - if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { + if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- add bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; @@ -74,7 +74,7 @@ public final class UserHistoryDictionaryBigramList { * Called when inserted to the SQL DB. */ public void updateBigram(String word1, String word2, byte fcValue) { - if (DynamicPredictionDictionaryBase.DBG_SAVE_RESTORE) { + if (DecayingExpandableBinaryDictionaryBase.DBG_SAVE_RESTORE) { Log.d(TAG, "--- update bigram: " + word1 + ", " + word2 + ", " + fcValue); } final HashMap<String, Byte> map; diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java index b140c919b..38e308a4e 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryPredictionDictionary.java @@ -26,7 +26,7 @@ import android.content.SharedPreferences; * Locally gathers stats about the words user types and various other signals like auto-correction * cancellation or manual picks. This allows the keyboard to adapt to the typist over time. */ -public class UserHistoryPredictionDictionary extends DynamicPredictionDictionaryBase { +public class UserHistoryPredictionDictionary extends DecayingExpandableBinaryDictionaryBase { /* package for tests */ static final String NAME = UserHistoryPredictionDictionary.class.getSimpleName(); /* package */ UserHistoryPredictionDictionary(final Context context, final String locale, diff --git a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java index 5dc0b5893..201a70d42 100644 --- a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java +++ b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java @@ -16,8 +16,11 @@ package com.android.inputmethod.latin.utils; -import java.util.ArrayDeque; import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * An object that executes submitted tasks using a thread. @@ -27,19 +30,20 @@ public class PrioritizedSerialExecutor { private final Object mLock = new Object(); - // The default value of capacities of task queues. - private static final int TASK_QUEUE_CAPACITY = 1000; private final Queue<Runnable> mTasks; private final Queue<Runnable> mPrioritizedTasks; private boolean mIsShutdown; + private final ThreadPoolExecutor mThreadPoolExecutor; // The task which is running now. private Runnable mActive; public PrioritizedSerialExecutor() { - mTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); - mPrioritizedTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); + mTasks = new ConcurrentLinkedQueue<Runnable>(); + mPrioritizedTasks = new ConcurrentLinkedQueue<Runnable>(); mIsShutdown = false; + mThreadPoolExecutor = new ThreadPoolExecutor(1 /* corePoolSize */, 1 /* maximumPoolSize */, + 0 /* keepAliveTime */, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<Runnable>(1)); } /** @@ -59,7 +63,16 @@ public class PrioritizedSerialExecutor { public void execute(final Runnable r) { synchronized(mLock) { if (!mIsShutdown) { - mTasks.offer(r); + mTasks.offer(new Runnable() { + @Override + public void run() { + try { + r.run(); + } finally { + scheduleNext(); + } + } + }); if (mActive == null) { scheduleNext(); } @@ -74,45 +87,36 @@ public class PrioritizedSerialExecutor { public void executePrioritized(final Runnable r) { synchronized(mLock) { if (!mIsShutdown) { - mPrioritizedTasks.offer(r); - if (mActive == null) { + mPrioritizedTasks.offer(new Runnable() { + @Override + public void run() { + try { + r.run(); + } finally { + scheduleNext(); + } + } + }); + if (mActive == null) { scheduleNext(); } } } } - private boolean fetchNextTasks() { - synchronized(mLock) { - mActive = mPrioritizedTasks.poll(); - if (mActive == null) { - mActive = mTasks.poll(); - } - return mActive != null; + private boolean fetchNextTasksLocked() { + mActive = mPrioritizedTasks.poll(); + if (mActive == null) { + mActive = mTasks.poll(); } + return mActive != null; } private void scheduleNext() { synchronized(mLock) { - if (!fetchNextTasks()) { - return; + if (fetchNextTasksLocked()) { + mThreadPoolExecutor.execute(mActive); } - new Thread(new Runnable() { - @Override - public void run() { - try { - do { - synchronized(mLock) { - if (mActive != null) { - mActive.run(); - } - } - } while (fetchNextTasks()); - } finally { - scheduleNext(); - } - } - }).start(); } } |