diff options
16 files changed, 257 insertions, 152 deletions
diff --git a/java/AndroidManifest.xml b/java/AndroidManifest.xml index a1ffe5a93..054c415a4 100644 --- a/java/AndroidManifest.xml +++ b/java/AndroidManifest.xml @@ -18,7 +18,7 @@ coreApp="true" package="com.android.inputmethod.latin"> - <uses-sdk android:minSdkVersion="14" android:targetSdkVersion="19" /> + <uses-sdk android:minSdkVersion="14" android:targetSdkVersion="21" /> <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" /> <uses-permission android:name="android.permission.DOWNLOAD_WITHOUT_NOTIFICATION" /> diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java index c20546607..eced45ea5 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitator.java @@ -61,9 +61,9 @@ public class DictionaryFacilitator { // dictionary. private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140; - private DictionaryGroup mDictionaryGroup = new DictionaryGroup(); + private DictionaryGroup[] mDictionaryGroups = new DictionaryGroup[] { new DictionaryGroup() }; private boolean mIsUserDictEnabled = false; - private volatile CountDownLatch mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0); + private volatile CountDownLatch mLatchForWaitingLoadingMainDictionaries = new CountDownLatch(0); // To synchronize assigning mDictionaryGroup to ensure closing dictionaries. private final Object mLock = new Object(); private final DistracterFilter mDistracterFilter; @@ -193,8 +193,9 @@ public class DictionaryFacilitator { mPersonalizationHelper.setIsMonolingualUser(isMonolingualUser); } + // TODO: remove this, replace with version returning multiple locales public Locale getLocale() { - return mDictionaryGroup.mLocale; + return mDictionaryGroups[0].mLocale; } private static ExpandableBinaryDictionary getSubDict(final String dictType, @@ -226,6 +227,21 @@ public class DictionaryFacilitator { usePersonalizedDicts, forceReloadMainDictionary, listener, "" /* dictNamePrefix */); } + private DictionaryGroup findDictionaryGroupWithLocale(final DictionaryGroup[] dictionaryGroups, + final Locale locale) { + for (int i = 0; i < dictionaryGroups.length; ++i) { + if (locale.equals(dictionaryGroups[i].mLocale)) { + return dictionaryGroups[i]; + } + } + return null; + } + + private DictionaryGroup getDictionaryGroupForActiveLanguage() { + // TODO: implement this + return mDictionaryGroups[0]; + } + public void resetDictionariesWithDictNamePrefix(final Context context, final Locale newLocaleToUse, final boolean useContactsDict, final boolean usePersonalizedDicts, @@ -252,7 +268,7 @@ public class DictionaryFacilitator { final ArrayList<String> dictsForLocale = new ArrayList<>(); existingDictsToCleanup.put(newLocale, dictsForLocale); final DictionaryGroup currentDictionaryGroupForLocale = - newLocale.equals(mDictionaryGroup.mLocale) ? mDictionaryGroup : null; + findDictionaryGroupWithLocale(mDictionaryGroups, newLocale); if (null == currentDictionaryGroupForLocale) { continue; } @@ -266,10 +282,11 @@ public class DictionaryFacilitator { } } - final HashMap<Locale, DictionaryGroup> newDictionaryGroups = new HashMap<>(); - for (final Locale newLocale : newLocales) { + final DictionaryGroup[] newDictionaryGroups = new DictionaryGroup[newLocales.length]; + for (int i = 0; i < newLocales.length; ++i) { + final Locale newLocale = newLocales[i]; final DictionaryGroup dictionaryGroupForLocale = - newLocale.equals(mDictionaryGroup.mLocale) ? mDictionaryGroup : null; + findDictionaryGroupWithLocale(mDictionaryGroups, newLocale); final ArrayList<String> dictsToCleanupForLocale = existingDictsToCleanup.get(newLocale); final boolean noExistingDictsForThisLocale = (null == dictionaryGroupForLocale); @@ -297,30 +314,29 @@ public class DictionaryFacilitator { } subDicts.put(subDictType, subDict); } - newDictionaryGroups.put(newLocale, new DictionaryGroup(newLocale, mainDict, subDicts)); + newDictionaryGroups[i] = new DictionaryGroup(newLocale, mainDict, subDicts); } // Replace Dictionaries. - // TODO: use multiple locales. - final DictionaryGroup newDictionaryGroup = newDictionaryGroups.get(newLocaleToUse); - final DictionaryGroup oldDictionaryGroup; + final DictionaryGroup[] oldDictionaryGroups; synchronized (mLock) { - oldDictionaryGroup = mDictionaryGroup; - mDictionaryGroup = newDictionaryGroup; + oldDictionaryGroups = mDictionaryGroups; + mDictionaryGroups = newDictionaryGroups; mIsUserDictEnabled = UserBinaryDictionary.isEnabled(context); - if (null == newDictionaryGroup.getDict(Dictionary.TYPE_MAIN)) { + if (hasAtLeastOneUninitializedMainDictionary()) { asyncReloadUninitializedMainDictionaries(context, newLocales, listener); } } if (listener != null) { - listener.onUpdateMainDictionaryAvailability(hasInitializedMainDictionary()); + listener.onUpdateMainDictionaryAvailability(hasAtLeastOneInitializedMainDictionary()); } // Clean up old dictionaries. for (final Locale localeToCleanUp : existingDictsToCleanup.keySet()) { final ArrayList<String> dictTypesToCleanUp = existingDictsToCleanup.get(localeToCleanUp); - final DictionaryGroup dictionarySetToCleanup = oldDictionaryGroup; + final DictionaryGroup dictionarySetToCleanup = + findDictionaryGroupWithLocale(oldDictionaryGroups, localeToCleanUp); for (final String dictType : dictTypesToCleanUp) { dictionarySetToCleanup.closeDict(dictType); } @@ -330,12 +346,18 @@ public class DictionaryFacilitator { private void asyncReloadUninitializedMainDictionaries(final Context context, final Locale[] locales, final DictionaryInitializationListener listener) { final CountDownLatch latchForWaitingLoadingMainDictionary = new CountDownLatch(1); - mLatchForWaitingLoadingMainDictionary = latchForWaitingLoadingMainDictionary; + mLatchForWaitingLoadingMainDictionaries = latchForWaitingLoadingMainDictionary; ExecutorUtils.getExecutor("InitializeBinaryDictionary").execute(new Runnable() { @Override public void run() { for (final Locale locale : locales) { - final DictionaryGroup dictionaryGroup = mDictionaryGroup; + final DictionaryGroup dictionaryGroup = + findDictionaryGroupWithLocale(mDictionaryGroups, locale); + if (null == dictionaryGroup) { + // This should never happen, but better safe than crashy + Log.w(TAG, "Expected a dictionary group for " + locale + " but none found"); + continue; + } final Dictionary mainDict = DictionaryFactory.createMainDictionaryFromManager(context, locale); synchronized (mLock) { @@ -348,7 +370,8 @@ public class DictionaryFacilitator { } } if (listener != null) { - listener.onUpdateMainDictionaryAvailability(hasInitializedMainDictionary()); + listener.onUpdateMainDictionaryAvailability( + hasAtLeastOneInitializedMainDictionary()); } latchForWaitingLoadingMainDictionary.countDown(); } @@ -381,17 +404,20 @@ public class DictionaryFacilitator { subDicts.put(dictType, dict); } } - mDictionaryGroup = new DictionaryGroup(locale, mainDictionary, subDicts); + mDictionaryGroups = new DictionaryGroup[] { + new DictionaryGroup(locale, mainDictionary, subDicts) }; } public void closeDictionaries() { - final DictionaryGroup dictionaryGroup; + final DictionaryGroup[] dictionaryGroups; synchronized (mLock) { - dictionaryGroup = mDictionaryGroup; - mDictionaryGroup = new DictionaryGroup(); + dictionaryGroups = mDictionaryGroups; + mDictionaryGroups = new DictionaryGroup[] { new DictionaryGroup() }; } - for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { - dictionaryGroup.closeDict(dictType); + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { + dictionaryGroup.closeDict(dictType); + } } mDistracterFilter.close(); if (mPersonalizationHelper != null) { @@ -401,40 +427,71 @@ public class DictionaryFacilitator { @UsedForTesting public ExpandableBinaryDictionary getSubDictForTesting(final String dictName) { - return mDictionaryGroup.getSubDict(dictName); + return mDictionaryGroups[0].getSubDict(dictName); } - // The main dictionary could have been loaded asynchronously. Don't cache the return value - // of this method. - public boolean hasInitializedMainDictionary() { - final Dictionary mainDict = mDictionaryGroup.getDict(Dictionary.TYPE_MAIN); - return mainDict != null && mainDict.isInitialized(); + // The main dictionaries are loaded asynchronously. Don't cache the return value + // of these methods. + public boolean hasAtLeastOneInitializedMainDictionary() { + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + final Dictionary mainDict = dictionaryGroup.getDict(Dictionary.TYPE_MAIN); + if (mainDict != null && mainDict.isInitialized()) { + return true; + } + } + return false; + } + + public boolean hasAtLeastOneUninitializedMainDictionary() { + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + final Dictionary mainDict = dictionaryGroup.getDict(Dictionary.TYPE_MAIN); + if (mainDict == null || !mainDict.isInitialized()) { + return true; + } + } + return false; } public boolean hasPersonalizationDictionary() { - return mDictionaryGroup.hasDict(Dictionary.TYPE_PERSONALIZATION); + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + if (dictionaryGroup.hasDict(Dictionary.TYPE_PERSONALIZATION)) { + return true; + } + } + return false; } public void flushPersonalizationDictionary() { - final ExpandableBinaryDictionary personalizationDictUsedForSuggestion = - mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION); + final HashSet<ExpandableBinaryDictionary> personalizationDictsUsedForSuggestion = + new HashSet<>(); + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + final ExpandableBinaryDictionary personalizationDictUsedForSuggestion = + dictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION); + personalizationDictsUsedForSuggestion.add(personalizationDictUsedForSuggestion); + } mPersonalizationHelper.flushPersonalizationDictionariesToUpdate( - personalizationDictUsedForSuggestion); + personalizationDictsUsedForSuggestion); mDistracterFilter.close(); } - public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit) + public void waitForLoadingMainDictionaries(final long timeout, final TimeUnit unit) throws InterruptedException { - mLatchForWaitingLoadingMainDictionary.await(timeout, unit); + mLatchForWaitingLoadingMainDictionaries.await(timeout, unit); } @UsedForTesting public void waitForLoadingDictionariesForTesting(final long timeout, final TimeUnit unit) throws InterruptedException { - waitForLoadingMainDictionary(timeout, unit); - final Map<String, ExpandableBinaryDictionary> dictMap = mDictionaryGroup.mSubDictMap; - for (final ExpandableBinaryDictionary dict : dictMap.values()) { - dict.waitAllTasksForTests(); + waitForLoadingMainDictionaries(timeout, unit); + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + for (final ExpandableBinaryDictionary dict : dictionaryGroup.mSubDictMap.values()) { + dict.waitAllTasksForTests(); + } } } @@ -453,7 +510,7 @@ public class DictionaryFacilitator { public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized, final PrevWordsInfo prevWordsInfo, final int timeStampInSeconds, final boolean blockPotentiallyOffensive) { - final DictionaryGroup dictionaryGroup = mDictionaryGroup; + final DictionaryGroup dictionaryGroup = getDictionaryGroupForActiveLanguage(); final String[] words = suggestion.split(Constants.WORD_SEPARATOR); PrevWordsInfo prevWordsInfoForCurrentWord = prevWordsInfo; for (int i = 0; i < words.length; i++) { @@ -520,7 +577,8 @@ public class DictionaryFacilitator { } private void removeWord(final String dictName, final String word) { - final ExpandableBinaryDictionary dictionary = mDictionaryGroup.getSubDict(dictName); + final ExpandableBinaryDictionary dictionary = + getDictionaryGroupForActiveLanguage().getSubDict(dictName); if (dictionary != null) { dictionary.removeUnigramEntryDynamically(word); } @@ -536,20 +594,22 @@ public class DictionaryFacilitator { public SuggestionResults getSuggestionResults(final WordComposer composer, final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId) { - final DictionaryGroup dictionaryGroup = mDictionaryGroup; + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; final SuggestionResults suggestionResults = new SuggestionResults(SuggestedWords.MAX_SUGGESTIONS); final float[] languageWeight = new float[] { Dictionary.NOT_A_LANGUAGE_WEIGHT }; - for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { - final Dictionary dictionary = dictionaryGroup.getDict(dictType); - if (null == dictionary) continue; - final ArrayList<SuggestedWordInfo> dictionarySuggestions = - dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, - settingsValuesForSuggestion, sessionId, languageWeight); - if (null == dictionarySuggestions) continue; - suggestionResults.addAll(dictionarySuggestions); - if (null != suggestionResults.mRawSuggestions) { - suggestionResults.mRawSuggestions.addAll(dictionarySuggestions); + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { + final Dictionary dictionary = dictionaryGroup.getDict(dictType); + if (null == dictionary) continue; + final ArrayList<SuggestedWordInfo> dictionarySuggestions = + dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, + settingsValuesForSuggestion, sessionId, languageWeight); + if (null == dictionarySuggestions) continue; + suggestionResults.addAll(dictionarySuggestions); + if (null != suggestionResults.mRawSuggestions) { + suggestionResults.mRawSuggestions.addAll(dictionarySuggestions); + } } } return suggestionResults; @@ -559,20 +619,22 @@ public class DictionaryFacilitator { if (TextUtils.isEmpty(word)) { return false; } - final DictionaryGroup dictionaryGroup = mDictionaryGroup; - if (dictionaryGroup.mLocale == null) { - return false; - } - final String lowerCasedWord = word.toLowerCase(dictionaryGroup.mLocale); - for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { - final Dictionary dictionary = dictionaryGroup.getDict(dictType); - // Ideally the passed map would come out of a {@link java.util.concurrent.Future} and - // would be immutable once it's finished initializing, but concretely a null test is - // probably good enough for the time being. - if (null == dictionary) continue; - if (dictionary.isValidWord(word) - || (ignoreCase && dictionary.isValidWord(lowerCasedWord))) { - return true; + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + if (dictionaryGroup.mLocale == null) { + continue; + } + final String lowerCasedWord = word.toLowerCase(dictionaryGroup.mLocale); + for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { + final Dictionary dictionary = dictionaryGroup.getDict(dictType); + // Ideally the passed map would come out of a {@link java.util.concurrent.Future} and + // would be immutable once it's finished initializing, but concretely a null test is + // probably good enough for the time being. + if (null == dictionary) continue; + if (dictionary.isValidWord(word) + || (ignoreCase && dictionary.isValidWord(lowerCasedWord))) { + return true; + } } } return false; @@ -584,18 +646,20 @@ public class DictionaryFacilitator { return Dictionary.NOT_A_PROBABILITY; } int maxFreq = Dictionary.NOT_A_PROBABILITY; - final DictionaryGroup dictionaryGroup = mDictionaryGroup; - for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { - final Dictionary dictionary = dictionaryGroup.getDict(dictType); - if (dictionary == null) continue; - final int tempFreq; - if (isGettingMaxFrequencyOfExactMatches) { - tempFreq = dictionary.getMaxFrequencyOfExactMatches(word); - } else { - tempFreq = dictionary.getFrequency(word); - } - if (tempFreq >= maxFreq) { - maxFreq = tempFreq; + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { + final Dictionary dictionary = dictionaryGroup.getDict(dictType); + if (dictionary == null) continue; + final int tempFreq; + if (isGettingMaxFrequencyOfExactMatches) { + tempFreq = dictionary.getMaxFrequencyOfExactMatches(word); + } else { + tempFreq = dictionary.getFrequency(word); + } + if (tempFreq >= maxFreq) { + maxFreq = tempFreq; + } } } return maxFreq; @@ -610,9 +674,12 @@ public class DictionaryFacilitator { } private void clearSubDictionary(final String dictName) { - final ExpandableBinaryDictionary dictionary = mDictionaryGroup.getSubDict(dictName); - if (dictionary != null) { - dictionary.clear(); + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictName); + if (dictionary != null) { + dictionary.clear(); + } } } @@ -641,8 +708,10 @@ public class DictionaryFacilitator { public void addPhraseToContextualDictionary(final String[] phrase, final int probability, final int bigramProbabilityForWords, final int bigramProbabilityForPhrases) { + // TODO: we're inserting the phrase into the dictionary for the active language. Rethink + // this a bit from a theoretical point of view. final ExpandableBinaryDictionary contextualDict = - mDictionaryGroup.getSubDict(Dictionary.TYPE_CONTEXTUAL); + getDictionaryGroupForActiveLanguage().getSubDict(Dictionary.TYPE_CONTEXTUAL); if (contextualDict == null) { return; } @@ -675,22 +744,27 @@ public class DictionaryFacilitator { } public void dumpDictionaryForDebug(final String dictName) { - final ExpandableBinaryDictionary dictToDump = mDictionaryGroup.getSubDict(dictName); - if (dictToDump == null) { - Log.e(TAG, "Cannot dump " + dictName + ". " - + "The dictionary is not being used for suggestion or cannot be dumped."); - return; + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + final ExpandableBinaryDictionary dictToDump = dictionaryGroup.getSubDict(dictName); + if (dictToDump == null) { + Log.e(TAG, "Cannot dump " + dictName + ". " + + "The dictionary is not being used for suggestion or cannot be dumped."); + return; + } + dictToDump.dumpAllWordsForDebug(); } - dictToDump.dumpAllWordsForDebug(); } public ArrayList<Pair<String, DictionaryStats>> getStatsOfEnabledSubDicts() { final ArrayList<Pair<String, DictionaryStats>> statsOfEnabledSubDicts = new ArrayList<>(); - final DictionaryGroup dictionaryGroup = mDictionaryGroup; - for (final String dictType : SUB_DICT_TYPES) { - final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictType); - if (dictionary == null) continue; - statsOfEnabledSubDicts.add(new Pair<>(dictType, dictionary.getDictionaryStats())); + final DictionaryGroup[] dictionaryGroups = mDictionaryGroups; + for (final DictionaryGroup dictionaryGroup : dictionaryGroups) { + for (final String dictType : SUB_DICT_TYPES) { + final ExpandableBinaryDictionary dictionary = dictionaryGroup.getSubDict(dictType); + if (dictionary == null) continue; + statsOfEnabledSubDicts.add(new Pair<>(dictType, dictionary.getDictionaryStats())); + } } return statsOfEnabledSubDicts; } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorLruCache.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorLruCache.java index fa0265d86..ff4a6bde1 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorLruCache.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorLruCache.java @@ -84,7 +84,7 @@ public class DictionaryFacilitatorLruCache { private void waitForLoadingMainDictionary(final DictionaryFacilitator dictionaryFacilitator) { for (int i = 0; i < MAX_RETRY_COUNT_FOR_WAITING_FOR_LOADING_DICT; i++) { try { - dictionaryFacilitator.waitForLoadingMainDictionary( + dictionaryFacilitator.waitForLoadingMainDictionaries( WAIT_FOR_LOADING_MAIN_DICT_IN_MILLISECONDS, TimeUnit.MILLISECONDS); return; } catch (final InterruptedException e) { diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index a1dd67f27..671ba6714 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -156,23 +156,25 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { } private void asyncExecuteTaskWithWriteLock(final Runnable task) { - asyncExecuteTaskWithLock(mLock.writeLock(), task); + asyncExecuteTaskWithLock(mLock.writeLock(), mDictName /* executorName */, task); } - private void asyncExecuteTaskWithLock(final Lock lock, final Runnable task) { - asyncPreCheckAndExecuteTaskWithLock(lock, null /* preCheckTask */, task); + private void asyncExecuteTaskWithLock(final Lock lock, final String executorName, + final Runnable task) { + asyncPreCheckAndExecuteTaskWithLock(lock, null /* preCheckTask */, executorName, task); } private void asyncPreCheckAndExecuteTaskWithWriteLock( final Callable<Boolean> preCheckTask, final Runnable task) { - asyncPreCheckAndExecuteTaskWithLock(mLock.writeLock(), preCheckTask, task); + asyncPreCheckAndExecuteTaskWithLock(mLock.writeLock(), preCheckTask, + mDictName /* executorName */, task); } // Execute task with lock when the result of preCheckTask is true or preCheckTask is null. private void asyncPreCheckAndExecuteTaskWithLock(final Lock lock, - final Callable<Boolean> preCheckTask, final Runnable task) { - ExecutorUtils.getExecutor(mDictName).execute(new Runnable() { + final Callable<Boolean> preCheckTask, final String executorName, final Runnable task) { + ExecutorUtils.getExecutor(executorName).execute(new Runnable() { @Override public void run() { if (preCheckTask != null) { @@ -676,10 +678,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { public void dumpAllWordsForDebug() { reloadDictionaryIfRequired(); - asyncExecuteTaskWithLock(mLock.readLock(), new Runnable() { + asyncExecuteTaskWithLock(mLock.readLock(), "dumpAllWordsForDebug", new Runnable() { @Override public void run() { - Log.d(TAG, "Dump dictionary: " + mDictName); + Log.d(TAG, "Dump dictionary: " + mDictName + " for " + mLocale); try { final DictionaryHeader header = mBinaryDictionary.getHeader(); Log.d(TAG, "Format version: " + mBinaryDictionary.getFormatVersion()); diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 0a64c4c26..475782042 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -1000,7 +1000,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen mHandler.cancelUpdateSuggestionStrip(); mainKeyboardView.setMainDictionaryAvailability( - mDictionaryFacilitator.hasInitializedMainDictionary()); + mDictionaryFacilitator.hasAtLeastOneInitializedMainDictionary()); mainKeyboardView.setKeyPreviewPopupEnabled(currentSettingsValues.mKeyPreviewPopupOn, currentSettingsValues.mKeyPreviewPopupDismissDelay); mainKeyboardView.setSlidingKeyInputPreviewEnabled( diff --git a/java/src/com/android/inputmethod/latin/PersonalizationHelperForDictionaryFacilitator.java b/java/src/com/android/inputmethod/latin/PersonalizationHelperForDictionaryFacilitator.java index 43cebdfa4..396d062f8 100644 --- a/java/src/com/android/inputmethod/latin/PersonalizationHelperForDictionaryFacilitator.java +++ b/java/src/com/android/inputmethod/latin/PersonalizationHelperForDictionaryFacilitator.java @@ -88,17 +88,17 @@ public class PersonalizationHelperForDictionaryFacilitator { /** * Flush personalization dictionaries to dictionary files. Close dictionaries after writing - * files except the dictionary that is used for generating suggestions. + * files except the dictionaries that is used for generating suggestions. * - * @param personalizationDictUsedForSuggestion the personalization dictionary used for + * @param personalizationDictsUsedForSuggestion the personalization dictionaries used for * generating suggestions that won't be closed. */ public void flushPersonalizationDictionariesToUpdate( - final ExpandableBinaryDictionary personalizationDictUsedForSuggestion) { + final HashSet<ExpandableBinaryDictionary> personalizationDictsUsedForSuggestion) { for (final ExpandableBinaryDictionary personalizationDict : mPersonalizationDictsToUpdate.values()) { personalizationDict.asyncFlushBinaryDictionary(); - if (personalizationDict != personalizationDictUsedForSuggestion) { + if (!personalizationDictsUsedForSuggestion.contains(personalizationDict)) { // Close if the dictionary is not being used for suggestion. personalizationDict.close(); } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 9e4aa40a2..9bf017578 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -157,7 +157,7 @@ public final class Suggest { if (!isCorrectionEnabled || !allowsToBeAutoCorrected || resultsArePredictions || suggestionResults.isEmpty() || wordComposer.hasDigits() || wordComposer.isMostlyCaps() || wordComposer.isResumed() - || !mDictionaryFacilitator.hasInitializedMainDictionary() + || !mDictionaryFacilitator.hasAtLeastOneInitializedMainDictionary() || suggestionResults.first().isKindOf(SuggestedWordInfo.KIND_SHORTCUT)) { // If we don't have a main dictionary, we never want to auto-correct. The reason for // this is, the user may have a contact whose name happens to match a valid word in diff --git a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java index 49b34d391..352391611 100644 --- a/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java +++ b/java/src/com/android/inputmethod/latin/spellcheck/AndroidSpellCheckerService.java @@ -185,7 +185,7 @@ public final class AndroidSpellCheckerService extends SpellCheckerService try { final DictionaryFacilitator dictionaryFacilitator = mDictionaryFacilitatorCache.get(locale); - return dictionaryFacilitator.hasInitializedMainDictionary(); + return dictionaryFacilitator.hasAtLeastOneInitializedMainDictionary(); } finally { mSemaphore.release(); } diff --git a/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h b/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h index cecfc7aa9..1b796b5d4 100644 --- a/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h +++ b/native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h @@ -32,7 +32,7 @@ class DicNodeProperties { public: AK_FORCE_INLINE DicNodeProperties() : mChildrenPtNodeArrayPos(NOT_A_DICT_POS), mDicNodeCodePoint(NOT_A_CODE_POINT), - mWordId(NOT_A_WORD_ID), mDepth(0), mLeavingDepth(0) {} + mWordId(NOT_A_WORD_ID), mDepth(0), mLeavingDepth(0), mPrevWordCount(0) {} ~DicNodeProperties() {} @@ -45,6 +45,7 @@ class DicNodeProperties { mDepth = depth; mLeavingDepth = leavingDepth; prevWordIds.copyToArray(&mPrevWordIds, 0 /* offset */); + mPrevWordCount = prevWordIds.size(); } // Init for root with prevWordsPtNodePos which is used for n-gram @@ -55,6 +56,7 @@ class DicNodeProperties { mDepth = 0; mLeavingDepth = 0; prevWordIds.copyToArray(&mPrevWordIds, 0 /* offset */); + mPrevWordCount = prevWordIds.size(); } void initByCopy(const DicNodeProperties *const dicNodeProp) { @@ -63,8 +65,9 @@ class DicNodeProperties { mWordId = dicNodeProp->mWordId; mDepth = dicNodeProp->mDepth; mLeavingDepth = dicNodeProp->mLeavingDepth; - WordIdArrayView::fromArray(dicNodeProp->mPrevWordIds) - .copyToArray(&mPrevWordIds, 0 /* offset */); + const WordIdArrayView prevWordIdArrayView = dicNodeProp->getPrevWordIds(); + prevWordIdArrayView.copyToArray(&mPrevWordIds, 0 /* offset */); + mPrevWordCount = prevWordIdArrayView.size(); } // Init as passing child @@ -74,8 +77,9 @@ class DicNodeProperties { mWordId = dicNodeProp->mWordId; mDepth = dicNodeProp->mDepth + 1; // Increment the depth of a passing child mLeavingDepth = dicNodeProp->mLeavingDepth; - WordIdArrayView::fromArray(dicNodeProp->mPrevWordIds) - .copyToArray(&mPrevWordIds, 0 /* offset */); + const WordIdArrayView prevWordIdArrayView = dicNodeProp->getPrevWordIds(); + prevWordIdArrayView.copyToArray(&mPrevWordIds, 0 /* offset */); + mPrevWordCount = prevWordIdArrayView.size(); } int getChildrenPtNodeArrayPos() const { @@ -104,7 +108,7 @@ class DicNodeProperties { } const WordIdArrayView getPrevWordIds() const { - return WordIdArrayView::fromArray(mPrevWordIds); + return WordIdArrayView::fromArray(mPrevWordIds).limit(mPrevWordCount); } int getWordId() const { @@ -121,6 +125,7 @@ class DicNodeProperties { uint16_t mDepth; uint16_t mLeavingDepth; WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> mPrevWordIds; + size_t mPrevWordCount; }; } // namespace latinime #endif // LATINIME_DIC_NODE_PROPERTIES_H diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp index f54bb151a..0675de6fa 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp @@ -39,7 +39,7 @@ bool LanguageModelDictContent::runGC( } int LanguageModelDictContent::getWordProbability(const WordIdArrayView prevWordIds, - const int wordId) const { + const int wordId, const HeaderPolicy *const headerPolicy) const { int bitmapEntryIndices[MAX_PREV_WORD_COUNT_FOR_N_GRAM + 1]; bitmapEntryIndices[0] = mTrieMap.getRootBitmapEntryIndex(); int maxLevel = 0; @@ -58,14 +58,15 @@ int LanguageModelDictContent::getWordProbability(const WordIdArrayView prevWordI if (!result.mIsValid) { continue; } - const int probability = - ProbabilityEntry::decode(result.mValue, mHasHistoricalInfo).getProbability(); + const ProbabilityEntry probabilityEntry = + ProbabilityEntry::decode(result.mValue, mHasHistoricalInfo); if (mHasHistoricalInfo) { - return std::min( - probability + ForgettingCurveUtils::getProbabilityBiasForNgram(i + 1 /* n */), - MAX_PROBABILITY); + const int probability = ForgettingCurveUtils::decodeProbability( + probabilityEntry.getHistoricalInfo(), headerPolicy) + + ForgettingCurveUtils::getProbabilityBiasForNgram(i + 1 /* n */); + return std::min(probability, MAX_PROBABILITY); } else { - return probability; + return probabilityEntry.getProbability(); } } // Cannot find the word. diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h index 4e0b47036..a793af4be 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h @@ -128,7 +128,8 @@ class LanguageModelDictContent { const LanguageModelDictContent *const originalContent, int *const outNgramCount); - int getWordProbability(const WordIdArrayView prevWordIds, const int wordId) const; + int getWordProbability(const WordIdArrayView prevWordIds, const int wordId, + const HeaderPolicy *const headerPolicy) const; ProbabilityEntry getProbabilityEntry(const int wordId) const { return getNgramProbabilityEntry(WordIdArrayView(), wordId); diff --git a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp index e624bf338..d537711b0 100644 --- a/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp +++ b/native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp @@ -121,9 +121,10 @@ const WordAttributes Ver4PatriciaTriePolicy::getWordAttributesInContext( mBuffers->getTerminalPositionLookupTable()->getTerminalPtNodePosition(wordId); const PtNodeParams ptNodeParams = mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos); // TODO: Support n-gram. - return WordAttributes(mBuffers->getLanguageModelDictContent()->getWordProbability( - prevWordIds.limit(1 /* maxSize */), wordId), ptNodeParams.isBlacklisted(), - ptNodeParams.isNotAWord(), ptNodeParams.getProbability() == 0); + const int probability = mBuffers->getLanguageModelDictContent()->getWordProbability( + prevWordIds.limit(1 /* maxSize */), wordId, mHeaderPolicy); + return WordAttributes(probability, ptNodeParams.isBlacklisted(), ptNodeParams.isNotAWord(), + probability == 0); } int Ver4PatriciaTriePolicy::getProbabilityOfWord(const WordIdArrayView prevWordIds, @@ -309,30 +310,32 @@ bool Ver4PatriciaTriePolicy::addNgramEntry(const PrevWordsInfo *const prevWordsI if (prevWordIds.empty()) { return false; } - // TODO: Support N-gram. - if (prevWordIds[0] == NOT_A_WORD_ID) { - if (prevWordsInfo->isNthPrevWordBeginningOfSentence(1 /* n */)) { - const std::vector<UnigramProperty::ShortcutProperty> shortcuts; - const UnigramProperty beginningOfSentenceUnigramProperty( - true /* representsBeginningOfSentence */, true /* isNotAWord */, - false /* isBlacklisted */, MAX_PROBABILITY /* probability */, - NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts); - if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */), - &beginningOfSentenceUnigramProperty)) { - AKLOGE("Cannot add unigram entry for the beginning-of-sentence."); - return false; - } - // Refresh word ids. - prevWordsInfo->getPrevWordIds(this, &prevWordIdArray, false /* tryLowerCaseSearch */); - } else { + for (size_t i = 0; i < prevWordIds.size(); ++i) { + if (prevWordIds[i] != NOT_A_WORD_ID) { + continue; + } + if (!prevWordsInfo->isNthPrevWordBeginningOfSentence(i + 1 /* n */)) { return false; } + const std::vector<UnigramProperty::ShortcutProperty> shortcuts; + const UnigramProperty beginningOfSentenceUnigramProperty( + true /* representsBeginningOfSentence */, true /* isNotAWord */, + false /* isBlacklisted */, MAX_PROBABILITY /* probability */, + NOT_A_TIMESTAMP /* timestamp */, 0 /* level */, 0 /* count */, &shortcuts); + if (!addUnigramEntry(prevWordsInfo->getNthPrevWordCodePoints(1 /* n */), + &beginningOfSentenceUnigramProperty)) { + AKLOGE("Cannot add unigram entry for the beginning-of-sentence."); + return false; + } + // Refresh word ids. + prevWordsInfo->getPrevWordIds(this, &prevWordIdArray, false /* tryLowerCaseSearch */); } const int wordId = getWordId(CodePointArrayView(*bigramProperty->getTargetCodePoints()), false /* forceLowerCaseSearch */); if (wordId == NOT_A_WORD_ID) { return false; } + // TODO: Support N-gram. bool addedNewEntry = false; WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordsPtNodePos; for (size_t i = 0; i < prevWordsPtNodePos.size(); ++i) { @@ -374,8 +377,7 @@ bool Ver4PatriciaTriePolicy::removeNgramEntry(const PrevWordsInfo *const prevWor WordIdArray<MAX_PREV_WORD_COUNT_FOR_N_GRAM> prevWordIdArray; const WordIdArrayView prevWordIds = prevWordsInfo->getPrevWordIds(this, &prevWordIdArray, false /* tryLowerCaseSerch */); - // TODO: Support N-gram. - if (prevWordIds.empty() || prevWordIds[0] == NOT_A_WORD_ID) { + if (prevWordIds.empty() || prevWordIds.contains(NOT_A_WORD_ID)) { return false; } const int wordId = getWordId(wordCodePoints, false /* forceLowerCaseSearch */); diff --git a/native/jni/src/utils/int_array_view.h b/native/jni/src/utils/int_array_view.h index caa13d976..cc5f328ba 100644 --- a/native/jni/src/utils/int_array_view.h +++ b/native/jni/src/utils/int_array_view.h @@ -17,6 +17,7 @@ #ifndef LATINIME_INT_ARRAY_VIEW_H #define LATINIME_INT_ARRAY_VIEW_H +#include <algorithm> #include <array> #include <cstdint> #include <cstring> @@ -92,12 +93,16 @@ class IntArrayView { return mPtr + mSize; } + AK_FORCE_INLINE bool contains(const int value) const { + return std::find(begin(), end(), value) != end(); + } + // Returns the view whose size is smaller than or equal to the given count. - const IntArrayView limit(const size_t maxSize) const { + AK_FORCE_INLINE const IntArrayView limit(const size_t maxSize) const { return IntArrayView(mPtr, std::min(maxSize, mSize)); } - const IntArrayView skip(const size_t n) const { + AK_FORCE_INLINE const IntArrayView skip(const size_t n) const { if (mSize <= n) { return IntArrayView(); } diff --git a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp index 7608b45c2..c5849d054 100644 --- a/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp +++ b/native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp @@ -107,13 +107,15 @@ TEST(LanguageModelDictContentTest, TestGetWordProbability) { languageModelDictContent.setProbabilityEntry(prevWordIds[0], &probabilityEntry); languageModelDictContent.setNgramProbabilityEntry(prevWordIds.limit(1), wordId, &bigramProbabilityEntry); - EXPECT_EQ(bigramProbability, languageModelDictContent.getWordProbability(prevWordIds, wordId)); + EXPECT_EQ(bigramProbability, languageModelDictContent.getWordProbability(prevWordIds, wordId, + nullptr /* headerPolicy */)); const ProbabilityEntry trigramProbabilityEntry(flag, trigramProbability); languageModelDictContent.setNgramProbabilityEntry(prevWordIds.limit(1), prevWordIds[1], &probabilityEntry); languageModelDictContent.setNgramProbabilityEntry(prevWordIds.limit(2), wordId, &trigramProbabilityEntry); - EXPECT_EQ(trigramProbability, languageModelDictContent.getWordProbability(prevWordIds, wordId)); + EXPECT_EQ(trigramProbability, languageModelDictContent.getWordProbability(prevWordIds, wordId, + nullptr /* headerPolicy */)); } } // namespace diff --git a/native/jni/tests/utils/int_array_view_test.cpp b/native/jni/tests/utils/int_array_view_test.cpp index 3bc294cdd..934e27e1c 100644 --- a/native/jni/tests/utils/int_array_view_test.cpp +++ b/native/jni/tests/utils/int_array_view_test.cpp @@ -58,6 +58,19 @@ TEST(IntArrayViewTest, TestConstructFromObject) { EXPECT_EQ(object, intArrayView[0]); } +TEST(IntArrayViewTest, TestContains) { + EXPECT_FALSE(IntArrayView().contains(0)); + EXPECT_FALSE(IntArrayView().contains(1)); + + const std::vector<int> intVector = {3, 2, 1, 0, -1, -2}; + IntArrayView intArrayView(intVector); + EXPECT_TRUE(intArrayView.contains(0)); + EXPECT_TRUE(intArrayView.contains(3)); + EXPECT_TRUE(intArrayView.contains(-2)); + EXPECT_FALSE(intArrayView.contains(-3)); + EXPECT_FALSE(intArrayView.limit(0).contains(3)); +} + TEST(IntArrayViewTest, TestLimit) { const std::vector<int> intVector = {3, 2, 1, 0, -1, -2}; IntArrayView intArrayView(intVector); diff --git a/tests/AndroidManifest.xml b/tests/AndroidManifest.xml index 4ca846be1..8a628cdac 100644 --- a/tests/AndroidManifest.xml +++ b/tests/AndroidManifest.xml @@ -17,7 +17,7 @@ <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.android.inputmethod.latin.tests"> - <uses-sdk android:minSdkVersion="14" android:targetSdkVersion="19" /> + <uses-sdk android:minSdkVersion="14" android:targetSdkVersion="21" /> <uses-permission android:name="android.permission.READ_CONTACTS" /> |