diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin')
20 files changed, 447 insertions, 305 deletions
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index b20bcd1f9..013f9220a 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -26,12 +26,12 @@ import com.android.inputmethod.latin.makedict.DictionaryHeader; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.StringUtils; -import com.android.inputmethod.latin.utils.WordProperty; import java.io.File; import java.util.ArrayList; @@ -217,9 +217,8 @@ public final class BinaryDictionary extends Dictionary { outAttributeValues.get(i)); attributes.put(attributeKey, attributeValue); } - final boolean hasHistoricalInfo = - attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY).equals( - DictionaryHeader.ATTRIBUTE_VALUE_TRUE); + final boolean hasHistoricalInfo = DictionaryHeader.ATTRIBUTE_VALUE_TRUE.equals( + attributes.get(DictionaryHeader.HAS_HISTORICAL_INFO_KEY)); return new DictionaryHeader(outHeaderSize[0], new DictionaryOptions(attributes), new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); } diff --git a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java index e68c6b771..259c1372e 100644 --- a/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java +++ b/java/src/com/android/inputmethod/latin/DictionaryFacilitatorForSuggest.java @@ -430,12 +430,19 @@ public class DictionaryFacilitatorForSuggest { public void getSuggestions(final WordComposer composer, final String prevWord, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, - final int sessionId, final Set<SuggestedWordInfo> suggestionSet) { + final int sessionId, final Set<SuggestedWordInfo> suggestionSet, + final ArrayList<SuggestedWordInfo> rawSuggestions) { for (final String key : mDictionaries.keySet()) { final Dictionary dictionary = mDictionaries.get(key); if (null == dictionary) continue; - suggestionSet.addAll(dictionary.getSuggestionsWithSessionId(composer, prevWord, - proximityInfo, blockOffensiveWords, additionalFeaturesOptions, sessionId)); + final ArrayList<SuggestedWordInfo> dictionarySuggestions = + dictionary.getSuggestionsWithSessionId(composer, prevWord, proximityInfo, + blockOffensiveWords, additionalFeaturesOptions, sessionId); + if (null == dictionarySuggestions) continue; + suggestionSet.addAll(dictionarySuggestions); + if (null != rawSuggestions) { + rawSuggestions.addAll(dictionarySuggestions); + } } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index 565d6a1f9..c2451ce8d 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -23,13 +23,15 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.makedict.DictionaryHeader; import com.android.inputmethod.latin.makedict.FormatSpec; +import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; import com.android.inputmethod.latin.utils.AsyncResultHolder; import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.CombinedFormatUtils; import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor; -import com.android.inputmethod.latin.utils.WordProperty; import java.io.File; import java.util.ArrayList; @@ -750,7 +752,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { // TODO: Implement BinaryDictionary.isInDictionary(). @UsedForTesting - public boolean isInDictionaryForTests(final String word) { + public boolean isInUnderlyingBinaryDictionaryForTests(final String word) { final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<Boolean>(); getExecutor(mDictName).executePrioritized(new Runnable() { @Override @@ -785,7 +787,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { getExecutor(mDictName).execute(new Runnable() { @Override public void run() { - Log.d(TAG, "dictionary=" + mDictName); + Log.d(TAG, "Dump dictionary: " + mDictName); + try { + final DictionaryHeader header = mBinaryDictionary.getHeader(); + Log.d(TAG, CombinedFormatUtils.formatAttributeMap( + header.mDictionaryOptions.mAttributes)); + } catch (final UnsupportedFormatException e) { + Log.d(TAG, "Cannot fetch header information.", e); + } int token = 0; do { final BinaryDictionary.GetNextWordPropertyResult result = diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index e55c08dae..3b427720a 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -620,6 +620,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ResearchLogger.getInstance().onDestroy(); } unregisterReceiver(mDictionaryPackInstallReceiver); + unregisterReceiver(mDictionaryDumpBroadcastReceiver); PersonalizationDictionarySessionRegistrar.close(this); LatinImeLogger.commit(); LatinImeLogger.onDestroy(); @@ -997,7 +998,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen SuggestedWords.getFromApplicationSpecifiedCompletions( applicationSpecifiedCompletions); final SuggestedWords suggestedWords = new SuggestedWords( - applicationSuggestedWords, + applicationSuggestedWords, null /* rawSuggestions */, false /* typedWordValid */, false /* willAutoCorrect */, false /* isPunctuationSuggestions */, @@ -1417,7 +1418,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen ? SuggestedWords.EMPTY : previousSuggestedWords; final ArrayList<SuggestedWords.SuggestedWordInfo> typedWordAndPreviousSuggestions = SuggestedWords.getTypedWordAndPreviousSuggestions(typedWord, oldSuggestedWords); - return new SuggestedWords(typedWordAndPreviousSuggestions, + return new SuggestedWords(typedWordAndPreviousSuggestions, null /* rawSuggestions */, false /* typedWordValid */, false /* hasAutoCorrectionCandidate */, false /* isPunctuationSuggestions */, diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 439c8562e..5e74d75b0 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -20,6 +20,7 @@ import android.text.TextUtils; import com.android.inputmethod.keyboard.ProximityInfo; import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; +import com.android.inputmethod.latin.define.ProductionFlag; import com.android.inputmethod.latin.utils.AutoCorrectionUtils; import com.android.inputmethod.latin.utils.BoundedTreeSet; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -51,7 +52,6 @@ public final class Suggest { private static final int SUPPRESS_SUGGEST_THRESHOLD = -2000000000; private static final boolean DBG = LatinImeLogger.sDBG; - public final DictionaryFacilitatorForSuggest mDictionaryFacilitator; private float mAutoCorrectionThreshold; @@ -124,24 +124,40 @@ public final class Suggest { } else { wordComposerForLookup = wordComposer; } + final ArrayList<SuggestedWordInfo> rawSuggestions; + if (ProductionFlag.INCLUDE_RAW_SUGGESTIONS) { + rawSuggestions = CollectionUtils.newArrayList(); + } else { + rawSuggestions = null; + } mDictionaryFacilitator.getSuggestions(wordComposerForLookup, prevWordForBigram, proximityInfo, blockOffensiveWords, additionalFeaturesOptions, SESSION_TYPING, - suggestionsSet); + suggestionsSet, rawSuggestions); + final String firstSuggestion; final String whitelistedWord; if (suggestionsSet.isEmpty()) { - whitelistedWord = null; - } else if (SuggestedWordInfo.KIND_WHITELIST != suggestionsSet.first().mKind) { - whitelistedWord = null; + whitelistedWord = firstSuggestion = null; } else { - whitelistedWord = suggestionsSet.first().mWord; + final SuggestedWordInfo firstSuggestedWordInfo = suggestionsSet.first(); + firstSuggestion = firstSuggestedWordInfo.mWord; + if (SuggestedWordInfo.KIND_WHITELIST != firstSuggestedWordInfo.mKind) { + whitelistedWord = null; + } else { + whitelistedWord = firstSuggestion; + } } - // The word can be auto-corrected if it has a whitelist entry that is not itself, - // or if it's a 2+ characters non-word (i.e. it's not in the dictionary). + // We allow auto-correction if we have a whitelisted word, or if the word is not a valid + // word of more than 1 char, except if the first suggestion is the same as the typed string + // because in this case if it's strong enough to auto-correct that will mistakenly designate + // the second candidate for auto-correction. + // TODO: stop relying on indices to find where is the auto-correction in the suggested + // words, and correct this test. final boolean allowsToBeAutoCorrected = (null != whitelistedWord && !whitelistedWord.equals(consideredWord)) || (consideredWord.length() > 1 && !mDictionaryFacilitator.isValidWord( - consideredWord, wordComposer.isFirstCharCapitalized())); + consideredWord, wordComposer.isFirstCharCapitalized()) + && !consideredWord.equals(firstSuggestion)); final boolean hasAutoCorrection; // TODO: using isCorrectionEnabled here is not very good. It's probably useless, because @@ -205,7 +221,7 @@ public final class Suggest { suggestionsList = suggestionsContainer; } - callback.onGetSuggestedWords(new SuggestedWords(suggestionsList, + callback.onGetSuggestedWords(new SuggestedWords(suggestionsList, rawSuggestions, // TODO: this first argument is lying. If this is a whitelisted word which is an // actual word, it says typedWordValid = false, which looks wrong. We should either // rename the attribute or change the value. @@ -225,8 +241,15 @@ public final class Suggest { final OnGetSuggestedWordsCallback callback) { final BoundedTreeSet suggestionsSet = new BoundedTreeSet(sSuggestedWordInfoComparator, SuggestedWords.MAX_SUGGESTIONS); + final ArrayList<SuggestedWordInfo> rawSuggestions; + if (ProductionFlag.INCLUDE_RAW_SUGGESTIONS) { + rawSuggestions = CollectionUtils.newArrayList(); + } else { + rawSuggestions = null; + } mDictionaryFacilitator.getSuggestions(wordComposer, prevWordForBigram, proximityInfo, - blockOffensiveWords, additionalFeaturesOptions, sessionId, suggestionsSet); + blockOffensiveWords, additionalFeaturesOptions, sessionId, suggestionsSet, + rawSuggestions); for (SuggestedWordInfo wordInfo : suggestionsSet) { LatinImeLogger.onAddSuggestedWord(wordInfo.mWord, wordInfo.mSourceDict.mDictType); } @@ -263,7 +286,7 @@ public final class Suggest { // In the batch input mode, the most relevant suggested word should act as a "typed word" // (typedWordValid=true), not as an "auto correct word" (willAutoCorrect=false). - callback.onGetSuggestedWords(new SuggestedWords(suggestionsContainer, + callback.onGetSuggestedWords(new SuggestedWords(suggestionsContainer, rawSuggestions, true /* typedWordValid */, false /* willAutoCorrect */, false /* isPunctuationSuggestions */, diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index dbaf822e4..b2efc4a86 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -37,7 +37,7 @@ public final class SuggestedWords { private static final ArrayList<SuggestedWordInfo> EMPTY_WORD_INFO_LIST = CollectionUtils.newArrayList(0); public static final SuggestedWords EMPTY = new SuggestedWords( - EMPTY_WORD_INFO_LIST, false, false, false, false, false); + EMPTY_WORD_INFO_LIST, null /* rawSuggestions */, false, false, false, false, false); public final String mTypedWord; public final boolean mTypedWordValid; @@ -50,25 +50,29 @@ public final class SuggestedWords { public final boolean mIsPrediction; public final int mSequenceNumber; // Sequence number for auto-commit. private final ArrayList<SuggestedWordInfo> mSuggestedWordInfoList; + public final ArrayList<SuggestedWordInfo> mRawSuggestions; public SuggestedWords(final ArrayList<SuggestedWordInfo> suggestedWordInfoList, + final ArrayList<SuggestedWordInfo> rawSuggestions, final boolean typedWordValid, final boolean willAutoCorrect, final boolean isPunctuationSuggestions, final boolean isObsoleteSuggestions, final boolean isPrediction) { - this(suggestedWordInfoList, typedWordValid, willAutoCorrect, isPunctuationSuggestions, - isObsoleteSuggestions, isPrediction, NOT_A_SEQUENCE_NUMBER); + this(suggestedWordInfoList, rawSuggestions, typedWordValid, willAutoCorrect, + isPunctuationSuggestions, isObsoleteSuggestions, isPrediction, + NOT_A_SEQUENCE_NUMBER); } public SuggestedWords(final ArrayList<SuggestedWordInfo> suggestedWordInfoList, + final ArrayList<SuggestedWordInfo> rawSuggestions, final boolean typedWordValid, final boolean willAutoCorrect, final boolean isPunctuationSuggestions, final boolean isObsoleteSuggestions, final boolean isPrediction, final int sequenceNumber) { - this(suggestedWordInfoList, + this(suggestedWordInfoList, rawSuggestions, suggestedWordInfoList.isEmpty() ? null : suggestedWordInfoList.get(INDEX_OF_TYPED_WORD).mWord, typedWordValid, willAutoCorrect, isPunctuationSuggestions, @@ -76,6 +80,7 @@ public final class SuggestedWords { } public SuggestedWords(final ArrayList<SuggestedWordInfo> suggestedWordInfoList, + final ArrayList<SuggestedWordInfo> rawSuggestions, final String typedWord, final boolean typedWordValid, final boolean willAutoCorrect, @@ -84,6 +89,7 @@ public final class SuggestedWords { final boolean isPrediction, final int sequenceNumber) { mSuggestedWordInfoList = suggestedWordInfoList; + mRawSuggestions = rawSuggestions; mTypedWordValid = typedWordValid; mWillAutoCorrect = willAutoCorrect; mIsPunctuationSuggestions = isPunctuationSuggestions; @@ -306,9 +312,9 @@ public final class SuggestedWords { } // We should never autocorrect, so we say the typed word is valid. Also, in this case, // no auto-correction should take place hence willAutoCorrect = false. - return new SuggestedWords(newSuggestions, typedWord, true /* typedWordValid */, - false /* willAutoCorrect */, mIsPunctuationSuggestions, mIsObsoleteSuggestions, - mIsPrediction, NOT_A_SEQUENCE_NUMBER); + return new SuggestedWords(newSuggestions, null /* rawSuggestions */, typedWord, + true /* typedWordValid */, false /* willAutoCorrect */, mIsPunctuationSuggestions, + mIsObsoleteSuggestions, mIsPrediction, NOT_A_SEQUENCE_NUMBER); } // Creates a new SuggestedWordInfo from the currently suggested words that removes all but the @@ -326,7 +332,7 @@ public final class SuggestedWords { info.mSourceDict, SuggestedWordInfo.NOT_AN_INDEX, SuggestedWordInfo.NOT_A_CONFIDENCE)); } - return new SuggestedWords(newSuggestions, mTypedWordValid, + return new SuggestedWords(newSuggestions, null /* rawSuggestions */, mTypedWordValid, mWillAutoCorrect, mIsPunctuationSuggestions, mIsObsoleteSuggestions, mIsPrediction); } diff --git a/java/src/com/android/inputmethod/latin/define/ProductionFlag.java b/java/src/com/android/inputmethod/latin/define/ProductionFlag.java index dc937fb25..e6fa1cdad 100644 --- a/java/src/com/android/inputmethod/latin/define/ProductionFlag.java +++ b/java/src/com/android/inputmethod/latin/define/ProductionFlag.java @@ -29,4 +29,7 @@ public final class ProductionFlag { public static final boolean USES_DEVELOPMENT_ONLY_DIAGNOSTICS_DEBUG = false; public static final boolean IS_HARDWARE_KEYBOARD_SUPPORTED = false; + + // Include all suggestions from all dictionaries in {@link SuggestedWords#mRawSuggestions}. + public static final boolean INCLUDE_RAW_SUGGESTIONS = false; } diff --git a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java index 3ecf5f0fb..6e9050593 100644 --- a/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java +++ b/java/src/com/android/inputmethod/latin/inputlogic/InputLogic.java @@ -1332,7 +1332,8 @@ public final class InputLogic { } else { // We found suggestion spans in the word. We'll create the SuggestedWords out of // them, and make willAutoCorrect false. - final SuggestedWords suggestedWords = new SuggestedWords(suggestions, typedWord, + final SuggestedWords suggestedWords = new SuggestedWords(suggestions, + null /* rawSuggestions */, typedWord, true /* typedWordValid */, false /* willAutoCorrect */, false /* isPunctuationSuggestions */, false /* isObsoleteSuggestions */, false /* isPrediction */, SuggestedWords.NOT_A_SEQUENCE_NUMBER); diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 369184573..782ada3f4 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -511,7 +511,7 @@ public final class BinaryDictDecoderUtils { final WeightedString word = getWordAtPosition(dictDecoder, headerSize, bigram.mAddress, options); final int reconstructedFrequency = - BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, + BinaryDictIOUtils.reconstructBigramFrequency(word.getProbability(), bigram.mFrequency); bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); } @@ -606,19 +606,21 @@ public final class BinaryDictDecoderUtils { FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); if (null != dict) { - for (final Word w : dict) { - if (w.mIsBlacklistEntry) { - newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); + for (final WordProperty wordProperty : dict) { + if (wordProperty.mIsBlacklistEntry) { + newDict.addBlacklistEntry(wordProperty.mWord, wordProperty.mShortcutTargets, + wordProperty.mIsNotAWord); } else { - newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); + newDict.add(wordProperty.mWord, wordProperty.getProbability(), + wordProperty.mShortcutTargets, wordProperty.mIsNotAWord); } } - for (final Word w : dict) { + for (final WordProperty wordProperty : dict) { // By construction a binary dictionary may not have bigrams pointing to // words that are not also registered as unigrams so we don't have to avoid // them explicitly here. - for (final WeightedString bigram : w.mBigrams) { - newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); + for (final WeightedString bigram : wordProperty.mBigrams) { + newDict.setBigram(wordProperty.mWord, bigram.mWord, bigram.getProbability()); } } } diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 5b0e8399a..e9561afd3 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -31,7 +31,7 @@ import java.util.LinkedList; * A dictionary that can fusion heads and tails of words for more compression. */ @UsedForTesting -public final class FusionDictionary implements Iterable<Word> { +public final class FusionDictionary implements Iterable<WordProperty> { private static final boolean DBG = MakedictLog.DBG; private static int CHARACTER_NOT_FOUND_INDEX = -1; @@ -67,29 +67,42 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * A string with a frequency. + * A string with a probability. * * This represents an "attribute", that is either a bigram or a shortcut. */ public static final class WeightedString { public final String mWord; - public int mFrequency; - public WeightedString(String word, int frequency) { + public ProbabilityInfo mProbabilityInfo; + + public WeightedString(final String word, final int probability) { + this(word, new ProbabilityInfo(probability)); + } + + public WeightedString(final String word, final ProbabilityInfo probabilityInfo) { mWord = word; - mFrequency = frequency; + mProbabilityInfo = probabilityInfo; + } + + public int getProbability() { + return mProbabilityInfo.mProbability; + } + + public void setProbability(final int probability) { + mProbabilityInfo = new ProbabilityInfo(probability); } @Override public int hashCode() { - return Arrays.hashCode(new Object[] { mWord, mFrequency }); + return Arrays.hashCode(new Object[] { mWord, mProbabilityInfo}); } @Override public boolean equals(Object o) { if (o == this) return true; if (!(o instanceof WeightedString)) return false; - WeightedString w = (WeightedString)o; - return mWord.equals(w.mWord) && mFrequency == w.mFrequency; + final WeightedString w = (WeightedString)o; + return mWord.equals(w.mWord) && mProbabilityInfo.equals(w.mProbabilityInfo); } } @@ -200,18 +213,18 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Adds a word to the bigram list. Updates the frequency if the word already + * Adds a word to the bigram list. Updates the probability if the word already * exists. */ - public void addBigram(final String word, final int frequency) { + public void addBigram(final String word, final int probability) { if (mBigrams == null) { mBigrams = new ArrayList<WeightedString>(); } WeightedString bigram = getBigram(word); if (bigram != null) { - bigram.mFrequency = frequency; + bigram.setProbability(probability); } else { - bigram = new WeightedString(word, frequency); + bigram = new WeightedString(word, probability); mBigrams.add(bigram); } } @@ -273,8 +286,8 @@ public final class FusionDictionary implements Iterable<Word> { final WeightedString existingShortcut = getShortcut(shortcut.mWord); if (existingShortcut == null) { mShortcutTargets.add(shortcut); - } else if (existingShortcut.mFrequency < shortcut.mFrequency) { - existingShortcut.mFrequency = shortcut.mFrequency; + } else if (existingShortcut.getProbability() < shortcut.getProbability()) { + existingShortcut.setProbability(shortcut.getProbability()); } } } @@ -289,8 +302,8 @@ public final class FusionDictionary implements Iterable<Word> { final WeightedString existingBigram = getBigram(bigram.mWord); if (existingBigram == null) { mBigrams.add(bigram); - } else if (existingBigram.mFrequency < bigram.mFrequency) { - existingBigram.mFrequency = bigram.mFrequency; + } else if (existingBigram.getProbability() < bigram.getProbability()) { + existingBigram.setProbability(bigram.getProbability()); } } } @@ -693,7 +706,7 @@ public final class FusionDictionary implements Iterable<Word> { * * This is purely for convenience. */ - public static final class DictionaryIterator implements Iterator<Word> { + public static final class DictionaryIterator implements Iterator<WordProperty> { private static final class Position { public Iterator<PtNode> pos; public int length; @@ -723,7 +736,7 @@ public final class FusionDictionary implements Iterable<Word> { } @Override - public Word next() { + public WordProperty next() { Position currentPos = mPositions.getLast(); mCurrentString.setLength(currentPos.length); @@ -740,7 +753,7 @@ public final class FusionDictionary implements Iterable<Word> { mPositions.addLast(currentPos); } if (currentPtNode.mFrequency >= 0) { - return new Word(mCurrentString.toString(), currentPtNode.mFrequency, + return new WordProperty(mCurrentString.toString(), currentPtNode.mFrequency, currentPtNode.mShortcutTargets, currentPtNode.mBigrams, currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry); } @@ -766,7 +779,7 @@ public final class FusionDictionary implements Iterable<Word> { * and say : for (Word w : x) {} */ @Override - public Iterator<Word> iterator() { + public Iterator<WordProperty> iterator() { return new DictionaryIterator(mRootNodeArray.mData); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java b/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java index c1a43cedf..dafbc04b9 100644 --- a/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java +++ b/java/src/com/android/inputmethod/latin/makedict/ProbabilityInfo.java @@ -17,6 +17,9 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.utils.CombinedFormatUtils; + +import java.util.Arrays; public final class ProbabilityInfo { public final int mProbability; @@ -39,8 +42,33 @@ public final class ProbabilityInfo { mCount = count; } + public boolean hasHistoricalInfo() { + return mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP; + } + + @Override + public int hashCode() { + if (hasHistoricalInfo()) { + return Arrays.hashCode(new Object[] { mProbability, mTimestamp, mLevel, mCount }); + } else { + return Arrays.hashCode(new Object[] { mProbability }); + } + } + @Override public String toString() { - return mTimestamp + ":" + mLevel + ":" + mCount; + return CombinedFormatUtils.formatProbabilityInfo(this); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof ProbabilityInfo)) return false; + final ProbabilityInfo p = (ProbabilityInfo)o; + if (!hasHistoricalInfo() && !p.hasHistoricalInfo()) { + return mProbability == p.mProbability; + } + return mProbability == p.mProbability && mTimestamp == p.mTimestamp && mLevel == p.mLevel + && mCount == p.mCount; } }
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java index a3a6c2c34..3de083ef3 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java @@ -197,7 +197,7 @@ public class Ver2DictEncoder implements DictEncoder { final WeightedString target = shortcutIterator.next(); final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), - target.mFrequency); + target.getProbability()); mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags, FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord); @@ -231,7 +231,7 @@ public class Ver2DictEncoder implements DictEncoder { final int offset = addressOfBigram - (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), - offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + offset, bigram.getProbability(), unigramFrequencyForThisWord, bigram.mWord); mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags, FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 4b0efbad0..147844fd8 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -71,27 +71,29 @@ public class Ver4DictEncoder implements DictEncoder { // Somehow createEmptyDictFile returned true, but the file was not created correctly throw new IOException("Cannot create dictionary file"); } - for (final Word word : dict) { + for (final WordProperty wordProperty : dict) { // TODO: switch to addMultipleDictionaryEntries when they support shortcuts - if (null == word.mShortcutTargets || word.mShortcutTargets.isEmpty()) { - binaryDict.addUnigramWord(word.mWord, word.mFrequency, + if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) { + binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(), null /* shortcutTarget */, 0 /* shortcutProbability */, - word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */); + wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry, + 0 /* timestamp */); } else { - for (final WeightedString shortcutTarget : word.mShortcutTargets) { - binaryDict.addUnigramWord(word.mWord, word.mFrequency, - shortcutTarget.mWord, shortcutTarget.mFrequency, - word.mIsNotAWord, word.mIsBlacklistEntry, 0 /* timestamp */); + for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + binaryDict.addUnigramWord(wordProperty.mWord, wordProperty.getProbability(), + shortcutTarget.mWord, shortcutTarget.getProbability(), + wordProperty.mIsNotAWord, wordProperty.mIsBlacklistEntry, + 0 /* timestamp */); } } if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { binaryDict.flushWithGC(); } } - for (final Word word0 : dict) { - if (null == word0.mBigrams) continue; - for (final WeightedString word1 : word0.mBigrams) { - binaryDict.addBigramWords(word0.mWord, word1.mWord, word1.mFrequency, + for (final WordProperty word0Property : dict) { + if (null == word0Property.mBigrams) continue; + for (final WeightedString word1 : word0Property.mBigrams) { + binaryDict.addBigramWords(word0Property.mWord, word1.mWord, word1.getProbability(), 0 /* timestamp */); if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { binaryDict.flushWithGC(); diff --git a/java/src/com/android/inputmethod/latin/makedict/Word.java b/java/src/com/android/inputmethod/latin/makedict/Word.java deleted file mode 100644 index 0eabb7bf3..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/Word.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; - -import java.util.ArrayList; -import java.util.Arrays; - -/** - * Utility class for a word with a frequency. - * - * This is chiefly used to iterate a dictionary. - */ -public final class Word implements Comparable<Word> { - public final String mWord; - public final int mFrequency; - public final ArrayList<WeightedString> mShortcutTargets; - public final ArrayList<WeightedString> mBigrams; - public final boolean mIsNotAWord; - public final boolean mIsBlacklistEntry; - - private int mHashCode = 0; - - public Word(final String word, final int frequency, - final ArrayList<WeightedString> shortcutTargets, - final ArrayList<WeightedString> bigrams, - final boolean isNotAWord, final boolean isBlacklistEntry) { - mWord = word; - mFrequency = frequency; - mShortcutTargets = shortcutTargets; - mBigrams = bigrams; - mIsNotAWord = isNotAWord; - mIsBlacklistEntry = isBlacklistEntry; - } - - private static int computeHashCode(Word word) { - return Arrays.hashCode(new Object[] { - word.mWord, - word.mFrequency, - word.mShortcutTargets.hashCode(), - word.mBigrams.hashCode(), - word.mIsNotAWord, - word.mIsBlacklistEntry - }); - } - - /** - * Three-way comparison. - * - * A Word x is greater than a word y if x has a higher frequency. If they have the same - * frequency, they are sorted in lexicographic order. - */ - @Override - public int compareTo(Word w) { - if (mFrequency < w.mFrequency) return 1; - if (mFrequency > w.mFrequency) return -1; - return mWord.compareTo(w.mWord); - } - - /** - * Equality test. - * - * Words are equal if they have the same frequency, the same spellings, and the same - * attributes. - */ - @Override - public boolean equals(Object o) { - if (o == this) return true; - if (!(o instanceof Word)) return false; - Word w = (Word)o; - return mFrequency == w.mFrequency && mWord.equals(w.mWord) - && mShortcutTargets.equals(w.mShortcutTargets) - && mBigrams.equals(w.mBigrams) - && mIsNotAWord == w.mIsNotAWord - && mIsBlacklistEntry == w.mIsBlacklistEntry; - } - - @Override - public int hashCode() { - if (mHashCode == 0) { - mHashCode = computeHashCode(this); - } - return mHashCode; - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/WordProperty.java b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java new file mode 100644 index 000000000..b93a0a525 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/WordProperty.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.BinaryDictionary; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.CollectionUtils; +import com.android.inputmethod.latin.utils.CombinedFormatUtils; +import com.android.inputmethod.latin.utils.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; + +/** + * Utility class for a word with a probability. + * + * This is chiefly used to iterate a dictionary. + */ +public final class WordProperty implements Comparable<WordProperty> { + public final String mWord; + public final ProbabilityInfo mProbabilityInfo; + public final ArrayList<WeightedString> mShortcutTargets; + public final ArrayList<WeightedString> mBigrams; + public final boolean mIsNotAWord; + public final boolean mIsBlacklistEntry; + public final boolean mHasShortcuts; + public final boolean mHasBigrams; + + private int mHashCode = 0; + + public WordProperty(final String word, final int probability, + final ArrayList<WeightedString> shortcutTargets, + final ArrayList<WeightedString> bigrams, + final boolean isNotAWord, final boolean isBlacklistEntry) { + mWord = word; + mProbabilityInfo = new ProbabilityInfo(probability); + mShortcutTargets = shortcutTargets; + mBigrams = bigrams; + mIsNotAWord = isNotAWord; + mIsBlacklistEntry = isBlacklistEntry; + mHasBigrams = bigrams != null && !bigrams.isEmpty(); + mHasShortcuts = shortcutTargets != null && !shortcutTargets.isEmpty(); + } + + private static ProbabilityInfo createProbabilityInfoFromArray(final int[] probabilityInfo) { + return new ProbabilityInfo( + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_PROBABILITY_INDEX], + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX], + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_LEVEL_INDEX], + probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_COUNT_INDEX]); + } + + // Construct word property using information from native code. + // This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY. + public WordProperty(final int[] codePoints, final boolean isNotAWord, + final boolean isBlacklisted, final boolean hasBigram, + final boolean hasShortcuts, final int[] probabilityInfo, + final ArrayList<int[]> bigramTargets, final ArrayList<int[]> bigramProbabilityInfo, + final ArrayList<int[]> shortcutTargets, + final ArrayList<Integer> shortcutProbabilities) { + mWord = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); + mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo); + mShortcutTargets = CollectionUtils.newArrayList(); + mBigrams = CollectionUtils.newArrayList(); + mIsNotAWord = isNotAWord; + mIsBlacklistEntry = isBlacklisted; + mHasShortcuts = hasShortcuts; + mHasBigrams = hasBigram; + + final int bigramTargetCount = bigramTargets.size(); + for (int i = 0; i < bigramTargetCount; i++) { + final String bigramTargetString = + StringUtils.getStringFromNullTerminatedCodePointArray(bigramTargets.get(i)); + mBigrams.add(new WeightedString(bigramTargetString, + createProbabilityInfoFromArray(bigramProbabilityInfo.get(i)))); + } + + final int shortcutTargetCount = shortcutTargets.size(); + for (int i = 0; i < shortcutTargetCount; i++) { + final String shortcutTargetString = + StringUtils.getStringFromNullTerminatedCodePointArray(shortcutTargets.get(i)); + mShortcutTargets.add( + new WeightedString(shortcutTargetString, shortcutProbabilities.get(i))); + } + } + + public int getProbability() { + return mProbabilityInfo.mProbability; + } + + private static int computeHashCode(WordProperty word) { + return Arrays.hashCode(new Object[] { + word.mWord, + word.mProbabilityInfo, + word.mShortcutTargets.hashCode(), + word.mBigrams.hashCode(), + word.mIsNotAWord, + word.mIsBlacklistEntry + }); + } + + /** + * Three-way comparison. + * + * A Word x is greater than a word y if x has a higher frequency. If they have the same + * frequency, they are sorted in lexicographic order. + */ + @Override + public int compareTo(final WordProperty w) { + if (getProbability() < w.getProbability()) return 1; + if (getProbability() > w.getProbability()) return -1; + return mWord.compareTo(w.mWord); + } + + /** + * Equality test. + * + * Words are equal if they have the same frequency, the same spellings, and the same + * attributes. + */ + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof WordProperty)) return false; + WordProperty w = (WordProperty)o; + return mProbabilityInfo.equals(w.mProbabilityInfo) && mWord.equals(w.mWord) + && mShortcutTargets.equals(w.mShortcutTargets) && mBigrams.equals(w.mBigrams) + && mIsNotAWord == w.mIsNotAWord && mIsBlacklistEntry == w.mIsBlacklistEntry + && mHasBigrams == w.mHasBigrams && mHasShortcuts && w.mHasBigrams; + } + + @Override + public int hashCode() { + if (mHashCode == 0) { + mHashCode = computeHashCode(this); + } + return mHashCode; + } + + @UsedForTesting + public boolean isValid() { + return getProbability() != BinaryDictionary.NOT_A_PROBABILITY; + } + + @Override + public String toString() { + return CombinedFormatUtils.formatWordProperty(this); + } +} diff --git a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java index 7c9b2a169..652614876 100644 --- a/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionary.java @@ -39,4 +39,10 @@ public class PersonalizationDictionary extends DecayingExpandableBinaryDictionar super(context, locale, Dictionary.TYPE_PERSONALIZATION, getDictNameWithLocale(NAME, locale), dictFile); } + + @Override + public boolean isValidWord(final String word) { + // Strings out of this dictionary should not be considered existing words. + return false; + } } diff --git a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java index c23bc9bc0..6778c2334 100644 --- a/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java +++ b/java/src/com/android/inputmethod/latin/personalization/UserHistoryDictionary.java @@ -45,4 +45,10 @@ public class UserHistoryDictionary extends DecayingExpandableBinaryDictionaryBas public void cancelAddingUserHistory(final String word0, final String word1) { removeBigramDynamically(word0, word1); } + + @Override + public boolean isValidWord(final String word) { + // Strings out of this dictionary should not be considered existing words. + return false; + } } diff --git a/java/src/com/android/inputmethod/latin/settings/SpacingAndPunctuations.java b/java/src/com/android/inputmethod/latin/settings/SpacingAndPunctuations.java index 70cb2b285..60ca5baab 100644 --- a/java/src/com/android/inputmethod/latin/settings/SpacingAndPunctuations.java +++ b/java/src/com/android/inputmethod/latin/settings/SpacingAndPunctuations.java @@ -75,7 +75,7 @@ public final class SpacingAndPunctuations { final ArrayList<SuggestedWordInfo> puncList = CollectionUtils.newArrayList(); if (puncs != null) { for (final String puncSpec : puncs) { - // TODO: Stop using KeySpceParser.getLabel(). + // TODO: Stop using KeySpecParser.getLabel(). // TODO: Punctuation suggestions should honor RTL languages. puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, @@ -84,7 +84,7 @@ public final class SpacingAndPunctuations { SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } } - return new SuggestedWords(puncList, + return new SuggestedWords(puncList, null /* rawSuggestions */, false /* typedWordValid */, false /* hasAutoCorrectionCandidate */, true /* isPunctuationSuggestions */, diff --git a/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java b/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java new file mode 100644 index 000000000..bb7ae2f9b --- /dev/null +++ b/java/src/com/android/inputmethod/latin/utils/CombinedFormatUtils.java @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.latin.utils; + +import com.android.inputmethod.latin.makedict.DictionaryHeader; +import com.android.inputmethod.latin.makedict.ProbabilityInfo; +import com.android.inputmethod.latin.makedict.WordProperty; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.util.HashMap; + +public class CombinedFormatUtils { + public static final String DICTIONARY_TAG = "dictionary"; + public static final String BIGRAM_TAG = "bigram"; + public static final String SHORTCUT_TAG = "shortcut"; + public static final String PROBABILITY_TAG = "f"; + public static final String HISTORICAL_INFO_TAG = "historicalInfo"; + public static final String HISTORICAL_INFO_SEPARATOR = ":"; + public static final String WORD_TAG = "word"; + public static final String NOT_A_WORD_TAG = "not_a_word"; + public static final String BLACKLISTED_TAG = "blacklisted"; + + public static String formatAttributeMap(final HashMap<String, String> attributeMap) { + final StringBuilder builder = new StringBuilder(); + builder.append(DICTIONARY_TAG + "="); + if (attributeMap.containsKey(DictionaryHeader.DICTIONARY_ID_KEY)) { + builder.append(attributeMap.get(DictionaryHeader.DICTIONARY_ID_KEY)); + } + for (final String key : attributeMap.keySet()) { + if (key.equals(DictionaryHeader.DICTIONARY_ID_KEY)) { + continue; + } + final String value = attributeMap.get(key); + builder.append("," + key + "=" + value); + } + builder.append("\n"); + return builder.toString(); + } + + public static String formatWordProperty(final WordProperty wordProperty) { + final StringBuilder builder = new StringBuilder(); + builder.append(" " + WORD_TAG + "=" + wordProperty.mWord); + builder.append(","); + builder.append(formatProbabilityInfo(wordProperty.mProbabilityInfo)); + if (wordProperty.mIsNotAWord) { + builder.append("," + NOT_A_WORD_TAG + "=true"); + } + if (wordProperty.mIsBlacklistEntry) { + builder.append("," + BLACKLISTED_TAG + "=true"); + } + builder.append("\n"); + if (wordProperty.mShortcutTargets != null) { + for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { + builder.append(" " + SHORTCUT_TAG + "=" + shortcutTarget.mWord); + builder.append(","); + builder.append(formatProbabilityInfo(shortcutTarget.mProbabilityInfo)); + builder.append("\n"); + } + } + if (wordProperty.mBigrams != null) { + for (final WeightedString bigram : wordProperty.mBigrams) { + builder.append(" " + BIGRAM_TAG + "=" + bigram.mWord); + builder.append(","); + builder.append(formatProbabilityInfo(bigram.mProbabilityInfo)); + builder.append("\n"); + } + } + return builder.toString(); + } + + public static String formatProbabilityInfo(final ProbabilityInfo probabilityInfo) { + final StringBuilder builder = new StringBuilder(); + builder.append(PROBABILITY_TAG + "=" + probabilityInfo.mProbability); + if (probabilityInfo.hasHistoricalInfo()) { + builder.append(","); + builder.append(HISTORICAL_INFO_TAG + "="); + builder.append(probabilityInfo.mTimestamp); + builder.append(HISTORICAL_INFO_SEPARATOR); + builder.append(probabilityInfo.mLevel); + builder.append(HISTORICAL_INFO_SEPARATOR); + builder.append(probabilityInfo.mCount); + } + return builder.toString(); + } +} diff --git a/java/src/com/android/inputmethod/latin/utils/WordProperty.java b/java/src/com/android/inputmethod/latin/utils/WordProperty.java deleted file mode 100644 index da56b213f..000000000 --- a/java/src/com/android/inputmethod/latin/utils/WordProperty.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package com.android.inputmethod.latin.utils; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.BinaryDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; -import com.android.inputmethod.latin.makedict.ProbabilityInfo; - -import java.util.ArrayList; - -// This has information that belong to a unigram. This class has some detailed attributes such as -// historical information but they have to be checked only for testing purpose. -@UsedForTesting -public class WordProperty { - public final String mCodePoints; - public final boolean mIsNotAWord; - public final boolean mIsBlacklisted; - public final boolean mHasBigrams; - public final boolean mHasShortcuts; - public final ProbabilityInfo mProbabilityInfo; - public final ArrayList<WeightedString> mBigramTargets = CollectionUtils.newArrayList(); - public final ArrayList<ProbabilityInfo> mBigramProbabilityInfo = CollectionUtils.newArrayList(); - public final ArrayList<WeightedString> mShortcutTargets = CollectionUtils.newArrayList(); - - private static ProbabilityInfo createProbabilityInfoFromArray(final int[] probabilityInfo) { - return new ProbabilityInfo( - probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_PROBABILITY_INDEX], - probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_TIMESTAMP_INDEX], - probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_LEVEL_INDEX], - probabilityInfo[BinaryDictionary.FORMAT_WORD_PROPERTY_COUNT_INDEX]); - } - - // This represents invalid word when the probability is BinaryDictionary.NOT_A_PROBABILITY. - public WordProperty(final int[] codePoints, final boolean isNotAWord, - final boolean isBlacklisted, final boolean hasBigram, - final boolean hasShortcuts, final int[] probabilityInfo, - final ArrayList<int[]> bigramTargets, final ArrayList<int[]> bigramProbabilityInfo, - final ArrayList<int[]> shortcutTargets, - final ArrayList<Integer> shortcutProbabilities) { - mCodePoints = StringUtils.getStringFromNullTerminatedCodePointArray(codePoints); - mIsNotAWord = isNotAWord; - mIsBlacklisted = isBlacklisted; - mHasBigrams = hasBigram; - mHasShortcuts = hasShortcuts; - mProbabilityInfo = createProbabilityInfoFromArray(probabilityInfo); - - final int bigramTargetCount = bigramTargets.size(); - for (int i = 0; i < bigramTargetCount; i++) { - final String bigramTargetString = - StringUtils.getStringFromNullTerminatedCodePointArray(bigramTargets.get(i)); - final ProbabilityInfo bigramProbability = - createProbabilityInfoFromArray(bigramProbabilityInfo.get(i)); - mBigramTargets.add( - new WeightedString(bigramTargetString, bigramProbability.mProbability)); - mBigramProbabilityInfo.add(bigramProbability); - } - - final int shortcutTargetCount = shortcutTargets.size(); - for (int i = 0; i < shortcutTargetCount; i++) { - final String shortcutTargetString = - StringUtils.getStringFromNullTerminatedCodePointArray(shortcutTargets.get(i)); - mShortcutTargets.add( - new WeightedString(shortcutTargetString, shortcutProbabilities.get(i))); - } - } - - @UsedForTesting - public boolean isValid() { - return mProbabilityInfo.mProbability != BinaryDictionary.NOT_A_PROBABILITY; - } - - @Override - public String toString() { - // TODO: Move this logic to CombinedInputOutput. - final StringBuffer builder = new StringBuffer(); - builder.append(" word=" + mCodePoints); - builder.append(","); - builder.append("f=" + mProbabilityInfo.mProbability); - if (mIsNotAWord) { - builder.append(","); - builder.append("not_a_word=true"); - } - if (mIsBlacklisted) { - builder.append(","); - builder.append("blacklisted=true"); - } - if (mProbabilityInfo.mTimestamp != BinaryDictionary.NOT_A_VALID_TIMESTAMP) { - builder.append(","); - builder.append("historicalInfo=" + mProbabilityInfo); - } - builder.append("\n"); - for (int i = 0; i < mBigramTargets.size(); i++) { - builder.append(" bigram=" + mBigramTargets.get(i).mWord); - builder.append(","); - builder.append("f=" + mBigramTargets.get(i).mFrequency); - if (mBigramProbabilityInfo.get(i).mTimestamp - != BinaryDictionary.NOT_A_VALID_TIMESTAMP) { - builder.append(","); - builder.append("historicalInfo=" + mBigramProbabilityInfo.get(i)); - } - builder.append("\n"); - } - for (int i = 0; i < mShortcutTargets.size(); i++) { - builder.append(" shortcut=" + mShortcutTargets.get(i).mWord); - builder.append(","); - builder.append("f=" + mShortcutTargets.get(i).mFrequency); - builder.append("\n"); - } - return builder.toString(); - } -}
\ No newline at end of file |