diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/BinaryDictionary.java')
| -rw-r--r-- | java/src/com/android/inputmethod/latin/BinaryDictionary.java | 234 |
1 files changed, 126 insertions, 108 deletions
diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index b88509fde..543f74fc4 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -30,7 +30,6 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException; import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.settings.NativeSuggestOptions; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; -import com.android.inputmethod.latin.utils.CollectionUtils; import com.android.inputmethod.latin.utils.FileUtils; import com.android.inputmethod.latin.utils.JniUtils; import com.android.inputmethod.latin.utils.LanguageModelParam; @@ -104,8 +103,7 @@ public final class BinaryDictionary extends Dictionary { private final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions(); - private final SparseArray<DicTraverseSession> mDicTraverseSessions = - CollectionUtils.newSparseArray(); + private final SparseArray<DicTraverseSession> mDicTraverseSessions = new SparseArray<>(); // TODO: There should be a way to remove used DicTraverseSession objects from // {@code mDicTraverseSessions}. @@ -113,11 +111,8 @@ public final class BinaryDictionary extends Dictionary { synchronized(mDicTraverseSessions) { DicTraverseSession traverseSession = mDicTraverseSessions.get(traverseSessionId); if (traverseSession == null) { - traverseSession = mDicTraverseSessions.get(traverseSessionId); - if (traverseSession == null) { - traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); - mDicTraverseSessions.put(traverseSessionId, traverseSession); - } + traverseSession = new DicTraverseSession(mLocale, mNativeDict, mDictSize); + mDicTraverseSessions.put(traverseSessionId, traverseSession); } return traverseSession; } @@ -188,13 +183,15 @@ public final class BinaryDictionary extends Dictionary { private static native void getHeaderInfoNative(long dict, int[] outHeaderSize, int[] outFormatVersion, ArrayList<int[]> outAttributeKeys, ArrayList<int[]> outAttributeValues); - private static native void flushNative(long dict, String filePath); + private static native boolean flushNative(long dict, String filePath); private static native boolean needsToRunGCNative(long dict, boolean mindsBlockByGC); - private static native void flushWithGCNative(long dict, String filePath); + private static native boolean flushWithGCNative(long dict, String filePath); private static native void closeNative(long dict); private static native int getFormatVersionNative(long dict); private static native int getProbabilityNative(long dict, int[] word); - private static native int getBigramProbabilityNative(long dict, int[] word0, int[] word1); + private static native int getMaxProbabilityOfExactMatchesNative(long dict, int[] word); + private static native int getBigramProbabilityNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1); private static native void getWordPropertyNative(long dict, int[] word, int[] outCodePoints, boolean[] outFlags, int[] outProbabilityInfo, ArrayList<int[]> outBigramTargets, ArrayList<int[]> outBigramProbabilityInfo, @@ -203,21 +200,24 @@ public final class BinaryDictionary extends Dictionary { private static native void getSuggestionsNative(long dict, long proximityInfo, long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds, int[] inputCodePoints, int inputSize, int[] suggestOptions, - int[] prevWordCodePointArray, int[] outputSuggestionCount, int[] outputCodePoints, - int[] outputScores, int[] outputIndices, int[] outputTypes, - int[] outputAutoCommitFirstWordConfidence, float[] inOutLanguageWeight); - private static native void addUnigramWordNative(long dict, int[] word, int probability, - int[] shortcutTarget, int shortcutProbability, boolean isNotAWord, - boolean isBlacklisted, int timestamp); - private static native void addBigramWordsNative(long dict, int[] word0, int[] word1, - int probability, int timestamp); - private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1); + int[] prevWordCodePointArray, boolean isBeginningOfSentence, + int[] outputSuggestionCount, int[] outputCodePoints, int[] outputScores, + int[] outputIndices, int[] outputTypes, int[] outputAutoCommitFirstWordConfidence, + float[] inOutLanguageWeight); + private static native boolean addUnigramWordNative(long dict, int[] word, int probability, + int[] shortcutTarget, int shortcutProbability, boolean isBeginningOfSentence, + boolean isNotAWord, boolean isBlacklisted, int timestamp); + private static native boolean removeUnigramWordNative(long dict, int[] word); + private static native boolean addBigramWordsNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1, int probability, int timestamp); + private static native boolean removeBigramWordsNative(long dict, int[] word0, + boolean isBeginningOfSentence, int[] word1); private static native int addMultipleDictionaryEntriesNative(long dict, LanguageModelParam[] languageModelParams, int startIndex); - private static native int calculateProbabilityNative(long dict, int unigramProbability, - int bigramProbability); private static native String getPropertyNative(long dict, String query); private static native boolean isCorruptedNative(long dict); + private static native boolean migrateNative(long dict, String dictFilePath, + long newFormatVersion); // TODO: Move native dict into session private final void loadDictionary(final String path, final long startOffset, @@ -248,11 +248,11 @@ public final class BinaryDictionary extends Dictionary { } final int[] outHeaderSize = new int[1]; final int[] outFormatVersion = new int[1]; - final ArrayList<int[]> outAttributeKeys = CollectionUtils.newArrayList(); - final ArrayList<int[]> outAttributeValues = CollectionUtils.newArrayList(); + final ArrayList<int[]> outAttributeKeys = new ArrayList<>(); + final ArrayList<int[]> outAttributeValues = new ArrayList<>(); getHeaderInfoNative(mNativeDict, outHeaderSize, outFormatVersion, outAttributeKeys, outAttributeValues); - final HashMap<String, String> attributes = new HashMap<String, String>(); + final HashMap<String, String> attributes = new HashMap<>(); for (int i = 0; i < outAttributeKeys.size(); i++) { final String attributeKey = StringUtils.getStringFromNullTerminatedCodePointArray( outAttributeKeys.get(i)); @@ -266,19 +266,9 @@ public final class BinaryDictionary extends Dictionary { new FormatSpec.FormatOptions(outFormatVersion[0], hasHistoricalInfo)); } - @Override public ArrayList<SuggestedWordInfo> getSuggestions(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, - final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, - final float[] inOutLanguageWeight) { - return getSuggestionsWithSessionId(composer, prevWord, proximityInfo, blockOffensiveWords, - additionalFeaturesOptions, 0 /* sessionId */, inOutLanguageWeight); - } - - @Override - public ArrayList<SuggestedWordInfo> getSuggestionsWithSessionId(final WordComposer composer, - final String prevWord, final ProximityInfo proximityInfo, + final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, final boolean blockOffensiveWords, final int[] additionalFeaturesOptions, final int sessionId, final float[] inOutLanguageWeight) { if (!isValidDictionary()) { @@ -287,8 +277,8 @@ public final class BinaryDictionary extends Dictionary { Arrays.fill(mInputCodePoints, Constants.NOT_A_CODE); // TODO: toLowerCase in the native code - final int[] prevWordCodePointArray = (null == prevWord) - ? null : StringUtils.toCodePointArray(prevWord); + final int[] prevWordCodePointArray = (null == prevWordsInfo.mPrevWord) + ? null : StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); final InputPointers inputPointers = composer.getInputPointers(); final boolean isGesture = composer.isBatchMode(); final int inputSize; @@ -303,6 +293,7 @@ public final class BinaryDictionary extends Dictionary { } mNativeSuggestOptions.setIsGesture(isGesture); + mNativeSuggestOptions.setBlockOffensiveWords(blockOffensiveWords); mNativeSuggestOptions.setAdditionalFeaturesOptions(additionalFeaturesOptions); if (inOutLanguageWeight != null) { mInputOutputLanguageWeight[0] = inOutLanguageWeight[0]; @@ -314,14 +305,15 @@ public final class BinaryDictionary extends Dictionary { getTraverseSession(sessionId).getSession(), inputPointers.getXCoordinates(), inputPointers.getYCoordinates(), inputPointers.getTimes(), inputPointers.getPointerIds(), mInputCodePoints, inputSize, - mNativeSuggestOptions.getOptions(), prevWordCodePointArray, mOutputSuggestionCount, + mNativeSuggestOptions.getOptions(), prevWordCodePointArray, + prevWordsInfo.mIsBeginningOfSentence, mOutputSuggestionCount, mOutputCodePoints, mOutputScores, mSpaceIndices, mOutputTypes, mOutputAutoCommitFirstWordConfidence, mInputOutputLanguageWeight); if (inOutLanguageWeight != null) { inOutLanguageWeight[0] = mInputOutputLanguageWeight[0]; } final int count = mOutputSuggestionCount[0]; - final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); + final ArrayList<SuggestedWordInfo> suggestions = new ArrayList<>(); for (int j = 0; j < count; ++j) { final int start = j * MAX_WORD_LENGTH; int len = 0; @@ -329,21 +321,8 @@ public final class BinaryDictionary extends Dictionary { ++len; } if (len > 0) { - final int flags = mOutputTypes[j] & SuggestedWordInfo.KIND_MASK_FLAGS; - if (blockOffensiveWords - && 0 != (flags & SuggestedWordInfo.KIND_FLAG_POSSIBLY_OFFENSIVE) - && 0 == (flags & SuggestedWordInfo.KIND_FLAG_EXACT_MATCH)) { - // If we block potentially offensive words, and if the word is possibly - // offensive, then we don't output it unless it's also an exact match. - continue; - } - final int kind = mOutputTypes[j] & SuggestedWordInfo.KIND_MASK_KIND; - final int score = SuggestedWordInfo.KIND_WHITELIST == kind - ? SuggestedWordInfo.MAX_SCORE : mOutputScores[j]; - // TODO: check that all users of the `kind' parameter are ready to accept - // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), - score, kind, this /* sourceDict */, + mOutputScores[j], mOutputTypes[j], this /* sourceDict */, mSpaceIndices[j] /* indexOfTouchPointOfSecondWord */, mOutputAutoCommitFirstWordConfidence[0])); } @@ -360,28 +339,37 @@ public final class BinaryDictionary extends Dictionary { } @Override - public boolean isValidWord(final String word) { + public boolean isInDictionary(final String word) { return getFrequency(word) != NOT_A_PROBABILITY; } @Override public int getFrequency(final String word) { - if (word == null) return NOT_A_PROBABILITY; + if (TextUtils.isEmpty(word)) return NOT_A_PROBABILITY; int[] codePoints = StringUtils.toCodePointArray(word); return getProbabilityNative(mNativeDict, codePoints); } - // TODO: Add a batch process version (isValidBigramMultiple?) to avoid excessive numbers of jni - // calls when checking for changes in an entire dictionary. - public boolean isValidBigram(final String word0, final String word1) { - return getBigramProbability(word0, word1) != NOT_A_PROBABILITY; + @Override + public int getMaxFrequencyOfExactMatches(final String word) { + if (TextUtils.isEmpty(word)) return NOT_A_PROBABILITY; + int[] codePoints = StringUtils.toCodePointArray(word); + return getMaxProbabilityOfExactMatchesNative(mNativeDict, codePoints); } - public int getBigramProbability(final String word0, final String word1) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) return NOT_A_PROBABILITY; - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1); + @UsedForTesting + public boolean isValidNgram(final PrevWordsInfo prevWordsInfo, final String word) { + return getNgramProbability(prevWordsInfo, word) != NOT_A_PROBABILITY; + } + + public int getNgramProbability(final PrevWordsInfo prevWordsInfo, final String word) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { + return NOT_A_PROBABILITY; + } + final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + final int[] codePoints1 = StringUtils.toCodePointArray(word); + return getBigramProbabilityNative(mNativeDict, codePoints0, + prevWordsInfo.mIsBeginningOfSentence, codePoints1); } public WordProperty getWordProperty(final String word) { @@ -393,10 +381,10 @@ public final class BinaryDictionary extends Dictionary { final boolean[] outFlags = new boolean[FORMAT_WORD_PROPERTY_OUTPUT_FLAG_COUNT]; final int[] outProbabilityInfo = new int[FORMAT_WORD_PROPERTY_OUTPUT_PROBABILITY_INFO_COUNT]; - final ArrayList<int[]> outBigramTargets = CollectionUtils.newArrayList(); - final ArrayList<int[]> outBigramProbabilityInfo = CollectionUtils.newArrayList(); - final ArrayList<int[]> outShortcutTargets = CollectionUtils.newArrayList(); - final ArrayList<Integer> outShortcutProbabilities = CollectionUtils.newArrayList(); + final ArrayList<int[]> outBigramTargets = new ArrayList<>(); + final ArrayList<int[]> outBigramProbabilityInfo = new ArrayList<>(); + final ArrayList<int[]> outShortcutTargets = new ArrayList<>(); + final ArrayList<Integer> outShortcutProbabilities = new ArrayList<>(); getWordPropertyNative(mNativeDict, codePoints, outCodePoints, outFlags, outProbabilityInfo, outBigramTargets, outBigramProbabilityInfo, outShortcutTargets, outShortcutProbabilities); @@ -413,8 +401,8 @@ public final class BinaryDictionary extends Dictionary { public WordProperty mWordProperty; public int mNextToken; - public GetNextWordPropertyResult(final WordProperty wordPreperty, final int nextToken) { - mWordProperty = wordPreperty; + public GetNextWordPropertyResult(final WordProperty wordProperty, final int nextToken) { + mWordProperty = wordProperty; mNextToken = nextToken; } } @@ -431,41 +419,66 @@ public final class BinaryDictionary extends Dictionary { } // Add a unigram entry to binary dictionary with unigram attributes in native code. - public void addUnigramWord(final String word, final int probability, - final String shortcutTarget, final int shortcutProbability, final boolean isNotAWord, + public boolean addUnigramEntry(final String word, final int probability, + final String shortcutTarget, final int shortcutProbability, + final boolean isBeginningOfSentence, final boolean isNotAWord, final boolean isBlacklisted, final int timestamp) { - if (TextUtils.isEmpty(word)) { - return; + if (word == null || (word.isEmpty() && !isBeginningOfSentence)) { + return false; } final int[] codePoints = StringUtils.toCodePointArray(word); final int[] shortcutTargetCodePoints = (shortcutTarget != null) ? StringUtils.toCodePointArray(shortcutTarget) : null; - addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, - shortcutProbability, isNotAWord, isBlacklisted, timestamp); + if (!addUnigramWordNative(mNativeDict, codePoints, probability, shortcutTargetCodePoints, + shortcutProbability, isBeginningOfSentence, isNotAWord, isBlacklisted, timestamp)) { + return false; + } mHasUpdated = true; + return true; } - // Add a bigram entry to binary dictionary with timestamp in native code. - public void addBigramWords(final String word0, final String word1, final int probability, - final int timestamp) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { - return; + // Remove a unigram entry from the binary dictionary in native code. + public boolean removeUnigramEntry(final String word) { + if (TextUtils.isEmpty(word)) { + return false; + } + final int[] codePoints = StringUtils.toCodePointArray(word); + if (!removeUnigramWordNative(mNativeDict, codePoints)) { + return false; } - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability, timestamp); mHasUpdated = true; + return true; } - // Remove a bigram entry form binary dictionary in native code. - public void removeBigramWords(final String word0, final String word1) { - if (TextUtils.isEmpty(word0) || TextUtils.isEmpty(word1)) { - return; + // Add an n-gram entry to the binary dictionary with timestamp in native code. + public boolean addNgramEntry(final PrevWordsInfo prevWordsInfo, final String word, + final int probability, final int timestamp) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { + return false; + } + final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + final int[] codePoints1 = StringUtils.toCodePointArray(word); + if (!addBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, + codePoints1, probability, timestamp)) { + return false; } - final int[] codePoints0 = StringUtils.toCodePointArray(word0); - final int[] codePoints1 = StringUtils.toCodePointArray(word1); - removeBigramWordsNative(mNativeDict, codePoints0, codePoints1); mHasUpdated = true; + return true; + } + + // Remove an n-gram entry from the binary dictionary in native code. + public boolean removeNgramEntry(final PrevWordsInfo prevWordsInfo, final String word) { + if (!prevWordsInfo.isValid() || TextUtils.isEmpty(word)) { + return false; + } + final int[] codePoints0 = StringUtils.toCodePointArray(prevWordsInfo.mPrevWord); + final int[] codePoints1 = StringUtils.toCodePointArray(word); + if (!removeBigramWordsNative(mNativeDict, codePoints0, prevWordsInfo.mIsBeginningOfSentence, + codePoints1)) { + return false; + } + mHasUpdated = true; + return true; } public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { @@ -495,26 +508,33 @@ public final class BinaryDictionary extends Dictionary { } // Flush to dict file if the dictionary has been updated. - public void flush() { - if (!isValidDictionary()) return; + public boolean flush() { + if (!isValidDictionary()) return false; if (mHasUpdated) { - flushNative(mNativeDict, mDictFilePath); + if (!flushNative(mNativeDict, mDictFilePath)) { + return false; + } reopen(); } + return true; } // Run GC and flush to dict file if the dictionary has been updated. - public void flushWithGCIfHasUpdated() { + public boolean flushWithGCIfHasUpdated() { if (mHasUpdated) { - flushWithGC(); + return flushWithGC(); } + return true; } // Run GC and flush to dict file. - public void flushWithGC() { - if (!isValidDictionary()) return; - flushWithGCNative(mNativeDict, mDictFilePath); + public boolean flushWithGC() { + if (!isValidDictionary()) return false; + if (!flushWithGCNative(mNativeDict, mDictFilePath)) { + return false; + } reopen(); + return true; } /** @@ -533,11 +553,15 @@ public final class BinaryDictionary extends Dictionary { return false; } final String tmpDictFilePath = mDictFilePath + DICT_FILE_NAME_SUFFIX_FOR_MIGRATION; - // TODO: Implement migrateNative(tmpDictFilePath, newFormatVersion). + if (!migrateNative(mNativeDict, tmpDictFilePath, newFormatVersion)) { + return false; + } close(); final File dictFile = new File(mDictFilePath); final File tmpDictFile = new File(tmpDictFilePath); - FileUtils.deleteRecursively(dictFile); + if (!FileUtils.deleteRecursively(dictFile)) { + return false; + } if (!BinaryDictionaryUtils.renameDict(tmpDictFile, dictFile)) { return false; } @@ -547,12 +571,6 @@ public final class BinaryDictionary extends Dictionary { } @UsedForTesting - public int calculateProbability(final int unigramProbability, final int bigramProbability) { - if (!isValidDictionary()) return NOT_A_PROBABILITY; - return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability); - } - - @UsedForTesting public String getPropertyForTest(final String query) { if (!isValidDictionary()) return ""; return getPropertyNative(mNativeDict, query); |