diff options
Diffstat (limited to 'java/src')
15 files changed, 426 insertions, 27 deletions
diff --git a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java index 7008b0619..de7f2e25c 100644 --- a/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java +++ b/java/src/com/android/inputmethod/keyboard/internal/KeyboardTextsSet.java @@ -2056,6 +2056,25 @@ public final class KeyboardTextsSet { /* 45 */ "\u0410\u0411\u0412", }; + /* Language lo: Lao */ + private static final String[] LANGUAGE_lo = { + /* 0~ */ + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + /* ~44 */ + // Label for "switch to alphabetic" key. + // U+0E81: "ກ" LAO LETTER KO + // U+0E82: "ຂ" LAO LETTER KHO SUNG + // U+0E84: "ຄ" LAO LETTER KHO TAM + /* 45 */ "\u0E81\u0E82\u0E84", + /* 46~ */ + null, null, null, null, null, + /* ~50 */ + // U+20AD: "₭" KIP SIGN + /* 51 */ "\u20AD", + }; + /* Language lt: Lithuanian */ private static final String[] LANGUAGE_lt = { // U+0105: "ą" LATIN SMALL LETTER A WITH OGONEK @@ -3332,6 +3351,7 @@ public final class KeyboardTextsSet { "ka", LANGUAGE_ka, /* Georgian */ "kk", LANGUAGE_kk, /* Kazakh */ "ky", LANGUAGE_ky, /* Kirghiz */ + "lo", LANGUAGE_lo, /* Lao */ "lt", LANGUAGE_lt, /* Lithuanian */ "lv", LANGUAGE_lv, /* Latvian */ "mk", LANGUAGE_mk, /* Macedonian */ diff --git a/java/src/com/android/inputmethod/latin/BinaryDictionary.java b/java/src/com/android/inputmethod/latin/BinaryDictionary.java index 834d3ed53..dacb8483c 100644 --- a/java/src/com/android/inputmethod/latin/BinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/BinaryDictionary.java @@ -40,6 +40,9 @@ public final class BinaryDictionary extends Dictionary { private static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH; // Must be equal to MAX_RESULTS in native/jni/src/defines.h private static final int MAX_RESULTS = 18; + // Required space count for auto commit. + // TODO: Remove this heuristic. + private static final int SPACE_COUNT_FOR_AUTO_COMMIT = 3; private long mNativeDict; private final Locale mLocale; @@ -49,6 +52,7 @@ public final class BinaryDictionary extends Dictionary { private final int[] mSpaceIndices = new int[MAX_RESULTS]; private final int[] mOutputScores = new int[MAX_RESULTS]; private final int[] mOutputTypes = new int[MAX_RESULTS]; + private final int[] mOutputAutoCommitFirstWordConfidence = new int[1]; // Only one result private final NativeSuggestOptions mNativeSuggestOptions = new NativeSuggestOptions(); @@ -104,7 +108,8 @@ public final class BinaryDictionary extends Dictionary { long traverseSession, int[] xCoordinates, int[] yCoordinates, int[] times, int[] pointerIds, int[] inputCodePoints, int inputSize, int commitPoint, int[] suggestOptions, int[] prevWordCodePointArray, - int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes); + int[] outputCodePoints, int[] outputScores, int[] outputIndices, int[] outputTypes, + int[] outputAutoCommitFirstWordConfidence); private static native float calcNormalizedScoreNative(int[] before, int[] after, int score); private static native int editDistanceNative(int[] before, int[] after); private static native void addUnigramWordNative(long dict, int[] word, int probability); @@ -157,7 +162,7 @@ public final class BinaryDictionary extends Dictionary { ips.getYCoordinates(), ips.getTimes(), ips.getPointerIds(), mInputCodePoints, inputSize, 0 /* commitPoint */, mNativeSuggestOptions.getOptions(), prevWordCodePointArray, mOutputCodePoints, mOutputScores, mSpaceIndices, - mOutputTypes); + mOutputTypes, mOutputAutoCommitFirstWordConfidence); final ArrayList<SuggestedWordInfo> suggestions = CollectionUtils.newArrayList(); for (int j = 0; j < count; ++j) { final int start = j * MAX_WORD_LENGTH; @@ -181,7 +186,8 @@ public final class BinaryDictionary extends Dictionary { // flags too and pass mOutputTypes[j] instead of kind suggestions.add(new SuggestedWordInfo(new String(mOutputCodePoints, start, len), score, kind, this /* sourceDict */, - mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */)); + mSpaceIndices[0] /* indexOfTouchPointOfSecondWord */, + mOutputAutoCommitFirstWordConfidence[0])); } } return suggestions; @@ -256,6 +262,22 @@ public final class BinaryDictionary extends Dictionary { } @Override + public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { + // TODO: actually use the confidence rather than use this completely broken heuristic + final String word = candidate.mWord; + final int length = word.length(); + int remainingSpaces = SPACE_COUNT_FOR_AUTO_COMMIT; + for (int i = 0; i < length; ++i) { + // This is okay because no low-surrogate and no high-surrogate can ever match the + // space character, so we don't need to take care of iterating on code points. + if (Constants.CODE_SPACE == word.charAt(i)) { + if (0 >= --remainingSpaces) return true; + } + } + return false; + } + + @Override public void close() { synchronized (mDicTraverseSessions) { final int sessionsSize = mDicTraverseSessions.size(); diff --git a/java/src/com/android/inputmethod/latin/Dictionary.java b/java/src/com/android/inputmethod/latin/Dictionary.java index 8a3a88438..fa79f5af7 100644 --- a/java/src/com/android/inputmethod/latin/Dictionary.java +++ b/java/src/com/android/inputmethod/latin/Dictionary.java @@ -137,7 +137,10 @@ public abstract class Dictionary { } /** - * Whether we think this suggestion should trigger an auto-commit. + * Whether we think this suggestion should trigger an auto-commit. prevWord is the word + * before the suggestion, so that we can use n-gram frequencies. + * @param candidate The candidate suggestion, in whole (not only the first part). + * @return whether we should auto-commit or not. */ public boolean shouldAutoCommit(final SuggestedWordInfo candidate) { // If we don't have support for auto-commit, or if we don't know, we return false to diff --git a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java index c884e7b1f..2a9076436 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java @@ -617,4 +617,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { }); return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS); } + + @UsedForTesting + public void shutdownExecutorForTests() { + getExecutor(mFilename).shutdown(); + } + + @UsedForTesting + public boolean isTerminatedForTests() { + return getExecutor(mFilename).isTerminated(); + } } diff --git a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java index ba7d1a2b0..d491f988a 100644 --- a/java/src/com/android/inputmethod/latin/ExpandableDictionary.java +++ b/java/src/com/android/inputmethod/latin/ExpandableDictionary.java @@ -344,7 +344,8 @@ public class ExpandableDictionary extends Dictionary { // in the future. suggestions.add(new SuggestedWordInfo(new String(word, 0, depth + 1), finalFreq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); if (suggestions.size() >= Suggest.MAX_SUGGESTIONS) return false; } if (null != node.mShortcutTargets) { @@ -353,7 +354,8 @@ public class ExpandableDictionary extends Dictionary { final char[] shortcut = node.mShortcutTargets.get(shortcutIndex); suggestions.add(new SuggestedWordInfo(new String(shortcut, 0, shortcut.length), finalFreq, SuggestedWordInfo.KIND_SHORTCUT, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); if (suggestions.size() > Suggest.MAX_SUGGESTIONS) return false; } } @@ -604,7 +606,8 @@ public class ExpandableDictionary extends Dictionary { suggestions.add(new SuggestedWordInfo(new String(mLookedUpString, index, Constants.DICTIONARY_MAX_WORD_LENGTH - index), freq, SuggestedWordInfo.KIND_CORRECTION, this /* sourceDict */, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } } } diff --git a/java/src/com/android/inputmethod/latin/LatinIME.java b/java/src/com/android/inputmethod/latin/LatinIME.java index 5657ed779..d8a47a307 100644 --- a/java/src/com/android/inputmethod/latin/LatinIME.java +++ b/java/src/com/android/inputmethod/latin/LatinIME.java @@ -2702,7 +2702,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen suggestions.add(new SuggestedWordInfo(s, SuggestionStripView.MAX_SUGGESTIONS - i, SuggestedWordInfo.KIND_RESUMED, Dictionary.DICTIONARY_RESUMED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE + /* autoCommitFirstWordConfidence */)); } } } diff --git a/java/src/com/android/inputmethod/latin/Suggest.java b/java/src/com/android/inputmethod/latin/Suggest.java index 9370757ec..7815f4d41 100644 --- a/java/src/com/android/inputmethod/latin/Suggest.java +++ b/java/src/com/android/inputmethod/latin/Suggest.java @@ -327,7 +327,8 @@ public final class Suggest { suggestionsContainer.add(0, new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } SuggestedWordInfo.removeDups(suggestionsContainer); @@ -474,7 +475,8 @@ public final class Suggest { sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE); } return new SuggestedWordInfo(sb.toString(), wordInfo.mScore, wordInfo.mKind, - wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord); + wordInfo.mSourceDict, wordInfo.mIndexOfTouchPointOfSecondWord, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); } public void close() { diff --git a/java/src/com/android/inputmethod/latin/SuggestedWords.java b/java/src/com/android/inputmethod/latin/SuggestedWords.java index b27fd81e9..17637054a 100644 --- a/java/src/com/android/inputmethod/latin/SuggestedWords.java +++ b/java/src/com/android/inputmethod/latin/SuggestedWords.java @@ -114,7 +114,8 @@ public final class SuggestedWords { final SuggestedWordInfo suggestedWordInfo = new SuggestedWordInfo(text.toString(), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_APP_DEFINED, Dictionary.DICTIONARY_APPLICATION_DEFINED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); result.add(suggestedWordInfo); } return result; @@ -128,7 +129,8 @@ public final class SuggestedWords { final HashSet<String> alreadySeen = CollectionUtils.newHashSet(); suggestionsList.add(new SuggestedWordInfo(typedWord, SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_TYPED, Dictionary.DICTIONARY_USER_TYPED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); alreadySeen.add(typedWord.toString()); final int previousSize = previousSuggestions.size(); for (int index = 1; index < previousSize; index++) { @@ -151,6 +153,7 @@ public final class SuggestedWords { public static final class SuggestedWordInfo { public static final int NOT_AN_INDEX = -1; + public static final int NOT_A_CONFIDENCE = -1; public static final int MAX_SCORE = Integer.MAX_VALUE; public static final int KIND_MASK_KIND = 0xFF; // Mask to get only the kind public static final int KIND_TYPED = 0; // What user typed @@ -180,16 +183,30 @@ public final class SuggestedWords { // passed to native code to get suggestions for a gesture that corresponds to the first // letter of the second word. public final int mIndexOfTouchPointOfSecondWord; + // For auto-commit. This is a measure of how confident we are that we can commit the + // first word of this suggestion. + public final int mAutoCommitFirstWordConfidence; private String mDebugString = ""; + /** + * Create a new suggested word info. + * @param word The string to suggest. + * @param score A measure of how likely this suggestion is. + * @param kind The kind of suggestion, as one of the above KIND_* constants. + * @param sourceDict What instance of Dictionary produced this suggestion. + * @param indexOfTouchPointOfSecondWord See mIndexOfTouchPointOfSecondWord. + * @param autoCommitFirstWordConfidence See mAutoCommitFirstWordConfidence. + */ public SuggestedWordInfo(final String word, final int score, final int kind, - final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord) { + final Dictionary sourceDict, final int indexOfTouchPointOfSecondWord, + final int autoCommitFirstWordConfidence) { mWord = word; mScore = score; mKind = kind; mSourceDict = sourceDict; mCodePointCount = StringUtils.codePointCount(mWord); mIndexOfTouchPointOfSecondWord = indexOfTouchPointOfSecondWord; + mAutoCommitFirstWordConfidence = autoCommitFirstWordConfidence; } public boolean isEligibleForAutoCommit() { diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 21e9811ef..f333b0d86 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -126,8 +126,14 @@ public class BinaryDictEncoderUtils { */ private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { int size = getNodeHeaderSize(ptNode, options); - // If terminal, one byte for the frequency - if (ptNode.isTerminal()) size += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (ptNode.isTerminal()) { + // If terminal, one byte for the frequency or four bytes for the terminal id. + if (options.mHasTerminalId) { + size += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + size += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address size += getShortcutListSize(ptNode.mShortcutTargets); if (null != ptNode.mBigrams) { @@ -345,7 +351,13 @@ public class BinaryDictEncoderUtils { changed = true; } int nodeSize = getNodeHeaderSize(ptNode, formatOptions); - if (ptNode.isTerminal()) nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + if (ptNode.isTerminal()) { + if (formatOptions.mHasTerminalId) { + nodeSize += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } if (formatOptions.mSupportsDynamicUpdate) { nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; } else if (null != ptNode.mChildren) { @@ -787,7 +799,6 @@ public class BinaryDictEncoderUtils { + FormatSpec.MAX_TERMINAL_FREQUENCY + " : " + ptNode.mFrequency); } - dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); } if (formatOptions.mSupportsDynamicUpdate) { diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index 44ae33de1..96ccd8e49 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -198,9 +198,12 @@ public final class FormatSpec { public static final int MAGIC_NUMBER = 0x9BC13AFE; static final int MINIMUM_SUPPORTED_VERSION = 2; - static final int MAXIMUM_SUPPORTED_VERSION = 3; + static final int MAXIMUM_SUPPORTED_VERSION = 4; static final int NOT_A_VERSION_NUMBER = -1; static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; + static final int FIRST_VERSION_WITH_TERMINAL_ID = 4; + static final int VERSION3 = 3; + static final int VERSION4 = 4; // These options need to be the same numeric values as the one in the native reading code. static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; @@ -251,11 +254,17 @@ public final class FormatSpec { static final int PTNODE_TERMINATOR_SIZE = 1; static final int PTNODE_FLAGS_SIZE = 1; static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_TERMINAL_ID_SIZE = 4; static final int PTNODE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; + // These values are used only by version 4 or later. + static final String TRIE_FILE_EXTENSION = ".trie"; + static final String FREQ_FILE_EXTENSION = ".freq"; + static final int FREQUENCY_AND_FLAGS_SIZE = 2; + static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; @@ -264,6 +273,7 @@ public final class FormatSpec { static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; + static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF; static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; @@ -287,6 +297,7 @@ public final class FormatSpec { public static final class FormatOptions { public final int mVersion; public final boolean mSupportsDynamicUpdate; + public final boolean mHasTerminalId; @UsedForTesting public FormatOptions(final int version) { this(version, false); @@ -300,6 +311,7 @@ public final class FormatSpec { + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); } mSupportsDynamicUpdate = supportsDynamicUpdate; + mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 3e685a3d7..be653feec 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -111,6 +111,7 @@ public final class FusionDictionary implements Iterable<Word> { ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal. + int mTerminalId; // NOT_A_TERMINAL == mTerminalId indicates this is not a terminal. PtNodeArray mChildren; boolean mIsNotAWord; // Only a shortcut boolean mIsBlacklistEntry; @@ -129,6 +130,7 @@ public final class FusionDictionary implements Iterable<Word> { final boolean isNotAWord, final boolean isBlacklistEntry) { mChars = chars; mFrequency = frequency; + mTerminalId = frequency; mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = null; @@ -156,6 +158,10 @@ public final class FusionDictionary implements Iterable<Word> { mChildren.mData.add(n); } + public int getTerminalId() { + return mTerminalId; + } + public boolean isTerminal() { return NOT_A_TERMINAL != mFrequency; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index 48a823d43..222a0f474 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -68,7 +68,7 @@ public class Ver3DictEncoder implements DictEncoder { @Override public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { - if (formatOptions.mVersion > 3) { + if (formatOptions.mVersion > FormatSpec.VERSION3) { throw new UnsupportedFormatException( "The given format options has wrong version number : " + formatOptions.mVersion); @@ -200,7 +200,7 @@ public class Ver3DictEncoder implements DictEncoder { mPosition += shortcutShift; } final int shortcutByteSize = mPosition - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { throw new RuntimeException("Shortcut list too large"); } BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java new file mode 100644 index 000000000..75b75ae2e --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -0,0 +1,269 @@ +/* +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * An implementation of DictEncoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictEncoder implements DictEncoder { + private final File mDictPlacedDir; + private byte[] mTrieBuf; + private byte[] mFreqBuf; + private int mTriePos; + private OutputStream mTrieOutStream; + private OutputStream mFreqOutStream; + + @UsedForTesting + public Ver4DictEncoder(final File dictPlacedDir) { + mDictPlacedDir = dictPlacedDir; + } + + private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions) + throws FileNotFoundException, IOException { + final FileHeader header = new FileHeader(0, dictOptions, formatOptions); + final String filename = header.getId() + "." + header.getVersion(); + final File mDictDir = new File(mDictPlacedDir, filename); + final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION); + final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION); + if (!mDictDir.isDirectory()) { + if (mDictDir.exists()) mDictDir.delete(); + mDictDir.mkdirs(); + } + if (!trieFile.exists()) trieFile.createNewFile(); + if (!freqFile.exists()) freqFile.createNewFile(); + mTrieOutStream = new FileOutputStream(trieFile); + mFreqOutStream = new FileOutputStream(freqFile); + } + + private void close() throws IOException { + try { + if (mTrieOutStream != null) { + mTrieOutStream.close(); + } + if (mFreqOutStream != null) { + mFreqOutStream.close(); + } + } finally { + mTrieOutStream = null; + mFreqOutStream = null; + } + } + + @Override + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion != FormatSpec.VERSION4) { + throw new UnsupportedFormatException("File header has a wrong version number : " + + formatOptions.mVersion); + } + if (!mDictPlacedDir.isDirectory()) { + throw new UnsupportedFormatException("Given path is not a directory."); + } + + if (mTrieOutStream == null) { + openStreams(formatOptions, dict.mOptions); + } + + BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, formatOptions); + + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + int terminalCount = 0; + for (final PtNodeArray array : flatNodes) { + for (final PtNode node : array.mData) { + if (node.isTerminal()) node.mTerminalId = terminalCount++; + } + } + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mTrieBuf = new byte[bufferSize]; + mFreqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE]; + + MakedictLog.i("Writing file..."); + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); + } + if (MakedictLog.DBG) { + BinaryDictEncoderUtils.showStatistics(flatNodes); + MakedictLog.i("has " + terminalCount + " terminals."); + } + mTrieOutStream.write(mTrieBuf); + mFreqOutStream.write(mFreqBuf); + + MakedictLog.i("Done"); + close(); + } + + @Override + public void setPosition(int position) { + if (mTrieBuf == null || position < 0 || position >- mTrieBuf.length) return; + mTriePos = position; + } + + @Override + public int getPosition() { + return mTriePos; + } + + @Override + public void writePtNodeCount(int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + // ptNodeCount must fit on one byte or two bytes. + // Please see comments in FormatSpec + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); + } + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, ptNodeCount, + countSize); + } + + private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mTriePos, childrenPos, + formatOptions), + FormatSpec.PTNODE_FLAGS_SIZE); + } + + private void writeParentPosition(int parentPos, final PtNode ptNode, + final FormatOptions formatOptions) { + if (parentPos != FormatSpec.NO_PARENT_ADDRESS) { + parentPos -= ptNode.mCachedAddressAfterUpdate; + } + mTriePos = BinaryDictEncoderUtils.writeParentAddress(mTrieBuf, mTriePos, parentPos, + formatOptions); + } + + private void writeCharacters(final int[] characters, final boolean hasSeveralChars) { + mTriePos = CharEncoding.writeCharArray(characters, mTrieBuf, mTriePos); + if (hasSeveralChars) { + mTrieBuf[mTriePos++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + private void writeTerminalId(final int terminalId) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, terminalId, + FormatSpec.PTNODE_TERMINAL_ID_SIZE); + } + + private void writeFrequency(final int frequency, final int terminalId) { + final int freqPos = terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE; + BinaryDictEncoderUtils.writeUIntToBuffer(mFreqBuf, freqPos, frequency, + FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + } + + private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + if (formatOptions.mSupportsDynamicUpdate) { + mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } else { + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } + } + + private void writeShortcuts(ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mTriePos; + mTriePos += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.mFrequency); + mTrieBuf[mTriePos++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos, + target.mWord); + mTriePos += shortcutShift; + } + final int shortcutByteSize = mTriePos - indexOfShortcutByteSize; + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { + throw new RuntimeException("Shortcut list too large : " + shortcutByteSize); + } + BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, indexOfShortcutByteSize, + shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + final int offset = addressOfBigram + - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + mTrieBuf[mTriePos++] = (byte) bigramFlags; + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(int forwardLinkAddress) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + forwardLinkAddress, FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict) { + writePtNodeFlags(ptNode, parentPosition, formatOptions); + writeParentPosition(parentPosition, ptNode, formatOptions); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + if (ptNode.isTerminal()) { + writeTerminalId(ptNode.mTerminalId); + writeFrequency(ptNode.mFrequency, ptNode.mTerminalId); + } + writeChildrenPosition(ptNode, formatOptions); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); + } +} diff --git a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java index 072bb8731..fc2d19298 100644 --- a/java/src/com/android/inputmethod/latin/settings/SettingsValues.java +++ b/java/src/com/android/inputmethod/latin/settings/SettingsValues.java @@ -295,7 +295,8 @@ public final class SettingsValues { puncList.add(new SuggestedWordInfo(KeySpecParser.getLabel(puncSpec), SuggestedWordInfo.MAX_SCORE, SuggestedWordInfo.KIND_HARDCODED, Dictionary.DICTIONARY_HARDCODED, - SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */)); + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */)); } } return new SuggestedWords(puncList, diff --git a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java index 3c1db6529..5dc0b5893 100644 --- a/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java +++ b/java/src/com/android/inputmethod/latin/utils/PrioritizedSerialExecutor.java @@ -31,6 +31,7 @@ public class PrioritizedSerialExecutor { private static final int TASK_QUEUE_CAPACITY = 1000; private final Queue<Runnable> mTasks; private final Queue<Runnable> mPrioritizedTasks; + private boolean mIsShutdown; // The task which is running now. private Runnable mActive; @@ -38,6 +39,7 @@ public class PrioritizedSerialExecutor { public PrioritizedSerialExecutor() { mTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); mPrioritizedTasks = new ArrayDeque<Runnable>(TASK_QUEUE_CAPACITY); + mIsShutdown = false; } /** @@ -56,9 +58,11 @@ public class PrioritizedSerialExecutor { */ public void execute(final Runnable r) { synchronized(mLock) { - mTasks.offer(r); - if (mActive == null) { - scheduleNext(); + if (!mIsShutdown) { + mTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } } } } @@ -69,9 +73,11 @@ public class PrioritizedSerialExecutor { */ public void executePrioritized(final Runnable r) { synchronized(mLock) { - mPrioritizedTasks.offer(r); - if (mActive == null) { - scheduleNext(); + if (!mIsShutdown) { + mPrioritizedTasks.offer(r); + if (mActive == null) { + scheduleNext(); + } } } } @@ -123,4 +129,19 @@ public class PrioritizedSerialExecutor { execute(newTask); } } + + public void shutdown() { + synchronized(mLock) { + mIsShutdown = true; + } + } + + public boolean isTerminated() { + synchronized(mLock) { + if (!mIsShutdown) { + return false; + } + return mPrioritizedTasks.isEmpty() && mTasks.isEmpty() && mActive == null; + } + } } |