diff options
Diffstat (limited to 'java/src')
5 files changed, 76 insertions, 235 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java index e7d1c98a9..1bc322724 100644 --- a/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/AbstractDictDecoder.java @@ -20,7 +20,6 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.FileNotFoundException; import java.io.IOException; @@ -117,94 +116,6 @@ public abstract class AbstractDictDecoder implements DictDecoder { } /** - * A utility class for reading a PtNode. - */ - protected static class PtNodeReader { - protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { - return dictBuffer.readUnsignedByte(); - } - - protected static int readParentAddress(final DictBuffer dictBuffer, - final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - return BinaryDictDecoderUtils.readSInt24(dictBuffer); - } else { - return FormatSpec.NO_PARENT_ADDRESS; - } - } - - protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, - final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); - if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; - return address; - } else { - switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: - return dictBuffer.readUnsignedByte(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: - return dictBuffer.readUnsignedShort(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: - return dictBuffer.readUnsignedInt24(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - } - - // Reads shortcuts and returns the read length. - protected static int readShortcut(final DictBuffer dictBuffer, - final ArrayList<WeightedString> shortcutTargets) { - final int pointerBefore = dictBuffer.position(); - dictBuffer.readUnsignedShort(); // skip the size - while (true) { - final int targetFlags = dictBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(dictBuffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return dictBuffer.position() - pointerBefore; - } - - protected static int readBigramAddresses(final DictBuffer dictBuffer, - final ArrayList<PendingAttribute> bigrams, final int baseAddress) { - int readLength = 0; - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - final int bigramFlags = dictBuffer.readUnsignedByte(); - ++readLength; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = baseAddress + readLength; - switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * dictBuffer.readUnsignedByte(); - readLength += 1; - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * dictBuffer.readUnsignedShort(); - readLength += 2; - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: - bigramAddress += sign * dictBuffer.readUnsignedInt24(); - readLength += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute( - bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return readLength; - } - } - - /** * Check whether the header contains the expected information. This is a no-error method, * that will return an error code and never throw a checked exception. * @return an error code, either ERROR_* or SUCCESS. @@ -264,9 +175,4 @@ public abstract class AbstractDictDecoder implements DictDecoder { public boolean hasNextPtNodeArray() { return false; } - - @Override - @UsedForTesting - public void skipPtNode(final FormatOptions formatOptions) { - } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 247d81ad3..fc5788de9 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -166,15 +166,6 @@ public final class BinaryDictDecoderUtils { return size; } - @UsedForTesting - static int getCharArraySize(final int[] chars, final int start, final int end) { - int size = 0; - for (int i = start; i < end; ++i) { - size += getCharSize(chars[i]); - } - return size; - } - /** * Writes a char array to a byte buffer. * @@ -253,41 +244,6 @@ public final class BinaryDictDecoderUtils { } /** - * Writes an array of code points with our character format to an OutputStream. - * - * This will also write the terminator byte. - * - * @param stream the OutputStream to write to. - * @param codePoints the array of code points - * @return the size written, in bytes. - */ - // TODO: Merge this method with writeCharArray and rename the various write* methods to - // make the difference clear. - @UsedForTesting - static int writeCodePoints(final OutputStream stream, final int[] codePoints, - final int startIndex, final int endIndex) - throws IOException { - int written = 0; - for (int i = startIndex; i < endIndex; ++i) { - final int codePoint = codePoints[i]; - final int charSize = getCharSize(codePoint); - if (1 == charSize) { - stream.write((byte) codePoint); - } else { - stream.write((byte) (0xFF & (codePoint >> 16))); - stream.write((byte) (0xFF & (codePoint >> 8))); - stream.write((byte) (0xFF & codePoint)); - } - written += charSize; - } - if (endIndex - startIndex > 1) { - stream.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); - written += FormatSpec.PTNODE_TERMINATOR_SIZE; - } - return written; - } - - /** * Reads a string from a DictBuffer. This is the converse of the above method. */ static String readString(final DictBuffer dictBuffer) { @@ -321,50 +277,6 @@ public final class BinaryDictDecoderUtils { } } - // Input methods: Read a binary dictionary to memory. - // readDictionaryBinary is the public entry point for them. - - static int readSInt24(final DictBuffer dictBuffer) { - final int retval = dictBuffer.readUnsignedInt24(); - final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; - return sign * (retval & FormatSpec.SINT24_MAX); - } - - static int readChildrenAddress(final DictBuffer dictBuffer, - final int optionFlags, final FormatOptions options) { - if (options.supportsDynamicUpdate()) { - final int address = dictBuffer.readUnsignedInt24(); - if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; - if ((address & FormatSpec.MSB24) != 0) { - return -(address & FormatSpec.SINT24_MAX); - } else { - return address; - } - } - switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: - return dictBuffer.readUnsignedByte(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: - return dictBuffer.readUnsignedShort(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: - return dictBuffer.readUnsignedInt24(); - case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - - static int readParentAddress(final DictBuffer dictBuffer, - final FormatOptions formatOptions) { - if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { - final int parentAddress = dictBuffer.readUnsignedInt24(); - final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; - return sign * (parentAddress & FormatSpec.SINT24_MAX); - } else { - return FormatSpec.NO_PARENT_ADDRESS; - } - } - /** * Reads and returns the PtNode count out of a buffer and forwards the pointer. */ diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 67f86d44e..54446df49 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -21,7 +21,6 @@ import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index bba1d434f..a93a8bbad 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -228,8 +228,6 @@ public interface DictDecoder { } } - public void skipPtNode(final FormatOptions formatOptions); - /** * @return whether this decoder has a valid binary dictionary that it can decode. */ diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java index f22b118d9..fd6138d41 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -20,7 +20,6 @@ import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import android.util.Log; @@ -38,11 +37,82 @@ import java.util.Arrays; public class Ver2DictDecoder extends AbstractDictDecoder { private static final String TAG = Ver2DictDecoder.class.getSimpleName(); - protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { + /** + * A utility class for reading a PtNode. + */ + protected static class PtNodeReader { private static ProbabilityInfo readProbabilityInfo(final DictBuffer dictBuffer) { // Ver2 dicts don't contain historical information. return new ProbabilityInfo(dictBuffer.readUnsignedByte()); } + + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, + final int ptNodeFlags) { + switch (ptNodeFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigramAddresses(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + bigramAddress += sign * dictBuffer.readUnsignedInt24(); + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return readLength; + } } protected final File mDictionaryBinaryFile; @@ -96,8 +166,8 @@ public class Ver2DictDecoder extends AbstractDictDecoder { } final DictionaryHeader header = super.readHeader(mDictBuffer); final int version = header.mFormatOptions.mVersion; - if (!(version >= 2 && version <= 3)) { - throw new UnsupportedFormatException("File header has a wrong version : " + version); + if (version != FormatSpec.VERSION2) { + throw new UnsupportedFormatException("File header has a wrong version : " + version); } return header; } @@ -109,12 +179,6 @@ public class Ver2DictDecoder extends AbstractDictDecoder { int addressPointer = ptNodePos; final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; - - final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); - if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += FormatSpec.PARENT_ADDRESS_SIZE; - } - final int characters[]; if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { int index = 0; @@ -141,7 +205,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder { } else { probabilityInfo = null; } - int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags); if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { childrenAddress += addressPointer; } @@ -168,7 +232,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder { bigrams = null; } return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, probabilityInfo, - parentAddress, childrenAddress, shortcutTargets, bigrams); + FormatSpec.NO_PARENT_ADDRESS, childrenAddress, shortcutTargets, bigrams); } @Override @@ -224,42 +288,4 @@ public class Ver2DictDecoder extends AbstractDictDecoder { public boolean hasNextPtNodeArray() { return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; } - - @Override - public void skipPtNode(final FormatOptions formatOptions) { - final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - PtNodeReader.readParentAddress(mDictBuffer, formatOptions); - BinaryDictIOUtils.skipString(mDictBuffer, - (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - PtNodeReader.readChildrenAddress(mDictBuffer, flags, formatOptions); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - PtNodeReader.readProbabilityInfo(mDictBuffer); - } - if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { - final int shortcutsSize = mDictBuffer.readUnsignedShort(); - mDictBuffer.position(mDictBuffer.position() + shortcutsSize - - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); - } - if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - final int bigramFlags = mDictBuffer.readUnsignedByte(); - switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: - mDictBuffer.readUnsignedByte(); - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: - mDictBuffer.readUnsignedShort(); - break; - case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: - mDictBuffer.readUnsignedInt24(); - break; - } - if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode."); - } - } - } } |