diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/makedict')
3 files changed, 80 insertions, 67 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 79f5ad8bd..1727382e8 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -501,52 +501,52 @@ public class BinaryDictEncoderUtils { } /** - * Helper method to write a variable-size address to a file. + * Helper method to write a children position to a file. * * @param buffer the buffer to write to. * @param index the index in the buffer to write the address to. - * @param address the address to write. + * @param position the position to write. * @return the size in bytes the address actually took. */ - private static int writeVariableAddress(final byte[] buffer, int index, final int address) { - switch (getByteSize(address)) { + private static int writeChildrenPosition(final byte[] buffer, int index, final int position) { + switch (getByteSize(position)) { case 1: - buffer[index++] = (byte)address; + buffer[index++] = (byte)position; return 1; case 2: - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); return 2; case 3: - buffer[index++] = (byte)(0xFF & (address >> 16)); - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); + buffer[index++] = (byte)(0xFF & (position >> 16)); + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); return 3; case 0: return 0; default: - throw new RuntimeException("Address " + address + " has a strange size"); + throw new RuntimeException("Position " + position + " has a strange size"); } } /** - * Helper method to write a variable-size signed address to a file. + * Helper method to write a signed children position to a file. * * @param buffer the buffer to write to. * @param index the index in the buffer to write the address to. - * @param address the address to write. + * @param position the position to write. * @return the size in bytes the address actually took. */ - private static int writeVariableSignedAddress(final byte[] buffer, int index, - final int address) { - if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + private static int writeSignedChildrenPosition(final byte[] buffer, int index, + final int position) { + if (!BinaryDictIOUtils.hasChildrenAddress(position)) { buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; } else { - final int absAddress = Math.abs(address); + final int absPosition = Math.abs(position); buffer[index++] = - (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16))); - buffer[index++] = (byte)(0xFF & (absAddress >> 8)); - buffer[index++] = (byte)(0xFF & absAddress); + (byte)((position < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absPosition >> 16))); + buffer[index++] = (byte)(0xFF & (absPosition >> 8)); + buffer[index++] = (byte)(0xFF & absPosition); } return 3; } @@ -721,6 +721,17 @@ public class BinaryDictEncoderUtils { } } + private static final int getChildrenPosition(final PtNode ptNode, + final FormatOptions formatOptions) { + int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate + + getNodeHeaderSize(ptNode, formatOptions); + if (ptNode.mFrequency >= 0) { + positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS + : ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField; + } + /** * Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached. * @@ -749,26 +760,22 @@ public class BinaryDictEncoderUtils { } else { throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); } - int ptNodeAddress = index; for (int i = 0; i < ptNodeCount; ++i) { final PtNode ptNode = ptNodeArray.mData.get(i); if (index != ptNode.mCachedAddressAfterUpdate) { throw new RuntimeException("Bug: write index is not the same as the cached address " + "of the node : " + index + " <> " + ptNode.mCachedAddressAfterUpdate); } - ptNodeAddress += getNodeHeaderSize(ptNode, formatOptions); // Sanity checks. if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { throw new RuntimeException("A node has a frequency > " + FormatSpec.MAX_TERMINAL_FREQUENCY + " : " + ptNode.mFrequency); } - if (ptNode.mFrequency >= 0) ptNodeAddress += FormatSpec.PTNODE_FREQUENCY_SIZE; - final int childrenOffset = null == ptNode.mChildren - ? FormatSpec.NO_CHILDREN_ADDRESS - : ptNode.mChildren.mCachedAddressAfterUpdate - ptNodeAddress; - buffer[index++] = - makePtNodeFlags(ptNode, ptNodeAddress, childrenOffset, formatOptions); + + final int childrenPosition = getChildrenPosition(ptNode, formatOptions); + buffer[index++] = makePtNodeFlags(ptNode, index, childrenPosition, + formatOptions); if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { index = writeParentAddress(buffer, index, parentAddress, formatOptions); @@ -787,31 +794,25 @@ public class BinaryDictEncoderUtils { buffer[index++] = (byte) ptNode.mFrequency; } - final int shift; if (formatOptions.mSupportsDynamicUpdate) { - shift = writeVariableSignedAddress(buffer, index, childrenOffset); + index += writeSignedChildrenPosition(buffer, index, childrenPosition); } else { - shift = writeVariableAddress(buffer, index, childrenOffset); + index += writeChildrenPosition(buffer, index, childrenPosition); } - index += shift; - ptNodeAddress += shift; // Write shortcuts if (null != ptNode.mShortcutTargets && !ptNode.mShortcutTargets.isEmpty()) { final int indexOfShortcutByteSize = index; index += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - ptNodeAddress += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; final Iterator<WeightedString> shortcutIterator = ptNode.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - ++ptNodeAddress; int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), target.mFrequency); buffer[index++] = (byte)shortcutFlags; final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); index += shortcutShift; - ptNodeAddress += shortcutShift; } final int shortcutByteSize = index - indexOfShortcutByteSize; if (shortcutByteSize > 0xFFFF) { @@ -829,14 +830,13 @@ public class BinaryDictEncoderUtils { FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); final int addressOfBigram = target.mCachedAddressAfterUpdate; final int unigramFrequencyForThisWord = target.mFrequency; - ++ptNodeAddress; - final int offset = addressOfBigram - ptNodeAddress; + final int offset = addressOfBigram + - (index + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); buffer[index++] = (byte)bigramFlags; - final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); + final int bigramShift = writeChildrenPosition(buffer, index, Math.abs(offset)); index += bigramShift; - ptNodeAddress += bigramShift; } } @@ -912,23 +912,15 @@ public class BinaryDictEncoderUtils { } /** - * Dumps a FusionDictionary to a file. + * Writes a file header to an output stream. * - * @param destination the stream to write the binary data to. + * @param destination the stream to write the file header to. * @param dict the dictionary to write. * @param formatOptions file format options. */ - /* package */ static void writeDictionaryBinary(final OutputStream destination, + /* package */ static void writeDictionaryHeader(final OutputStream destination, final FusionDictionary dict, final FormatOptions formatOptions) - throws IOException, UnsupportedFormatException { - - // Addresses are limited to 3 bytes, but since addresses can be relative to each node - // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding - // the order of the PtNode arrays becomes a quite complicated problem, because though the - // dictionary itself does not have a size limit, each node array must still be within 16MB - // of all its children and parents. As long as this is ensured, the dictionary file may - // grow to any size. - + throws IOException, UnsupportedFormatException { final int version = formatOptions.mVersion; if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { @@ -975,6 +967,24 @@ public class BinaryDictEncoderUtils { destination.write(bytes); headerBuffer.close(); + } + + /** + * Dumps a FusionDictionary to a file. + * + * @param destination the stream to write the dictionary body to. + * @param dict the dictionary to write. + * @param formatOptions file format options. + */ + /* package */ static void writeDictionaryBody(final OutputStream destination, + final FusionDictionary dict, final FormatOptions formatOptions) throws IOException { + + // Addresses are limited to 3 bytes, but since addresses can be relative to each node + // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding + // the order of the PtNode arrays becomes a quite complicated problem, because though the + // dictionary itself does not have a size limit, each node array must still be within 16MB + // of all its children and parents. As long as this is ensured, the dictionary file may + // grow to any size. // Leave the choice of the optimal node order to the flattenTree function. MakedictLog.i("Flattening the tree..."); @@ -989,7 +999,6 @@ public class BinaryDictEncoderUtils { final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; final byte[] buffer = new byte[bufferSize]; - int index = 0; MakedictLog.i("Writing file..."); int dataEndOffset = 0; diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index b8ef57696..bf35f6a8a 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -112,8 +112,10 @@ public final class FormatSpec { * e | 1 byte = bbbbbbbb match * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte - * a | - * ddress + * a | This address is relative to the head of the PtNode. + * d | If the node doesn't have a parent, this field is set to 0. + * d | + * ress * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers @@ -132,17 +134,18 @@ public final class FormatSpec { * i | 1 byte = bbbbbbbb match * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte - * r | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType - * e | // nothing - * n | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType - * A | children address, 1 byte - * d | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType - * d | children address, 2 bytes - * r | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType - * e | children address, 3 bytes - * s | END - * s - * ress + * r | if this node doesn't have children, this field is set to 0. + * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) + * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType + * a | // nothing + * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType + * d | children address, 1 byte + * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType + * e | children address, 2 bytes + * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType + * s | children address, 3 bytes + * | END + * | This address is relative to the position of this field. * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index e81fd4514..4e28feac8 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -62,7 +62,8 @@ public class Ver3DictEncoder implements DictEncoder { if (mOutStream == null) { openStream(); } - BinaryDictEncoderUtils.writeDictionaryBinary(mOutStream, dict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryHeader(mOutStream, dict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryBody(mOutStream, dict, formatOptions); close(); } } |