diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/makedict')
4 files changed, 285 insertions, 186 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index 79f5ad8bd..5a213415a 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Iterator; /** * Encodes binary files for a FusionDictionary. @@ -432,7 +431,7 @@ public class BinaryDictEncoderUtils { * @param formatOptions file format options. * @return the same array it was passed. The nodes have been updated for address and size. */ - private static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, + /* package */ static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) { // First get the worst possible sizes and offsets for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions); @@ -484,7 +483,7 @@ public class BinaryDictEncoderUtils { * * @param arrays the list of node arrays to check */ - private static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) { + /* package */ static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) { int offset = 0; int index = 0; for (final PtNodeArray ptNodeArray : arrays) { @@ -501,52 +500,53 @@ public class BinaryDictEncoderUtils { } /** - * Helper method to write a variable-size address to a file. + * Helper method to write a children position to a file. * * @param buffer the buffer to write to. * @param index the index in the buffer to write the address to. - * @param address the address to write. + * @param position the position to write. * @return the size in bytes the address actually took. */ - private static int writeVariableAddress(final byte[] buffer, int index, final int address) { - switch (getByteSize(address)) { + /* package */ static int writeChildrenPosition(final byte[] buffer, int index, + final int position) { + switch (getByteSize(position)) { case 1: - buffer[index++] = (byte)address; + buffer[index++] = (byte)position; return 1; case 2: - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); return 2; case 3: - buffer[index++] = (byte)(0xFF & (address >> 16)); - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); + buffer[index++] = (byte)(0xFF & (position >> 16)); + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); return 3; case 0: return 0; default: - throw new RuntimeException("Address " + address + " has a strange size"); + throw new RuntimeException("Position " + position + " has a strange size"); } } /** - * Helper method to write a variable-size signed address to a file. + * Helper method to write a signed children position to a file. * * @param buffer the buffer to write to. * @param index the index in the buffer to write the address to. - * @param address the address to write. + * @param position the position to write. * @return the size in bytes the address actually took. */ - private static int writeVariableSignedAddress(final byte[] buffer, int index, - final int address) { - if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + /* package */ static int writeSignedChildrenPosition(final byte[] buffer, int index, + final int position) { + if (!BinaryDictIOUtils.hasChildrenAddress(position)) { buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; } else { - final int absAddress = Math.abs(address); + final int absPosition = Math.abs(position); buffer[index++] = - (byte)((address < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absAddress >> 16))); - buffer[index++] = (byte)(0xFF & (absAddress >> 8)); - buffer[index++] = (byte)(0xFF & absAddress); + (byte)((position < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absPosition >> 16))); + buffer[index++] = (byte)(0xFF & (absPosition >> 8)); + buffer[index++] = (byte)(0xFF & absPosition); } return 3; } @@ -598,7 +598,7 @@ public class BinaryDictEncoderUtils { return flags; } - private static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress, + /* package */ static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress, final int childrenOffset, final FormatOptions formatOptions) { return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0, getByteSize(childrenOffset), @@ -616,7 +616,7 @@ public class BinaryDictEncoderUtils { * @param word the second bigram, for debugging purposes * @return the flags */ - private static final int makeBigramFlags(final boolean more, final int offset, + /* package */ static final int makeBigramFlags(final boolean more, final int offset, int bigramFrequency, final int unigramFrequency, final String word) { int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); @@ -702,7 +702,7 @@ public class BinaryDictEncoderUtils { + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); } - private static final int writeParentAddress(final byte[] buffer, final int index, + /* package */ static final int writeParentAddress(final byte[] buffer, final int index, final int address, final FormatOptions formatOptions) { if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { if (address == FormatSpec.NO_PARENT_ADDRESS) { @@ -721,137 +721,69 @@ public class BinaryDictEncoderUtils { } } + /* package */ static final int getChildrenPosition(final PtNode ptNode, + final FormatOptions formatOptions) { + int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate + + getNodeHeaderSize(ptNode, formatOptions); + if (ptNode.mFrequency >= 0) { + positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS + : ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField; + } + /** * Write a PtNodeArray to memory. The PtNodeArray is expected to have its final position cached. * * @param dict the dictionary the node array is a part of (for relative offsets). - * @param buffer the memory buffer to write to. + * @param dictEncoder the dictionary encoder. * @param ptNodeArray the node array to write. * @param formatOptions file format options. - * @return the address of the END of the node. */ @SuppressWarnings("unused") - private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, - final PtNodeArray ptNodeArray, final FormatOptions formatOptions) { + /* package */ static void writePlacedNode(final FusionDictionary dict, + final DictEncoder dictEncoder, final PtNodeArray ptNodeArray, + final FormatOptions formatOptions) { // TODO: Make the code in common with BinaryDictIOUtils#writePtNode - int index = ptNodeArray.mCachedAddressAfterUpdate; + dictEncoder.setPosition(ptNodeArray.mCachedAddressAfterUpdate); final int ptNodeCount = ptNodeArray.mData.size(); - final int countSize = getPtNodeCountSize(ptNodeArray); - final int parentAddress = ptNodeArray.mCachedParentAddress; - if (1 == countSize) { - buffer[index++] = (byte)ptNodeCount; - } else if (2 == countSize) { - // We need to signal 2-byte size by setting the top bit of the MSB to 1, so - // we | 0x80 to do this. - buffer[index++] = (byte)((ptNodeCount >> 8) | 0x80); - buffer[index++] = (byte)(ptNodeCount & 0xFF); - } else { - throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); - } - int ptNodeAddress = index; + dictEncoder.writePtNodeCount(ptNodeCount); + final int parentPosition = + (ptNodeArray.mCachedParentAddress == FormatSpec.NO_PARENT_ADDRESS) + ? FormatSpec.NO_PARENT_ADDRESS + : ptNodeArray.mCachedParentAddress + ptNodeArray.mCachedAddressAfterUpdate; for (int i = 0; i < ptNodeCount; ++i) { final PtNode ptNode = ptNodeArray.mData.get(i); - if (index != ptNode.mCachedAddressAfterUpdate) { + if (dictEncoder.getPosition() != ptNode.mCachedAddressAfterUpdate) { throw new RuntimeException("Bug: write index is not the same as the cached address " - + "of the node : " + index + " <> " + ptNode.mCachedAddressAfterUpdate); + + "of the node : " + dictEncoder.getPosition() + " <> " + + ptNode.mCachedAddressAfterUpdate); } - ptNodeAddress += getNodeHeaderSize(ptNode, formatOptions); // Sanity checks. if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { throw new RuntimeException("A node has a frequency > " + FormatSpec.MAX_TERMINAL_FREQUENCY + " : " + ptNode.mFrequency); } - if (ptNode.mFrequency >= 0) ptNodeAddress += FormatSpec.PTNODE_FREQUENCY_SIZE; - final int childrenOffset = null == ptNode.mChildren - ? FormatSpec.NO_CHILDREN_ADDRESS - : ptNode.mChildren.mCachedAddressAfterUpdate - ptNodeAddress; - buffer[index++] = - makePtNodeFlags(ptNode, ptNodeAddress, childrenOffset, formatOptions); - - if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { - index = writeParentAddress(buffer, index, parentAddress, formatOptions); - } else { - index = writeParentAddress(buffer, index, parentAddress - + (ptNodeArray.mCachedAddressAfterUpdate - - ptNode.mCachedAddressAfterUpdate), - formatOptions); - } - - index = CharEncoding.writeCharArray(ptNode.mChars, buffer, index); - if (ptNode.hasSeveralChars()) { - buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; - } - if (ptNode.mFrequency >= 0) { - buffer[index++] = (byte) ptNode.mFrequency; - } - - final int shift; - if (formatOptions.mSupportsDynamicUpdate) { - shift = writeVariableSignedAddress(buffer, index, childrenOffset); - } else { - shift = writeVariableAddress(buffer, index, childrenOffset); - } - index += shift; - ptNodeAddress += shift; - - // Write shortcuts - if (null != ptNode.mShortcutTargets && !ptNode.mShortcutTargets.isEmpty()) { - final int indexOfShortcutByteSize = index; - index += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - ptNodeAddress += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = - ptNode.mShortcutTargets.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - ++ptNodeAddress; - int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), - target.mFrequency); - buffer[index++] = (byte)shortcutFlags; - final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); - index += shortcutShift; - ptNodeAddress += shortcutShift; - } - final int shortcutByteSize = index - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { - throw new RuntimeException("Shortcut list too large"); - } - buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); - buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); - } - // Write bigrams - if (null != ptNode.mBigrams) { - final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator(); - while (bigramIterator.hasNext()) { - final WeightedString bigram = bigramIterator.next(); - final PtNode target = - FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); - final int addressOfBigram = target.mCachedAddressAfterUpdate; - final int unigramFrequencyForThisWord = target.mFrequency; - ++ptNodeAddress; - final int offset = addressOfBigram - ptNodeAddress; - int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, - bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); - buffer[index++] = (byte)bigramFlags; - final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); - index += bigramShift; - ptNodeAddress += bigramShift; - } - } + dictEncoder.writePtNodeFlags(ptNode, parentPosition, formatOptions); + dictEncoder.writeParentPosition(parentPosition, ptNode, formatOptions); + dictEncoder.writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + dictEncoder.writeFrequency(ptNode.mFrequency); + dictEncoder.writeChildrenPosition(ptNode, formatOptions); + dictEncoder.writeShortcuts(ptNode.mShortcutTargets); + dictEncoder.writeBigrams(ptNode.mBigrams, dict); } if (formatOptions.mSupportsDynamicUpdate) { - buffer[index] = buffer[index + 1] = buffer[index + 2] - = FormatSpec.NO_FORWARD_LINK_ADDRESS; - index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); } - if (index != ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize) { - throw new RuntimeException( - "Not the same size : written " + (index - ptNodeArray.mCachedAddressAfterUpdate) + if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate + + ptNodeArray.mCachedSize) { + throw new RuntimeException("Not the same size : written " + + (dictEncoder.getPosition() - ptNodeArray.mCachedAddressAfterUpdate) + " bytes from a node that should have " + ptNodeArray.mCachedSize + " bytes"); } - return index; } /** @@ -862,7 +794,7 @@ public class BinaryDictEncoderUtils { * * @param ptNodeArrays the list of PtNode arrays. */ - private static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) { + /* package */ static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) { int firstTerminalAddress = Integer.MAX_VALUE; int lastTerminalAddress = Integer.MIN_VALUE; int size = 0; @@ -912,23 +844,15 @@ public class BinaryDictEncoderUtils { } /** - * Dumps a FusionDictionary to a file. + * Writes a file header to an output stream. * - * @param destination the stream to write the binary data to. + * @param destination the stream to write the file header to. * @param dict the dictionary to write. * @param formatOptions file format options. */ - /* package */ static void writeDictionaryBinary(final OutputStream destination, + /* package */ static void writeDictionaryHeader(final OutputStream destination, final FusionDictionary dict, final FormatOptions formatOptions) - throws IOException, UnsupportedFormatException { - - // Addresses are limited to 3 bytes, but since addresses can be relative to each node - // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding - // the order of the PtNode arrays becomes a quite complicated problem, because though the - // dictionary itself does not have a size limit, each node array must still be within 16MB - // of all its children and parents. As long as this is ensured, the dictionary file may - // grow to any size. - + throws IOException, UnsupportedFormatException { final int version = formatOptions.mVersion; if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { @@ -975,33 +899,5 @@ public class BinaryDictEncoderUtils { destination.write(bytes); headerBuffer.close(); - - // Leave the choice of the optimal node order to the flattenTree function. - MakedictLog.i("Flattening the tree..."); - ArrayList<PtNodeArray> flatNodes = flattenTree(dict.mRootNodeArray); - - MakedictLog.i("Computing addresses..."); - computeAddresses(dict, flatNodes, formatOptions); - MakedictLog.i("Checking PtNode array..."); - if (DBG) checkFlatPtNodeArrayList(flatNodes); - - // Create a buffer that matches the final dictionary size. - final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); - final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; - final byte[] buffer = new byte[bufferSize]; - int index = 0; - - MakedictLog.i("Writing file..."); - int dataEndOffset = 0; - for (PtNodeArray nodeArray : flatNodes) { - dataEndOffset = writePlacedNode(dict, buffer, nodeArray, formatOptions); - } - - if (DBG) showStatistics(flatNodes); - - destination.write(buffer, 0, dataEndOffset); - - destination.close(); - MakedictLog.i("Done"); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java index 89c982e7b..d1589a30e 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -17,8 +17,11 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.IOException; +import java.util.ArrayList; /** * An interface of binary dictionary encoder. @@ -26,4 +29,32 @@ import java.io.IOException; public interface DictEncoder { public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException; + + public void setPosition(final int position); + public int getPosition(); + public void writePtNodeCount(final int ptNodeCount); + public void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions); + public void writeParentPosition(final int parentPosition, final PtNode ptNode, + final FormatOptions formatOptions); + public void writeCharacters(final int[] characters, final boolean hasSeveralChars); + public void writeFrequency(final int frequency); + public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions); + + /** + * Write a shortcut attributes list to memory. + * + * @param shortcuts the shortcut attributes list. + */ + public void writeShortcuts(final ArrayList<WeightedString> shortcuts); + + /** + * Write a bigram attributes list to memory. + * + * @param bigrams the bigram attributes list. + * @param dict the dictionary the node array is a part of (for relative offsets). + */ + public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict); + + public void writeForwardLinkAddress(final int forwardLinkAddress); } diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index b8ef57696..bf35f6a8a 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -112,8 +112,10 @@ public final class FormatSpec { * e | 1 byte = bbbbbbbb match * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte - * a | - * ddress + * a | This address is relative to the head of the PtNode. + * d | If the node doesn't have a parent, this field is set to 0. + * d | + * ress * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers @@ -132,17 +134,18 @@ public final class FormatSpec { * i | 1 byte = bbbbbbbb match * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte - * r | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType - * e | // nothing - * n | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType - * A | children address, 1 byte - * d | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType - * d | children address, 2 bytes - * r | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType - * e | children address, 3 bytes - * s | END - * s - * ress + * r | if this node doesn't have children, this field is set to 0. + * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) + * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType + * a | // nothing + * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType + * d | children address, 1 byte + * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType + * e | children address, 2 bytes + * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType + * s | children address, 3 bytes + * | END + * | This address is relative to the position of this field. * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java index e81fd4514..3f26ff378 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -16,13 +16,19 @@ package com.android.inputmethod.latin.makedict; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; /** * An implementation of DictEncoder for version 3 binary dictionary. @@ -31,10 +37,13 @@ public class Ver3DictEncoder implements DictEncoder { private final File mDictFile; private OutputStream mOutStream; + private byte[] mBuffer; + private int mPosition; public Ver3DictEncoder(final File dictFile) { mDictFile = dictFile; mOutStream = null; + mBuffer = null; } // This constructor is used only by BinaryDictOffdeviceUtilsTests. @@ -57,12 +66,172 @@ public class Ver3DictEncoder implements DictEncoder { } @Override - public void writeDictionary(FusionDictionary dict, FormatOptions formatOptions) + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion > 3) { + throw new UnsupportedFormatException( + "The given format options has wrong version number : " + + formatOptions.mVersion); + } + if (mOutStream == null) { openStream(); } - BinaryDictEncoderUtils.writeDictionaryBinary(mOutStream, dict, formatOptions); + BinaryDictEncoderUtils.writeDictionaryHeader(mOutStream, dict, formatOptions); + + // Addresses are limited to 3 bytes, but since addresses can be relative to each node + // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding + // the order of the PtNode arrays becomes a quite complicated problem, because though the + // dictionary itself does not have a size limit, each node array must still be within 16MB + // of all its children and parents. As long as this is ensured, the dictionary file may + // grow to any size. + + // Leave the choice of the optimal node order to the flattenTree function. + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + MakedictLog.i("Checking PtNode array..."); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + // Create a buffer that matches the final dictionary size. + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mBuffer = new byte[bufferSize]; + + MakedictLog.i("Writing file..."); + + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedNode(dict, this, nodeArray, formatOptions); + } + if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes); + mOutStream.write(mBuffer, 0, mPosition); + + MakedictLog.i("Done"); close(); } + + @Override + public void setPosition(final int position) { + if (mBuffer == null || position < 0 || position >= mBuffer.length) return; + mPosition = position; + } + + @Override + public int getPosition() { + return mPosition; + } + + @Override + public void writePtNodeCount(final int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + if (1 == countSize) { + mBuffer[mPosition++] = (byte) ptNodeCount; + } else if (2 == countSize) { + mBuffer[mPosition++] = (byte) ((ptNodeCount >> 8) & 0xFF); + mBuffer[mPosition++] = (byte) (ptNodeCount & 0xFF); + } else { + throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); + } + } + + @Override + public void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + mBuffer[mPosition++] = BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, + childrenPos, formatOptions); + } + + @Override + public void writeParentPosition(final int parentPosition, final PtNode ptNode, + final FormatOptions formatOptions) { + if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) { + mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, + parentPosition, formatOptions); + } else { + mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, + parentPosition - ptNode.mCachedAddressAfterUpdate, formatOptions); + } + } + + @Override + public void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) { + mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition); + if (hasSeveralChars) { + mBuffer[mPosition++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + @Override + public void writeFrequency(final int frequency) { + if (frequency >= 0) { + mBuffer[mPosition++] = (byte) frequency; + } + } + + @Override + public void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + if (formatOptions.mSupportsDynamicUpdate) { + mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition, + childrenPos); + } else { + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + childrenPos); + } + } + + @Override + public void writeShortcuts(final ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mPosition; + mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.mFrequency); + mBuffer[mPosition++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord); + mPosition += shortcutShift; + } + final int shortcutByteSize = mPosition - indexOfShortcutByteSize; + if (shortcutByteSize > 0xFFFF) { + throw new RuntimeException("Shortcut list too large"); + } + mBuffer[indexOfShortcutByteSize] = (byte)((shortcutByteSize >> 8) & 0xFF); + mBuffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); + } + + @Override + public void writeBigrams(final ArrayList<WeightedString> bigrams, final FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + final int offset = addressOfBigram + - (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + mBuffer[mPosition++] = (byte) bigramFlags; + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(final int forwardLinkAddress) { + mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 16) & 0xFF); + mBuffer[mPosition++] = (byte) ((forwardLinkAddress >> 8) & 0xFF); + mBuffer[mPosition++] = (byte) (forwardLinkAddress & 0xFF); + } } |