diff options
Diffstat (limited to 'java/src/com/android/inputmethod/latin/makedict')
15 files changed, 4327 insertions, 2727 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java new file mode 100644 index 000000000..665c7a27c --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -0,0 +1,624 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; + +/** + * Decodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + * + * TODO: Remove calls from classes except Ver3DictDecoder + * TODO: Move this file to makedict/internal. + * TODO: Rename this class to DictDecoderUtils. + */ +public final class BinaryDictDecoderUtils { + + private static final boolean DBG = MakedictLog.DBG; + + private BinaryDictDecoderUtils() { + // This utility class is not publicly instantiable. + } + + private static final int MAX_JUMPS = 12; + + @UsedForTesting + public interface DictBuffer { + public int readUnsignedByte(); + public int readUnsignedShort(); + public int readUnsignedInt24(); + public int readInt(); + public int position(); + public void position(int newPosition); + public void put(final byte b); + public int limit(); + @UsedForTesting + public int capacity(); + } + + public static final class ByteBufferDictBuffer implements DictBuffer { + private ByteBuffer mBuffer; + + public ByteBufferDictBuffer(final ByteBuffer buffer) { + mBuffer = buffer; + } + + @Override + public int readUnsignedByte() { + return mBuffer.get() & 0xFF; + } + + @Override + public int readUnsignedShort() { + return mBuffer.getShort() & 0xFFFF; + } + + @Override + public int readUnsignedInt24() { + final int retval = readUnsignedByte(); + return (retval << 16) + readUnsignedShort(); + } + + @Override + public int readInt() { + return mBuffer.getInt(); + } + + @Override + public int position() { + return mBuffer.position(); + } + + @Override + public void position(int newPos) { + mBuffer.position(newPos); + } + + @Override + public void put(final byte b) { + mBuffer.put(b); + } + + @Override + public int limit() { + return mBuffer.limit(); + } + + @Override + public int capacity() { + return mBuffer.capacity(); + } + } + + /** + * A class grouping utility function for our specific character encoding. + */ + static final class CharEncoding { + private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; + private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; + + /** + * Helper method to find out whether this code fits on one byte + */ + private static boolean fitsOnOneByte(final int character) { + return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE + && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; + } + + /** + * Compute the size of a character given its character code. + * + * Char format is: + * 1 byte = bbbbbbbb match + * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte + * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because + * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with + * 00011111 would be outside unicode. + * else: iso-latin-1 code + * This allows for the whole unicode range to be encoded, including chars outside of + * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control + * characters which should never happen anyway (and still work, but take 3 bytes). + * + * @param character the character code. + * @return the size in binary encoded-form, either 1 or 3 bytes. + */ + static int getCharSize(final int character) { + // See char encoding in FusionDictionary.java + if (fitsOnOneByte(character)) return 1; + if (FormatSpec.INVALID_CHARACTER == character) return 1; + return 3; + } + + /** + * Compute the byte size of a character array. + */ + static int getCharArraySize(final int[] chars) { + int size = 0; + for (int character : chars) size += getCharSize(character); + return size; + } + + /** + * Writes a char array to a byte buffer. + * + * @param codePoints the code point array to write. + * @param buffer the byte buffer to write to. + * @param index the index in buffer to write the character array to. + * @return the index after the last character. + */ + static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { + for (int codePoint : codePoints) { + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + return index; + } + + /** + * Writes a string with our character format to a byte buffer. + * + * This will also write the terminator byte. + * + * @param buffer the byte buffer to write to. + * @param origin the offset to write from. + * @param word the string to write. + * @return the size written, in bytes. + */ + static int writeString(final byte[] buffer, final int origin, + final String word) { + final int length = word.length(); + int index = origin; + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer[index++] = (byte)codePoint; + } else { + buffer[index++] = (byte)(0xFF & (codePoint >> 16)); + buffer[index++] = (byte)(0xFF & (codePoint >> 8)); + buffer[index++] = (byte)(0xFF & codePoint); + } + } + buffer[index++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + return index - origin; + } + + /** + * Writes a string with our character format to a ByteArrayOutputStream. + * + * This will also write the terminator byte. + * + * @param buffer the ByteArrayOutputStream to write to. + * @param word the string to write. + */ + static void writeString(final ByteArrayOutputStream buffer, final String word) { + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + if (1 == getCharSize(codePoint)) { + buffer.write((byte) codePoint); + } else { + buffer.write((byte) (0xFF & (codePoint >> 16))); + buffer.write((byte) (0xFF & (codePoint >> 8))); + buffer.write((byte) (0xFF & codePoint)); + } + } + buffer.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + } + + /** + * Reads a string from a DictBuffer. This is the converse of the above method. + */ + static String readString(final DictBuffer dictBuffer) { + final StringBuilder s = new StringBuilder(); + int character = readChar(dictBuffer); + while (character != FormatSpec.INVALID_CHARACTER) { + s.appendCodePoint(character); + character = readChar(dictBuffer); + } + return s.toString(); + } + + /** + * Reads a character from the buffer. + * + * This follows the character format documented earlier in this source file. + * + * @param dictBuffer the buffer, positioned over an encoded character. + * @return the character code. + */ + static int readChar(final DictBuffer dictBuffer) { + int character = dictBuffer.readUnsignedByte(); + if (!fitsOnOneByte(character)) { + if (FormatSpec.PTNODE_CHARACTERS_TERMINATOR == character) { + return FormatSpec.INVALID_CHARACTER; + } + character <<= 16; + character += dictBuffer.readUnsignedShort(); + } + return character; + } + } + + // Input methods: Read a binary dictionary to memory. + // readDictionaryBinary is the public entry point for them. + + static int readSInt24(final DictBuffer dictBuffer) { + final int retval = dictBuffer.readUnsignedInt24(); + final int sign = ((retval & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (retval & FormatSpec.SINT24_MAX); + } + + static int readChildrenAddress(final DictBuffer dictBuffer, + final int optionFlags, final FormatOptions options) { + if (options.mSupportsDynamicUpdate) { + final int address = dictBuffer.readUnsignedInt24(); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + if ((address & FormatSpec.MSB24) != 0) { + return -(address & FormatSpec.SINT24_MAX); + } else { + return address; + } + } + int address; + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + + static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int parentAddress = dictBuffer.readUnsignedInt24(); + final int sign = ((parentAddress & FormatSpec.MSB24) != 0) ? -1 : 1; + return sign * (parentAddress & FormatSpec.SINT24_MAX); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + /** + * Reads and returns the PtNode count out of a buffer and forwards the pointer. + */ + /* package */ static int readPtNodeCount(final DictBuffer dictBuffer) { + final int msb = dictBuffer.readUnsignedByte(); + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= msb) { + return msb; + } else { + return ((FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT & msb) << 8) + + dictBuffer.readUnsignedByte(); + } + } + + /** + * Finds, as a string, the word at the position passed as an argument. + * + * @param dictDecoder the dict decoder. + * @param headerSize the size of the header. + * @param pos the position to seek. + * @param formatOptions file format options. + * @return the word with its frequency, as a weighted string. + */ + /* package for tests */ static WeightedString getWordAtPosition(final DictDecoder dictDecoder, + final int headerSize, final int pos, final FormatOptions formatOptions) { + final WeightedString result; + final int originalPos = dictDecoder.getPosition(); + dictDecoder.setPosition(pos); + + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + result = getWordAtPositionWithParentAddress(dictDecoder, pos, formatOptions); + } else { + result = getWordAtPositionWithoutParentAddress(dictDecoder, headerSize, pos, + formatOptions); + } + + dictDecoder.setPosition(originalPos); + return result; + } + + @SuppressWarnings("unused") + private static WeightedString getWordAtPositionWithParentAddress(final DictDecoder dictDecoder, + final int pos, final FormatOptions options) { + int currentPos = pos; + int frequency = Integer.MIN_VALUE; + final StringBuilder builder = new StringBuilder(); + // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH + for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { + PtNodeInfo currentInfo; + int loopCounter = 0; + do { + dictDecoder.setPosition(currentPos); + currentInfo = dictDecoder.readPtNode(currentPos, options); + if (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)) { + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + } + if (DBG && loopCounter++ > MAX_JUMPS) { + MakedictLog.d("Too many jumps - probably a bug"); + } + } while (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options)); + if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; + builder.insert(0, + new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length)); + if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; + currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress; + } + return new WeightedString(builder.toString(), frequency); + } + + private static WeightedString getWordAtPositionWithoutParentAddress( + final DictDecoder dictDecoder, final int headerSize, final int pos, + final FormatOptions options) { + dictDecoder.setPosition(headerSize); + final int count = dictDecoder.readPtNodeCount(); + int groupPos = headerSize + BinaryDictIOUtils.getPtNodeCountSize(count); + final StringBuilder builder = new StringBuilder(); + WeightedString result = null; + + PtNodeInfo last = null; + for (int i = count - 1; i >= 0; --i) { + PtNodeInfo info = dictDecoder.readPtNode(groupPos, options); + groupPos = info.mEndAddress; + if (info.mOriginalAddress == pos) { + builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); + result = new WeightedString(builder.toString(), info.mFrequency); + break; // and return + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + if (info.mChildrenAddress > pos) { + if (null == last) continue; + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + dictDecoder.setPosition(last.mChildrenAddress); + i = dictDecoder.readPtNodeCount(); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); + last = null; + continue; + } + last = info; + } + if (0 == i && BinaryDictIOUtils.hasChildrenAddress(last.mChildrenAddress)) { + builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); + dictDecoder.setPosition(last.mChildrenAddress); + i = dictDecoder.readPtNodeCount(); + groupPos = last.mChildrenAddress + BinaryDictIOUtils.getPtNodeCountSize(i); + last = null; + continue; + } + } + return result; + } + + /** + * Reads a single node array from a buffer. + * + * This methods reads the file at the current position. A node array is fully expected to start + * at the current position. + * This will recursively read other node arrays into the structure, populating the reverse + * maps on the fly and using them to keep track of already read nodes. + * + * @param dictDecoder the dict decoder, correctly positioned at the start of a node array. + * @param headerSize the size, in bytes, of the file header. + * @param reverseNodeArrayMap a mapping from addresses to already read node arrays. + * @param reversePtNodeMap a mapping from addresses to already read PtNodes. + * @param options file format options. + * @return the read node array with all his children already read. + */ + private static PtNodeArray readNodeArray(final DictDecoder dictDecoder, + final int headerSize, final Map<Integer, PtNodeArray> reverseNodeArrayMap, + final Map<Integer, PtNode> reversePtNodeMap, final FormatOptions options) + throws IOException { + final ArrayList<PtNode> nodeArrayContents = new ArrayList<PtNode>(); + final int nodeArrayOriginPos = dictDecoder.getPosition(); + + do { // Scan the linked-list node. + final int nodeArrayHeadPos = dictDecoder.getPosition(); + final int count = dictDecoder.readPtNodeCount(); + int groupOffsetPos = nodeArrayHeadPos + BinaryDictIOUtils.getPtNodeCountSize(count); + for (int i = count; i > 0; --i) { // Scan the array of PtNode. + PtNodeInfo info = dictDecoder.readPtNode(groupOffsetPos, options); + if (BinaryDictIOUtils.isMovedPtNode(info.mFlags, options)) continue; + ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; + ArrayList<WeightedString> bigrams = null; + if (null != info.mBigrams) { + bigrams = new ArrayList<WeightedString>(); + for (PendingAttribute bigram : info.mBigrams) { + final WeightedString word = getWordAtPosition(dictDecoder, headerSize, + bigram.mAddress, options); + final int reconstructedFrequency = + BinaryDictIOUtils.reconstructBigramFrequency(word.mFrequency, + bigram.mFrequency); + bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); + } + } + if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) { + PtNodeArray children = reverseNodeArrayMap.get(info.mChildrenAddress); + if (null == children) { + final int currentPosition = dictDecoder.getPosition(); + dictDecoder.setPosition(info.mChildrenAddress); + children = readNodeArray(dictDecoder, headerSize, reverseNodeArrayMap, + reversePtNodeMap, options); + dictDecoder.setPosition(currentPosition); + } + nodeArrayContents.add( + new PtNode(info.mCharacters, shortcutTargets, bigrams, + info.mFrequency, + 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), + 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); + } else { + nodeArrayContents.add( + new PtNode(info.mCharacters, shortcutTargets, bigrams, + info.mFrequency, + 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), + 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); + } + groupOffsetPos = info.mEndAddress; + } + + // reach the end of the array. + if (options.mSupportsDynamicUpdate) { + final boolean hasValidForwardLink = dictDecoder.readAndFollowForwardLink(); + if (!hasValidForwardLink) break; + } + } while (options.mSupportsDynamicUpdate && dictDecoder.hasNextPtNodeArray()); + + final PtNodeArray nodeArray = new PtNodeArray(nodeArrayContents); + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOriginPos; + nodeArray.mCachedAddressAfterUpdate = nodeArrayOriginPos; + reverseNodeArrayMap.put(nodeArray.mCachedAddressAfterUpdate, nodeArray); + return nodeArray; + } + + /** + * Helper function to get the binary format version from the header. + * @throws IOException + */ + private static int getFormatVersion(final DictBuffer dictBuffer) + throws IOException { + final int magic = dictBuffer.readInt(); + if (FormatSpec.MAGIC_NUMBER == magic) return dictBuffer.readUnsignedShort(); + return FormatSpec.NOT_A_VERSION_NUMBER; + } + + /** + * Helper function to get and validate the binary format version. + * @throws UnsupportedFormatException + * @throws IOException + */ + static int checkFormatVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + final int version = getFormatVersion(dictBuffer); + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("This file has version " + version + + ", but this implementation does not support versions above " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + return version; + } + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. The optional dict argument is an existing dictionary to + * which words from the buffer should be added. If it is null, a new dictionary is created. + * + * @param dictDecoder the dict decoder. + * @param dict an optional dictionary to add words to, or null. + * @return the created (or merged) dictionary. + */ + @UsedForTesting + /* package */ static FusionDictionary readDictionaryBinary(final DictDecoder dictDecoder, + final FusionDictionary dict) throws IOException, UnsupportedFormatException { + // Read header + final FileHeader fileHeader = dictDecoder.readHeader(); + + Map<Integer, PtNodeArray> reverseNodeArrayMapping = new TreeMap<Integer, PtNodeArray>(); + Map<Integer, PtNode> reversePtNodeMapping = new TreeMap<Integer, PtNode>(); + final PtNodeArray root = readNodeArray(dictDecoder, fileHeader.mHeaderSize, + reverseNodeArrayMapping, reversePtNodeMapping, fileHeader.mFormatOptions); + + FusionDictionary newDict = new FusionDictionary(root, fileHeader.mDictionaryOptions); + if (null != dict) { + for (final Word w : dict) { + if (w.mIsBlacklistEntry) { + newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); + } else { + newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); + } + } + for (final Word w : dict) { + // By construction a binary dictionary may not have bigrams pointing to + // words that are not also registered as unigrams so we don't have to avoid + // them explicitly here. + for (final WeightedString bigram : w.mBigrams) { + newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); + } + } + } + + return newDict; + } + + /** + * Helper method to pass a file name instead of a File object to isBinaryDictionary. + */ + public static boolean isBinaryDictionary(final String filename) { + final File file = new File(filename); + return isBinaryDictionary(file); + } + + /** + * Basic test to find out whether the file is a binary dictionary or not. + * + * Concretely this only tests the magic number. + * + * @param file The file to test. + * @return true if it's a binary dictionary, false otherwise + */ + public static boolean isBinaryDictionary(final File file) { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final ByteBuffer buffer = inStream.getChannel().map( + FileChannel.MapMode.READ_ONLY, 0, file.length()); + final int version = getFormatVersion(new ByteBufferDictBuffer(buffer)); + return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION + && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } catch (FileNotFoundException e) { + return false; + } catch (IOException e) { + return false; + } finally { + if (inStream != null) { + try { + inStream.close(); + } catch (IOException e) { + // do nothing + } + } + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java new file mode 100644 index 000000000..6cc0bfb76 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -0,0 +1,958 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; + +/** + * Encodes binary files for a FusionDictionary. + * + * All the methods in this class are static. + * + * TODO: Rename this class to DictEncoderUtils. + */ +public class BinaryDictEncoderUtils { + + private static final boolean DBG = MakedictLog.DBG; + + private BinaryDictEncoderUtils() { + // This utility class is not publicly instantiable. + } + + // Arbitrary limit to how much passes we consider address size compression should + // terminate in. At the time of this writing, our largest dictionary completes + // compression in five passes. + // If the number of passes exceeds this number, makedict bails with an exception on + // suspicion that a bug might be causing an infinite loop. + private static final int MAX_PASSES = 24; + + /** + * Compute the binary size of the character array. + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param characters the character array + * @return the size of the char array, including the terminator if any + */ + static int getPtNodeCharactersSize(final int[] characters) { + int size = CharEncoding.getCharArraySize(characters); + if (characters.length > 1) size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the binary size of the character array in a PtNode + * + * If only one character, this is the size of this character. If many, it's the sum of their + * sizes + 1 byte for the terminator. + * + * @param ptNode the PtNode + * @return the size of the char array, including the terminator if any + */ + private static int getPtNodeCharactersSize(final PtNode ptNode) { + return getPtNodeCharactersSize(ptNode.mChars); + } + + /** + * Compute the binary size of the PtNode count for a node array. + * @param nodeArray the nodeArray + * @return the size of the PtNode count, either 1 or 2 bytes. + */ + private static int getPtNodeCountSize(final PtNodeArray nodeArray) { + return BinaryDictIOUtils.getPtNodeCountSize(nodeArray.mData.size()); + } + + /** + * Compute the size of a shortcut in bytes. + */ + private static int getShortcutSize(final WeightedString shortcut) { + int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + final String word = shortcut.mWord; + final int length = word.length(); + for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { + final int codePoint = word.codePointAt(i); + size += CharEncoding.getCharSize(codePoint); + } + size += FormatSpec.PTNODE_TERMINATOR_SIZE; + return size; + } + + /** + * Compute the size of a shortcut list in bytes. + * + * This is known in advance and does not change according to position in the file + * like address lists do. + */ + static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { + if (null == shortcutList || shortcutList.isEmpty()) return 0; + int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + for (final WeightedString shortcut : shortcutList) { + size += getShortcutSize(shortcut); + } + return size; + } + + /** + * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. + * + * @param ptNode the PtNode to compute the size of. + * @param options file format options. + * @return the maximum size of the PtNode. + */ + private static int getPtNodeMaximumSize(final PtNode ptNode, final FormatOptions options) { + int size = getNodeHeaderSize(ptNode, options); + if (ptNode.isTerminal()) { + // If terminal, one byte for the frequency or four bytes for the terminal id. + if (options.mHasTerminalId) { + size += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + size += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } + size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address + size += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE + + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE) + * ptNode.mBigrams.size(); + } + return size; + } + + /** + * Compute the maximum size of each PtNode of a PtNode array, assuming 3-byte addresses for + * everything, and caches it in the `mCachedSize' member of the nodes; deduce the size of + * the containing node array, and cache it it its 'mCachedSize' member. + * + * @param ptNodeArray the node array to compute the maximum size of. + * @param options file format options. + */ + private static void calculatePtNodeArrayMaximumSize(final PtNodeArray ptNodeArray, + final FormatOptions options) { + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode node : ptNodeArray.mData) { + final int nodeSize = getPtNodeMaximumSize(node, options); + node.mCachedSize = nodeSize; + size += nodeSize; + } + if (options.mSupportsDynamicUpdate) { + size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + ptNodeArray.mCachedSize = size; + } + + /** + * Compute the size of the header (flag + [parent address] + characters size) of a PtNode. + * + * @param ptNode the PtNode of which to compute the size of the header + * @param options file format options. + */ + private static int getNodeHeaderSize(final PtNode ptNode, final FormatOptions options) { + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + return FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + getPtNodeCharactersSize(ptNode); + } else { + return FormatSpec.PTNODE_FLAGS_SIZE + getPtNodeCharactersSize(ptNode); + } + } + + /** + * Compute the size, in bytes, that an address will occupy. + * + * This can be used either for children addresses (which are always positive) or for + * attribute, which may be positive or negative but + * store their sign bit separately. + * + * @param address the address + * @return the byte size. + */ + static int getByteSize(final int address) { + assert(address <= FormatSpec.UINT24_MAX); + if (!BinaryDictIOUtils.hasChildrenAddress(address)) { + return 0; + } else if (Math.abs(address) <= FormatSpec.UINT8_MAX) { + return 1; + } else if (Math.abs(address) <= FormatSpec.UINT16_MAX) { + return 2; + } else { + return 3; + } + } + + static int writeUIntToBuffer(final byte[] buffer, int position, final int value, + final int size) { + switch(size) { + case 4: + buffer[position++] = (byte) ((value >> 24) & 0xFF); + /* fall through */ + case 3: + buffer[position++] = (byte) ((value >> 16) & 0xFF); + /* fall through */ + case 2: + buffer[position++] = (byte) ((value >> 8) & 0xFF); + /* fall through */ + case 1: + buffer[position++] = (byte) (value & 0xFF); + break; + default: + /* nop */ + } + return position; + } + + static void writeUIntToStream(final OutputStream stream, final int value, final int size) + throws IOException { + switch(size) { + case 4: + stream.write((value >> 24) & 0xFF); + /* fall through */ + case 3: + stream.write((value >> 16) & 0xFF); + /* fall through */ + case 2: + stream.write((value >> 8) & 0xFF); + /* fall through */ + case 1: + stream.write(value & 0xFF); + break; + default: + /* nop */ + } + } + + // End utility methods + + // This method is responsible for finding a nice ordering of the nodes that favors run-time + // cache performance and dictionary size. + /* package for tests */ static ArrayList<PtNodeArray> flattenTree( + final PtNodeArray rootNodeArray) { + final int treeSize = FusionDictionary.countPtNodes(rootNodeArray); + MakedictLog.i("Counted nodes : " + treeSize); + final ArrayList<PtNodeArray> flatTree = new ArrayList<PtNodeArray>(treeSize); + return flattenTreeInner(flatTree, rootNodeArray); + } + + private static ArrayList<PtNodeArray> flattenTreeInner(final ArrayList<PtNodeArray> list, + final PtNodeArray ptNodeArray) { + // Removing the node is necessary if the tails are merged, because we would then + // add the same node several times when we only want it once. A number of places in + // the code also depends on any node being only once in the list. + // Merging tails can only be done if there are no attributes. Searching for attributes + // in LatinIME code depends on a total breadth-first ordering, which merging tails + // breaks. If there are no attributes, it should be fine (and reduce the file size) + // to merge tails, and removing the node from the list would be necessary. However, + // we don't merge tails because breaking the breadth-first ordering would result in + // extreme overhead at bigram lookup time (it would make the search function O(n) instead + // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty + // high). + // If no nodes are ever merged, we can't have the same node twice in the list, hence + // searching for duplicates in unnecessary. It is also very performance consuming, + // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making + // this simple list.remove operation O(n*n) overall. On Android this overhead is very + // high. + // For future reference, the code to remove duplicate is a simple : list.remove(node); + list.add(ptNodeArray); + final ArrayList<PtNode> branches = ptNodeArray.mData; + final int nodeSize = branches.size(); + for (PtNode ptNode : branches) { + if (null != ptNode.mChildren) flattenTreeInner(list, ptNode.mChildren); + } + return list; + } + + /** + * Get the offset from a position inside a current node array to a target node array, during + * update. + * + * If the current node array is before the target node array, the target node array has not + * been updated yet, so we should return the offset from the old position of the current node + * array to the old position of the target node array. If on the other hand the target is + * before the current node array, it already has been updated, so we should return the offset + * from the new position in the current node array to the new position in the target node + * array. + * + * @param currentNodeArray node array containing the PtNode where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetNodeArray the target node array to get the offset to + * @return the offset to the target node array + */ + private static int getOffsetToTargetNodeArrayDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNodeArray targetNodeArray) { + final boolean isTargetBeforeCurrent = (targetNodeArray.mCachedAddressBeforeUpdate + < currentNodeArray.mCachedAddressBeforeUpdate); + if (isTargetBeforeCurrent) { + return targetNodeArray.mCachedAddressAfterUpdate + - (currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray); + } else { + return targetNodeArray.mCachedAddressBeforeUpdate + - (currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray); + } + } + + /** + * Get the offset from a position inside a current node array to a target PtNode, during + * update. + * + * @param currentNodeArray node array containing the PtNode where the offset will be written + * @param offsetFromStartOfCurrentNodeArray offset, in bytes, from the start of currentNodeArray + * @param targetPtNode the target PtNode to get the offset to + * @return the offset to the target PtNode + */ + // TODO: is there any way to factorize this method with the one above? + private static int getOffsetToTargetPtNodeDuringUpdate(final PtNodeArray currentNodeArray, + final int offsetFromStartOfCurrentNodeArray, final PtNode targetPtNode) { + final int oldOffsetBasePoint = currentNodeArray.mCachedAddressBeforeUpdate + + offsetFromStartOfCurrentNodeArray; + final boolean isTargetBeforeCurrent = (targetPtNode.mCachedAddressBeforeUpdate + < oldOffsetBasePoint); + // If the target is before the current node array, then its address has already been + // updated. We can use the AfterUpdate member, and compare it to our own member after + // update. Otherwise, the AfterUpdate member is not updated yet, so we need to use the + // BeforeUpdate member, and of course we have to compare this to our own address before + // update. + if (isTargetBeforeCurrent) { + final int newOffsetBasePoint = currentNodeArray.mCachedAddressAfterUpdate + + offsetFromStartOfCurrentNodeArray; + return targetPtNode.mCachedAddressAfterUpdate - newOffsetBasePoint; + } else { + return targetPtNode.mCachedAddressBeforeUpdate - oldOffsetBasePoint; + } + } + + /** + * Computes the actual node array size, based on the cached addresses of the children nodes. + * + * Each node array stores its tentative address. During dictionary address computing, these + * are not final, but they can be used to compute the node array size (the node array size + * depends on the address of the children because the number of bytes necessary to store an + * address depends on its numeric value. The return value indicates whether the node array + * contents (as in, any of the addresses stored in the cache fields) have changed with + * respect to their previous value. + * + * @param ptNodeArray the node array to compute the size of. + * @param dict the dictionary in which the word/attributes are to be found. + * @param formatOptions file format options. + * @return false if none of the cached addresses inside the node array changed, true otherwise. + */ + private static boolean computeActualPtNodeArraySize(final PtNodeArray ptNodeArray, + final FusionDictionary dict, final FormatOptions formatOptions) { + boolean changed = false; + int size = getPtNodeCountSize(ptNodeArray); + for (PtNode ptNode : ptNodeArray.mData) { + ptNode.mCachedAddressAfterUpdate = ptNodeArray.mCachedAddressAfterUpdate + size; + if (ptNode.mCachedAddressAfterUpdate != ptNode.mCachedAddressBeforeUpdate) { + changed = true; + } + int nodeSize = getNodeHeaderSize(ptNode, formatOptions); + if (ptNode.isTerminal()) { + if (formatOptions.mHasTerminalId) { + nodeSize += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + nodeSize += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } + if (formatOptions.mSupportsDynamicUpdate) { + nodeSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + } else if (null != ptNode.mChildren) { + nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, + nodeSize + size, ptNode.mChildren)); + } + nodeSize += getShortcutListSize(ptNode.mShortcutTargets); + if (null != ptNode.mBigrams) { + for (WeightedString bigram : ptNode.mBigrams) { + final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, + nodeSize + size + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE, + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord)); + nodeSize += getByteSize(offset) + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; + } + } + ptNode.mCachedSize = nodeSize; + size += nodeSize; + } + if (formatOptions.mSupportsDynamicUpdate) { + size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + if (ptNodeArray.mCachedSize != size) { + ptNodeArray.mCachedSize = size; + changed = true; + } + return changed; + } + + /** + * Initializes the cached addresses of node arrays and their containing nodes from their size. + * + * @param flatNodes the list of node arrays. + * @param formatOptions file format options. + * @return the byte size of the entire stack. + */ + private static int initializePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes, + final FormatOptions formatOptions) { + int nodeArrayOffset = 0; + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArrayOffset; + int nodeCountSize = getPtNodeCountSize(nodeArray); + int nodeffset = 0; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate = + nodeCountSize + nodeArrayOffset + nodeffset; + nodeffset += ptNode.mCachedSize; + } + final int nodeSize = nodeCountSize + nodeffset + + (formatOptions.mSupportsDynamicUpdate + ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); + nodeArrayOffset += nodeArray.mCachedSize; + } + return nodeArrayOffset; + } + + /** + * Updates the cached addresses of node arrays after recomputing their new positions. + * + * @param flatNodes the list of node arrays. + */ + private static void updatePtNodeArraysCachedAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + nodeArray.mCachedAddressBeforeUpdate = nodeArray.mCachedAddressAfterUpdate; + for (final PtNode ptNode : nodeArray.mData) { + ptNode.mCachedAddressBeforeUpdate = ptNode.mCachedAddressAfterUpdate; + } + } + } + + /** + * Compute the cached parent addresses after all has been updated. + * + * The parent addresses are used by some binary formats at write-to-disk time. Not all formats + * need them. In particular, version 2 does not need them, and version 3 does. + * + * @param flatNodes the flat array of node arrays to fill in + */ + private static void computeParentAddresses(final ArrayList<PtNodeArray> flatNodes) { + for (final PtNodeArray nodeArray : flatNodes) { + for (final PtNode ptNode : nodeArray.mData) { + if (null != ptNode.mChildren) { + // Assign my address to children's parent address + // Here BeforeUpdate and AfterUpdate addresses have the same value, so it + // does not matter which we use. + ptNode.mChildren.mCachedParentAddress = ptNode.mCachedAddressAfterUpdate + - ptNode.mChildren.mCachedAddressAfterUpdate; + } + } + } + } + + /** + * Compute the addresses and sizes of an ordered list of PtNode arrays. + * + * This method takes a list of PtNode arrays and will update their cached address and size + * values so that they can be written into a file. It determines the smallest size each of the + * PtNode arrays can be given the addresses of its children and attributes, and store that into + * each PtNode. + * The order of the PtNode is given by the order of the array. This method makes no effort + * to find a good order; it only mechanically computes the size this order results in. + * + * @param dict the dictionary + * @param flatNodes the ordered list of PtNode arrays + * @param formatOptions file format options. + * @return the same array it was passed. The nodes have been updated for address and size. + */ + /* package */ static ArrayList<PtNodeArray> computeAddresses(final FusionDictionary dict, + final ArrayList<PtNodeArray> flatNodes, final FormatOptions formatOptions) { + // First get the worst possible sizes and offsets + for (final PtNodeArray n : flatNodes) calculatePtNodeArrayMaximumSize(n, formatOptions); + final int offset = initializePtNodeArraysCachedAddresses(flatNodes, formatOptions); + + MakedictLog.i("Compressing the array addresses. Original size : " + offset); + MakedictLog.i("(Recursively seen size : " + offset + ")"); + + int passes = 0; + boolean changesDone = false; + do { + changesDone = false; + int ptNodeArrayStartOffset = 0; + for (final PtNodeArray ptNodeArray : flatNodes) { + ptNodeArray.mCachedAddressAfterUpdate = ptNodeArrayStartOffset; + final int oldNodeArraySize = ptNodeArray.mCachedSize; + final boolean changed = + computeActualPtNodeArraySize(ptNodeArray, dict, formatOptions); + final int newNodeArraySize = ptNodeArray.mCachedSize; + if (oldNodeArraySize < newNodeArraySize) { + throw new RuntimeException("Increased size ?!"); + } + ptNodeArrayStartOffset += newNodeArraySize; + changesDone |= changed; + } + updatePtNodeArraysCachedAddresses(flatNodes); + ++passes; + if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); + } while (changesDone); + + if (formatOptions.mSupportsDynamicUpdate) { + computeParentAddresses(flatNodes); + } + final PtNodeArray lastPtNodeArray = flatNodes.get(flatNodes.size() - 1); + MakedictLog.i("Compression complete in " + passes + " passes."); + MakedictLog.i("After address compression : " + + (lastPtNodeArray.mCachedAddressAfterUpdate + lastPtNodeArray.mCachedSize)); + + return flatNodes; + } + + /** + * Sanity-checking method. + * + * This method checks a list of PtNode arrays for juxtaposition, that is, it will do + * nothing if each node array's cached address is actually the previous node array's address + * plus the previous node's size. + * If this is not the case, it will throw an exception. + * + * @param arrays the list of node arrays to check + */ + /* package */ static void checkFlatPtNodeArrayList(final ArrayList<PtNodeArray> arrays) { + int offset = 0; + int index = 0; + for (final PtNodeArray ptNodeArray : arrays) { + // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter + // which we use. + if (ptNodeArray.mCachedAddressAfterUpdate != offset) { + throw new RuntimeException("Wrong address for node " + index + + " : expected " + offset + ", got " + + ptNodeArray.mCachedAddressAfterUpdate); + } + ++index; + offset += ptNodeArray.mCachedSize; + } + } + + /** + * Helper method to write a children position to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param position the position to write. + * @return the size in bytes the address actually took. + */ + /* package */ static int writeChildrenPosition(final byte[] buffer, int index, + final int position) { + switch (getByteSize(position)) { + case 1: + buffer[index++] = (byte)position; + return 1; + case 2: + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); + return 2; + case 3: + buffer[index++] = (byte)(0xFF & (position >> 16)); + buffer[index++] = (byte)(0xFF & (position >> 8)); + buffer[index++] = (byte)(0xFF & position); + return 3; + case 0: + return 0; + default: + throw new RuntimeException("Position " + position + " has a strange size"); + } + } + + /** + * Helper method to write a signed children position to a file. + * + * @param buffer the buffer to write to. + * @param index the index in the buffer to write the address to. + * @param position the position to write. + * @return the size in bytes the address actually took. + */ + /* package */ static int writeSignedChildrenPosition(final byte[] buffer, int index, + final int position) { + if (!BinaryDictIOUtils.hasChildrenAddress(position)) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absPosition = Math.abs(position); + buffer[index++] = + (byte)((position < 0 ? FormatSpec.MSB8 : 0) | (0xFF & (absPosition >> 16))); + buffer[index++] = (byte)(0xFF & (absPosition >> 8)); + buffer[index++] = (byte)(0xFF & absPosition); + } + return 3; + } + + /** + * Makes the flag value for a PtNode. + * + * @param hasMultipleChars whether the PtNode has multiple chars. + * @param isTerminal whether the PtNode is terminal. + * @param childrenAddressSize the size of a children address. + * @param hasShortcuts whether the PtNode has shortcuts. + * @param hasBigrams whether the PtNode has bigrams. + * @param isNotAWord whether the PtNode is not a word. + * @param isBlackListEntry whether the PtNode is a blacklist entry. + * @param formatOptions file format options. + * @return the flags + */ + static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, + final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, + final boolean isNotAWord, final boolean isBlackListEntry, + final FormatOptions formatOptions) { + byte flags = 0; + if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; + if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; + if (formatOptions.mSupportsDynamicUpdate) { + flags |= FormatSpec.FLAG_IS_NOT_MOVED; + } else if (true) { + switch (childrenAddressSize) { + case 1: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES; + break; + case 0: + flags |= FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS; + break; + default: + throw new RuntimeException("Node with a strange address"); + } + } + if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; + if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; + if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; + if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; + return flags; + } + + /* package */ static byte makePtNodeFlags(final PtNode node, final int ptNodeAddress, + final int childrenOffset, final FormatOptions formatOptions) { + return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0, + getByteSize(childrenOffset), + node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), + node.mBigrams != null, node.mIsNotAWord, node.mIsBlacklistEntry, formatOptions); + } + + /** + * Makes the flag value for a bigram. + * + * @param more whether there are more bigrams after this one. + * @param offset the offset of the bigram. + * @param bigramFrequency the frequency of the bigram, 0..255. + * @param unigramFrequency the unigram frequency of the same word, 0..255. + * @param word the second bigram, for debugging purposes + * @return the flags + */ + /* package */ static final int makeBigramFlags(final boolean more, final int offset, + int bigramFrequency, final int unigramFrequency, final String word) { + int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); + switch (getByteSize(offset)) { + case 1: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; + break; + case 2: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; + break; + case 3: + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; + break; + default: + throw new RuntimeException("Strange offset size"); + } + if (unigramFrequency > bigramFrequency) { + MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word + + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " + + word + " is " + unigramFrequency); + bigramFrequency = unigramFrequency; + } + // We compute the difference between 255 (which means probability = 1) and the + // unigram score. We split this into a number of discrete steps. + // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 + // represents an increase of 16 steps: a value of 15 will be interpreted as the median + // value of the 16th step. In all justice, if the bigram frequency is low enough to be + // rounded below the first step (which means it is less than half a step higher than the + // unigram frequency) then the unigram frequency itself is the best approximation of the + // bigram freq that we could possibly supply, hence we should *not* include this bigram + // in the file at all. + // until this is done, we'll write 0 and slightly overestimate this case. + // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step + // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to + // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the + // step size. Then we compute the start of the first step (the one where value 0 starts) + // by adding half-a-step to the unigramFrequency. From there, we compute the integer + // number of steps to the bigramFrequency. One last thing: we want our steps to include + // their lower bound and exclude their higher bound so we need to have the first step + // start at exactly 1 unit higher than floor(unigramFreq + half a step). + // Note : to reconstruct the score, the dictionary reader will need to divide + // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, + // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best + // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the + // step pointed by the discretized frequency. + final float stepSize = + (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); + final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); + // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 + // here. The best approximation would be the unigram freq itself, so we should not + // include this bigram in the dictionary. For now, register as 0, and live with the + // small over-estimation that we get in this case. TODO: actually remove this bigram + // if discretizedFrequency < 0. + final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; + bigramFlags += finalBigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; + return bigramFlags; + } + + /** + * Makes the 2-byte value for options flags. + */ + private static final int makeOptionsValue(final FusionDictionary dictionary, + final FormatOptions formatOptions) { + final DictionaryOptions options = dictionary.mOptions; + final boolean hasBigrams = dictionary.hasBigrams(); + return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) + + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) + + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) + + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); + } + + /** + * Makes the flag value for a shortcut. + * + * @param more whether there are more attributes after this one. + * @param frequency the frequency of the attribute, 0..15 + * @return the flags + */ + static final int makeShortcutFlags(final boolean more, final int frequency) { + return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); + } + + /* package */ static final int writeParentAddress(final byte[] buffer, final int index, + final int address, final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + if (address == FormatSpec.NO_PARENT_ADDRESS) { + buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; + } else { + final int absAddress = Math.abs(address); + assert(absAddress <= FormatSpec.SINT24_MAX); + buffer[index] = (byte)((address < 0 ? FormatSpec.MSB8 : 0) + | ((absAddress >> 16) & 0xFF)); + buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); + buffer[index + 2] = (byte)(absAddress & 0xFF); + } + return index + 3; + } else { + return index; + } + } + + /* package */ static final int getChildrenPosition(final PtNode ptNode, + final FormatOptions formatOptions) { + int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate + + getNodeHeaderSize(ptNode, formatOptions); + if (ptNode.isTerminal()) { + // A terminal node has either the terminal id or the frequency. + // If positionOfChildrenPosField is incorrect, we may crash when jumping to the children + // position. + if (formatOptions.mHasTerminalId) { + positionOfChildrenPosField += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + positionOfChildrenPosField += FormatSpec.PTNODE_FREQUENCY_SIZE; + } + } + return null == ptNode.mChildren ? FormatSpec.NO_CHILDREN_ADDRESS + : ptNode.mChildren.mCachedAddressAfterUpdate - positionOfChildrenPosField; + } + + /** + * Write a PtNodeArray. The PtNodeArray is expected to have its final position cached. + * + * @param dict the dictionary the node array is a part of (for relative offsets). + * @param dictEncoder the dictionary encoder. + * @param ptNodeArray the node array to write. + * @param formatOptions file format options. + */ + @SuppressWarnings("unused") + /* package */ static void writePlacedPtNodeArray(final FusionDictionary dict, + final DictEncoder dictEncoder, final PtNodeArray ptNodeArray, + final FormatOptions formatOptions) { + // TODO: Make the code in common with BinaryDictIOUtils#writePtNode + dictEncoder.setPosition(ptNodeArray.mCachedAddressAfterUpdate); + + final int ptNodeCount = ptNodeArray.mData.size(); + dictEncoder.writePtNodeCount(ptNodeCount); + final int parentPosition = + (ptNodeArray.mCachedParentAddress == FormatSpec.NO_PARENT_ADDRESS) + ? FormatSpec.NO_PARENT_ADDRESS + : ptNodeArray.mCachedParentAddress + ptNodeArray.mCachedAddressAfterUpdate; + for (int i = 0; i < ptNodeCount; ++i) { + final PtNode ptNode = ptNodeArray.mData.get(i); + if (dictEncoder.getPosition() != ptNode.mCachedAddressAfterUpdate) { + throw new RuntimeException("Bug: write index is not the same as the cached address " + + "of the node : " + dictEncoder.getPosition() + " <> " + + ptNode.mCachedAddressAfterUpdate); + } + // Sanity checks. + if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { + throw new RuntimeException("A node has a frequency > " + + FormatSpec.MAX_TERMINAL_FREQUENCY + + " : " + ptNode.mFrequency); + } + dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict); + } + if (formatOptions.mSupportsDynamicUpdate) { + dictEncoder.writeForwardLinkAddress(FormatSpec.NO_FORWARD_LINK_ADDRESS); + } + if (dictEncoder.getPosition() != ptNodeArray.mCachedAddressAfterUpdate + + ptNodeArray.mCachedSize) { + throw new RuntimeException("Not the same size : written " + + (dictEncoder.getPosition() - ptNodeArray.mCachedAddressAfterUpdate) + + " bytes from a node that should have " + ptNodeArray.mCachedSize + " bytes"); + } + } + + /** + * Dumps a collection of useful statistics about a list of PtNode arrays. + * + * This prints purely informative stuff, like the total estimated file size, the + * number of PtNode arrays, of PtNodes, the repartition of each address size, etc + * + * @param ptNodeArrays the list of PtNode arrays. + */ + /* package */ static void showStatistics(ArrayList<PtNodeArray> ptNodeArrays) { + int firstTerminalAddress = Integer.MAX_VALUE; + int lastTerminalAddress = Integer.MIN_VALUE; + int size = 0; + int ptNodes = 0; + int maxNodes = 0; + int maxRuns = 0; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + if (maxNodes < ptNodeArray.mData.size()) maxNodes = ptNodeArray.mData.size(); + for (final PtNode ptNode : ptNodeArray.mData) { + ++ptNodes; + if (ptNode.mChars.length > maxRuns) maxRuns = ptNode.mChars.length; + if (ptNode.mFrequency >= 0) { + if (ptNodeArray.mCachedAddressAfterUpdate < firstTerminalAddress) + firstTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + if (ptNodeArray.mCachedAddressAfterUpdate > lastTerminalAddress) + lastTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate; + } + } + if (ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize > size) { + size = ptNodeArray.mCachedAddressAfterUpdate + ptNodeArray.mCachedSize; + } + } + final int[] ptNodeCounts = new int[maxNodes + 1]; + final int[] runCounts = new int[maxRuns + 1]; + for (final PtNodeArray ptNodeArray : ptNodeArrays) { + ++ptNodeCounts[ptNodeArray.mData.size()]; + for (final PtNode ptNode : ptNodeArray.mData) { + ++runCounts[ptNode.mChars.length]; + } + } + + MakedictLog.i("Statistics:\n" + + " total file size " + size + "\n" + + " " + ptNodeArrays.size() + " node arrays\n" + + " " + ptNodes + " PtNodes (" + ((float)ptNodes / ptNodeArrays.size()) + + " PtNodes per node)\n" + + " first terminal at " + firstTerminalAddress + "\n" + + " last terminal at " + lastTerminalAddress + "\n" + + " PtNode stats : max = " + maxNodes); + for (int i = 0; i < ptNodeCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + ptNodeCounts[i]); + } + MakedictLog.i(" Character run stats : max = " + maxRuns); + for (int i = 0; i < runCounts.length; ++i) { + MakedictLog.i(" " + i + " : " + runCounts[i]); + } + } + + /** + * Writes a file header to an output stream. + * + * @param destination the stream to write the file header to. + * @param dict the dictionary to write. + * @param formatOptions file format options. + * @return the size of the header. + */ + /* package */ static int writeDictionaryHeader(final OutputStream destination, + final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + final int version = formatOptions.mVersion; + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("Requested file format version " + version + + ", but this implementation only supports versions " + + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " + + FormatSpec.MAXIMUM_SUPPORTED_VERSION); + } + + ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); + + // The magic number in big-endian order. + // Magic number for all versions. + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 24))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 16))); + headerBuffer.write((byte) (0xFF & (FormatSpec.MAGIC_NUMBER >> 8))); + headerBuffer.write((byte) (0xFF & FormatSpec.MAGIC_NUMBER)); + // Dictionary version. + headerBuffer.write((byte) (0xFF & (version >> 8))); + headerBuffer.write((byte) (0xFF & version)); + + // Options flags + final int options = makeOptionsValue(dict, formatOptions); + headerBuffer.write((byte) (0xFF & (options >> 8))); + headerBuffer.write((byte) (0xFF & options)); + final int headerSizeOffset = headerBuffer.size(); + // Placeholder to be written later with header size. + for (int i = 0; i < 4; ++i) { + headerBuffer.write(0); + } + // Write out the options. + for (final String key : dict.mOptions.mAttributes.keySet()) { + final String value = dict.mOptions.mAttributes.get(key); + CharEncoding.writeString(headerBuffer, key); + CharEncoding.writeString(headerBuffer, value); + } + final int size = headerBuffer.size(); + final byte[] bytes = headerBuffer.toByteArray(); + // Write out the header size. + bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); + bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); + bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); + bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); + destination.write(bytes); + + headerBuffer.close(); + return size; + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 167c6915c..a282f595c 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -18,56 +18,52 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; -import java.nio.channels.FileChannel; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.Map; import java.util.Stack; public final class BinaryDictIOUtils { private static final boolean DBG = false; - private static final int MSB24 = 0x800000; - private static final int SINT24_MAX = 0x7FFFFF; - private static final int MAX_JUMPS = 10000; private BinaryDictIOUtils() { // This utility class is not publicly instantiable. } private static final class Position { - public static final int NOT_READ_GROUPCOUNT = -1; + public static final int NOT_READ_PTNODE_COUNT = -1; public int mAddress; - public int mNumOfCharGroup; + public int mNumOfPtNode; public int mPosition; public int mLength; public Position(int address, int length) { mAddress = address; mLength = length; - mNumOfCharGroup = NOT_READ_GROUPCOUNT; + mNumOfPtNode = NOT_READ_PTNODE_COUNT; } } /** - * Tours all node without recursive call. + * Retrieves all node arrays without recursive call. */ - private static void readUnigramsAndBigramsBinaryInner( - final FusionDictionaryBufferInterface buffer, final int headerSize, - final Map<Integer, String> words, final Map<Integer, Integer> frequencies, + private static void readUnigramsAndBigramsBinaryInner(final DictDecoder dictDecoder, + final int headerSize, final Map<Integer, String> words, + final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams, final FormatOptions formatOptions) { int[] pushedChars = new int[FormatSpec.MAX_WORD_LENGTH + 1]; @@ -82,47 +78,47 @@ public final class BinaryDictIOUtils { Position p = stack.peek(); if (DBG) { - MakedictLog.d("read: address=" + p.mAddress + ", numOfCharGroup=" + - p.mNumOfCharGroup + ", position=" + p.mPosition + ", length=" + p.mLength); + MakedictLog.d("read: address=" + p.mAddress + ", numOfPtNode=" + + p.mNumOfPtNode + ", position=" + p.mPosition + ", length=" + p.mLength); } - if (buffer.position() != p.mAddress) buffer.position(p.mAddress); + if (dictDecoder.getPosition() != p.mAddress) dictDecoder.setPosition(p.mAddress); if (index != p.mLength) index = p.mLength; - if (p.mNumOfCharGroup == Position.NOT_READ_GROUPCOUNT) { - p.mNumOfCharGroup = BinaryDictInputOutput.readCharGroupCount(buffer); - p.mAddress += BinaryDictInputOutput.getGroupCountSize(p.mNumOfCharGroup); + if (p.mNumOfPtNode == Position.NOT_READ_PTNODE_COUNT) { + p.mNumOfPtNode = dictDecoder.readPtNodeCount(); + p.mAddress += getPtNodeCountSize(p.mNumOfPtNode); p.mPosition = 0; } - if (p.mNumOfCharGroup == 0) { + if (p.mNumOfPtNode == 0) { stack.pop(); continue; } - CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer, - p.mAddress - headerSize, formatOptions); + PtNodeInfo info = dictDecoder.readPtNode(p.mAddress, formatOptions); for (int i = 0; i < info.mCharacters.length; ++i) { pushedChars[index++] = info.mCharacters[i]; } p.mPosition++; - final boolean isMovedGroup = BinaryDictInputOutput.isMovedGroup(info.mFlags, + final boolean isMovedPtNode = isMovedPtNode(info.mFlags, formatOptions); - final boolean isDeletedGroup = BinaryDictInputOutput.isDeletedGroup(info.mFlags, + final boolean isDeletedPtNode = isDeletedPtNode(info.mFlags, formatOptions); - if (!isMovedGroup && !isDeletedGroup - && info.mFrequency != FusionDictionary.CharGroup.NOT_A_TERMINAL) {// found word + if (!isMovedPtNode && !isDeletedPtNode + && info.mFrequency != FusionDictionary.PtNode.NOT_A_TERMINAL) {// found word words.put(info.mOriginalAddress, new String(pushedChars, 0, index)); frequencies.put(info.mOriginalAddress, info.mFrequency); if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams); } - if (p.mPosition == p.mNumOfCharGroup) { + if (p.mPosition == p.mNumOfPtNode) { if (formatOptions.mSupportsDynamicUpdate) { - final int forwardLinkAddress = buffer.readUnsignedInt24(); - if (forwardLinkAddress != FormatSpec.NO_FORWARD_LINK_ADDRESS) { - // the node has a forward link. - p.mNumOfCharGroup = Position.NOT_READ_GROUPCOUNT; - p.mAddress = forwardLinkAddress; + final boolean hasValidForwardLinkAddress = + dictDecoder.readAndFollowForwardLink(); + if (hasValidForwardLinkAddress && dictDecoder.hasNextPtNodeArray()) { + // The node array has a forward link. + p.mNumOfPtNode = Position.NOT_READ_PTNODE_COUNT; + p.mAddress = dictDecoder.getPosition(); } else { stack.pop(); } @@ -130,12 +126,12 @@ public final class BinaryDictIOUtils { stack.pop(); } } else { - // the node has more groups. - p.mAddress = buffer.position(); + // The Ptnode array has more PtNodes. + p.mAddress = dictDecoder.getPosition(); } - if (!isMovedGroup && BinaryDictInputOutput.hasChildrenAddress(info.mChildrenAddress)) { - Position childrenPos = new Position(info.mChildrenAddress + headerSize, index); + if (!isMovedPtNode && hasChildrenAddress(info.mChildrenAddress)) { + final Position childrenPos = new Position(info.mChildrenAddress, index); stack.push(childrenPos); } } @@ -143,61 +139,59 @@ public final class BinaryDictIOUtils { /** * Reads unigrams and bigrams from the binary file. - * Doesn't make the memory representation of the dictionary. + * Doesn't store a full memory representation of the dictionary. * - * @param buffer the buffer to read. + * @param dictDecoder the dict decoder. * @param words the map to store the address as a key and the word as a value. * @param frequencies the map to store the address as a key and the frequency as a value. * @param bigrams the map to store the address as a key and the list of address as a value. - * @throws IOException - * @throws UnsupportedFormatException + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. */ - public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer, + /* package */ static void readUnigramsAndBigramsBinary(final DictDecoder dictDecoder, final Map<Integer, String> words, final Map<Integer, Integer> frequencies, final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException, UnsupportedFormatException { // Read header - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams, - header.mFormatOptions); + final FileHeader header = dictDecoder.readHeader(); + readUnigramsAndBigramsBinaryInner(dictDecoder, header.mHeaderSize, words, + frequencies, bigrams, header.mFormatOptions); } /** - * Gets the address of the last CharGroup of the exact matching word in the dictionary. + * Gets the address of the last PtNode of the exact matching word in the dictionary. * If no match is found, returns NOT_VALID_WORD. * - * @param buffer the buffer to read. + * @param dictDecoder the dict decoder. * @param word the word we search for. * @return the address of the terminal node. - * @throws IOException - * @throws UnsupportedFormatException + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. */ @UsedForTesting - public static int getTerminalPosition(final FusionDictionaryBufferInterface buffer, + /* package */ static int getTerminalPosition(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { if (word == null) return FormatSpec.NOT_VALID_WORD; - if (buffer.position() != 0) buffer.position(0); + dictDecoder.setPosition(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); + final FileHeader header = dictDecoder.readHeader(); int wordPos = 0; final int wordLen = word.codePointCount(0, word.length()); for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { if (wordPos >= wordLen) return FormatSpec.NOT_VALID_WORD; do { - final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); - boolean foundNextCharGroup = false; - for (int i = 0; i < charGroupCount; ++i) { - final int charGroupPos = buffer.position(); - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, - buffer.position(), header.mFormatOptions); - final boolean isMovedGroup = - BinaryDictInputOutput.isMovedGroup(currentInfo.mFlags, - header.mFormatOptions); - final boolean isDeletedGroup = - BinaryDictInputOutput.isDeletedGroup(currentInfo.mFlags, - header.mFormatOptions); - if (isMovedGroup) continue; + final int ptNodeCount = dictDecoder.readPtNodeCount(); + boolean foundNextPtNode = false; + for (int i = 0; i < ptNodeCount; ++i) { + final int ptNodePos = dictDecoder.getPosition(); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(ptNodePos, + header.mFormatOptions); + final boolean isMovedNode = isMovedPtNode(currentInfo.mFlags, + header.mFormatOptions); + final boolean isDeletedNode = isDeletedPtNode(currentInfo.mFlags, + header.mFormatOptions); + if (isMovedNode) continue; boolean same = true; for (int p = 0, j = word.offsetByCodePoints(0, wordPos); p < currentInfo.mCharacters.length; @@ -210,86 +204,60 @@ public final class BinaryDictIOUtils { } if (same) { - // found the group matches the word. + // found the PtNode matches the word. if (wordPos + currentInfo.mCharacters.length == wordLen) { - if (currentInfo.mFrequency == CharGroup.NOT_A_TERMINAL - || isDeletedGroup) { + if (currentInfo.mFrequency == PtNode.NOT_A_TERMINAL + || isDeletedNode) { return FormatSpec.NOT_VALID_WORD; } else { - return charGroupPos; + return ptNodePos; } } wordPos += currentInfo.mCharacters.length; if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { return FormatSpec.NOT_VALID_WORD; } - foundNextCharGroup = true; - buffer.position(currentInfo.mChildrenAddress); + foundNextPtNode = true; + dictDecoder.setPosition(currentInfo.mChildrenAddress); break; } } - // If we found the next char group, it is under the file pointer. - // But if not, we are at the end of this node so we expect to have + // If we found the next PtNode, it is under the file pointer. + // But if not, we are at the end of this node array so we expect to have // a forward link address that we need to consult and possibly resume - // search on the next node in the linked list. - if (foundNextCharGroup) break; + // search on the next node array in the linked list. + if (foundNextPtNode) break; if (!header.mFormatOptions.mSupportsDynamicUpdate) { return FormatSpec.NOT_VALID_WORD; } - final int forwardLinkAddress = buffer.readUnsignedInt24(); - if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { + final boolean hasValidForwardLinkAddress = + dictDecoder.readAndFollowForwardLink(); + if (!hasValidForwardLinkAddress || !dictDecoder.hasNextPtNodeArray()) { return FormatSpec.NOT_VALID_WORD; } - buffer.position(forwardLinkAddress); } while(true); } return FormatSpec.NOT_VALID_WORD; } - private static int markAsDeleted(final int flags) { - return (flags & (~FormatSpec.MASK_GROUP_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; - } - - /** - * Delete the word from the binary file. - * - * @param buffer the buffer to write. - * @param word the word we delete - * @throws IOException - * @throws UnsupportedFormatException - */ - @UsedForTesting - public static void deleteWord(final FusionDictionaryBufferInterface buffer, - final String word) throws IOException, UnsupportedFormatException { - buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - final int wordPosition = getTerminalPosition(buffer, word); - if (wordPosition == FormatSpec.NOT_VALID_WORD) return; - - buffer.position(wordPosition); - final int flags = buffer.readUnsignedByte(); - buffer.position(wordPosition); - buffer.put((byte)markAsDeleted(flags)); - } - /** * @return the size written, in bytes. Always 3 bytes. */ - private static int writeSInt24ToBuffer(final FusionDictionaryBufferInterface buffer, + static int writeSInt24ToBuffer(final DictBuffer dictBuffer, final int value) { final int absValue = Math.abs(value); - buffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); - buffer.put((byte)((absValue >> 8) & 0xFF)); - buffer.put((byte)(absValue & 0xFF)); + dictBuffer.put((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); + dictBuffer.put((byte)((absValue >> 8) & 0xFF)); + dictBuffer.put((byte)(absValue & 0xFF)); return 3; } /** * @return the size written, in bytes. Always 3 bytes. */ - private static int writeSInt24ToStream(final OutputStream destination, final int value) + static int writeSInt24ToStream(final OutputStream destination, final int value) throws IOException { final int absValue = Math.abs(value); destination.write((byte)(((value < 0 ? 0x80 : 0) | (absValue >> 16)) & 0xFF)); @@ -303,7 +271,7 @@ public final class BinaryDictIOUtils { */ private static int writeVariableAddress(final OutputStream destination, final int value) throws IOException { - switch (BinaryDictInputOutput.getByteSize(value)) { + switch (BinaryDictEncoderUtils.getByteSize(value)) { case 1: destination.write((byte)value); break; @@ -317,111 +285,52 @@ public final class BinaryDictIOUtils { destination.write((byte)(0xFF & value)); break; } - return BinaryDictInputOutput.getByteSize(value); + return BinaryDictEncoderUtils.getByteSize(value); } - /** - * Update a parent address in a CharGroup that is referred to by groupOriginAddress. - * - * @param buffer the buffer to write. - * @param groupOriginAddress the address of the group. - * @param newParentAddress the absolute address of the parent. - * @param formatOptions file format options. - */ - public static void updateParentAddress(final FusionDictionaryBufferInterface buffer, - final int groupOriginAddress, final int newParentAddress, - final FormatOptions formatOptions) { - final int originalPosition = buffer.position(); - buffer.position(groupOriginAddress); - if (!formatOptions.mSupportsDynamicUpdate) { - throw new RuntimeException("this file format does not support parent addresses"); - } - final int flags = buffer.readUnsignedByte(); - if (BinaryDictInputOutput.isMovedGroup(flags, formatOptions)) { - // if the group is moved, the parent address is stored in the destination group. - // We are guaranteed to process the destination group later, so there is no need to - // update anything here. - buffer.position(originalPosition); - return; - } - if (DBG) { - MakedictLog.d("update parent address flags=" + flags + ", " + groupOriginAddress); - } - final int parentOffset = newParentAddress - groupOriginAddress; - writeSInt24ToBuffer(buffer, parentOffset); - buffer.position(originalPosition); - } - - private static void skipCharGroup(final FusionDictionaryBufferInterface buffer, - final FormatOptions formatOptions) { - final int flags = buffer.readUnsignedByte(); - BinaryDictInputOutput.readParentAddress(buffer, formatOptions); - skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - BinaryDictInputOutput.readChildrenAddress(buffer, flags, formatOptions); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); + static void skipPtNode(final DictBuffer dictBuffer, final FormatOptions formatOptions) { + final int flags = dictBuffer.readUnsignedByte(); + BinaryDictDecoderUtils.readParentAddress(dictBuffer, formatOptions); + skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + BinaryDictDecoderUtils.readChildrenAddress(dictBuffer, flags, formatOptions); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); if ((flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) != 0) { - final int shortcutsSize = buffer.readUnsignedShort(); - buffer.position(buffer.position() + shortcutsSize - - FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE); + final int shortcutsSize = dictBuffer.readUnsignedShort(); + dictBuffer.position(dictBuffer.position() + shortcutsSize + - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); } if ((flags & FormatSpec.FLAG_HAS_BIGRAMS) != 0) { int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = buffer.readUnsignedByte(); - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - buffer.readUnsignedByte(); + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + dictBuffer.readUnsignedByte(); break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - buffer.readUnsignedShort(); + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + dictBuffer.readUnsignedShort(); break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - buffer.readUnsignedInt24(); + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + dictBuffer.readUnsignedInt24(); break; } - if ((bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT) == 0) break; + if ((bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) == 0) break; } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - throw new RuntimeException("Too many bigrams in a group."); + if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode."); } } } - /** - * Update parent addresses in a Node that is referred to by nodeOriginAddress. - * - * @param buffer the buffer to be modified. - * @param nodeOriginAddress the address of a modified Node. - * @param newParentAddress the address to be written. - * @param formatOptions file format options. - */ - public static void updateParentAddresses(final FusionDictionaryBufferInterface buffer, - final int nodeOriginAddress, final int newParentAddress, - final FormatOptions formatOptions) { - final int originalPosition = buffer.position(); - buffer.position(nodeOriginAddress); - do { - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); - for (int i = 0; i < count; ++i) { - updateParentAddress(buffer, buffer.position(), newParentAddress, formatOptions); - skipCharGroup(buffer, formatOptions); - } - final int forwardLinkAddress = buffer.readUnsignedInt24(); - buffer.position(forwardLinkAddress); - } while (formatOptions.mSupportsDynamicUpdate - && buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); - buffer.position(originalPosition); - } - - private static void skipString(final FusionDictionaryBufferInterface buffer, + static void skipString(final DictBuffer dictBuffer, final boolean hasMultipleChars) { if (hasMultipleChars) { - int character = CharEncoding.readChar(buffer); + int character = CharEncoding.readChar(dictBuffer); while (character != FormatSpec.INVALID_CHARACTER) { - character = CharEncoding.readChar(buffer); + character = CharEncoding.readChar(dictBuffer); } } else { - CharEncoding.readChar(buffer); + CharEncoding.readChar(dictBuffer); } } @@ -449,46 +358,24 @@ public final class BinaryDictIOUtils { size += 3; } } - destination.write((byte)FormatSpec.GROUP_CHARACTERS_TERMINATOR); - size += FormatSpec.GROUP_TERMINATOR_SIZE; + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + size += FormatSpec.PTNODE_TERMINATOR_SIZE; return size; } /** - * Update a children address in a CharGroup that is addressed by groupOriginAddress. - * - * @param buffer the buffer to write. - * @param groupOriginAddress the address of the group. - * @param newChildrenAddress the absolute address of the child. - * @param formatOptions file format options. - */ - public static void updateChildrenAddress(final FusionDictionaryBufferInterface buffer, - final int groupOriginAddress, final int newChildrenAddress, - final FormatOptions formatOptions) { - final int originalPosition = buffer.position(); - buffer.position(groupOriginAddress); - final int flags = buffer.readUnsignedByte(); - final int parentAddress = BinaryDictInputOutput.readParentAddress(buffer, formatOptions); - skipString(buffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); - if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) buffer.readUnsignedByte(); - final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS - ? FormatSpec.NO_CHILDREN_ADDRESS : newChildrenAddress - buffer.position(); - writeSInt24ToBuffer(buffer, childrenOffset); - buffer.position(originalPosition); - } - - /** - * Write a char group to an output stream. - * A char group is an in-memory representation of a node in trie. - * A char group info is an on-disk representation of a node. + * Write a PtNode to an output stream from a PtNodeInfo. + * A PtNode is an in-memory representation of a node in the patricia trie. + * A PtNode info is a container for low-level information about how the + * PtNode is stored in the binary format. * * @param destination the stream to write. - * @param info the char group info to be written. + * @param info the PtNode info to be written. * @return the size written, in bytes. */ - public static int writeCharGroup(final OutputStream destination, final CharGroupInfo info) + private static int writePtNode(final OutputStream destination, final PtNodeInfo info) throws IOException { - int size = FormatSpec.GROUP_FLAGS_SIZE; + int size = FormatSpec.PTNODE_FLAGS_SIZE; destination.write((byte)info.mFlags); final int parentOffset = info.mParentAddress == FormatSpec.NO_PARENT_ADDRESS ? FormatSpec.NO_PARENT_ADDRESS : info.mParentAddress - info.mOriginalAddress; @@ -503,7 +390,7 @@ public final class BinaryDictIOUtils { } } if (info.mCharacters.length > 1) { - destination.write((byte)FormatSpec.GROUP_CHARACTERS_TERMINATOR); + destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); size++; } @@ -513,7 +400,7 @@ public final class BinaryDictIOUtils { } if (DBG) { - MakedictLog.d("writeCharGroup origin=" + info.mOriginalAddress + ", size=" + size + MakedictLog.d("writePtNode origin=" + info.mOriginalAddress + ", size=" + size + ", child=" + info.mChildrenAddress + ", characters =" + new String(info.mCharacters, 0, info.mCharacters.length)); } @@ -524,14 +411,14 @@ public final class BinaryDictIOUtils { if (info.mShortcutTargets != null && info.mShortcutTargets.size() > 0) { final int shortcutListSize = - BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); + BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); destination.write((byte)(shortcutListSize >> 8)); destination.write((byte)(shortcutListSize & 0xFF)); size += 2; final Iterator<WeightedString> shortcutIterator = info.mShortcutTargets.iterator(); while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); - destination.write((byte)BinaryDictInputOutput.makeShortcutFlags( + destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; size += writeString(destination, target.mWord); @@ -540,28 +427,28 @@ public final class BinaryDictIOUtils { if (info.mBigrams != null) { // TODO: Consolidate this code with the code that computes the size of the bigram list - // in BinaryDictionaryInputOutput#computeActualNodeSize + // in BinaryDictEncoderUtils#computeActualNodeArraySize for (int i = 0; i < info.mBigrams.size(); ++i) { final int bigramFrequency = info.mBigrams.get(i).mFrequency; int bigramFlags = (i < info.mBigrams.size() - 1) - ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0; + ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0; size++; final int bigramOffset = info.mBigrams.get(i).mAddress - (info.mOriginalAddress + size); - bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0; - switch (BinaryDictInputOutput.getByteSize(bigramOffset)) { + bigramFlags |= (bigramOffset < 0) ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0; + switch (BinaryDictEncoderUtils.getByteSize(bigramOffset)) { case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE; break; case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES; break; case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; + bigramFlags |= FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES; break; } - bigramFlags |= bigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; + bigramFlags |= bigramFrequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY; destination.write((byte)bigramFlags); size += writeVariableAddress(destination, Math.abs(bigramOffset)); } @@ -569,82 +456,40 @@ public final class BinaryDictIOUtils { return size; } - @SuppressWarnings("unused") - private static void updateForwardLink(final FusionDictionaryBufferInterface buffer, - final int nodeOriginAddress, final int newNodeAddress, - final FormatOptions formatOptions) { - buffer.position(nodeOriginAddress); - int jumpCount = 0; - while (jumpCount++ < MAX_JUMPS) { - final int count = BinaryDictInputOutput.readCharGroupCount(buffer); - for (int i = 0; i < count; ++i) skipCharGroup(buffer, formatOptions); - final int forwardLinkAddress = buffer.readUnsignedInt24(); - if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { - buffer.position(buffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE); - writeSInt24ToBuffer(buffer, newNodeAddress); - return; - } - buffer.position(forwardLinkAddress); - } - if (DBG && jumpCount >= MAX_JUMPS) { - throw new RuntimeException("too many jumps, probably a bug."); - } - } - /** - * Helper method to move a char group to the tail of the file. + * Compute the size of the PtNode. */ - private static int moveCharGroup(final OutputStream destination, - final FusionDictionaryBufferInterface buffer, final CharGroupInfo info, - final int nodeOriginAddress, final int oldGroupAddress, - final FormatOptions formatOptions) throws IOException { - updateParentAddress(buffer, oldGroupAddress, buffer.limit() + 1, formatOptions); - buffer.position(oldGroupAddress); - final int currentFlags = buffer.readUnsignedByte(); - buffer.position(oldGroupAddress); - buffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags - & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); - int size = FormatSpec.GROUP_FLAGS_SIZE; - updateForwardLink(buffer, nodeOriginAddress, buffer.limit(), formatOptions); - size += writeNode(destination, new CharGroupInfo[] { info }); - return size; - } - - /** - * Compute the size of the char group. - */ - private static int computeGroupSize(final CharGroupInfo info, - final FormatOptions formatOptions) { - int size = FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + BinaryDictInputOutput.getGroupCharactersSize(info.mCharacters) - + BinaryDictInputOutput.getChildrenAddressSize(info.mFlags, formatOptions); + static int computePtNodeSize(final PtNodeInfo info, final FormatOptions formatOptions) { + int size = FormatSpec.PTNODE_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE + + BinaryDictEncoderUtils.getPtNodeCharactersSize(info.mCharacters) + + getChildrenAddressSize(info.mFlags, formatOptions); if ((info.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0) { - size += FormatSpec.GROUP_FREQUENCY_SIZE; + size += FormatSpec.PTNODE_FREQUENCY_SIZE; } if (info.mShortcutTargets != null && !info.mShortcutTargets.isEmpty()) { - size += BinaryDictInputOutput.getShortcutListSize(info.mShortcutTargets); + size += BinaryDictEncoderUtils.getShortcutListSize(info.mShortcutTargets); } if (info.mBigrams != null) { for (final PendingAttribute attr : info.mBigrams) { - size += FormatSpec.GROUP_FLAGS_SIZE; - size += BinaryDictInputOutput.getByteSize(attr.mAddress); + size += FormatSpec.PTNODE_FLAGS_SIZE; + size += BinaryDictEncoderUtils.getByteSize(attr.mAddress); } } return size; } /** - * Write a node to the stream. + * Write a node array to the stream. * * @param destination the stream to write. - * @param infos groups to be written. + * @param infos an array of PtNodeInfo to be written. * @return the size written, in bytes. * @throws IOException */ - private static int writeNode(final OutputStream destination, final CharGroupInfo[] infos) + static int writeNodes(final OutputStream destination, final PtNodeInfo[] infos) throws IOException { - int size = BinaryDictInputOutput.getGroupCountSize(infos.length); - switch (BinaryDictInputOutput.getGroupCountSize(infos.length)) { + int size = getPtNodeCountSize(infos.length); + switch (getPtNodeCountSize(infos.length)) { case 1: destination.write((byte)infos.length); break; @@ -653,335 +498,13 @@ public final class BinaryDictIOUtils { destination.write((byte)(infos.length & 0xFF)); break; default: - throw new RuntimeException("Invalid group count size."); + throw new RuntimeException("Invalid node count size."); } - for (final CharGroupInfo info : infos) size += writeCharGroup(destination, info); + for (final PtNodeInfo info : infos) size += writePtNode(destination, info); writeSInt24ToStream(destination, FormatSpec.NO_FORWARD_LINK_ADDRESS); return size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; } - /** - * Move a group that is referred to by oldGroupOrigin to the tail of the file. - * And set the children address to the byte after the group. - * - * @param nodeOrigin the address of the tail of the file. - * @param characters - * @param length - * @param flags - * @param frequency - * @param parentAddress - * @param shortcutTargets - * @param bigrams - * @param destination the stream representing the tail of the file. - * @param buffer the buffer representing the (constant-size) body of the file. - * @param oldNodeOrigin - * @param oldGroupOrigin - * @param formatOptions - * @return the size written, in bytes. - * @throws IOException - */ - private static int moveGroup(final int nodeOrigin, final int[] characters, final int length, - final int flags, final int frequency, final int parentAddress, - final ArrayList<WeightedString> shortcutTargets, - final ArrayList<PendingAttribute> bigrams, final OutputStream destination, - final FusionDictionaryBufferInterface buffer, final int oldNodeOrigin, - final int oldGroupOrigin, final FormatOptions formatOptions) throws IOException { - int size = 0; - final int newGroupOrigin = nodeOrigin + 1; - final int[] writtenCharacters = Arrays.copyOfRange(characters, 0, length); - final CharGroupInfo tmpInfo = new CharGroupInfo(newGroupOrigin, -1 /* endAddress */, - flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, - shortcutTargets, bigrams); - size = computeGroupSize(tmpInfo, formatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newGroupOrigin, newGroupOrigin + size, - flags, writtenCharacters, frequency, parentAddress, - nodeOrigin + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, - bigrams); - moveCharGroup(destination, buffer, newInfo, oldNodeOrigin, oldGroupOrigin, formatOptions); - return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - - /** - * Insert a word into a binary dictionary. - * - * @param buffer - * @param destination - * @param word - * @param frequency - * @param bigramStrings - * @param shortcuts - * @throws IOException - * @throws UnsupportedFormatException - */ - // TODO: Support batch insertion. - // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. - @UsedForTesting - public static void insertWord(final FusionDictionaryBufferInterface buffer, - final OutputStream destination, final String word, final int frequency, - final ArrayList<WeightedString> bigramStrings, - final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, - final boolean isBlackListEntry) - throws IOException, UnsupportedFormatException { - final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); - if (bigramStrings != null) { - for (final WeightedString bigram : bigramStrings) { - int position = getTerminalPosition(buffer, bigram.mWord); - if (position == FormatSpec.NOT_VALID_WORD) { - // TODO: figure out what is the correct thing to do here. - } else { - bigrams.add(new PendingAttribute(bigram.mFrequency, position)); - } - } - } - - final boolean isTerminal = true; - final boolean hasBigrams = !bigrams.isEmpty(); - final boolean hasShortcuts = shortcuts != null && !shortcuts.isEmpty(); - - // find the insert position of the word. - if (buffer.position() != 0) buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - - int wordPos = 0, address = buffer.position(), nodeOriginAddress = buffer.position(); - final int[] codePoints = FusionDictionary.getCodePoints(word); - final int wordLen = codePoints.length; - - for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { - if (wordPos >= wordLen) break; - nodeOriginAddress = buffer.position(); - int nodeParentAddress = -1; - final int charGroupCount = BinaryDictInputOutput.readCharGroupCount(buffer); - boolean foundNextGroup = false; - - for (int i = 0; i < charGroupCount; ++i) { - address = buffer.position(); - final CharGroupInfo currentInfo = BinaryDictInputOutput.readCharGroup(buffer, - buffer.position(), header.mFormatOptions); - final boolean isMovedGroup = BinaryDictInputOutput.isMovedGroup(currentInfo.mFlags, - header.mFormatOptions); - if (isMovedGroup) continue; - nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) - ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; - boolean matched = true; - for (int p = 0; p < currentInfo.mCharacters.length; ++p) { - if (wordPos + p >= wordLen) { - /* - * splitting - * before - * abcd - ef - * - * insert "abc" - * - * after - * abc - d - ef - */ - final int newNodeAddress = buffer.limit(); - final int flags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, - isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, - false /* isBlackListEntry */, header.mFormatOptions); - int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, flags, - frequency, nodeParentAddress, shortcuts, bigrams, destination, - buffer, nodeOriginAddress, address, header.mFormatOptions); - - final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, - currentInfo.mCharacters.length); - if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(buffer, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, header.mFormatOptions); - } - final CharGroupInfo newInfo2 = new CharGroupInfo( - newNodeAddress + written + 1, -1 /* endAddress */, - currentInfo.mFlags, characters2, currentInfo.mFrequency, - newNodeAddress + 1, currentInfo.mChildrenAddress, - currentInfo.mShortcutTargets, currentInfo.mBigrams); - writeNode(destination, new CharGroupInfo[] { newInfo2 }); - return; - } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { - if (p > 0) { - /* - * splitting - * before - * ab - cd - * - * insert "ac" - * - * after - * a - b - cd - * | - * - c - */ - - final int newNodeAddress = buffer.limit(); - final int childrenAddress = currentInfo.mChildrenAddress; - - // move prefix - final int prefixFlags = BinaryDictInputOutput.makeCharGroupFlags(p > 1, - false /* isTerminal */, 0 /* childrenAddressSize*/, - false /* hasShortcut */, false /* hasBigrams */, - false /* isNotAWord */, false /* isBlackListEntry */, - header.mFormatOptions); - int written = moveGroup(newNodeAddress, currentInfo.mCharacters, p, - prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, - destination, buffer, nodeOriginAddress, address, - header.mFormatOptions); - - final int[] suffixCharacters = Arrays.copyOfRange( - currentInfo.mCharacters, p, currentInfo.mCharacters.length); - if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - updateParentAddresses(buffer, currentInfo.mChildrenAddress, - newNodeAddress + written + 1, header.mFormatOptions); - } - final int suffixFlags = BinaryDictInputOutput.makeCharGroupFlags( - suffixCharacters.length > 1, - (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, - 0 /* childrenAddressSize */, - (currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) - != 0, - (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo suffixInfo = new CharGroupInfo( - newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, - suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, - currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, - currentInfo.mBigrams); - written += computeGroupSize(suffixInfo, header.mFormatOptions) + 1; - - final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, - codePoints.length); - final int flags = BinaryDictInputOutput.makeCharGroupFlags( - newCharacters.length > 1, isTerminal, - 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo( - newNodeAddress + written, -1 /* endAddress */, flags, - newCharacters, frequency, newNodeAddress + 1, - FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - writeNode(destination, new CharGroupInfo[] { suffixInfo, newInfo }); - return; - } - matched = false; - break; - } - } - - if (matched) { - if (wordPos + currentInfo.mCharacters.length == wordLen) { - // the word exists in the dictionary. - // only update group. - final int newNodeAddress = buffer.limit(); - final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; - final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, - -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, - nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, - bigrams); - moveCharGroup(destination, buffer, newInfo, nodeOriginAddress, address, - header.mFormatOptions); - return; - } - wordPos += currentInfo.mCharacters.length; - if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { - /* - * found the prefix of the word. - * make new node and link to the node from this group. - * - * before - * ab - cd - * - * insert "abcde" - * - * after - * ab - cd - e - */ - final int newNodeAddress = buffer.limit(); - updateChildrenAddress(buffer, address, newNodeAddress, - header.mFormatOptions); - final int newGroupAddress = newNodeAddress + 1; - final boolean hasMultipleChars = (wordLen - wordPos) > 1; - final int flags = BinaryDictInputOutput.makeCharGroupFlags(hasMultipleChars, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final CharGroupInfo newInfo = new CharGroupInfo(newGroupAddress, -1, flags, - characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, - shortcuts, bigrams); - writeNode(destination, new CharGroupInfo[] { newInfo }); - return; - } - buffer.position(currentInfo.mChildrenAddress); - foundNextGroup = true; - break; - } - } - - if (foundNextGroup) continue; - - // reached the end of the array. - final int linkAddressPosition = buffer.position(); - int nextLink = buffer.readUnsignedInt24(); - if ((nextLink & MSB24) != 0) { - nextLink = -(nextLink & SINT24_MAX); - } - if (nextLink == FormatSpec.NO_FORWARD_LINK_ADDRESS) { - /* - * expand this node. - * - * before - * ab - cd - * - * insert "abef" - * - * after - * ab - cd - * | - * - ef - */ - - // change the forward link address. - final int newNodeAddress = buffer.limit(); - buffer.position(linkAddressPosition); - writeSInt24ToBuffer(buffer, newNodeAddress); - - final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); - final int flags = BinaryDictInputOutput.makeCharGroupFlags(characters.length > 1, - isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, - isNotAWord, isBlackListEntry, header.mFormatOptions); - final CharGroupInfo newInfo = new CharGroupInfo(newNodeAddress + 1, - -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, - FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); - writeNode(destination, new CharGroupInfo[]{ newInfo }); - return; - } else { - depth--; - buffer.position(nextLink); - } - } - } - - /** - * Find a word from the buffer. - * - * @param buffer the buffer representing the body of the dictionary file. - * @param word the word searched - * @return the found group - * @throws IOException - * @throws UnsupportedFormatException - */ - @UsedForTesting - public static CharGroupInfo findWordFromBuffer(final FusionDictionaryBufferInterface buffer, - final String word) throws IOException, UnsupportedFormatException { - int position = getTerminalPosition(buffer, word); - if (position != FormatSpec.NOT_VALID_WORD) { - buffer.position(0); - final FileHeader header = BinaryDictInputOutput.readHeader(buffer); - buffer.position(position); - return BinaryDictInputOutput.readCharGroup(buffer, position, header.mFormatOptions); - } - return null; - } - private static final int HEADER_READING_BUFFER_SIZE = 16384; /** * Convenience method to read the header of a binary file. @@ -992,20 +515,27 @@ public final class BinaryDictIOUtils { * @param offset The offset in the file where to start reading the data. * @param length The length of the data file. */ - public static FileHeader getDictionaryFileHeader( + private static FileHeader getDictionaryFileHeader( final File file, final long offset, final long length) throws FileNotFoundException, IOException, UnsupportedFormatException { final byte[] buffer = new byte[HEADER_READING_BUFFER_SIZE]; - final FileInputStream inStream = new FileInputStream(file); - try { - inStream.read(buffer); - final BinaryDictInputOutput.ByteBufferWrapper wrapper = - new BinaryDictInputOutput.ByteBufferWrapper(inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, offset, length)); - return BinaryDictInputOutput.readHeader(wrapper); - } finally { - inStream.close(); - } + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, + new DictDecoder.DictionaryBufferFactory() { + @Override + public DictBuffer getDictionaryBuffer(File file) + throws FileNotFoundException, IOException { + final FileInputStream inStream = new FileInputStream(file); + try { + inStream.skip(offset); + inStream.read(buffer); + return new ByteArrayDictBuffer(buffer); + } finally { + inStream.close(); + } + } + } + ); + return dictDecoder.readHeader(); } public static FileHeader getDictionaryFileHeaderOrNull(final File file, final long offset, @@ -1019,4 +549,83 @@ public final class BinaryDictIOUtils { return null; } } + + /** + * Helper method to hide the actual value of the no children address. + */ + public static boolean hasChildrenAddress(final int address) { + return FormatSpec.NO_CHILDREN_ADDRESS != address; + } + + /** + * Helper method to check whether the node is moved. + */ + public static boolean isMovedPtNode(final int flags, final FormatOptions options) { + return options.mSupportsDynamicUpdate + && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); + } + + /** + * Helper method to check whether the dictionary can be updated dynamically. + */ + public static boolean supportsDynamicUpdate(final FormatOptions options) { + return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE + && options.mSupportsDynamicUpdate; + } + + /** + * Helper method to check whether the node is deleted. + */ + public static boolean isDeletedPtNode(final int flags, final FormatOptions formatOptions) { + return formatOptions.mSupportsDynamicUpdate + && ((flags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); + } + + /** + * Compute the binary size of the node count + * @param count the node count + * @return the size of the node count, either 1 or 2 bytes. + */ + public static int getPtNodeCountSize(final int count) { + if (FormatSpec.MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT >= count) { + return 1; + } else if (FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY >= count) { + return 2; + } else { + throw new RuntimeException("Can't have more than " + + FormatSpec.MAX_PTNODES_IN_A_PT_NODE_ARRAY + " PtNode in a PtNodeArray (found " + + count + ")"); + } + } + + static int getChildrenAddressSize(final int optionFlags, + final FormatOptions formatOptions) { + if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return 1; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return 2; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return 3; + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return 0; + } + } + + /** + * Calculate bigram frequency from compressed value + * + * @param unigramFrequency + * @param bigramFrequency compressed frequency + * @return approximate bigram frequency + */ + public static int reconstructBigramFrequency(final int unigramFrequency, + final int bigramFrequency) { + final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) + / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); + final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); + return (int)resultFreqFloat; + } } diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java deleted file mode 100644 index 1b187d85d..000000000 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java +++ /dev/null @@ -1,1838 +0,0 @@ -/* - * Copyright (C) 2011 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.inputmethod.latin.makedict; - -import com.android.inputmethod.annotations.UsedForTesting; -import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; -import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; -import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - -/** - * Reads and writes XML files for a FusionDictionary. - * - * All the methods in this class are static. - */ -public final class BinaryDictInputOutput { - - private static final boolean DBG = MakedictLog.DBG; - - // Arbitrary limit to how much passes we consider address size compression should - // terminate in. At the time of this writing, our largest dictionary completes - // compression in five passes. - // If the number of passes exceeds this number, makedict bails with an exception on - // suspicion that a bug might be causing an infinite loop. - private static final int MAX_PASSES = 24; - private static final int MAX_JUMPS = 12; - - @UsedForTesting - public interface FusionDictionaryBufferInterface { - public int readUnsignedByte(); - public int readUnsignedShort(); - public int readUnsignedInt24(); - public int readInt(); - public int position(); - public void position(int newPosition); - public void put(final byte b); - public int limit(); - public int capacity(); - } - - public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface { - private ByteBuffer mBuffer; - - public ByteBufferWrapper(final ByteBuffer buffer) { - mBuffer = buffer; - } - - @Override - public int readUnsignedByte() { - return mBuffer.get() & 0xFF; - } - - @Override - public int readUnsignedShort() { - return mBuffer.getShort() & 0xFFFF; - } - - @Override - public int readUnsignedInt24() { - final int retval = readUnsignedByte(); - return (retval << 16) + readUnsignedShort(); - } - - @Override - public int readInt() { - return mBuffer.getInt(); - } - - @Override - public int position() { - return mBuffer.position(); - } - - @Override - public void position(int newPos) { - mBuffer.position(newPos); - } - - @Override - public void put(final byte b) { - mBuffer.put(b); - } - - @Override - public int limit() { - return mBuffer.limit(); - } - - @Override - public int capacity() { - return mBuffer.capacity(); - } - } - - /** - * A class grouping utility function for our specific character encoding. - */ - static final class CharEncoding { - private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; - private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; - - /** - * Helper method to find out whether this code fits on one byte - */ - private static boolean fitsOnOneByte(final int character) { - return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE - && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; - } - - /** - * Compute the size of a character given its character code. - * - * Char format is: - * 1 byte = bbbbbbbb match - * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte - * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because - * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with - * 00011111 would be outside unicode. - * else: iso-latin-1 code - * This allows for the whole unicode range to be encoded, including chars outside of - * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control - * characters which should never happen anyway (and still work, but take 3 bytes). - * - * @param character the character code. - * @return the size in binary encoded-form, either 1 or 3 bytes. - */ - static int getCharSize(final int character) { - // See char encoding in FusionDictionary.java - if (fitsOnOneByte(character)) return 1; - if (FormatSpec.INVALID_CHARACTER == character) return 1; - return 3; - } - - /** - * Compute the byte size of a character array. - */ - private static int getCharArraySize(final int[] chars) { - int size = 0; - for (int character : chars) size += getCharSize(character); - return size; - } - - /** - * Writes a char array to a byte buffer. - * - * @param codePoints the code point array to write. - * @param buffer the byte buffer to write to. - * @param index the index in buffer to write the character array to. - * @return the index after the last character. - */ - private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { - for (int codePoint : codePoints) { - if (1 == getCharSize(codePoint)) { - buffer[index++] = (byte)codePoint; - } else { - buffer[index++] = (byte)(0xFF & (codePoint >> 16)); - buffer[index++] = (byte)(0xFF & (codePoint >> 8)); - buffer[index++] = (byte)(0xFF & codePoint); - } - } - return index; - } - - /** - * Writes a string with our character format to a byte buffer. - * - * This will also write the terminator byte. - * - * @param buffer the byte buffer to write to. - * @param origin the offset to write from. - * @param word the string to write. - * @return the size written, in bytes. - */ - private static int writeString(final byte[] buffer, final int origin, - final String word) { - final int length = word.length(); - int index = origin; - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer[index++] = (byte)codePoint; - } else { - buffer[index++] = (byte)(0xFF & (codePoint >> 16)); - buffer[index++] = (byte)(0xFF & (codePoint >> 8)); - buffer[index++] = (byte)(0xFF & codePoint); - } - } - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; - return index - origin; - } - - /** - * Writes a string with our character format to a ByteArrayOutputStream. - * - * This will also write the terminator byte. - * - * @param buffer the ByteArrayOutputStream to write to. - * @param word the string to write. - */ - private static void writeString(final ByteArrayOutputStream buffer, final String word) { - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { - buffer.write((byte) codePoint); - } else { - buffer.write((byte) (0xFF & (codePoint >> 16))); - buffer.write((byte) (0xFF & (codePoint >> 8))); - buffer.write((byte) (0xFF & codePoint)); - } - } - buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); - } - - /** - * Reads a string from a buffer. This is the converse of the above method. - */ - private static String readString(final FusionDictionaryBufferInterface buffer) { - final StringBuilder s = new StringBuilder(); - int character = readChar(buffer); - while (character != FormatSpec.INVALID_CHARACTER) { - s.appendCodePoint(character); - character = readChar(buffer); - } - return s.toString(); - } - - /** - * Reads a character from the buffer. - * - * This follows the character format documented earlier in this source file. - * - * @param buffer the buffer, positioned over an encoded character. - * @return the character code. - */ - static int readChar(final FusionDictionaryBufferInterface buffer) { - int character = buffer.readUnsignedByte(); - if (!fitsOnOneByte(character)) { - if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { - return FormatSpec.INVALID_CHARACTER; - } - character <<= 16; - character += buffer.readUnsignedShort(); - } - return character; - } - } - - /** - * Compute the binary size of the character array. - * - * If only one character, this is the size of this character. If many, it's the sum of their - * sizes + 1 byte for the terminator. - * - * @param characters the character array - * @return the size of the char array, including the terminator if any - */ - static int getGroupCharactersSize(final int[] characters) { - int size = CharEncoding.getCharArraySize(characters); - if (characters.length > 1) size += FormatSpec.GROUP_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the binary size of the character array in a group - * - * If only one character, this is the size of this character. If many, it's the sum of their - * sizes + 1 byte for the terminator. - * - * @param group the group - * @return the size of the char array, including the terminator if any - */ - private static int getGroupCharactersSize(final CharGroup group) { - return getGroupCharactersSize(group.mChars); - } - - /** - * Compute the binary size of the group count - * @param count the group count - * @return the size of the group count, either 1 or 2 bytes. - */ - public static int getGroupCountSize(final int count) { - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { - return 1; - } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) { - return 2; - } else { - throw new RuntimeException("Can't have more than " - + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count - + ")"); - } - } - - /** - * Compute the binary size of the group count for a node - * @param node the node - * @return the size of the group count, either 1 or 2 bytes. - */ - private static int getGroupCountSize(final Node node) { - return getGroupCountSize(node.mData.size()); - } - - /** - * Compute the size of a shortcut in bytes. - */ - private static int getShortcutSize(final WeightedString shortcut) { - int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE; - final String word = shortcut.mWord; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - size += CharEncoding.getCharSize(codePoint); - } - size += FormatSpec.GROUP_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the size of a shortcut list in bytes. - * - * This is known in advance and does not change according to position in the file - * like address lists do. - */ - static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { - if (null == shortcutList) return 0; - int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - for (final WeightedString shortcut : shortcutList) { - size += getShortcutSize(shortcut); - } - return size; - } - - /** - * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. - * - * @param group the CharGroup to compute the size of. - * @param options file format options. - * @return the maximum size of the group. - */ - private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) { - int size = getGroupHeaderSize(group, options); - // If terminal, one byte for the frequency - if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE; - size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address - size += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE - + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) - * group.mBigrams.size(); - } - return size; - } - - /** - * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches - * it in the 'actualSize' member of the node. - * - * @param node the node to compute the maximum size of. - * @param options file format options. - */ - private static void calculateNodeMaximumSize(final Node node, final FormatOptions options) { - int size = getGroupCountSize(node); - for (CharGroup g : node.mData) { - final int groupSize = getCharGroupMaximumSize(g, options); - g.mCachedSize = groupSize; - size += groupSize; - } - if (options.mSupportsDynamicUpdate) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - node.mCachedSize = size; - } - - /** - * Helper method to hide the actual value of the no children address. - */ - public static boolean hasChildrenAddress(final int address) { - return FormatSpec.NO_CHILDREN_ADDRESS != address; - } - - /** - * Helper method to check whether the group is moved. - */ - public static boolean isMovedGroup(final int flags, final FormatOptions options) { - return options.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_MOVED); - } - - /** - * Helper method to check whether the group is deleted. - */ - public static boolean isDeletedGroup(final int flags, final FormatOptions formatOptions) { - return formatOptions.mSupportsDynamicUpdate - && ((flags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) == FormatSpec.FLAG_IS_DELETED); - } - - /** - * Helper method to check whether the dictionary can be updated dynamically. - */ - public static boolean supportsDynamicUpdate(final FormatOptions options) { - return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE - && options.mSupportsDynamicUpdate; - } - - /** - * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. - * - * @param group the group of which to compute the size of the header - * @param options file format options. - */ - private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { - if (supportsDynamicUpdate(options)) { - return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE - + getGroupCharactersSize(group); - } else { - return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group); - } - } - - private static final int UINT8_MAX = 0xFF; - private static final int UINT16_MAX = 0xFFFF; - private static final int UINT24_MAX = 0xFFFFFF; - - /** - * Compute the size, in bytes, that an address will occupy. - * - * This can be used either for children addresses (which are always positive) or for - * attribute, which may be positive or negative but - * store their sign bit separately. - * - * @param address the address - * @return the byte size. - */ - static int getByteSize(final int address) { - assert(address <= UINT24_MAX); - if (!hasChildrenAddress(address)) { - return 0; - } else if (Math.abs(address) <= UINT8_MAX) { - return 1; - } else if (Math.abs(address) <= UINT16_MAX) { - return 2; - } else { - return 3; - } - } - - private static final int SINT24_MAX = 0x7FFFFF; - private static final int MSB8 = 0x80; - private static final int MSB24 = 0x800000; - - // End utility methods. - - // This method is responsible for finding a nice ordering of the nodes that favors run-time - // cache performance and dictionary size. - /* package for tests */ static ArrayList<Node> flattenTree(final Node root) { - final int treeSize = FusionDictionary.countCharGroups(root); - MakedictLog.i("Counted nodes : " + treeSize); - final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize); - return flattenTreeInner(flatTree, root); - } - - private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) { - // Removing the node is necessary if the tails are merged, because we would then - // add the same node several times when we only want it once. A number of places in - // the code also depends on any node being only once in the list. - // Merging tails can only be done if there are no attributes. Searching for attributes - // in LatinIME code depends on a total breadth-first ordering, which merging tails - // breaks. If there are no attributes, it should be fine (and reduce the file size) - // to merge tails, and removing the node from the list would be necessary. However, - // we don't merge tails because breaking the breadth-first ordering would result in - // extreme overhead at bigram lookup time (it would make the search function O(n) instead - // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty - // high). - // If no nodes are ever merged, we can't have the same node twice in the list, hence - // searching for duplicates in unnecessary. It is also very performance consuming, - // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making - // this simple list.remove operation O(n*n) overall. On Android this overhead is very - // high. - // For future reference, the code to remove duplicate is a simple : list.remove(node); - list.add(node); - final ArrayList<CharGroup> branches = node.mData; - final int nodeSize = branches.size(); - for (CharGroup group : branches) { - if (null != group.mChildren) flattenTreeInner(list, group.mChildren); - } - return list; - } - - /** - * Get the offset from a position inside a current node to a target node, during update. - * - * If the current node is before the target node, the target node has not been updated yet, - * so we should return the offset from the old position of the current node to the old position - * of the target node. If on the other hand the target is before the current node, it already - * has been updated, so we should return the offset from the new position in the current node - * to the new position in the target node. - * @param currentNode the node containing the CharGroup where the offset will be written - * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode - * @param targetNode the target node to get the offset to - * @return the offset to the target node - */ - private static int getOffsetToTargetNodeDuringUpdate(final Node currentNode, - final int offsetFromStartOfCurrentNode, final Node targetNode) { - final boolean isTargetBeforeCurrent = (targetNode.mCachedAddressBeforeUpdate - < currentNode.mCachedAddressBeforeUpdate); - if (isTargetBeforeCurrent) { - return targetNode.mCachedAddressAfterUpdate - - (currentNode.mCachedAddressAfterUpdate + offsetFromStartOfCurrentNode); - } else { - return targetNode.mCachedAddressBeforeUpdate - - (currentNode.mCachedAddressBeforeUpdate + offsetFromStartOfCurrentNode); - } - } - - /** - * Get the offset from a position inside a current node to a target CharGroup, during update. - * @param currentNode the node containing the CharGroup where the offset will be written - * @param offsetFromStartOfCurrentNode the offset, in bytes, from the start of currentNode - * @param targetCharGroup the target CharGroup to get the offset to - * @return the offset to the target CharGroup - */ - // TODO: is there any way to factorize this method with the one above? - private static int getOffsetToTargetCharGroupDuringUpdate(final Node currentNode, - final int offsetFromStartOfCurrentNode, final CharGroup targetCharGroup) { - final int oldOffsetBasePoint = currentNode.mCachedAddressBeforeUpdate - + offsetFromStartOfCurrentNode; - final boolean isTargetBeforeCurrent = (targetCharGroup.mCachedAddressBeforeUpdate - < oldOffsetBasePoint); - // If the target is before the current node, then its address has already been updated. - // We can use the AfterUpdate member, and compare it to our own member after update. - // Otherwise, the AfterUpdate member is not updated yet, so we need to use the BeforeUpdate - // member, and of course we have to compare this to our own address before update. - if (isTargetBeforeCurrent) { - final int newOffsetBasePoint = currentNode.mCachedAddressAfterUpdate - + offsetFromStartOfCurrentNode; - return targetCharGroup.mCachedAddressAfterUpdate - newOffsetBasePoint; - } else { - return targetCharGroup.mCachedAddressBeforeUpdate - oldOffsetBasePoint; - } - } - - /** - * Computes the actual node size, based on the cached addresses of the children nodes. - * - * Each node stores its tentative address. During dictionary address computing, these - * are not final, but they can be used to compute the node size (the node size depends - * on the address of the children because the number of bytes necessary to store an - * address depends on its numeric value. The return value indicates whether the node - * contents (as in, any of the addresses stored in the cache fields) have changed with - * respect to their previous value. - * - * @param node the node to compute the size of. - * @param dict the dictionary in which the word/attributes are to be found. - * @param formatOptions file format options. - * @return false if none of the cached addresses inside the node changed, true otherwise. - */ - private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict, - final FormatOptions formatOptions) { - boolean changed = false; - int size = getGroupCountSize(node); - for (CharGroup group : node.mData) { - group.mCachedAddressAfterUpdate = node.mCachedAddressAfterUpdate + size; - if (group.mCachedAddressAfterUpdate != group.mCachedAddressBeforeUpdate) { - changed = true; - } - int groupSize = getGroupHeaderSize(group, formatOptions); - if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE; - if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else if (null != group.mChildren) { - if (formatOptions.mSupportsDynamicUpdate) { - groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - } else { - groupSize += getByteSize(getOffsetToTargetNodeDuringUpdate(node, - groupSize + size, group.mChildren)); - } - } - groupSize += getShortcutListSize(group.mShortcutTargets); - if (null != group.mBigrams) { - for (WeightedString bigram : group.mBigrams) { - final int offset = getOffsetToTargetCharGroupDuringUpdate(node, - groupSize + size + FormatSpec.GROUP_FLAGS_SIZE, - FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord)); - groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE; - } - } - group.mCachedSize = groupSize; - size += groupSize; - } - if (formatOptions.mSupportsDynamicUpdate) { - size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - if (node.mCachedSize != size) { - node.mCachedSize = size; - changed = true; - } - return changed; - } - - /** - * Initializes the cached addresses of nodes from their size. - * - * @param flatNodes the array of nodes. - * @param formatOptions file format options. - * @return the byte size of the entire stack. - */ - private static int initializeNodesCachedAddresses(final ArrayList<Node> flatNodes, - final FormatOptions formatOptions) { - int nodeOffset = 0; - for (final Node n : flatNodes) { - n.mCachedAddressBeforeUpdate = nodeOffset; - int groupCountSize = getGroupCountSize(n); - int groupOffset = 0; - for (final CharGroup g : n.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate = - groupCountSize + nodeOffset + groupOffset; - groupOffset += g.mCachedSize; - } - final int nodeSize = groupCountSize + groupOffset - + (formatOptions.mSupportsDynamicUpdate - ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); - nodeOffset += n.mCachedSize; - } - return nodeOffset; - } - - /** - * Updates the cached addresses of nodes after recomputing their new positions. - * - * @param flatNodes the array of nodes. - */ - private static void updateNodeCachedAddresses(final ArrayList<Node> flatNodes) { - for (final Node n : flatNodes) { - n.mCachedAddressBeforeUpdate = n.mCachedAddressAfterUpdate; - for (final CharGroup g : n.mData) { - g.mCachedAddressBeforeUpdate = g.mCachedAddressAfterUpdate; - } - } - } - - /** - * Compute the cached parent addresses after all has been updated. - * - * The parent addresses are used by some binary formats at write-to-disk time. Not all formats - * need them. In particular, version 2 does not need them, and version 3 does. - * - * @param flatNodes the flat array of nodes to fill in - */ - private static void computeParentAddresses(final ArrayList<Node> flatNodes) { - for (final Node node : flatNodes) { - for (final CharGroup group : node.mData) { - if (null != group.mChildren) { - // Assign my address to children's parent address - // Here BeforeUpdate and AfterUpdate addresses have the same value, so it - // does not matter which we use. - group.mChildren.mCachedParentAddress = group.mCachedAddressAfterUpdate - - group.mChildren.mCachedAddressAfterUpdate; - } - } - } - } - - /** - * Compute the addresses and sizes of an ordered node array. - * - * This method takes a node array and will update its cached address and size values - * so that they can be written into a file. It determines the smallest size each of the - * nodes can be given the addresses of its children and attributes, and store that into - * each node. - * The order of the node is given by the order of the array. This method makes no effort - * to find a good order; it only mechanically computes the size this order results in. - * - * @param dict the dictionary - * @param flatNodes the ordered array of nodes - * @param formatOptions file format options. - * @return the same array it was passed. The nodes have been updated for address and size. - */ - private static ArrayList<Node> computeAddresses(final FusionDictionary dict, - final ArrayList<Node> flatNodes, final FormatOptions formatOptions) { - // First get the worst possible sizes and offsets - for (final Node n : flatNodes) calculateNodeMaximumSize(n, formatOptions); - final int offset = initializeNodesCachedAddresses(flatNodes, formatOptions); - - MakedictLog.i("Compressing the array addresses. Original size : " + offset); - MakedictLog.i("(Recursively seen size : " + offset + ")"); - - int passes = 0; - boolean changesDone = false; - do { - changesDone = false; - int nodeStartOffset = 0; - for (final Node n : flatNodes) { - n.mCachedAddressAfterUpdate = nodeStartOffset; - final int oldNodeSize = n.mCachedSize; - final boolean changed = computeActualNodeSize(n, dict, formatOptions); - final int newNodeSize = n.mCachedSize; - if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!"); - nodeStartOffset += newNodeSize; - changesDone |= changed; - } - updateNodeCachedAddresses(flatNodes); - ++passes; - if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); - } while (changesDone); - - if (formatOptions.mSupportsDynamicUpdate) { - computeParentAddresses(flatNodes); - } - final Node lastNode = flatNodes.get(flatNodes.size() - 1); - MakedictLog.i("Compression complete in " + passes + " passes."); - MakedictLog.i("After address compression : " - + (lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize)); - - return flatNodes; - } - - /** - * Sanity-checking method. - * - * This method checks an array of node for juxtaposition, that is, it will do - * nothing if each node's cached address is actually the previous node's address - * plus the previous node's size. - * If this is not the case, it will throw an exception. - * - * @param array the array node to check - */ - private static void checkFlatNodeArray(final ArrayList<Node> array) { - int offset = 0; - int index = 0; - for (final Node n : array) { - // BeforeUpdate and AfterUpdate addresses are the same here, so it does not matter - // which we use. - if (n.mCachedAddressAfterUpdate != offset) { - throw new RuntimeException("Wrong address for node " + index - + " : expected " + offset + ", got " + n.mCachedAddressAfterUpdate); - } - ++index; - offset += n.mCachedSize; - } - } - - /** - * Helper method to write a variable-size address to a file. - * - * @param buffer the buffer to write to. - * @param index the index in the buffer to write the address to. - * @param address the address to write. - * @return the size in bytes the address actually took. - */ - private static int writeVariableAddress(final byte[] buffer, int index, final int address) { - switch (getByteSize(address)) { - case 1: - buffer[index++] = (byte)address; - return 1; - case 2: - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); - return 2; - case 3: - buffer[index++] = (byte)(0xFF & (address >> 16)); - buffer[index++] = (byte)(0xFF & (address >> 8)); - buffer[index++] = (byte)(0xFF & address); - return 3; - case 0: - return 0; - default: - throw new RuntimeException("Address " + address + " has a strange size"); - } - } - - /** - * Helper method to write a variable-size signed address to a file. - * - * @param buffer the buffer to write to. - * @param index the index in the buffer to write the address to. - * @param address the address to write. - * @return the size in bytes the address actually took. - */ - private static int writeVariableSignedAddress(final byte[] buffer, int index, - final int address) { - if (!hasChildrenAddress(address)) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - buffer[index++] = (byte)((address < 0 ? MSB8 : 0) | (0xFF & (absAddress >> 16))); - buffer[index++] = (byte)(0xFF & (absAddress >> 8)); - buffer[index++] = (byte)(0xFF & absAddress); - } - return 3; - } - - /** - * Makes the flag value for a char group. - * - * @param hasMultipleChars whether the group has multiple chars. - * @param isTerminal whether the group is terminal. - * @param childrenAddressSize the size of a children address. - * @param hasShortcuts whether the group has shortcuts. - * @param hasBigrams whether the group has bigrams. - * @param isNotAWord whether the group is not a word. - * @param isBlackListEntry whether the group is a blacklist entry. - * @param formatOptions file format options. - * @return the flags - */ - static int makeCharGroupFlags(final boolean hasMultipleChars, final boolean isTerminal, - final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, - final boolean isNotAWord, final boolean isBlackListEntry, - final FormatOptions formatOptions) { - byte flags = 0; - if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; - if (isTerminal) flags |= FormatSpec.FLAG_IS_TERMINAL; - if (formatOptions.mSupportsDynamicUpdate) { - flags |= FormatSpec.FLAG_IS_NOT_MOVED; - } else if (true) { - switch (childrenAddressSize) { - case 1: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; - break; - case 0: - flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS; - break; - default: - throw new RuntimeException("Node with a strange address"); - } - } - if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; - if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; - if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; - if (isBlackListEntry) flags |= FormatSpec.FLAG_IS_BLACKLISTED; - return flags; - } - - private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress, - final int childrenOffset, final FormatOptions formatOptions) { - return (byte) makeCharGroupFlags(group.mChars.length > 1, group.mFrequency >= 0, - getByteSize(childrenOffset), group.mShortcutTargets != null, group.mBigrams != null, - group.mIsNotAWord, group.mIsBlacklistEntry, formatOptions); - } - - /** - * Makes the flag value for a bigram. - * - * @param more whether there are more bigrams after this one. - * @param offset the offset of the bigram. - * @param bigramFrequency the frequency of the bigram, 0..255. - * @param unigramFrequency the unigram frequency of the same word, 0..255. - * @param word the second bigram, for debugging purposes - * @return the flags - */ - private static final int makeBigramFlags(final boolean more, final int offset, - int bigramFrequency, final int unigramFrequency, final String word) { - int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0); - switch (getByteSize(offset)) { - case 1: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; - break; - case 2: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; - break; - case 3: - bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; - break; - default: - throw new RuntimeException("Strange offset size"); - } - if (unigramFrequency > bigramFrequency) { - MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word - + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " - + word + " is " + unigramFrequency); - bigramFrequency = unigramFrequency; - } - // We compute the difference between 255 (which means probability = 1) and the - // unigram score. We split this into a number of discrete steps. - // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 - // represents an increase of 16 steps: a value of 15 will be interpreted as the median - // value of the 16th step. In all justice, if the bigram frequency is low enough to be - // rounded below the first step (which means it is less than half a step higher than the - // unigram frequency) then the unigram frequency itself is the best approximation of the - // bigram freq that we could possibly supply, hence we should *not* include this bigram - // in the file at all. - // until this is done, we'll write 0 and slightly overestimate this case. - // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step - // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to - // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the - // step size. Then we compute the start of the first step (the one where value 0 starts) - // by adding half-a-step to the unigramFrequency. From there, we compute the integer - // number of steps to the bigramFrequency. One last thing: we want our steps to include - // their lower bound and exclude their higher bound so we need to have the first step - // start at exactly 1 unit higher than floor(unigramFreq + half a step). - // Note : to reconstruct the score, the dictionary reader will need to divide - // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, - // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best - // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the - // step pointed by the discretized frequency. - final float stepSize = - (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) - / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); - final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); - final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); - // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 - // here. The best approximation would be the unigram freq itself, so we should not - // include this bigram in the dictionary. For now, register as 0, and live with the - // small over-estimation that we get in this case. TODO: actually remove this bigram - // if discretizedFrequency < 0. - final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; - bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; - return bigramFlags; - } - - /** - * Makes the 2-byte value for options flags. - */ - private static final int makeOptionsValue(final FusionDictionary dictionary, - final FormatOptions formatOptions) { - final DictionaryOptions options = dictionary.mOptions; - final boolean hasBigrams = dictionary.hasBigrams(); - return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) - + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) - + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) - + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); - } - - /** - * Makes the flag value for a shortcut. - * - * @param more whether there are more attributes after this one. - * @param frequency the frequency of the attribute, 0..15 - * @return the flags - */ - static final int makeShortcutFlags(final boolean more, final int frequency) { - return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) - + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY); - } - - private static final int writeParentAddress(final byte[] buffer, final int index, - final int address, final FormatOptions formatOptions) { - if (supportsDynamicUpdate(formatOptions)) { - if (address == FormatSpec.NO_PARENT_ADDRESS) { - buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; - } else { - final int absAddress = Math.abs(address); - assert(absAddress <= SINT24_MAX); - buffer[index] = (byte)((address < 0 ? MSB8 : 0) - | ((absAddress >> 16) & 0xFF)); - buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); - buffer[index + 2] = (byte)(absAddress & 0xFF); - } - return index + 3; - } else { - return index; - } - } - - /** - * Write a node to memory. The node is expected to have its final position cached. - * - * This can be an empty map, but the more is inside the faster the lookups will be. It can - * be carried on as long as nodes do not move. - * - * @param dict the dictionary the node is a part of (for relative offsets). - * @param buffer the memory buffer to write to. - * @param node the node to write. - * @param formatOptions file format options. - * @return the address of the END of the node. - */ - @SuppressWarnings("unused") - private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, - final Node node, final FormatOptions formatOptions) { - // TODO: Make the code in common with BinaryDictIOUtils#writeCharGroup - int index = node.mCachedAddressAfterUpdate; - - final int groupCount = node.mData.size(); - final int countSize = getGroupCountSize(node); - final int parentAddress = node.mCachedParentAddress; - if (1 == countSize) { - buffer[index++] = (byte)groupCount; - } else if (2 == countSize) { - // We need to signal 2-byte size by setting the top bit of the MSB to 1, so - // we | 0x80 to do this. - buffer[index++] = (byte)((groupCount >> 8) | 0x80); - buffer[index++] = (byte)(groupCount & 0xFF); - } else { - throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); - } - int groupAddress = index; - for (int i = 0; i < groupCount; ++i) { - final CharGroup group = node.mData.get(i); - if (index != group.mCachedAddressAfterUpdate) { - throw new RuntimeException("Bug: write index is not the same as the cached address " - + "of the group : " + index + " <> " + group.mCachedAddressAfterUpdate); - } - groupAddress += getGroupHeaderSize(group, formatOptions); - // Sanity checks. - if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { - throw new RuntimeException("A node has a frequency > " - + FormatSpec.MAX_TERMINAL_FREQUENCY - + " : " + group.mFrequency); - } - if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE; - final int childrenOffset = null == group.mChildren - ? FormatSpec.NO_CHILDREN_ADDRESS - : group.mChildren.mCachedAddressAfterUpdate - groupAddress; - buffer[index++] = - makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions); - - if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { - index = writeParentAddress(buffer, index, parentAddress, formatOptions); - } else { - index = writeParentAddress(buffer, index, parentAddress - + (node.mCachedAddressAfterUpdate - group.mCachedAddressAfterUpdate), - formatOptions); - } - - index = CharEncoding.writeCharArray(group.mChars, buffer, index); - if (group.hasSeveralChars()) { - buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; - } - if (group.mFrequency >= 0) { - buffer[index++] = (byte) group.mFrequency; - } - - final int shift; - if (formatOptions.mSupportsDynamicUpdate) { - shift = writeVariableSignedAddress(buffer, index, childrenOffset); - } else { - shift = writeVariableAddress(buffer, index, childrenOffset); - } - index += shift; - groupAddress += shift; - - // Write shortcuts - if (null != group.mShortcutTargets) { - final int indexOfShortcutByteSize = index; - index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - ++groupAddress; - int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), - target.mFrequency); - buffer[index++] = (byte)shortcutFlags; - final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); - index += shortcutShift; - groupAddress += shortcutShift; - } - final int shortcutByteSize = index - indexOfShortcutByteSize; - if (shortcutByteSize > 0xFFFF) { - throw new RuntimeException("Shortcut list too large"); - } - buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); - buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); - } - // Write bigrams - if (null != group.mBigrams) { - final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); - while (bigramIterator.hasNext()) { - final WeightedString bigram = bigramIterator.next(); - final CharGroup target = - FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord); - final int addressOfBigram = target.mCachedAddressAfterUpdate; - final int unigramFrequencyForThisWord = target.mFrequency; - ++groupAddress; - final int offset = addressOfBigram - groupAddress; - int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, - bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); - buffer[index++] = (byte)bigramFlags; - final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); - index += bigramShift; - groupAddress += bigramShift; - } - } - - } - if (formatOptions.mSupportsDynamicUpdate) { - buffer[index] = buffer[index + 1] = buffer[index + 2] - = FormatSpec.NO_FORWARD_LINK_ADDRESS; - index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; - } - if (index != node.mCachedAddressAfterUpdate + node.mCachedSize) throw new RuntimeException( - "Not the same size : written " - + (index - node.mCachedAddressAfterUpdate) + " bytes from a node that should have " - + node.mCachedSize + " bytes"); - return index; - } - - /** - * Dumps a collection of useful statistics about a node array. - * - * This prints purely informative stuff, like the total estimated file size, the - * number of nodes, of character groups, the repartition of each address size, etc - * - * @param nodes the node array. - */ - private static void showStatistics(ArrayList<Node> nodes) { - int firstTerminalAddress = Integer.MAX_VALUE; - int lastTerminalAddress = Integer.MIN_VALUE; - int size = 0; - int charGroups = 0; - int maxGroups = 0; - int maxRuns = 0; - for (final Node n : nodes) { - if (maxGroups < n.mData.size()) maxGroups = n.mData.size(); - for (final CharGroup cg : n.mData) { - ++charGroups; - if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length; - if (cg.mFrequency >= 0) { - if (n.mCachedAddressAfterUpdate < firstTerminalAddress) - firstTerminalAddress = n.mCachedAddressAfterUpdate; - if (n.mCachedAddressAfterUpdate > lastTerminalAddress) - lastTerminalAddress = n.mCachedAddressAfterUpdate; - } - } - if (n.mCachedAddressAfterUpdate + n.mCachedSize > size) { - size = n.mCachedAddressAfterUpdate + n.mCachedSize; - } - } - final int[] groupCounts = new int[maxGroups + 1]; - final int[] runCounts = new int[maxRuns + 1]; - for (final Node n : nodes) { - ++groupCounts[n.mData.size()]; - for (final CharGroup cg : n.mData) { - ++runCounts[cg.mChars.length]; - } - } - - MakedictLog.i("Statistics:\n" - + " total file size " + size + "\n" - + " " + nodes.size() + " nodes\n" - + " " + charGroups + " groups (" + ((float)charGroups / nodes.size()) - + " groups per node)\n" - + " first terminal at " + firstTerminalAddress + "\n" - + " last terminal at " + lastTerminalAddress + "\n" - + " Group stats : max = " + maxGroups); - for (int i = 0; i < groupCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + groupCounts[i]); - } - MakedictLog.i(" Character run stats : max = " + maxRuns); - for (int i = 0; i < runCounts.length; ++i) { - MakedictLog.i(" " + i + " : " + runCounts[i]); - } - } - - /** - * Dumps a FusionDictionary to a file. - * - * This is the public entry point to write a dictionary to a file. - * - * @param destination the stream to write the binary data to. - * @param dict the dictionary to write. - * @param formatOptions file format options. - */ - public static void writeDictionaryBinary(final OutputStream destination, - final FusionDictionary dict, final FormatOptions formatOptions) - throws IOException, UnsupportedFormatException { - - // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the - // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order - // of the nodes becomes a quite complicated problem, because though the dictionary itself - // does not have a size limit, each node must still be within 16MB of all its children and - // parents. As long as this is ensured, the dictionary file may grow to any size. - - final int version = formatOptions.mVersion; - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("Requested file format version " + version - + ", but this implementation only supports versions " - + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - - ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); - - // The magic number in big-endian order. - if (version >= FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { - // Magic number for version 2+. - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 24))); - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 16))); - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 8))); - headerBuffer.write((byte) (0xFF & FormatSpec.VERSION_2_MAGIC_NUMBER)); - // Dictionary version. - headerBuffer.write((byte) (0xFF & (version >> 8))); - headerBuffer.write((byte) (0xFF & version)); - } else { - // Magic number for version 1. - headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_1_MAGIC_NUMBER >> 8))); - headerBuffer.write((byte) (0xFF & FormatSpec.VERSION_1_MAGIC_NUMBER)); - // Dictionary version. - headerBuffer.write((byte) (0xFF & version)); - } - // Options flags - final int options = makeOptionsValue(dict, formatOptions); - headerBuffer.write((byte) (0xFF & (options >> 8))); - headerBuffer.write((byte) (0xFF & options)); - if (version >= FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { - final int headerSizeOffset = headerBuffer.size(); - // Placeholder to be written later with header size. - for (int i = 0; i < 4; ++i) { - headerBuffer.write(0); - } - // Write out the options. - for (final String key : dict.mOptions.mAttributes.keySet()) { - final String value = dict.mOptions.mAttributes.get(key); - CharEncoding.writeString(headerBuffer, key); - CharEncoding.writeString(headerBuffer, value); - } - final int size = headerBuffer.size(); - final byte[] bytes = headerBuffer.toByteArray(); - // Write out the header size. - bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); - bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); - bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); - bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); - destination.write(bytes); - } else { - headerBuffer.writeTo(destination); - } - - headerBuffer.close(); - - // Leave the choice of the optimal node order to the flattenTree function. - MakedictLog.i("Flattening the tree..."); - ArrayList<Node> flatNodes = flattenTree(dict.mRoot); - - MakedictLog.i("Computing addresses..."); - computeAddresses(dict, flatNodes, formatOptions); - MakedictLog.i("Checking array..."); - if (DBG) checkFlatNodeArray(flatNodes); - - // Create a buffer that matches the final dictionary size. - final Node lastNode = flatNodes.get(flatNodes.size() - 1); - final int bufferSize = lastNode.mCachedAddressAfterUpdate + lastNode.mCachedSize; - final byte[] buffer = new byte[bufferSize]; - int index = 0; - - MakedictLog.i("Writing file..."); - int dataEndOffset = 0; - for (Node n : flatNodes) { - dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions); - } - - if (DBG) showStatistics(flatNodes); - - destination.write(buffer, 0, dataEndOffset); - - destination.close(); - MakedictLog.i("Done"); - } - - - // Input methods: Read a binary dictionary to memory. - // readDictionaryBinary is the public entry point for them. - - static int getChildrenAddressSize(final int optionFlags, - final FormatOptions formatOptions) { - if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - return 1; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - return 2; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - return 3; - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: - default: - return 0; - } - } - - static int readChildrenAddress(final FusionDictionaryBufferInterface buffer, - final int optionFlags, final FormatOptions options) { - if (options.mSupportsDynamicUpdate) { - final int address = buffer.readUnsignedInt24(); - if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; - if ((address & MSB24) != 0) { - return -(address & SINT24_MAX); - } else { - return address; - } - } - int address; - switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: - return buffer.readUnsignedByte(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: - return buffer.readUnsignedShort(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: - return buffer.readUnsignedInt24(); - case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: - default: - return FormatSpec.NO_CHILDREN_ADDRESS; - } - } - - static int readParentAddress(final FusionDictionaryBufferInterface buffer, - final FormatOptions formatOptions) { - if (supportsDynamicUpdate(formatOptions)) { - final int parentAddress = buffer.readUnsignedInt24(); - final int sign = ((parentAddress & MSB24) != 0) ? -1 : 1; - return sign * (parentAddress & SINT24_MAX); - } else { - return FormatSpec.NO_PARENT_ADDRESS; - } - } - - private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; - public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer, - final int originalGroupAddress, final FormatOptions options) { - int addressPointer = originalGroupAddress; - final int flags = buffer.readUnsignedByte(); - ++addressPointer; - - final int parentAddress = readParentAddress(buffer, options); - if (supportsDynamicUpdate(options)) { - addressPointer += 3; - } - - final int characters[]; - if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { - int index = 0; - int character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - while (-1 != character) { - // FusionDictionary is making sure that the length of the word is smaller than - // MAX_WORD_LENGTH. - // So we'll never write past the end of CHARACTER_BUFFER. - CHARACTER_BUFFER[index++] = character; - character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - } - characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); - } else { - final int character = CharEncoding.readChar(buffer); - addressPointer += CharEncoding.getCharSize(character); - characters = new int[] { character }; - } - final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - ++addressPointer; - frequency = buffer.readUnsignedByte(); - } else { - frequency = CharGroup.NOT_A_TERMINAL; - } - int childrenAddress = readChildrenAddress(buffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += getChildrenAddressSize(flags, options); - ArrayList<WeightedString> shortcutTargets = null; - if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { - final int pointerBefore = buffer.position(); - shortcutTargets = new ArrayList<WeightedString>(); - buffer.readUnsignedShort(); // Skip the size - while (true) { - final int targetFlags = buffer.readUnsignedByte(); - final String word = CharEncoding.readString(buffer); - shortcutTargets.add(new WeightedString(word, - targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); - if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - addressPointer += buffer.position() - pointerBefore; - } - ArrayList<PendingAttribute> bigrams = null; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - int bigramCount = 0; - while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - final int bigramFlags = buffer.readUnsignedByte(); - ++addressPointer; - final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) - ? 1 : -1; - int bigramAddress = addressPointer; - switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: - bigramAddress += sign * buffer.readUnsignedByte(); - addressPointer += 1; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: - bigramAddress += sign * buffer.readUnsignedShort(); - addressPointer += 2; - break; - case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: - final int offset = (buffer.readUnsignedByte() << 16) - + buffer.readUnsignedShort(); - bigramAddress += sign * offset; - addressPointer += 3; - break; - default: - throw new RuntimeException("Has bigrams with no address"); - } - bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, - bigramAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; - } - if (bigramCount >= FormatSpec.MAX_BIGRAMS_IN_A_GROUP) { - MakedictLog.d("too many bigrams in a group."); - } - } - return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); - } - - /** - * Reads and returns the char group count out of a buffer and forwards the pointer. - */ - public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) { - final int msb = buffer.readUnsignedByte(); - if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { - return msb; - } else { - return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) - + buffer.readUnsignedByte(); - } - } - - // The word cache here is a stopgap bandaid to help the catastrophic performance - // of this method. Since it performs direct, unbuffered random access to the file and - // may be called hundreds of thousands of times, the resulting performance is not - // reasonable without some kind of cache. Thus: - private static TreeMap<Integer, WeightedString> wordCache = - new TreeMap<Integer, WeightedString>(); - /** - * Finds, as a string, the word at the address passed as an argument. - * - * @param buffer the buffer to read from. - * @param headerSize the size of the header. - * @param address the address to seek. - * @param formatOptions file format options. - * @return the word with its frequency, as a weighted string. - */ - /* package for tests */ static WeightedString getWordAtAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions formatOptions) { - final WeightedString cachedString = wordCache.get(address); - if (null != cachedString) return cachedString; - - final WeightedString result; - final int originalPointer = buffer.position(); - buffer.position(address); - - if (supportsDynamicUpdate(formatOptions)) { - result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions); - } else { - result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address, - formatOptions); - } - - wordCache.put(address, result); - buffer.position(originalPointer); - return result; - } - - // TODO: static!? This will behave erratically when used in multi-threaded code. - // We need to fix this - private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; - @SuppressWarnings("unused") - private static WeightedString getWordAtAddressWithParentAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions options) { - int currentAddress = address; - int index = FormatSpec.MAX_WORD_LENGTH - 1; - int frequency = Integer.MIN_VALUE; - // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH - for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { - CharGroupInfo currentInfo; - int loopCounter = 0; - do { - buffer.position(currentAddress + headerSize); - currentInfo = readCharGroup(buffer, currentAddress, options); - if (isMovedGroup(currentInfo.mFlags, options)) { - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - if (DBG && loopCounter++ > MAX_JUMPS) { - MakedictLog.d("Too many jumps - probably a bug"); - } - } while (isMovedGroup(currentInfo.mFlags, options)); - if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency; - for (int i = 0; i < currentInfo.mCharacters.length; ++i) { - sGetWordBuffer[index--] = - currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1]; - } - if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; - currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; - } - - return new WeightedString( - new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1), - frequency); - } - - private static WeightedString getWordAtAddressWithoutParentAddress( - final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, - final FormatOptions options) { - buffer.position(headerSize); - final int count = readCharGroupCount(buffer); - int groupOffset = getGroupCountSize(count); - final StringBuilder builder = new StringBuilder(); - WeightedString result = null; - - CharGroupInfo last = null; - for (int i = count - 1; i >= 0; --i) { - CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - groupOffset = info.mEndAddress; - if (info.mOriginalAddress == address) { - builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); - result = new WeightedString(builder.toString(), info.mFrequency); - break; // and return - } - if (hasChildrenAddress(info.mChildrenAddress)) { - if (info.mChildrenAddress > address) { - if (null == last) continue; - builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - buffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + getGroupCountSize(i); - last = null; - continue; - } - last = info; - } - if (0 == i && hasChildrenAddress(last.mChildrenAddress)) { - builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); - buffer.position(last.mChildrenAddress + headerSize); - i = readCharGroupCount(buffer); - groupOffset = last.mChildrenAddress + getGroupCountSize(i); - last = null; - continue; - } - } - return result; - } - - /** - * Reads a single node from a buffer. - * - * This methods reads the file at the current position. A node is fully expected to start at - * the current position. - * This will recursively read other nodes into the structure, populating the reverse - * maps on the fly and using them to keep track of already read nodes. - * - * @param buffer the buffer, correctly positioned at the start of a node. - * @param headerSize the size, in bytes, of the file header. - * @param reverseNodeMap a mapping from addresses to already read nodes. - * @param reverseGroupMap a mapping from addresses to already read character groups. - * @param options file format options. - * @return the read node with all his children already read. - */ - private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize, - final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap, - final FormatOptions options) - throws IOException { - final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>(); - final int nodeOrigin = buffer.position() - headerSize; - - do { // Scan the linked-list node. - final int nodeHeadPosition = buffer.position() - headerSize; - final int count = readCharGroupCount(buffer); - int groupOffset = nodeHeadPosition + getGroupCountSize(count); - for (int i = count; i > 0; --i) { // Scan the array of CharGroup. - CharGroupInfo info = readCharGroup(buffer, groupOffset, options); - if (isMovedGroup(info.mFlags, options)) continue; - ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; - ArrayList<WeightedString> bigrams = null; - if (null != info.mBigrams) { - bigrams = new ArrayList<WeightedString>(); - for (PendingAttribute bigram : info.mBigrams) { - final WeightedString word = getWordAtAddress( - buffer, headerSize, bigram.mAddress, options); - final int reconstructedFrequency = - reconstructBigramFrequency(word.mFrequency, bigram.mFrequency); - bigrams.add(new WeightedString(word.mWord, reconstructedFrequency)); - } - } - if (hasChildrenAddress(info.mChildrenAddress)) { - Node children = reverseNodeMap.get(info.mChildrenAddress); - if (null == children) { - final int currentPosition = buffer.position(); - buffer.position(info.mChildrenAddress + headerSize); - children = readNode( - buffer, headerSize, reverseNodeMap, reverseGroupMap, options); - buffer.position(currentPosition); - } - nodeContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, - info.mFrequency, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); - } else { - nodeContents.add( - new CharGroup(info.mCharacters, shortcutTargets, bigrams, - info.mFrequency, - 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), - 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); - } - groupOffset = info.mEndAddress; - } - - // reach the end of the array. - if (options.mSupportsDynamicUpdate) { - final int nextAddress = buffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < buffer.limit()) { - buffer.position(nextAddress); - } else { - break; - } - } - } while (options.mSupportsDynamicUpdate && - buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); - - final Node node = new Node(nodeContents); - node.mCachedAddressBeforeUpdate = nodeOrigin; - node.mCachedAddressAfterUpdate = nodeOrigin; - reverseNodeMap.put(node.mCachedAddressAfterUpdate, node); - return node; - } - - /** - * Helper function to get the binary format version from the header. - * @throws IOException - */ - private static int getFormatVersion(final FusionDictionaryBufferInterface buffer) - throws IOException { - final int magic_v1 = buffer.readUnsignedShort(); - if (FormatSpec.VERSION_1_MAGIC_NUMBER == magic_v1) return buffer.readUnsignedByte(); - final int magic_v2 = (magic_v1 << 16) + buffer.readUnsignedShort(); - if (FormatSpec.VERSION_2_MAGIC_NUMBER == magic_v2) return buffer.readUnsignedShort(); - return FormatSpec.NOT_A_VERSION_NUMBER; - } - - /** - * Helper function to get and validate the binary format version. - * @throws UnsupportedFormatException - * @throws IOException - */ - private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer) - throws IOException, UnsupportedFormatException { - final int version = getFormatVersion(buffer); - if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION - || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { - throw new UnsupportedFormatException("This file has version " + version - + ", but this implementation does not support versions above " - + FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } - return version; - } - - /** - * Reads a header from a buffer. - * @param buffer the buffer to read. - * @throws IOException - * @throws UnsupportedFormatException - */ - public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer) - throws IOException, UnsupportedFormatException { - final int version = checkFormatVersion(buffer); - final int optionsFlags = buffer.readUnsignedShort(); - - final HashMap<String, String> attributes = new HashMap<String, String>(); - final int headerSize; - if (version < FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { - headerSize = buffer.position(); - } else { - headerSize = buffer.readInt(); - populateOptions(buffer, headerSize, attributes); - buffer.position(headerSize); - } - - if (headerSize < 0) { - throw new UnsupportedFormatException("header size can't be negative."); - } - - final FileHeader header = new FileHeader(headerSize, - new FusionDictionary.DictionaryOptions(attributes, - 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), - 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), - new FormatOptions(version, - 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); - return header; - } - - /** - * Reads options from a buffer and populate a map with their contents. - * - * The buffer is read at the current position, so the caller must take care the pointer - * is in the right place before calling this. - */ - public static void populateOptions(final FusionDictionaryBufferInterface buffer, - final int headerSize, final HashMap<String, String> options) { - while (buffer.position() < headerSize) { - final String key = CharEncoding.readString(buffer); - final String value = CharEncoding.readString(buffer); - options.put(key, value); - } - } - - /** - * Reads a buffer and returns the memory representation of the dictionary. - * - * This high-level method takes a buffer and reads its contents, populating a - * FusionDictionary structure. The optional dict argument is an existing dictionary to - * which words from the buffer should be added. If it is null, a new dictionary is created. - * - * @param buffer the buffer to read. - * @param dict an optional dictionary to add words to, or null. - * @return the created (or merged) dictionary. - */ - @UsedForTesting - public static FusionDictionary readDictionaryBinary( - final FusionDictionaryBufferInterface buffer, final FusionDictionary dict) - throws IOException, UnsupportedFormatException { - // clear cache - wordCache.clear(); - - // Read header - final FileHeader header = readHeader(buffer); - - Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>(); - Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); - final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping, - reverseGroupMapping, header.mFormatOptions); - - FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions); - if (null != dict) { - for (final Word w : dict) { - if (w.mIsBlacklistEntry) { - newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); - } else { - newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); - } - } - for (final Word w : dict) { - // By construction a binary dictionary may not have bigrams pointing to - // words that are not also registered as unigrams so we don't have to avoid - // them explicitly here. - for (final WeightedString bigram : w.mBigrams) { - newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); - } - } - } - - return newDict; - } - - /** - * Helper method to pass a file name instead of a File object to isBinaryDictionary. - */ - public static boolean isBinaryDictionary(final String filename) { - final File file = new File(filename); - return isBinaryDictionary(file); - } - - /** - * Basic test to find out whether the file is a binary dictionary or not. - * - * Concretely this only tests the magic number. - * - * @param file The file to test. - * @return true if it's a binary dictionary, false otherwise - */ - public static boolean isBinaryDictionary(final File file) { - FileInputStream inStream = null; - try { - inStream = new FileInputStream(file); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - final int version = getFormatVersion(new ByteBufferWrapper(buffer)); - return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION - && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); - } catch (FileNotFoundException e) { - return false; - } catch (IOException e) { - return false; - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } - } - } - - /** - * Calculate bigram frequency from compressed value - * - * @see #makeBigramFlags - * - * @param unigramFrequency - * @param bigramFrequency compressed frequency - * @return approximate bigram frequency - */ - public static int reconstructBigramFrequency(final int unigramFrequency, - final int bigramFrequency) { - final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) - / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); - final float resultFreqFloat = unigramFrequency + stepSize * (bigramFrequency + 1.0f); - return (int)resultFreqFloat; - } -} diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java new file mode 100644 index 000000000..3796a466c --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -0,0 +1,394 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.ByteArrayDictBuffer; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.TreeMap; + +/** + * The base class of binary dictionary decoders. + */ +public abstract class DictDecoder { + + protected FileHeader readHeader(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + if (dictBuffer == null) { + openDictBuffer(); + } + + final int version = HeaderReader.readVersion(dictBuffer); + if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION + || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { + throw new UnsupportedFormatException("Unsupported version : " + version); + } + // TODO: Remove this field. + final int optionsFlags = HeaderReader.readOptionFlags(dictBuffer); + + final int headerSize = HeaderReader.readHeaderSize(dictBuffer); + + if (headerSize < 0) { + throw new UnsupportedFormatException("header size can't be negative."); + } + + final HashMap<String, String> attributes = HeaderReader.readAttributes(dictBuffer, + headerSize); + + final FileHeader header = new FileHeader(headerSize, + new FusionDictionary.DictionaryOptions(attributes, + 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), + 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), + new FormatOptions(version, + 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); + return header; + } + + /** + * Reads and returns the file header. + */ + public abstract FileHeader readHeader() throws IOException, UnsupportedFormatException; + + /** + * Reads PtNode from nodeAddress. + * @param ptNodePos the position of PtNode. + * @param formatOptions the format options. + * @return PtNodeInfo. + */ + public abstract PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions formatOptions); + + /** + * Reads a buffer and returns the memory representation of the dictionary. + * + * This high-level method takes a buffer and reads its contents, populating a + * FusionDictionary structure. The optional dict argument is an existing dictionary to + * which words from the buffer should be added. If it is null, a new dictionary is created. + * + * @param dict an optional dictionary to add words to, or null. + * @param deleteDictIfBroken a flag indicating whether this method should remove the broken + * dictionary or not. + * @return the created (or merged) dictionary. + */ + @UsedForTesting + public abstract FusionDictionary readDictionaryBinary(final FusionDictionary dict, + final boolean deleteDictIfBroken) + throws FileNotFoundException, IOException, UnsupportedFormatException; + + /** + * Gets the address of the last PtNode of the exact matching word in the dictionary. + * If no match is found, returns NOT_VALID_WORD. + * + * @param word the word we search for. + * @return the address of the terminal node. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + public int getTerminalPosition(final String word) + throws IOException, UnsupportedFormatException { + if (!isDictBufferOpen()) { + openDictBuffer(); + } + return BinaryDictIOUtils.getTerminalPosition(this, word); + } + + /** + * Reads unigrams and bigrams from the binary file. + * Doesn't store a full memory representation of the dictionary. + * + * @param words the map to store the address as a key and the word as a value. + * @param frequencies the map to store the address as a key and the frequency as a value. + * @param bigrams the map to store the address as a key and the list of address as a value. + * @throws IOException if the file can't be read. + * @throws UnsupportedFormatException if the format of the file is not recognized. + */ + @UsedForTesting + public void readUnigramsAndBigramsBinary(final TreeMap<Integer, String> words, + final TreeMap<Integer, Integer> frequencies, + final TreeMap<Integer, ArrayList<PendingAttribute>> bigrams) + throws IOException, UnsupportedFormatException { + if (!isDictBufferOpen()) { + openDictBuffer(); + } + BinaryDictIOUtils.readUnigramsAndBigramsBinary(this, words, frequencies, bigrams); + } + + /** + * Sets the position of the buffer to the given value. + * + * @param newPos the new position + */ + public abstract void setPosition(final int newPos); + + /** + * Gets the position of the buffer. + * + * @return the position + */ + public abstract int getPosition(); + + /** + * Reads and returns the PtNode count out of a buffer and forwards the pointer. + */ + public abstract int readPtNodeCount(); + + /** + * Reads the forward link and advances the position. + * + * @return true if this method moves the file pointer, false otherwise. + */ + public abstract boolean readAndFollowForwardLink(); + public abstract boolean hasNextPtNodeArray(); + + /** + * Opens the dictionary file and makes DictBuffer. + */ + @UsedForTesting + public abstract void openDictBuffer() throws FileNotFoundException, IOException; + @UsedForTesting + public abstract boolean isDictBufferOpen(); + + // Constants for DictionaryBufferFactory. + public static final int USE_READONLY_BYTEBUFFER = 0x01000000; + public static final int USE_BYTEARRAY = 0x02000000; + public static final int USE_WRITABLE_BYTEBUFFER = 0x03000000; + public static final int MASK_DICTBUFFER = 0x0F000000; + + public interface DictionaryBufferFactory { + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException; + } + + /** + * Creates DictionaryBuffer using a ByteBuffer + * + * This class uses less memory than DictionaryBufferFromByteArrayFactory, + * but doesn't perform as fast. + * When operating on a big dictionary, this class is preferred. + */ + public static final class DictionaryBufferFromReadOnlyByteBufferFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + ByteBuffer buffer = null; + try { + inStream = new FileInputStream(file); + buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY, + 0, file.length()); + } finally { + if (inStream != null) { + inStream.close(); + } + } + if (buffer != null) { + return new BinaryDictDecoderUtils.ByteBufferDictBuffer(buffer); + } + return null; + } + } + + /** + * Creates DictionaryBuffer using a byte array + * + * This class performs faster than other classes, but consumes more memory. + * When operating on a small dictionary, this class is preferred. + */ + public static final class DictionaryBufferFromByteArrayFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + FileInputStream inStream = null; + try { + inStream = new FileInputStream(file); + final byte[] array = new byte[(int) file.length()]; + inStream.read(array); + return new ByteArrayDictBuffer(array); + } finally { + if (inStream != null) { + inStream.close(); + } + } + } + } + + /** + * Creates DictionaryBuffer using a writable ByteBuffer and a RandomAccessFile. + * + * This class doesn't perform as fast as other classes, + * but this class is the only option available for destructive operations (insert or delete) + * on a dictionary. + */ + @UsedForTesting + public static final class DictionaryBufferFromWritableByteBufferFactory + implements DictionaryBufferFactory { + @Override + public DictBuffer getDictionaryBuffer(final File file) + throws FileNotFoundException, IOException { + RandomAccessFile raFile = null; + ByteBuffer buffer = null; + try { + raFile = new RandomAccessFile(file, "rw"); + buffer = raFile.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, file.length()); + } finally { + if (raFile != null) { + raFile.close(); + } + } + if (buffer != null) { + return new BinaryDictDecoderUtils.ByteBufferDictBuffer(buffer); + } + return null; + } + } + + /** + * A utility class for reading a file header. + */ + protected static class HeaderReader { + protected static int readVersion(final DictBuffer dictBuffer) + throws IOException, UnsupportedFormatException { + return BinaryDictDecoderUtils.checkFormatVersion(dictBuffer); + } + + protected static int readOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedShort(); + } + + protected static int readHeaderSize(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + + protected static HashMap<String, String> readAttributes(final DictBuffer dictBuffer, + final int headerSize) { + final HashMap<String, String> attributes = new HashMap<String, String>(); + while (dictBuffer.position() < headerSize) { + // We can avoid an infinite loop here since dictBuffer.position() is always + // increased by calling CharEncoding.readString. + final String key = CharEncoding.readString(dictBuffer); + final String value = CharEncoding.readString(dictBuffer); + attributes.put(key, value); + } + dictBuffer.position(headerSize); + return attributes; + } + } + + /** + * A utility class for reading a PtNode. + */ + protected static class PtNodeReader { + protected static int readPtNodeOptionFlags(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + + protected static int readParentAddress(final DictBuffer dictBuffer, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + return BinaryDictDecoderUtils.readSInt24(dictBuffer); + } else { + return FormatSpec.NO_PARENT_ADDRESS; + } + } + + protected static int readChildrenAddress(final DictBuffer dictBuffer, final int optionFlags, + final FormatOptions formatOptions) { + if (BinaryDictIOUtils.supportsDynamicUpdate(formatOptions)) { + final int address = BinaryDictDecoderUtils.readSInt24(dictBuffer); + if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; + return address; + } else { + switch (optionFlags & FormatSpec.MASK_CHILDREN_ADDRESS_TYPE) { + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE: + return dictBuffer.readUnsignedByte(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES: + return dictBuffer.readUnsignedShort(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES: + return dictBuffer.readUnsignedInt24(); + case FormatSpec.FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS: + default: + return FormatSpec.NO_CHILDREN_ADDRESS; + } + } + } + + // Reads shortcuts and returns the read length. + protected static int readShortcut(final DictBuffer dictBuffer, + final ArrayList<WeightedString> shortcutTargets) { + final int pointerBefore = dictBuffer.position(); + dictBuffer.readUnsignedShort(); // skip the size + while (true) { + final int targetFlags = dictBuffer.readUnsignedByte(); + final String word = CharEncoding.readString(dictBuffer); + shortcutTargets.add(new WeightedString(word, + targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (targetFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return dictBuffer.position() - pointerBefore; + } + + protected static int readBigramAddresses(final DictBuffer dictBuffer, + final ArrayList<PendingAttribute> bigrams, final int baseAddress) { + int readLength = 0; + int bigramCount = 0; + while (bigramCount++ < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + final int bigramFlags = dictBuffer.readUnsignedByte(); + ++readLength; + final int sign = 0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) + ? 1 : -1; + int bigramAddress = baseAddress + readLength; + switch (bigramFlags & FormatSpec.MASK_BIGRAM_ATTR_ADDRESS_TYPE) { + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE: + bigramAddress += sign * dictBuffer.readUnsignedByte(); + readLength += 1; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES: + bigramAddress += sign * dictBuffer.readUnsignedShort(); + readLength += 2; + break; + case FormatSpec.FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES: + bigramAddress += sign * dictBuffer.readUnsignedInt24(); + readLength += 3; + break; + default: + throw new RuntimeException("Has bigrams with no address"); + } + bigrams.add(new PendingAttribute( + bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + bigramAddress)); + if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; + } + return readLength; + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java new file mode 100644 index 000000000..ea5d492d8 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DictEncoder.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; + +import java.io.IOException; + +/** + * An interface of binary dictionary encoder. + */ +public interface DictEncoder { + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException; + + public void setPosition(final int position); + public int getPosition(); + public void writePtNodeCount(final int ptNodeCount); + public void writeForwardLinkAddress(final int forwardLinkAddress); + + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict); +} diff --git a/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java new file mode 100644 index 000000000..bf3d19101 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/DynamicBinaryDictIOUtils.java @@ -0,0 +1,502 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * The utility class to help dynamic updates on the binary dictionary. + * + * All the methods in this class are static. + */ +@UsedForTesting +public final class DynamicBinaryDictIOUtils { + private static final boolean DBG = false; + private static final int MAX_JUMPS = 10000; + + private DynamicBinaryDictIOUtils() { + // This utility class is not publicly instantiable. + } + + private static int markAsDeleted(final int flags) { + return (flags & (~FormatSpec.MASK_CHILDREN_ADDRESS_TYPE)) | FormatSpec.FLAG_IS_DELETED; + } + + /** + * Delete the word from the binary file. + * + * @param dictDecoder the dict decoder. + * @param word the word we delete + * @throws IOException + * @throws UnsupportedFormatException + */ + @UsedForTesting + public static void deleteWord(final Ver3DictDecoder dictDecoder, final String word) + throws IOException, UnsupportedFormatException { + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + dictBuffer.position(0); + final FileHeader header = dictDecoder.readHeader(); + final int wordPosition = dictDecoder.getTerminalPosition(word); + if (wordPosition == FormatSpec.NOT_VALID_WORD) return; + + dictBuffer.position(wordPosition); + final int flags = dictBuffer.readUnsignedByte(); + dictBuffer.position(wordPosition); + dictBuffer.put((byte)markAsDeleted(flags)); + } + + /** + * Update a parent address in a PtNode that is referred to by ptNodeOriginAddress. + * + * @param dictBuffer the DictBuffer to write. + * @param ptNodeOriginAddress the address of the PtNode. + * @param newParentAddress the absolute address of the parent. + * @param formatOptions file format options. + */ + public static void updateParentAddress(final DictBuffer dictBuffer, + final int ptNodeOriginAddress, final int newParentAddress, + final FormatOptions formatOptions) { + final int originalPosition = dictBuffer.position(); + dictBuffer.position(ptNodeOriginAddress); + if (!formatOptions.mSupportsDynamicUpdate) { + throw new RuntimeException("this file format does not support parent addresses"); + } + final int flags = dictBuffer.readUnsignedByte(); + if (BinaryDictIOUtils.isMovedPtNode(flags, formatOptions)) { + // If the node is moved, the parent address is stored in the destination node. + // We are guaranteed to process the destination node later, so there is no need to + // update anything here. + dictBuffer.position(originalPosition); + return; + } + if (DBG) { + MakedictLog.d("update parent address flags=" + flags + ", " + ptNodeOriginAddress); + } + final int parentOffset = newParentAddress - ptNodeOriginAddress; + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, parentOffset); + dictBuffer.position(originalPosition); + } + + /** + * Update parent addresses in a node array stored at ptNodeOriginAddress. + * + * @param dictBuffer the DictBuffer to be modified. + * @param ptNodeOriginAddress the address of the node array to update. + * @param newParentAddress the address to be written. + * @param formatOptions file format options. + */ + public static void updateParentAddresses(final DictBuffer dictBuffer, + final int ptNodeOriginAddress, final int newParentAddress, + final FormatOptions formatOptions) { + final int originalPosition = dictBuffer.position(); + dictBuffer.position(ptNodeOriginAddress); + do { + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + for (int i = 0; i < count; ++i) { + updateParentAddress(dictBuffer, dictBuffer.position(), newParentAddress, + formatOptions); + BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); + } + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); + dictBuffer.position(forwardLinkAddress); + } while (formatOptions.mSupportsDynamicUpdate + && dictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); + dictBuffer.position(originalPosition); + } + + /** + * Update a children address in a PtNode that is addressed by ptNodeOriginAddress. + * + * @param dictBuffer the DictBuffer to write. + * @param ptNodeOriginAddress the address of the PtNode. + * @param newChildrenAddress the absolute address of the child. + * @param formatOptions file format options. + */ + public static void updateChildrenAddress(final DictBuffer dictBuffer, + final int ptNodeOriginAddress, final int newChildrenAddress, + final FormatOptions formatOptions) { + final int originalPosition = dictBuffer.position(); + dictBuffer.position(ptNodeOriginAddress); + final int flags = dictBuffer.readUnsignedByte(); + final int parentAddress = BinaryDictDecoderUtils.readParentAddress(dictBuffer, + formatOptions); + BinaryDictIOUtils.skipString(dictBuffer, (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS) != 0); + if ((flags & FormatSpec.FLAG_IS_TERMINAL) != 0) dictBuffer.readUnsignedByte(); + final int childrenOffset = newChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS + ? FormatSpec.NO_CHILDREN_ADDRESS : newChildrenAddress - dictBuffer.position(); + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, childrenOffset); + dictBuffer.position(originalPosition); + } + + /** + * Helper method to move a PtNode to the tail of the file. + */ + private static int movePtNode(final OutputStream destination, + final DictBuffer dictBuffer, final PtNodeInfo info, + final int nodeArrayOriginAddress, final int oldNodeAddress, + final FormatOptions formatOptions) throws IOException { + updateParentAddress(dictBuffer, oldNodeAddress, dictBuffer.limit() + 1, formatOptions); + dictBuffer.position(oldNodeAddress); + final int currentFlags = dictBuffer.readUnsignedByte(); + dictBuffer.position(oldNodeAddress); + dictBuffer.put((byte)(FormatSpec.FLAG_IS_MOVED | (currentFlags + & (~FormatSpec.MASK_MOVE_AND_DELETE_FLAG)))); + int size = FormatSpec.PTNODE_FLAGS_SIZE; + updateForwardLink(dictBuffer, nodeArrayOriginAddress, dictBuffer.limit(), formatOptions); + size += BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { info }); + return size; + } + + @SuppressWarnings("unused") + private static void updateForwardLink(final DictBuffer dictBuffer, + final int nodeArrayOriginAddress, final int newNodeArrayAddress, + final FormatOptions formatOptions) { + dictBuffer.position(nodeArrayOriginAddress); + int jumpCount = 0; + while (jumpCount++ < MAX_JUMPS) { + final int count = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + for (int i = 0; i < count; ++i) { + BinaryDictIOUtils.skipPtNode(dictBuffer, formatOptions); + } + final int forwardLinkAddress = dictBuffer.readUnsignedInt24(); + if (forwardLinkAddress == FormatSpec.NO_FORWARD_LINK_ADDRESS) { + dictBuffer.position(dictBuffer.position() - FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeArrayAddress); + return; + } + dictBuffer.position(forwardLinkAddress); + } + if (DBG && jumpCount >= MAX_JUMPS) { + throw new RuntimeException("too many jumps, probably a bug."); + } + } + + /** + * Move a PtNode that is referred to by oldPtNodeOrigin to the tail of the file, and set the + * children address to the byte after the PtNode. + * + * @param fileEndAddress the address of the tail of the file. + * @param codePoints the characters to put inside the PtNode. + * @param length how many code points to read from codePoints. + * @param flags the flags for this PtNode. + * @param frequency the frequency of this terminal. + * @param parentAddress the address of the parent PtNode of this PtNode. + * @param shortcutTargets the shortcut targets for this PtNode. + * @param bigrams the bigrams for this PtNode. + * @param destination the stream representing the tail of the file. + * @param dictBuffer the DictBuffer representing the (constant-size) body of the file. + * @param oldPtNodeArrayOrigin the origin of the old PtNode array this PtNode was a part of. + * @param oldPtNodeOrigin the old origin where this PtNode used to be stored. + * @param formatOptions format options for this dictionary. + * @return the size written, in bytes. + * @throws IOException if the file can't be accessed + */ + private static int movePtNode(final int fileEndAddress, final int[] codePoints, + final int length, final int flags, final int frequency, final int parentAddress, + final ArrayList<WeightedString> shortcutTargets, + final ArrayList<PendingAttribute> bigrams, final OutputStream destination, + final DictBuffer dictBuffer, final int oldPtNodeArrayOrigin, + final int oldPtNodeOrigin, final FormatOptions formatOptions) throws IOException { + int size = 0; + final int newPtNodeOrigin = fileEndAddress + 1; + final int[] writtenCharacters = Arrays.copyOfRange(codePoints, 0, length); + final PtNodeInfo tmpInfo = new PtNodeInfo(newPtNodeOrigin, -1 /* endAddress */, + flags, writtenCharacters, frequency, parentAddress, FormatSpec.NO_CHILDREN_ADDRESS, + shortcutTargets, bigrams); + size = BinaryDictIOUtils.computePtNodeSize(tmpInfo, formatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newPtNodeOrigin, newPtNodeOrigin + size, + flags, writtenCharacters, frequency, parentAddress, + fileEndAddress + 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE, shortcutTargets, + bigrams); + movePtNode(destination, dictBuffer, newInfo, oldPtNodeArrayOrigin, oldPtNodeOrigin, + formatOptions); + return 1 + size + FormatSpec.FORWARD_LINK_ADDRESS_SIZE; + } + + /** + * Insert a word into a binary dictionary. + * + * @param dictDecoder the dict decoder. + * @param destination a stream to the underlying file, with the pointer at the end of the file. + * @param word the word to insert. + * @param frequency the frequency of the new word. + * @param bigramStrings bigram list, or null if none. + * @param shortcuts shortcut list, or null if none. + * @param isBlackListEntry whether this should be a blacklist entry. + * @throws IOException if the file can't be accessed. + * @throws UnsupportedFormatException if the existing dictionary is in an unexpected format. + */ + // TODO: Support batch insertion. + // TODO: Remove @UsedForTesting once UserHistoryDictionary is implemented by BinaryDictionary. + @UsedForTesting + public static void insertWord(final Ver3DictDecoder dictDecoder, + final OutputStream destination, final String word, final int frequency, + final ArrayList<WeightedString> bigramStrings, + final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, + final boolean isBlackListEntry) + throws IOException, UnsupportedFormatException { + final ArrayList<PendingAttribute> bigrams = new ArrayList<PendingAttribute>(); + final DictBuffer dictBuffer = dictDecoder.getDictBuffer(); + if (bigramStrings != null) { + for (final WeightedString bigram : bigramStrings) { + int position = dictDecoder.getTerminalPosition(bigram.mWord); + if (position == FormatSpec.NOT_VALID_WORD) { + // TODO: figure out what is the correct thing to do here. + } else { + bigrams.add(new PendingAttribute(bigram.mFrequency, position)); + } + } + } + + final boolean isTerminal = true; + final boolean hasBigrams = !bigrams.isEmpty(); + final boolean hasShortcuts = shortcuts != null && !shortcuts.isEmpty(); + + // find the insert position of the word. + if (dictBuffer.position() != 0) dictBuffer.position(0); + final FileHeader fileHeader = dictDecoder.readHeader(); + + int wordPos = 0, address = dictBuffer.position(), nodeOriginAddress = dictBuffer.position(); + final int[] codePoints = FusionDictionary.getCodePoints(word); + final int wordLen = codePoints.length; + + for (int depth = 0; depth < Constants.DICTIONARY_MAX_WORD_LENGTH; ++depth) { + if (wordPos >= wordLen) break; + nodeOriginAddress = dictBuffer.position(); + int nodeParentAddress = -1; + final int ptNodeCount = BinaryDictDecoderUtils.readPtNodeCount(dictBuffer); + boolean foundNextNode = false; + + for (int i = 0; i < ptNodeCount; ++i) { + address = dictBuffer.position(); + final PtNodeInfo currentInfo = dictDecoder.readPtNode(address, + fileHeader.mFormatOptions); + final boolean isMovedNode = BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, + fileHeader.mFormatOptions); + if (isMovedNode) continue; + nodeParentAddress = (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) + ? FormatSpec.NO_PARENT_ADDRESS : currentInfo.mParentAddress + address; + boolean matched = true; + for (int p = 0; p < currentInfo.mCharacters.length; ++p) { + if (wordPos + p >= wordLen) { + /* + * splitting + * before + * abcd - ef + * + * insert "abc" + * + * after + * abc - d - ef + */ + final int newNodeAddress = dictBuffer.limit(); + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, + isTerminal, 0, hasShortcuts, hasBigrams, false /* isNotAWord */, + false /* isBlackListEntry */, fileHeader.mFormatOptions); + int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, flags, + frequency, nodeParentAddress, shortcuts, bigrams, destination, + dictBuffer, nodeOriginAddress, address, fileHeader.mFormatOptions); + + final int[] characters2 = Arrays.copyOfRange(currentInfo.mCharacters, p, + currentInfo.mCharacters.length); + if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, + newNodeAddress + written + 1, fileHeader.mFormatOptions); + } + final PtNodeInfo newInfo2 = new PtNodeInfo( + newNodeAddress + written + 1, -1 /* endAddress */, + currentInfo.mFlags, characters2, currentInfo.mFrequency, + newNodeAddress + 1, currentInfo.mChildrenAddress, + currentInfo.mShortcutTargets, currentInfo.mBigrams); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo2 }); + return; + } else if (codePoints[wordPos + p] != currentInfo.mCharacters[p]) { + if (p > 0) { + /* + * splitting + * before + * ab - cd + * + * insert "ac" + * + * after + * a - b - cd + * | + * - c + */ + + final int newNodeAddress = dictBuffer.limit(); + final int childrenAddress = currentInfo.mChildrenAddress; + + // move prefix + final int prefixFlags = BinaryDictEncoderUtils.makePtNodeFlags(p > 1, + false /* isTerminal */, 0 /* childrenAddressSize*/, + false /* hasShortcut */, false /* hasBigrams */, + false /* isNotAWord */, false /* isBlackListEntry */, + fileHeader.mFormatOptions); + int written = movePtNode(newNodeAddress, currentInfo.mCharacters, p, + prefixFlags, -1 /* frequency */, nodeParentAddress, null, null, + destination, dictBuffer, nodeOriginAddress, address, + fileHeader.mFormatOptions); + + final int[] suffixCharacters = Arrays.copyOfRange( + currentInfo.mCharacters, p, currentInfo.mCharacters.length); + if (currentInfo.mChildrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + updateParentAddresses(dictBuffer, currentInfo.mChildrenAddress, + newNodeAddress + written + 1, fileHeader.mFormatOptions); + } + final int suffixFlags = BinaryDictEncoderUtils.makePtNodeFlags( + suffixCharacters.length > 1, + (currentInfo.mFlags & FormatSpec.FLAG_IS_TERMINAL) != 0, + 0 /* childrenAddressSize */, + (currentInfo.mFlags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS) + != 0, + (currentInfo.mFlags & FormatSpec.FLAG_HAS_BIGRAMS) != 0, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo suffixInfo = new PtNodeInfo( + newNodeAddress + written + 1, -1 /* endAddress */, suffixFlags, + suffixCharacters, currentInfo.mFrequency, newNodeAddress + 1, + currentInfo.mChildrenAddress, currentInfo.mShortcutTargets, + currentInfo.mBigrams); + written += BinaryDictIOUtils.computePtNodeSize(suffixInfo, + fileHeader.mFormatOptions) + 1; + + final int[] newCharacters = Arrays.copyOfRange(codePoints, wordPos + p, + codePoints.length); + final int flags = BinaryDictEncoderUtils.makePtNodeFlags( + newCharacters.length > 1, isTerminal, + 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo newInfo = new PtNodeInfo( + newNodeAddress + written, -1 /* endAddress */, flags, + newCharacters, frequency, newNodeAddress + 1, + FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); + BinaryDictIOUtils.writeNodes(destination, + new PtNodeInfo[] { suffixInfo, newInfo }); + return; + } + matched = false; + break; + } + } + + if (matched) { + if (wordPos + currentInfo.mCharacters.length == wordLen) { + // the word exists in the dictionary. + // only update the PtNode. + final int newNodeAddress = dictBuffer.limit(); + final boolean hasMultipleChars = currentInfo.mCharacters.length > 1; + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, + isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, + -1 /* endAddress */, flags, currentInfo.mCharacters, frequency, + nodeParentAddress, currentInfo.mChildrenAddress, shortcuts, + bigrams); + movePtNode(destination, dictBuffer, newInfo, nodeOriginAddress, address, + fileHeader.mFormatOptions); + return; + } + wordPos += currentInfo.mCharacters.length; + if (currentInfo.mChildrenAddress == FormatSpec.NO_CHILDREN_ADDRESS) { + /* + * found the prefix of the word. + * make new PtNode and link to the PtNode from this PtNode. + * + * before + * ab - cd + * + * insert "abcde" + * + * after + * ab - cd - e + */ + final int newNodeArrayAddress = dictBuffer.limit(); + updateChildrenAddress(dictBuffer, address, newNodeArrayAddress, + fileHeader.mFormatOptions); + final int newNodeAddress = newNodeArrayAddress + 1; + final boolean hasMultipleChars = (wordLen - wordPos) > 1; + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(hasMultipleChars, + isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress, -1, flags, + characters, frequency, address, FormatSpec.NO_CHILDREN_ADDRESS, + shortcuts, bigrams); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[] { newInfo }); + return; + } + dictBuffer.position(currentInfo.mChildrenAddress); + foundNextNode = true; + break; + } + } + + if (foundNextNode) continue; + + // reached the end of the array. + final int linkAddressPosition = dictBuffer.position(); + int nextLink = dictBuffer.readUnsignedInt24(); + if ((nextLink & FormatSpec.MSB24) != 0) { + nextLink = -(nextLink & FormatSpec.SINT24_MAX); + } + if (nextLink == FormatSpec.NO_FORWARD_LINK_ADDRESS) { + /* + * expand this node. + * + * before + * ab - cd + * + * insert "abef" + * + * after + * ab - cd + * | + * - ef + */ + + // change the forward link address. + final int newNodeAddress = dictBuffer.limit(); + dictBuffer.position(linkAddressPosition); + BinaryDictIOUtils.writeSInt24ToBuffer(dictBuffer, newNodeAddress); + + final int[] characters = Arrays.copyOfRange(codePoints, wordPos, wordLen); + final int flags = BinaryDictEncoderUtils.makePtNodeFlags(characters.length > 1, + isTerminal, 0 /* childrenAddressSize */, hasShortcuts, hasBigrams, + isNotAWord, isBlackListEntry, fileHeader.mFormatOptions); + final PtNodeInfo newInfo = new PtNodeInfo(newNodeAddress + 1, + -1 /* endAddress */, flags, characters, frequency, nodeParentAddress, + FormatSpec.NO_CHILDREN_ADDRESS, shortcuts, bigrams); + BinaryDictIOUtils.writeNodes(destination, new PtNodeInfo[]{ newInfo }); + return; + } else { + depth--; + dictBuffer.position(nextLink); + } + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java index feadcda76..849bff050 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java +++ b/java/src/com/android/inputmethod/latin/makedict/FormatSpec.java @@ -16,30 +16,68 @@ package com.android.inputmethod.latin.makedict; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.Constants; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import java.io.File; + /** * Dictionary File Format Specification. */ public final class FormatSpec { /* - * Array of Node(FusionDictionary.Node) layout is as follows: + * File header layout is as follows: + * + * v | + * e | MAGIC_NUMBER + version of the file format, 2 bytes. + * r | + * sion + * + * o | + * p | not used 4 bits + * t | has bigrams ? 1 bit, 1 = yes, 0 = no : CONTAINS_BIGRAMS_FLAG + * i | FRENCH_LIGATURE_PROCESSING_FLAG + * o | supports dynamic updates ? 1 bit, 1 = yes, 0 = no : SUPPORTS_DYNAMIC_UPDATE + * n | GERMAN_UMLAUT_PROCESSING_FLAG + * f | + * lags * - * g | - * r | the number of groups, 1 or 2 bytes. - * o | 1 byte = bbbbbbbb match - * u | case 1xxxxxxx => xxxxxxx << 8 + next byte - * p | otherwise => bbbbbbbb - * c | - * ount + * h | + * e | size of the file header, 4bytes + * a | including the size of the magic number, the option flags and the header size + * d | + * ersize + * + * | attributes list * - * g | - * r | sequence of groups, - * o | the layout of each group is described below. - * u | - * ps + * attributes list is: + * <key> = | string of characters at the char format described below, with the terminator used + * | to signal the end of the string. + * <value> = | string of characters at the char format described below, with the terminator used + * | to signal the end of the string. + * if the size of already read < headersize, goto key. + * + */ + + /* + * Node array (FusionDictionary.PtNodeArray) layout is as follows: + * + * n | + * o | the number of PtNodes, 1 or 2 bytes. + * d | 1 byte = bbbbbbbb match + * e | case 1xxxxxxx => xxxxxxx << 8 + next byte + * c | otherwise => bbbbbbbb + * o | + * unt + * + * n | + * o | sequence of PtNodes, + * d | the layout of each PtNode is described below. + * e | + * s * * f | * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) @@ -51,19 +89,19 @@ public final class FormatSpec { * linkaddress */ - /* Node(CharGroup) layout is as follows: + /* Node (FusionDictionary.PtNode) layout is as follows: * | IF !SUPPORTS_DYNAMIC_UPDATE - * | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE - * | 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS - * f | 01 = 1 byte : FLAG_GROUP_ADDRESS_TYPE_ONEBYTE - * l | 10 = 2 bytes : FLAG_GROUP_ADDRESS_TYPE_TWOBYTES - * a | 11 = 3 bytes : FLAG_GROUP_ADDRESS_TYPE_THREEBYTES + * | addressType xx : mask with MASK_CHILDREN_ADDRESS_TYPE + * | 2 bits, 00 = no children : FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS + * f | 01 = 1 byte : FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE + * l | 10 = 2 bytes : FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES + * a | 11 = 3 bytes : FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES * g | ELSE - * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED - * | This must be the same as FLAG_GROUP_ADDRESS_TYPE_THREEBYTES - * | 01 = yes : FLAG_IS_MOVED + * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED + * | This must be the same as FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES + * | 01 = yes : FLAG_IS_MOVED * | the new address is stored in the same place as the parent address - * | is deleted? 10 = yes : FLAG_IS_DELETED + * | is deleted? 10 = yes : FLAG_IS_DELETED * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS @@ -77,11 +115,13 @@ public final class FormatSpec { * e | 1 byte = bbbbbbbb match * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte - * a | - * ddress + * a | This address is relative to the head of the PtNode. + * d | If the node doesn't have a parent, this field is set to 0. + * d | + * ress * * c | IF FLAG_HAS_MULTIPLE_CHARS - * h | char, char, char, char n * (1 or 3 bytes) : use CharGroupInfo for i/o helpers + * h | char, char, char, char n * (1 or 3 bytes) : use PtNodeInfo for i/o helpers * a | end 1 byte, = 0 * r | ELSE * s | char 1 or 3 bytes @@ -92,17 +132,23 @@ public final class FormatSpec { * e | frequency 1 byte * q | * - * c | IF 00 = FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = addressType - * h | // nothing - * i | ELSIF 01 = FLAG_GROUP_ADDRESS_TYPE_ONEBYTE == addressType - * l | children address, 1 byte - * d | ELSIF 10 = FLAG_GROUP_ADDRESS_TYPE_TWOBYTES == addressType - * r | children address, 2 bytes - * e | ELSE // 11 = FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = addressType - * n | children address, 3 bytes - * A | END - * d - * dress + * c | IF SUPPORTS_DYNAMIC_UPDATE + * h | children address, 3 bytes + * i | 1 byte = bbbbbbbb match + * l | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) + * d | otherwise => (bbbbbbbb<<16) + (next byte << 8) + next byte + * r | if this node doesn't have children, this field is set to 0. + * e | (see BinaryDictEncoderUtils#writeVariableSignedAddress) + * n | ELSIF 00 = FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS == addressType + * a | // nothing + * d | ELSIF 01 = FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE == addressType + * d | children address, 1 byte + * r | ELSIF 10 = FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES == addressType + * e | children address, 2 bytes + * s | ELSE // 11 = FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = addressType + * s | children address, 3 bytes + * | END + * | This address is relative to the position of this field. * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list @@ -121,42 +167,43 @@ public final class FormatSpec { * characters which should never happen anyway (and still work, but take 3 bytes). * * bigram address list is: - * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT - * | addressSign = 1 bit, : FLAG_ATTRIBUTE_OFFSET_NEGATIVE + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT + * | addressSign = 1 bit, : FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE * | 1 = must take -address, 0 = must take +address - * | xx : mask with MASK_ATTRIBUTE_ADDRESS_TYPE - * | addressFormat = 2 bits, 00 = unused : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE - * | 01 = 1 byte : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE - * | 10 = 2 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES - * | 11 = 3 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES - * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY - * <address> | IF (01 == FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE == addressFormat) + * | xx : mask with MASK_BIGRAM_ATTR_ADDRESS_TYPE + * | addressFormat = 2 bits, 00 = unused : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 01 = 1 byte : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE + * | 10 = 2 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES + * | 11 = 3 bytes : FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY + * <address> | IF (01 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE == addressFormat) * | read 1 byte, add top 4 bits - * | ELSIF (10 == FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES == addressFormat) + * | ELSIF (10 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES == addressFormat) * | read 2 bytes, add top 4 bits - * | ELSE // 11 == FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES == addressFormat + * | ELSE // 11 == FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES == addressFormat * | read 3 bytes, add top 4 bits * | END - * | if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE) then address = -address - * if (FLAG_ATTRIBUTE_HAS_NEXT) goto bigram_and_shortcut_address_list_is + * | if (FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE) then address = -address + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT) goto bigram_and_shortcut_address_list_is * * shortcut string list is: - * <byte size> = GROUP_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. - * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT + * <byte size> = PTNODE_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. + * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT * | reserved = 3 bits, must be 0 - * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY + * | 4 bits : frequency : mask with FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY * <shortcut> = | string of characters at the char format described above, with the terminator * | used to signal the end of the string. - * if (FLAG_ATTRIBUTE_HAS_NEXT goto flags + * if (FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT goto flags */ - static final int VERSION_1_MAGIC_NUMBER = 0x78B1; - public static final int VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; - static final int MINIMUM_SUPPORTED_VERSION = 1; - static final int MAXIMUM_SUPPORTED_VERSION = 3; + public static final int MAGIC_NUMBER = 0x9BC13AFE; + static final int MINIMUM_SUPPORTED_VERSION = 2; + static final int MAXIMUM_SUPPORTED_VERSION = 4; static final int NOT_A_VERSION_NUMBER = -1; - static final int FIRST_VERSION_WITH_HEADER_SIZE = 2; static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; + static final int FIRST_VERSION_WITH_TERMINAL_ID = 4; + static final int VERSION3 = 3; + static final int VERSION4 = 4; // These options need to be the same numeric values as the one in the native reading code. static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; @@ -173,11 +220,11 @@ public final class FormatSpec { static final int FORWARD_LINK_ADDRESS_SIZE = 3; // These flags are used only in the static dictionary. - static final int MASK_GROUP_ADDRESS_TYPE = 0xC0; - static final int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; - static final int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; - static final int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; - static final int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; + static final int MASK_CHILDREN_ADDRESS_TYPE = 0xC0; + static final int FLAG_CHILDREN_ADDRESS_TYPE_NOADDRESS = 0x00; + static final int FLAG_CHILDREN_ADDRESS_TYPE_ONEBYTE = 0x40; + static final int FLAG_CHILDREN_ADDRESS_TYPE_TWOBYTES = 0x80; + static final int FLAG_CHILDREN_ADDRESS_TYPE_THREEBYTES = 0xC0; static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; @@ -194,32 +241,42 @@ public final class FormatSpec { static final int FLAG_IS_NOT_MOVED = 0x80 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE; static final int FLAG_IS_DELETED = 0x80; - static final int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; - static final int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; - static final int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; - static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; - static final int FLAG_ATTRIBUTE_FREQUENCY = 0x0F; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT = 0x80; + static final int FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE = 0x40; + static final int MASK_BIGRAM_ATTR_ADDRESS_TYPE = 0x30; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_ONEBYTE = 0x10; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_TWOBYTES = 0x20; + static final int FLAG_BIGRAM_ATTR_ADDRESS_TYPE_THREEBYTES = 0x30; + static final int FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY = 0x0F; + + static final int PTNODE_CHARACTERS_TERMINATOR = 0x1F; - static final int GROUP_CHARACTERS_TERMINATOR = 0x1F; + static final int PTNODE_TERMINATOR_SIZE = 1; + static final int PTNODE_FLAGS_SIZE = 1; + static final int PTNODE_FREQUENCY_SIZE = 1; + static final int PTNODE_TERMINAL_ID_SIZE = 4; + static final int PTNODE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_ATTRIBUTE_FLAGS_SIZE = 1; + static final int PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; + static final int PTNODE_SHORTCUT_LIST_SIZE_SIZE = 2; - static final int GROUP_TERMINATOR_SIZE = 1; - static final int GROUP_FLAGS_SIZE = 1; - static final int GROUP_FREQUENCY_SIZE = 1; - static final int GROUP_MAX_ADDRESS_SIZE = 3; - static final int GROUP_ATTRIBUTE_FLAGS_SIZE = 1; - static final int GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; - static final int GROUP_SHORTCUT_LIST_SIZE_SIZE = 2; + // These values are used only by version 4 or later. + static final String TRIE_FILE_EXTENSION = ".trie"; + static final String FREQ_FILE_EXTENSION = ".freq"; + // tat = Terminal Address Table + static final String TERMINAL_ADDRESS_TABLE_FILE_EXTENSION = ".tat"; + static final int FREQUENCY_AND_FLAGS_SIZE = 2; + static final int TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE = 3; static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; static final int INVALID_CHARACTER = -1; - static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127 - static final int MAX_CHARGROUPS_IN_A_NODE = 0x7FFF; // 32767 - static final int MAX_BIGRAMS_IN_A_GROUP = 10000; + static final int MAX_PTNODES_FOR_ONE_BYTE_PTNODE_COUNT = 0x7F; // 127 + static final int MAX_PTNODES_IN_A_PT_NODE_ARRAY = 0x7FFF; // 32767 + static final int MAX_BIGRAMS_IN_A_PTNODE = 10000; + static final int MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE = 0xFFFF; static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; @@ -230,15 +287,26 @@ public final class FormatSpec { static final int NOT_VALID_WORD = -99; static final int SIGNED_CHILDREN_ADDRESS_SIZE = 3; + static final int UINT8_MAX = 0xFF; + static final int UINT16_MAX = 0xFFFF; + static final int UINT24_MAX = 0xFFFFFF; + static final int SINT24_MAX = 0x7FFFFF; + static final int MSB8 = 0x80; + static final int MSB24 = 0x800000; + /** * Options about file format. */ public static final class FormatOptions { public final int mVersion; public final boolean mSupportsDynamicUpdate; + public final boolean mHasTerminalId; + @UsedForTesting public FormatOptions(final int version) { this(version, false); } + + @UsedForTesting public FormatOptions(final int version, final boolean supportsDynamicUpdate) { mVersion = version; if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) { @@ -246,6 +314,7 @@ public final class FormatSpec { + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); } mSupportsDynamicUpdate = supportsDynamicUpdate; + mHasTerminalId = (version >= FIRST_VERSION_WITH_TERMINAL_ID); } } @@ -256,6 +325,12 @@ public final class FormatSpec { public final int mHeaderSize; public final DictionaryOptions mDictionaryOptions; public final FormatOptions mFormatOptions; + // Note that these are corresponding definitions in native code in latinime::HeaderPolicy + // and latinime::HeaderReadWriteUtils. + public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE"; + public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE"; + public static final String ATTRIBUTE_VALUE_TRUE = "1"; + private static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale"; private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary"; @@ -290,6 +365,36 @@ public final class FormatSpec { } } + /** + * Returns new dictionary decoder. + * + * @param dictFile the dictionary file. + * @param bufferType The type of buffer, as one of USE_* in DictDecoder. + * @return new dictionary decoder if the dictionary file exists, otherwise null. + */ + public static DictDecoder getDictDecoder(final File dictFile, final int bufferType) { + if (dictFile.isDirectory()) { + return new Ver4DictDecoder(dictFile, bufferType); + } else if (dictFile.isFile()) { + return new Ver3DictDecoder(dictFile, bufferType); + } + return null; + } + + public static DictDecoder getDictDecoder(final File dictFile, + final DictionaryBufferFactory factory) { + if (dictFile.isDirectory()) { + return new Ver4DictDecoder(dictFile, factory); + } else if (dictFile.isFile()) { + return new Ver3DictDecoder(dictFile, factory); + } + return null; + } + + public static DictDecoder getDictDecoder(final File dictFile) { + return getDictDecoder(dictFile, DictDecoder.USE_READONLY_BYTEBUFFER); + } + private FormatSpec() { // This utility class is not publicly instantiable. } diff --git a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index 118dc22b8..be653feec 100644 --- a/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -37,15 +37,15 @@ public final class FusionDictionary implements Iterable<Word> { private static int CHARACTER_NOT_FOUND_INDEX = -1; /** - * A node of the dictionary, containing several CharGroups. + * A node array of the dictionary, containing several PtNodes. * - * A node is but an ordered array of CharGroups, which essentially contain all the + * A PtNodeArray is but an ordered array of PtNodes, which essentially contain all the * real information. * This class also contains fields to cache size and address, to help with binary * generation. */ - public static final class Node { - ArrayList<CharGroup> mData; + public static final class PtNodeArray { + ArrayList<PtNode> mData; // To help with binary generation int mCachedSize = Integer.MIN_VALUE; // mCachedAddressBefore/AfterUpdate are helpers for binary dictionary generation. They @@ -57,10 +57,10 @@ public final class FusionDictionary implements Iterable<Word> { int mCachedAddressAfterUpdate = Integer.MIN_VALUE; int mCachedParentAddress = 0; - public Node() { - mData = new ArrayList<CharGroup>(); + public PtNodeArray() { + mData = new ArrayList<PtNode>(); } - public Node(ArrayList<CharGroup> data) { + public PtNodeArray(ArrayList<PtNode> data) { mData = data; } } @@ -93,24 +93,26 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * A group of characters, with a frequency, shortcut targets, bigrams, and children. + * PtNode is a group of characters, with a frequency, shortcut targets, bigrams, and children + * (Pt means Patricia Trie). * - * This is the central class of the in-memory representation. A CharGroup is what can + * This is the central class of the in-memory representation. A PtNode is what can * be seen as a traditional "trie node", except it can hold several characters at the - * same time. A CharGroup essentially represents one or several characters in the middle - * of the trie trie; as such, it can be a terminal, and it can have children. - * In this in-memory representation, whether the CharGroup is a terminal or not is represented + * same time. A PtNode essentially represents one or several characters in the middle + * of the trie tree; as such, it can be a terminal, and it can have children. + * In this in-memory representation, whether the PtNode is a terminal or not is represented * in the frequency, where NOT_A_TERMINAL (= -1) means this is not a terminal and any other * value is the frequency of this terminal. A terminal may have non-null shortcuts and/or * bigrams, but a non-terminal may not. Moreover, children, if present, are null. */ - public static final class CharGroup { + public static final class PtNode { public static final int NOT_A_TERMINAL = -1; final int mChars[]; ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; int mFrequency; // NOT_A_TERMINAL == mFrequency indicates this is not a terminal. - Node mChildren; + int mTerminalId; // NOT_A_TERMINAL == mTerminalId indicates this is not a terminal. + PtNodeArray mChildren; boolean mIsNotAWord; // Only a shortcut boolean mIsBlacklistEntry; // mCachedSize and mCachedAddressBefore/AfterUpdate are helpers for binary dictionary @@ -119,15 +121,16 @@ public final class FusionDictionary implements Iterable<Word> { // same time. Updating will update the AfterUpdate value, and the code will move them // to BeforeUpdate before the next update pass. // The update process does not need two versions of mCachedSize. - int mCachedSize; // The size, in bytes, of this char group. - int mCachedAddressBeforeUpdate; // The address of this char group (before update) - int mCachedAddressAfterUpdate; // The address of this char group (after update) + int mCachedSize; // The size, in bytes, of this PtNode. + int mCachedAddressBeforeUpdate; // The address of this PtNode (before update) + int mCachedAddressAfterUpdate; // The address of this PtNode (after update) - public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, final boolean isNotAWord, final boolean isBlacklistEntry) { mChars = chars; mFrequency = frequency; + mTerminalId = frequency; mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = null; @@ -135,9 +138,10 @@ public final class FusionDictionary implements Iterable<Word> { mIsBlacklistEntry = isBlacklistEntry; } - public CharGroup(final int[] chars, final ArrayList<WeightedString> shortcutTargets, + public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final int frequency, - final boolean isNotAWord, final boolean isBlacklistEntry, final Node children) { + final boolean isNotAWord, final boolean isBlacklistEntry, + final PtNodeArray children) { mChars = chars; mFrequency = frequency; mShortcutTargets = shortcutTargets; @@ -147,13 +151,17 @@ public final class FusionDictionary implements Iterable<Word> { mIsBlacklistEntry = isBlacklistEntry; } - public void addChild(CharGroup n) { + public void addChild(PtNode n) { if (null == mChildren) { - mChildren = new Node(); + mChildren = new PtNodeArray(); } mChildren.mData.add(n); } + public int getTerminalId() { + return mTerminalId; + } + public boolean isTerminal() { return NOT_A_TERMINAL != mFrequency; } @@ -244,7 +252,7 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Updates the CharGroup with the given properties. Adds the shortcut and bigram lists to + * Updates the PtNode with the given properties. Adds the shortcut and bigram lists to * the existing ones if any. Note: unigram, bigram, and shortcut frequencies are only * updated if they are higher than the existing ones. */ @@ -344,10 +352,10 @@ public final class FusionDictionary implements Iterable<Word> { } public final DictionaryOptions mOptions; - public final Node mRoot; + public final PtNodeArray mRootNodeArray; - public FusionDictionary(final Node root, final DictionaryOptions options) { - mRoot = root; + public FusionDictionary(final PtNodeArray rootNodeArray, final DictionaryOptions options) { + mRootNodeArray = rootNodeArray; mOptions = options; } @@ -406,13 +414,13 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Sanity check for a node. + * Sanity check for a PtNode array. * - * This method checks that all CharGroups in a node are ordered as expected. + * This method checks that all PtNodes in a node array are ordered as expected. * If they are, nothing happens. If they aren't, an exception is thrown. */ - private void checkStack(Node node) { - ArrayList<CharGroup> stack = node.mData; + private void checkStack(PtNodeArray ptNodeArray) { + ArrayList<PtNode> stack = ptNodeArray.mData; int lastValue = -1; for (int i = 0; i < stack.size(); ++i) { int currentValue = stack.get(i).mChars[0]; @@ -431,18 +439,18 @@ public final class FusionDictionary implements Iterable<Word> { * @param frequency the bigram frequency */ public void setBigram(final String word1, final String word2, final int frequency) { - CharGroup charGroup = findWordInTree(mRoot, word1); - if (charGroup != null) { - final CharGroup charGroup2 = findWordInTree(mRoot, word2); - if (charGroup2 == null) { + PtNode ptNode = findWordInTree(mRootNodeArray, word1); + if (ptNode != null) { + final PtNode ptNode2 = findWordInTree(mRootNodeArray, word2); + if (ptNode2 == null) { add(getCodePoints(word2), 0, null, false /* isNotAWord */, false /* isBlacklistEntry */); - // The chargroup for the first word may have moved by the above insertion, + // The PtNode for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been - // a cutting point until now. In this case, we need to refresh charGroup. - charGroup = findWordInTree(mRoot, word1); + // a cutting point until now. In this case, we need to refresh ptNode. + ptNode = findWordInTree(mRootNodeArray, word1); } - charGroup.addBigram(word2, frequency); + ptNode.addBigram(word2, frequency); } else { throw new RuntimeException("First word of bigram not found"); } @@ -469,92 +477,91 @@ public final class FusionDictionary implements Iterable<Word> { return; } - Node currentNode = mRoot; + PtNodeArray currentNodeArray = mRootNodeArray; int charIndex = 0; - CharGroup currentGroup = null; + PtNode currentPtNode = null; int differentCharIndex = 0; // Set by the loop to the index of the char that differs - int nodeIndex = findIndexOfChar(mRoot, word[charIndex]); + int nodeIndex = findIndexOfChar(mRootNodeArray, word[charIndex]); while (CHARACTER_NOT_FOUND_INDEX != nodeIndex) { - currentGroup = currentNode.mData.get(nodeIndex); - differentCharIndex = compareArrays(currentGroup.mChars, word, charIndex); + currentPtNode = currentNodeArray.mData.get(nodeIndex); + differentCharIndex = compareCharArrays(currentPtNode.mChars, word, charIndex); if (ARRAYS_ARE_EQUAL != differentCharIndex - && differentCharIndex < currentGroup.mChars.length) break; - if (null == currentGroup.mChildren) break; - charIndex += currentGroup.mChars.length; + && differentCharIndex < currentPtNode.mChars.length) break; + if (null == currentPtNode.mChildren) break; + charIndex += currentPtNode.mChars.length; if (charIndex >= word.length) break; - currentNode = currentGroup.mChildren; - nodeIndex = findIndexOfChar(currentNode, word[charIndex]); + currentNodeArray = currentPtNode.mChildren; + nodeIndex = findIndexOfChar(currentNodeArray, word[charIndex]); } if (CHARACTER_NOT_FOUND_INDEX == nodeIndex) { // No node at this point to accept the word. Create one. - final int insertionIndex = findInsertionIndex(currentNode, word[charIndex]); - final CharGroup newGroup = new CharGroup( - Arrays.copyOfRange(word, charIndex, word.length), + final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); + final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentNode.mData.add(insertionIndex, newGroup); - if (DBG) checkStack(currentNode); + currentNodeArray.mData.add(insertionIndex, newPtNode); + if (DBG) checkStack(currentNodeArray); } else { // There is a word with a common prefix. - if (differentCharIndex == currentGroup.mChars.length) { + if (differentCharIndex == currentPtNode.mChars.length) { if (charIndex + differentCharIndex >= word.length) { // The new word is a prefix of an existing word, but the node on which it - // should end already exists as is. Since the old CharNode was not a terminal, + // should end already exists as is. Since the old PtNode was not a terminal, // make it one by filling in its frequency and other attributes - currentGroup.update(frequency, shortcutTargets, null, isNotAWord, + currentPtNode.update(frequency, shortcutTargets, null, isNotAWord, isBlacklistEntry); } else { // The new word matches the full old word and extends past it. // We only have to create a new node and add it to the end of this. - final CharGroup newNode = new CharGroup( + final PtNode newNode = new PtNode( Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); - currentGroup.mChildren = new Node(); - currentGroup.mChildren.mData.add(newNode); + currentPtNode.mChildren = new PtNodeArray(); + currentPtNode.mChildren.mData.add(newNode); } } else { if (0 == differentCharIndex) { // Exact same word. Update the frequency if higher. This will also add the // new shortcuts to the existing shortcut list if it already exists. - currentGroup.update(frequency, shortcutTargets, null, - currentGroup.mIsNotAWord && isNotAWord, - currentGroup.mIsBlacklistEntry || isBlacklistEntry); + currentPtNode.update(frequency, shortcutTargets, null, + currentPtNode.mIsNotAWord && isNotAWord, + currentPtNode.mIsBlacklistEntry || isBlacklistEntry); } else { // Partial prefix match only. We have to replace the current node with a node // containing the current prefix and create two new ones for the tails. - Node newChildren = new Node(); - final CharGroup newOldWord = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, differentCharIndex, - currentGroup.mChars.length), currentGroup.mShortcutTargets, - currentGroup.mBigrams, currentGroup.mFrequency, - currentGroup.mIsNotAWord, currentGroup.mIsBlacklistEntry, - currentGroup.mChildren); + PtNodeArray newChildren = new PtNodeArray(); + final PtNode newOldWord = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex, + currentPtNode.mChars.length), currentPtNode.mShortcutTargets, + currentPtNode.mBigrams, currentPtNode.mFrequency, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry, + currentPtNode.mChildren); newChildren.mData.add(newOldWord); - final CharGroup newParent; + final PtNode newParent; if (charIndex + differentCharIndex >= word.length) { - newParent = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, 0, differentCharIndex), + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry, newChildren); } else { - newParent = new CharGroup( - Arrays.copyOfRange(currentGroup.mChars, 0, differentCharIndex), + newParent = new PtNode( + Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), null /* shortcutTargets */, null /* bigrams */, -1, false /* isNotAWord */, false /* isBlacklistEntry */, newChildren); - final CharGroup newWord = new CharGroup(Arrays.copyOfRange(word, + final PtNode newWord = new PtNode(Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), shortcutTargets, null /* bigrams */, frequency, isNotAWord, isBlacklistEntry); final int addIndex = word[charIndex + differentCharIndex] - > currentGroup.mChars[differentCharIndex] ? 1 : 0; + > currentPtNode.mChars[differentCharIndex] ? 1 : 0; newChildren.mData.add(addIndex, newWord); } - currentNode.mData.set(nodeIndex, newParent); + currentNodeArray.mData.set(nodeIndex, newParent); } - if (DBG) checkStack(currentNode); + if (DBG) checkStack(currentNodeArray); } } } @@ -576,7 +583,7 @@ public final class FusionDictionary implements Iterable<Word> { * @param dstOffset the offset in the right-hand side string. * @return the index at which the strings differ, or ARRAYS_ARE_EQUAL = 0 if they don't. */ - private static int compareArrays(final int[] src, final int[] dst, int dstOffset) { + private static int compareCharArrays(final int[] src, final int[] dst, int dstOffset) { // We do NOT test the first char, because we come from a method that already // tested it. for (int i = 1; i < src.length; ++i) { @@ -588,43 +595,43 @@ public final class FusionDictionary implements Iterable<Word> { } /** - * Helper class that compares and sorts two chargroups according to their + * Helper class that compares and sorts two PtNodes according to their * first element only. I repeat: ONLY the first element is considered, the rest * is ignored. * This comparator imposes orderings that are inconsistent with equals. */ - static private final class CharGroupComparator implements java.util.Comparator<CharGroup> { + static private final class PtNodeComparator implements java.util.Comparator<PtNode> { @Override - public int compare(CharGroup c1, CharGroup c2) { - if (c1.mChars[0] == c2.mChars[0]) return 0; - return c1.mChars[0] < c2.mChars[0] ? -1 : 1; + public int compare(PtNode p1, PtNode p2) { + if (p1.mChars[0] == p2.mChars[0]) return 0; + return p1.mChars[0] < p2.mChars[0] ? -1 : 1; } } - final static private CharGroupComparator CHARGROUP_COMPARATOR = new CharGroupComparator(); + final static private PtNodeComparator PTNODE_COMPARATOR = new PtNodeComparator(); /** - * Finds the insertion index of a character within a node. + * Finds the insertion index of a character within a node array. */ - private static int findInsertionIndex(final Node node, int character) { - final ArrayList<CharGroup> data = node.mData; - final CharGroup reference = new CharGroup(new int[] { character }, + private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { + final ArrayList<PtNode> data = nodeArray.mData; + final PtNode reference = new PtNode(new int[] { character }, null /* shortcutTargets */, null /* bigrams */, 0, false /* isNotAWord */, false /* isBlacklistEntry */); - int result = Collections.binarySearch(data, reference, CHARGROUP_COMPARATOR); + int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR); return result >= 0 ? result : -result - 1; } /** - * Find the index of a char in a node, if it exists. + * Find the index of a char in a node array, if it exists. * - * @param node the node to search in. + * @param nodeArray the node array to search in. * @param character the character to search for. * @return the position of the character if it's there, or CHARACTER_NOT_FOUND_INDEX = -1 else. */ - private static int findIndexOfChar(final Node node, int character) { - final int insertionIndex = findInsertionIndex(node, character); - if (node.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; - return character == node.mData.get(insertionIndex).mChars[0] ? insertionIndex + private static int findIndexOfChar(final PtNodeArray nodeArray, int character) { + final int insertionIndex = findInsertionIndex(nodeArray, character); + if (nodeArray.mData.size() <= insertionIndex) return CHARACTER_NOT_FOUND_INDEX; + return character == nodeArray.mData.get(insertionIndex).mChars[0] ? insertionIndex : CHARACTER_NOT_FOUND_INDEX; } @@ -632,35 +639,37 @@ public final class FusionDictionary implements Iterable<Word> { * Helper method to find a word in a given branch. */ @SuppressWarnings("unused") - public static CharGroup findWordInTree(Node node, final String string) { + public static PtNode findWordInTree(PtNodeArray nodeArray, final String string) { int index = 0; final StringBuilder checker = DBG ? new StringBuilder() : null; final int[] codePoints = getCodePoints(string); - CharGroup currentGroup; + PtNode currentPtNode; do { - int indexOfGroup = findIndexOfChar(node, codePoints[index]); + int indexOfGroup = findIndexOfChar(nodeArray, codePoints[index]); if (CHARACTER_NOT_FOUND_INDEX == indexOfGroup) return null; - currentGroup = node.mData.get(indexOfGroup); + currentPtNode = nodeArray.mData.get(indexOfGroup); - if (codePoints.length - index < currentGroup.mChars.length) return null; + if (codePoints.length - index < currentPtNode.mChars.length) return null; int newIndex = index; - while (newIndex < codePoints.length && newIndex - index < currentGroup.mChars.length) { - if (currentGroup.mChars[newIndex - index] != codePoints[newIndex]) return null; + while (newIndex < codePoints.length && newIndex - index < currentPtNode.mChars.length) { + if (currentPtNode.mChars[newIndex - index] != codePoints[newIndex]) return null; newIndex++; } index = newIndex; - if (DBG) checker.append(new String(currentGroup.mChars, 0, currentGroup.mChars.length)); + if (DBG) { + checker.append(new String(currentPtNode.mChars, 0, currentPtNode.mChars.length)); + } if (index < codePoints.length) { - node = currentGroup.mChildren; + nodeArray = currentPtNode.mChildren; } - } while (null != node && index < codePoints.length); + } while (null != nodeArray && index < codePoints.length); if (index < codePoints.length) return null; - if (!currentGroup.isTerminal()) return null; + if (!currentPtNode.isTerminal()) return null; if (DBG && !string.equals(checker.toString())) return null; - return currentGroup; + return currentPtNode; } /** @@ -670,22 +679,22 @@ public final class FusionDictionary implements Iterable<Word> { if (null == s || "".equals(s)) { throw new RuntimeException("Can't search for a null or empty string"); } - return null != findWordInTree(mRoot, s); + return null != findWordInTree(mRootNodeArray, s); } /** - * Recursively count the number of character groups in a given branch of the trie. + * Recursively count the number of PtNodes in a given branch of the trie. * - * @param node the parent node. - * @return the number of char groups in all the branch under this node. + * @param nodeArray the parent node. + * @return the number of PtNodes in all the branch under this node. */ - public static int countCharGroups(final Node node) { - final int nodeSize = node.mData.size(); + public static int countPtNodes(final PtNodeArray nodeArray) { + final int nodeSize = nodeArray.mData.size(); int size = nodeSize; for (int i = nodeSize - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); - if (null != group.mChildren) - size += countCharGroups(group.mChildren); + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countPtNodes(ptNode.mChildren); } return size; } @@ -693,15 +702,15 @@ public final class FusionDictionary implements Iterable<Word> { /** * Recursively count the number of nodes in a given branch of the trie. * - * @param node the node to count. + * @param nodeArray the node array to count. * @return the number of nodes in this branch. */ - public static int countNodes(final Node node) { + public static int countNodeArrays(final PtNodeArray nodeArray) { int size = 1; - for (int i = node.mData.size() - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); - if (null != group.mChildren) - size += countNodes(group.mChildren); + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mChildren) + size += countNodeArrays(ptNode.mChildren); } return size; } @@ -709,12 +718,12 @@ public final class FusionDictionary implements Iterable<Word> { // Recursively find out whether there are any bigrams. // This can be pretty expensive especially if there aren't any (we return as soon // as we find one, so it's much cheaper if there are bigrams) - private static boolean hasBigramsInternal(final Node node) { - if (null == node) return false; - for (int i = node.mData.size() - 1; i >= 0; --i) { - CharGroup group = node.mData.get(i); - if (null != group.mBigrams) return true; - if (hasBigramsInternal(group.mChildren)) return true; + private static boolean hasBigramsInternal(final PtNodeArray nodeArray) { + if (null == nodeArray) return false; + for (int i = nodeArray.mData.size() - 1; i >= 0; --i) { + PtNode ptNode = nodeArray.mData.get(i); + if (null != ptNode.mBigrams) return true; + if (hasBigramsInternal(ptNode.mChildren)) return true; } return false; } @@ -729,7 +738,7 @@ public final class FusionDictionary implements Iterable<Word> { // find a more efficient way of doing this, without compromising too much on memory // and ease of use. public boolean hasBigrams() { - return hasBigramsInternal(mRoot); + return hasBigramsInternal(mRootNodeArray); } // Historically, the tails of the words were going to be merged to save space. @@ -747,16 +756,16 @@ public final class FusionDictionary implements Iterable<Word> { MakedictLog.i("Do not merge tails"); return; -// MakedictLog.i("Merging nodes. Number of nodes : " + countNodes(root)); -// MakedictLog.i("Number of groups : " + countCharGroups(root)); +// MakedictLog.i("Merging PtNodes. Number of PtNodes : " + countPtNodes(root)); +// MakedictLog.i("Number of PtNodes : " + countPtNodes(root)); // -// final HashMap<String, ArrayList<Node>> repository = -// new HashMap<String, ArrayList<Node>>(); +// final HashMap<String, ArrayList<PtNodeArray>> repository = +// new HashMap<String, ArrayList<PtNodeArray>>(); // mergeTailsInner(repository, root); // // MakedictLog.i("Number of different pseudohashes : " + repository.size()); // int size = 0; -// for (ArrayList<Node> a : repository.values()) { +// for (ArrayList<PtNodeArray> a : repository.values()) { // size += a.size(); // } // MakedictLog.i("Number of nodes after merge : " + (1 + size)); @@ -764,58 +773,58 @@ public final class FusionDictionary implements Iterable<Word> { } // The following methods are used by the deactivated mergeTails() -// private static boolean isEqual(Node a, Node b) { +// private static boolean isEqual(PtNodeArray a, PtNodeArray b) { // if (null == a && null == b) return true; // if (null == a || null == b) return false; // if (a.data.size() != b.data.size()) return false; // final int size = a.data.size(); // for (int i = size - 1; i >= 0; --i) { -// CharGroup aGroup = a.data.get(i); -// CharGroup bGroup = b.data.get(i); -// if (aGroup.frequency != bGroup.frequency) return false; -// if (aGroup.alternates == null && bGroup.alternates != null) return false; -// if (aGroup.alternates != null && !aGroup.equals(bGroup.alternates)) return false; -// if (!Arrays.equals(aGroup.chars, bGroup.chars)) return false; -// if (!isEqual(aGroup.children, bGroup.children)) return false; +// PtNode aPtNode = a.data.get(i); +// PtNode bPtNode = b.data.get(i); +// if (aPtNode.frequency != bPtNode.frequency) return false; +// if (aPtNode.alternates == null && bPtNode.alternates != null) return false; +// if (aPtNode.alternates != null && !aPtNode.equals(bPtNode.alternates)) return false; +// if (!Arrays.equals(aPtNode.chars, bPtNode.chars)) return false; +// if (!isEqual(aPtNode.children, bPtNode.children)) return false; // } // return true; // } -// static private HashMap<String, ArrayList<Node>> mergeTailsInner( -// final HashMap<String, ArrayList<Node>> map, final Node node) { -// final ArrayList<CharGroup> branches = node.data; +// static private HashMap<String, ArrayList<PtNodeArray>> mergeTailsInner( +// final HashMap<String, ArrayList<PtNodeArray>> map, final PtNodeArray nodeArray) { +// final ArrayList<PtNode> branches = nodeArray.data; // final int nodeSize = branches.size(); // for (int i = 0; i < nodeSize; ++i) { -// CharGroup group = branches.get(i); -// if (null != group.children) { -// String pseudoHash = getPseudoHash(group.children); -// ArrayList<Node> similarList = map.get(pseudoHash); +// PtNode ptNode = branches.get(i); +// if (null != ptNode.children) { +// String pseudoHash = getPseudoHash(ptNode.children); +// ArrayList<PtNodeArray> similarList = map.get(pseudoHash); // if (null == similarList) { -// similarList = new ArrayList<Node>(); +// similarList = new ArrayList<PtNodeArray>(); // map.put(pseudoHash, similarList); // } // boolean merged = false; -// for (Node similar : similarList) { -// if (isEqual(group.children, similar)) { -// group.children = similar; +// for (PtNodeArray similar : similarList) { +// if (isEqual(ptNode.children, similar)) { +// ptNode.children = similar; // merged = true; // break; // } // } // if (!merged) { -// similarList.add(group.children); +// similarList.add(ptNode.children); // } -// mergeTailsInner(map, group.children); +// mergeTailsInner(map, ptNode.children); // } // } // return map; // } -// private static String getPseudoHash(final Node node) { +// private static String getPseudoHash(final PtNodeArray nodeArray) { // StringBuilder s = new StringBuilder(); -// for (CharGroup g : node.data) { -// s.append(g.frequency); -// for (int ch : g.chars) { +// for (PtNode ptNode : nodeArray.data) { +// s.append(ptNode.frequency); +// for (int ch : ptNode.chars) { // s.append(Character.toChars(ch)); // } // } @@ -829,20 +838,20 @@ public final class FusionDictionary implements Iterable<Word> { */ public static final class DictionaryIterator implements Iterator<Word> { private static final class Position { - public Iterator<CharGroup> pos; + public Iterator<PtNode> pos; public int length; - public Position(ArrayList<CharGroup> groups) { - pos = groups.iterator(); + public Position(ArrayList<PtNode> ptNodes) { + pos = ptNodes.iterator(); length = 0; } } final StringBuilder mCurrentString; final LinkedList<Position> mPositions; - public DictionaryIterator(ArrayList<CharGroup> root) { + public DictionaryIterator(ArrayList<PtNode> ptRoot) { mCurrentString = new StringBuilder(); mPositions = new LinkedList<Position>(); - final Position rootPos = new Position(root); + final Position rootPos = new Position(ptRoot); mPositions.add(rootPos); } @@ -863,20 +872,20 @@ public final class FusionDictionary implements Iterable<Word> { do { if (currentPos.pos.hasNext()) { - final CharGroup currentGroup = currentPos.pos.next(); + final PtNode currentPtNode = currentPos.pos.next(); currentPos.length = mCurrentString.length(); - for (int i : currentGroup.mChars) { + for (int i : currentPtNode.mChars) { mCurrentString.append(Character.toChars(i)); } - if (null != currentGroup.mChildren) { - currentPos = new Position(currentGroup.mChildren.mData); + if (null != currentPtNode.mChildren) { + currentPos = new Position(currentPtNode.mChildren.mData); currentPos.length = mCurrentString.length(); mPositions.addLast(currentPos); } - if (currentGroup.mFrequency >= 0) { - return new Word(mCurrentString.toString(), currentGroup.mFrequency, - currentGroup.mShortcutTargets, currentGroup.mBigrams, - currentGroup.mIsNotAWord, currentGroup.mIsBlacklistEntry); + if (currentPtNode.mFrequency >= 0) { + return new Word(mCurrentString.toString(), currentPtNode.mFrequency, + currentPtNode.mShortcutTargets, currentPtNode.mBigrams, + currentPtNode.mIsNotAWord, currentPtNode.mIsBlacklistEntry); } } else { mPositions.removeLast(); @@ -901,6 +910,6 @@ public final class FusionDictionary implements Iterable<Word> { */ @Override public Iterator<Word> iterator() { - return new DictionaryIterator(mRoot.mData); + return new DictionaryIterator(mRootNodeArray.mData); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/CharGroupInfo.java b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java index b3617443e..188de7a0f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/CharGroupInfo.java +++ b/java/src/com/android/inputmethod/latin/makedict/PtNodeInfo.java @@ -21,9 +21,9 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.util.ArrayList; /** - * Raw char group info straight out of a file. This will contain numbers for addresses. + * Raw PtNode info straight out of a file. This will contain numbers for addresses. */ -public final class CharGroupInfo { +public final class PtNodeInfo { public final int mOriginalAddress; public final int mEndAddress; @@ -35,7 +35,7 @@ public final class CharGroupInfo { public final ArrayList<WeightedString> mShortcutTargets; public final ArrayList<PendingAttribute> mBigrams; - public CharGroupInfo(final int originalAddress, final int endAddress, final int flags, + public PtNodeInfo(final int originalAddress, final int endAddress, final int flags, final int[] characters, final int frequency, final int parentAddress, final int childrenAddress, final ArrayList<WeightedString> shortcutTargets, final ArrayList<PendingAttribute> bigrams) { diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java new file mode 100644 index 000000000..0b9cf91d2 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; + +/** + * SparseTable is an extensible map from integer to integer. + * This holds one value for every mBlockSize keys, so it uses 1/mBlockSize'th of the full index + * memory. + */ +@UsedForTesting +public class SparseTable { + + /** + * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize + * terminals. + * It contains at index i = j / mBlockSize the index in mContentsTable where the values for + * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array. + */ + private final ArrayList<Integer> mLookupTable; + private final ArrayList<Integer> mContentTable; + + private final int mBlockSize; + public static final int NOT_EXIST = -1; + + @UsedForTesting + public SparseTable(final int initialCapacity, final int blockSize) { + mBlockSize = blockSize; + final int lookupTableSize = initialCapacity / mBlockSize + + (initialCapacity % mBlockSize > 0 ? 1 : 0); + mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST)); + mContentTable = new ArrayList<Integer>(); + } + + @UsedForTesting + public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) { + mBlockSize = blockSize; + mLookupTable = new ArrayList<Integer>(lookupTable.length); + for (int i = 0; i < lookupTable.length; ++i) { + mLookupTable.add(lookupTable[i]); + } + mContentTable = new ArrayList<Integer>(contentTable.length); + for (int i = 0; i < contentTable.length; ++i) { + mContentTable.add(contentTable[i]); + } + } + + /** + * Converts an byte array to an int array considering each set of 4 bytes is an int stored in + * big-endian. + * The length of byteArray must be a multiple of four. + * Otherwise, IndexOutOfBoundsException will be raised. + */ + @UsedForTesting + private static void convertByteArrayToIntegerArray(final byte[] byteArray, + final ArrayList<Integer> integerArray) { + for (int i = 0; i < byteArray.length; i += 4) { + int value = 0; + for (int j = i; j < i + 4; ++j) { + value <<= 8; + value |= byteArray[j] & 0xFF; + } + integerArray.add(value); + } + } + + @UsedForTesting + public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) { + mBlockSize = blockSize; + mLookupTable = new ArrayList<Integer>(lookupTable.length / 4); + mContentTable = new ArrayList<Integer>(contentTable.length / 4); + convertByteArrayToIntegerArray(lookupTable, mLookupTable); + convertByteArrayToIntegerArray(contentTable, mContentTable); + } + + @UsedForTesting + public int get(final int index) { + if (index < 0 || index / mBlockSize >= mLookupTable.size() + || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + return NOT_EXIST; + } + return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); + } + + @UsedForTesting + public void set(final int index, final int value) { + if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + mLookupTable.set(index / mBlockSize, mContentTable.size()); + for (int i = 0; i < mBlockSize; ++i) { + mContentTable.add(NOT_EXIST); + } + } + mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); + } + + public void remove(final int index) { + set(index, NOT_EXIST); + } + + @UsedForTesting + public int size() { + return mLookupTable.size() * mBlockSize; + } + + @UsedForTesting + /* package */ int getContentTableSize() { + return mContentTable.size(); + } + + @UsedForTesting + /* package */ int getLookupTableSize() { + return mLookupTable.size(); + } + + public boolean contains(final int index) { + return get(index) != NOT_EXIST; + } + + @UsedForTesting + public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream) + throws IOException { + for (final int index : mLookupTable) { + BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4); + } + + for (final int index : mContentTable) { + BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4); + } + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java new file mode 100644 index 000000000..848277cd4 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictDecoder.java @@ -0,0 +1,234 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; +import com.android.inputmethod.latin.utils.JniUtils; + +import android.util.Log; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * An implementation of DictDecoder for version 3 binary dictionary. + */ +@UsedForTesting +public class Ver3DictDecoder extends DictDecoder { + private static final String TAG = Ver3DictDecoder.class.getSimpleName(); + + static { + JniUtils.loadNativeLibrary(); + } + + // TODO: implement something sensical instead of just a phony method + private static native int doNothing(); + + protected static class PtNodeReader extends DictDecoder.PtNodeReader { + private static int readFrequency(final DictBuffer dictBuffer) { + return dictBuffer.readUnsignedByte(); + } + } + + private final File mDictionaryBinaryFile; + private final DictionaryBufferFactory mBufferFactory; + private DictBuffer mDictBuffer; + + /* package */ Ver3DictDecoder(final File file, final int factoryFlag) { + mDictionaryBinaryFile = file; + mDictBuffer = null; + + if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { + mBufferFactory = new DictionaryBufferFromByteArrayFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); + } else { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } + } + + /* package */ Ver3DictDecoder(final File file, final DictionaryBufferFactory factory) { + mDictionaryBinaryFile = file; + mBufferFactory = factory; + } + + @Override + public void openDictBuffer() throws FileNotFoundException, IOException { + mDictBuffer = mBufferFactory.getDictionaryBuffer(mDictionaryBinaryFile); + } + + @Override + public boolean isDictBufferOpen() { + return mDictBuffer != null; + } + + /* package */ DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @UsedForTesting + /* package */ DictBuffer openAndGetDictBuffer() throws FileNotFoundException, IOException { + openDictBuffer(); + return getDictBuffer(); + } + + @Override + public FileHeader readHeader() throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + final FileHeader header = super.readHeader(mDictBuffer); + final int version = header.mFormatOptions.mVersion; + if (!(version >= 2 && version <= 3)) { + throw new UnsupportedFormatException("File header has a wrong version : " + version); + } + return header; + } + + // TODO: Make this buffer multi thread safe. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public PtNodeInfo readPtNode(final int ptNodePos, final FormatOptions options) { + int addressPointer = ptNodePos; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += FormatSpec.PARENT_ADDRESS_SIZE; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (FormatSpec.INVALID_CHARACTER != character) { + // FusionDictionary is making sure that the length of the word is smaller than + // MAX_WORD_LENGTH. + // So we'll never write past the end of mCharacterBuffer. + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + frequency = PtNodeReader.readFrequency(mDictBuffer); + addressPointer += FormatSpec.PTNODE_FREQUENCY_SIZE; + } else { + frequency = PtNode.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + final ArrayList<WeightedString> shortcutTargets; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + // readShortcut will add shortcuts to shortcutTargets. + shortcutTargets = new ArrayList<WeightedString>(); + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } else { + shortcutTargets = null; + } + + final ArrayList<PendingAttribute> bigrams; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, + addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + MakedictLog.d("too many bigrams in a PtNode."); + } + } else { + bigrams = null; + } + return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } + + @Override + public FusionDictionary readDictionaryBinary(final FusionDictionary dict, + final boolean deleteDictIfBroken) + throws FileNotFoundException, IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + try { + return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); + } catch (IOException e) { + Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); + if (deleteDictIfBroken && !mDictionaryBinaryFile.delete()) { + Log.e(TAG, "Failed to delete the broken dictionary."); + } + throw e; + } catch (UnsupportedFormatException e) { + Log.e(TAG, "The dictionary " + mDictionaryBinaryFile.getName() + " is broken.", e); + if (deleteDictIfBroken && !mDictionaryBinaryFile.delete()) { + Log.e(TAG, "Failed to delete the broken dictionary."); + } + throw e; + } + } + + @Override + public void setPosition(int newPos) { + mDictBuffer.position(newPos); + } + + @Override + public int getPosition() { + return mDictBuffer.position(); + } + + @Override + public int readPtNodeCount() { + return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); + } + + @Override + public boolean readAndFollowForwardLink() { + final int nextAddress = mDictBuffer.readUnsignedInt24(); + if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { + mDictBuffer.position(nextAddress); + return true; + } + return false; + } + + @Override + public boolean hasNextPtNodeArray() { + return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java new file mode 100644 index 000000000..76f0f4052 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictEncoder.java @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * An implementation of DictEncoder for version 3 binary dictionary. + */ +public class Ver3DictEncoder implements DictEncoder { + + private final File mDictFile; + private OutputStream mOutStream; + private byte[] mBuffer; + private int mPosition; + + public Ver3DictEncoder(final File dictFile) { + mDictFile = dictFile; + mOutStream = null; + mBuffer = null; + } + + // This constructor is used only by BinaryDictOffdeviceUtilsTests. + // If you want to use this in the production code, you should consider keeping consistency of + // the interface of Ver3DictDecoder by using factory. + public Ver3DictEncoder(final OutputStream outStream) { + mDictFile = null; + mOutStream = outStream; + } + + private void openStream() throws FileNotFoundException { + mOutStream = new FileOutputStream(mDictFile); + } + + private void close() throws IOException { + if (mOutStream != null) { + mOutStream.close(); + mOutStream = null; + } + } + + @Override + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion > FormatSpec.VERSION3) { + throw new UnsupportedFormatException( + "The given format options has wrong version number : " + + formatOptions.mVersion); + } + + if (mOutStream == null) { + openStream(); + } + BinaryDictEncoderUtils.writeDictionaryHeader(mOutStream, dict, formatOptions); + + // Addresses are limited to 3 bytes, but since addresses can be relative to each node + // array, the structure itself is not limited to 16MB. However, if it is over 16MB deciding + // the order of the PtNode arrays becomes a quite complicated problem, because though the + // dictionary itself does not have a size limit, each node array must still be within 16MB + // of all its children and parents. As long as this is ensured, the dictionary file may + // grow to any size. + + // Leave the choice of the optimal node order to the flattenTree function. + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + MakedictLog.i("Checking PtNode array..."); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + // Create a buffer that matches the final dictionary size. + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mBuffer = new byte[bufferSize]; + + MakedictLog.i("Writing file..."); + + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); + } + if (MakedictLog.DBG) BinaryDictEncoderUtils.showStatistics(flatNodes); + mOutStream.write(mBuffer, 0, mPosition); + + MakedictLog.i("Done"); + close(); + } + + @Override + public void setPosition(final int position) { + if (mBuffer == null || position < 0 || position >= mBuffer.length) return; + mPosition = position; + } + + @Override + public int getPosition() { + return mPosition; + } + + @Override + public void writePtNodeCount(final int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); + } + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, ptNodeCount, + countSize); + } + + private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mPosition, childrenPos, + formatOptions), + FormatSpec.PTNODE_FLAGS_SIZE); + } + + private void writeParentPosition(final int parentPosition, final PtNode ptNode, + final FormatOptions formatOptions) { + if (parentPosition == FormatSpec.NO_PARENT_ADDRESS) { + mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, + parentPosition, formatOptions); + } else { + mPosition = BinaryDictEncoderUtils.writeParentAddress(mBuffer, mPosition, + parentPosition - ptNode.mCachedAddressAfterUpdate, formatOptions); + } + } + + private void writeCharacters(final int[] codePoints, final boolean hasSeveralChars) { + mPosition = CharEncoding.writeCharArray(codePoints, mBuffer, mPosition); + if (hasSeveralChars) { + mBuffer[mPosition++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + private void writeFrequency(final int frequency) { + if (frequency >= 0) { + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, frequency, + FormatSpec.PTNODE_FREQUENCY_SIZE); + } + } + + private void writeChildrenPosition(final PtNode ptNode, final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + if (formatOptions.mSupportsDynamicUpdate) { + mPosition += BinaryDictEncoderUtils.writeSignedChildrenPosition(mBuffer, mPosition, + childrenPos); + } else { + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + childrenPos); + } + } + + /** + * Write a shortcut attributes list to mBuffer. + * + * @param shortcuts the shortcut attributes list. + */ + private void writeShortcuts(final ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mPosition; + mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.mFrequency); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord); + mPosition += shortcutShift; + } + final int shortcutByteSize = mPosition - indexOfShortcutByteSize; + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { + throw new RuntimeException("Shortcut list too large"); + } + BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, + FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + /** + * Write a bigram attributes list to mBuffer. + * + * @param bigrams the bigram attributes list. + * @param dict the dictionary the node array is a part of (for relative offsets). + */ + private void writeBigrams(final ArrayList<WeightedString> bigrams, + final FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + final int offset = addressOfBigram + - (mPosition + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + final int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, bigramFlags, + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + mPosition += BinaryDictEncoderUtils.writeChildrenPosition(mBuffer, mPosition, + Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(final int forwardLinkAddress) { + mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, forwardLinkAddress, + FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict) { + writePtNodeFlags(ptNode, parentPosition, formatOptions); + writeParentPosition(parentPosition, ptNode, formatOptions); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + writeFrequency(ptNode.mFrequency); + writeChildrenPosition(ptNode, formatOptions); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java new file mode 100644 index 000000000..4c8ff8ea4 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import android.util.Log; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + +/** + * An implementation of binary dictionary decoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictDecoder extends DictDecoder { + private static final String TAG = Ver4DictDecoder.class.getSimpleName(); + + private static final int FILETYPE_TRIE = 1; + private static final int FILETYPE_FREQUENCY = 2; + private static final int FILETYPE_TERMINAL_ADDRESS_TABLE = 3; + + private final File mDictDirectory; + private final DictionaryBufferFactory mBufferFactory; + private DictBuffer mDictBuffer; + private DictBuffer mFrequencyBuffer; + private DictBuffer mTerminalAddressTableBuffer; + + @UsedForTesting + /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { + mDictDirectory = dictDirectory; + mDictBuffer = mFrequencyBuffer = null; + + if ((factoryFlag & MASK_DICTBUFFER) == USE_READONLY_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_BYTEARRAY) { + mBufferFactory = new DictionaryBufferFromByteArrayFactory(); + } else if ((factoryFlag & MASK_DICTBUFFER) == USE_WRITABLE_BYTEBUFFER) { + mBufferFactory = new DictionaryBufferFromWritableByteBufferFactory(); + } else { + mBufferFactory = new DictionaryBufferFromReadOnlyByteBufferFactory(); + } + } + + @UsedForTesting + /* package */ Ver4DictDecoder(final File dictDirectory, final DictionaryBufferFactory factory) { + mDictDirectory = dictDirectory; + mBufferFactory = factory; + mDictBuffer = mFrequencyBuffer = null; + } + + private File getFile(final int fileType) { + if (fileType == FILETYPE_TRIE) { + return new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.TRIE_FILE_EXTENSION); + } else if (fileType == FILETYPE_FREQUENCY) { + return new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.FREQ_FILE_EXTENSION); + } else if (fileType == FILETYPE_TERMINAL_ADDRESS_TABLE) { + return new File(mDictDirectory, + mDictDirectory.getName() + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + } else { + throw new RuntimeException("Unsupported kind of file : " + fileType); + } + } + + @Override + public void openDictBuffer() throws FileNotFoundException, IOException { + final String filename = mDictDirectory.getName(); + mDictBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_TRIE)); + mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); + mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( + getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); + } + + @Override + public boolean isDictBufferOpen() { + return mDictBuffer != null; + } + + /* package */ DictBuffer getDictBuffer() { + return mDictBuffer; + } + + @Override + public FileHeader readHeader() throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + final FileHeader header = super.readHeader(mDictBuffer); + final int version = header.mFormatOptions.mVersion; + if (version != 4) { + throw new UnsupportedFormatException("File header has a wrong version : " + version); + } + return header; + } + + protected static class PtNodeReader extends DictDecoder.PtNodeReader { + protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) { + frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1); + return frequencyBuffer.readUnsignedByte(); + } + + protected static int readTerminalId(final DictBuffer dictBuffer) { + return dictBuffer.readInt(); + } + } + + // TODO: Make this buffer thread safe. + // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH. + private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; + @Override + public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { + int addressPointer = ptNodePos; + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; + + final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { + addressPointer += FormatSpec.PARENT_ADDRESS_SIZE; + } + + final int characters[]; + if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { + int index = 0; + int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + while (FormatSpec.INVALID_CHARACTER != character + && index < FormatSpec.MAX_WORD_LENGTH) { + mCharacterBuffer[index++] = character; + character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + } + characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + } else { + final int character = CharEncoding.readChar(mDictBuffer); + addressPointer += CharEncoding.getCharSize(character); + characters = new int[] { character }; + } + final int terminalId; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + terminalId = PtNodeReader.readTerminalId(mDictBuffer); + addressPointer += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + } else { + terminalId = PtNode.NOT_A_TERMINAL; + } + + final int frequency; + if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { + frequency = PtNodeReader.readFrequency(mFrequencyBuffer, terminalId); + } else { + frequency = PtNode.NOT_A_TERMINAL; + } + int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenAddress += addressPointer; + } + addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + final ArrayList<WeightedString> shortcutTargets; + if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { + // readShortcut will add shortcuts to shortcutTargets. + shortcutTargets = new ArrayList<WeightedString>(); + addressPointer += PtNodeReader.readShortcut(mDictBuffer, shortcutTargets); + } else { + shortcutTargets = null; + } + + final ArrayList<PendingAttribute> bigrams; + if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { + bigrams = new ArrayList<PendingAttribute>(); + addressPointer += PtNodeReader.readBigramAddresses(mDictBuffer, bigrams, + addressPointer); + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + MakedictLog.d("too many bigrams in a node."); + } + } else { + bigrams = null; + } + return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, + parentAddress, childrenAddress, shortcutTargets, bigrams); + } + + private void deleteDictFiles() { + final File[] files = mDictDirectory.listFiles(); + for (int i = 0; i < files.length; ++i) { + files[i].delete(); + } + } + + @Override + public FusionDictionary readDictionaryBinary(final FusionDictionary dict, + final boolean deleteDictIfBroken) + throws FileNotFoundException, IOException, UnsupportedFormatException { + if (mDictBuffer == null) { + openDictBuffer(); + } + try { + return BinaryDictDecoderUtils.readDictionaryBinary(this, dict); + } catch (IOException e) { + Log.e(TAG, "The dictionary " + mDictDirectory.getName() + " is broken.", e); + if (deleteDictIfBroken) { + deleteDictFiles(); + } + throw e; + } catch (UnsupportedFormatException e) { + Log.e(TAG, "The dictionary " + mDictDirectory.getName() + " is broken.", e); + if (deleteDictIfBroken) { + deleteDictFiles(); + } + throw e; + } + } + + @Override + public void setPosition(int newPos) { + mDictBuffer.position(newPos); + } + + @Override + public int getPosition() { + return mDictBuffer.position(); + } + + @Override + public int readPtNodeCount() { + return BinaryDictDecoderUtils.readPtNodeCount(mDictBuffer); + } + + @Override + public boolean readAndFollowForwardLink() { + final int nextAddress = mDictBuffer.readUnsignedInt24(); + if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { + mDictBuffer.position(nextAddress); + return true; + } + return false; + } + + @Override + public boolean hasNextPtNodeArray() { + return mDictBuffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS; + } +} diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java new file mode 100644 index 000000000..4fb89671f --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -0,0 +1,294 @@ +/* +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; +import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Iterator; + +/** + * An implementation of DictEncoder for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictEncoder implements DictEncoder { + private final File mDictPlacedDir; + private byte[] mTrieBuf; + private int mTriePos; + private int mHeaderSize; + private OutputStream mTrieOutStream; + private OutputStream mFreqOutStream; + private OutputStream mTerminalAddressTableOutStream; + + @UsedForTesting + public Ver4DictEncoder(final File dictPlacedDir) { + mDictPlacedDir = dictPlacedDir; + } + + private void openStreams(final FormatOptions formatOptions, final DictionaryOptions dictOptions) + throws FileNotFoundException, IOException { + final FileHeader header = new FileHeader(0, dictOptions, formatOptions); + final String filename = header.getId() + "." + header.getVersion(); + final File mDictDir = new File(mDictPlacedDir, filename); + final File trieFile = new File(mDictDir, filename + FormatSpec.TRIE_FILE_EXTENSION); + final File freqFile = new File(mDictDir, filename + FormatSpec.FREQ_FILE_EXTENSION); + final File terminalAddressTableFile = new File(mDictDir, + filename + FormatSpec.TERMINAL_ADDRESS_TABLE_FILE_EXTENSION); + if (!mDictDir.isDirectory()) { + if (mDictDir.exists()) mDictDir.delete(); + mDictDir.mkdirs(); + } + if (!trieFile.exists()) trieFile.createNewFile(); + if (!freqFile.exists()) freqFile.createNewFile(); + if (!terminalAddressTableFile.exists()) terminalAddressTableFile.createNewFile(); + mTrieOutStream = new FileOutputStream(trieFile); + mFreqOutStream = new FileOutputStream(freqFile); + mTerminalAddressTableOutStream = new FileOutputStream(terminalAddressTableFile); + } + + private void close() throws IOException { + try { + if (mTrieOutStream != null) { + mTrieOutStream.close(); + } + if (mFreqOutStream != null) { + mFreqOutStream.close(); + } + if (mTerminalAddressTableOutStream != null) { + mTerminalAddressTableOutStream.close(); + } + } finally { + mTrieOutStream = null; + mFreqOutStream = null; + mTerminalAddressTableOutStream = null; + } + } + + @Override + public void writeDictionary(final FusionDictionary dict, final FormatOptions formatOptions) + throws IOException, UnsupportedFormatException { + if (formatOptions.mVersion != FormatSpec.VERSION4) { + throw new UnsupportedFormatException("File header has a wrong version number : " + + formatOptions.mVersion); + } + if (!mDictPlacedDir.isDirectory()) { + throw new UnsupportedFormatException("Given path is not a directory."); + } + + if (mTrieOutStream == null) { + openStreams(formatOptions, dict.mOptions); + } + + mHeaderSize = BinaryDictEncoderUtils.writeDictionaryHeader(mTrieOutStream, dict, + formatOptions); + + MakedictLog.i("Flattening the tree..."); + ArrayList<PtNodeArray> flatNodes = BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); + int terminalCount = 0; + for (final PtNodeArray array : flatNodes) { + for (final PtNode node : array.mData) { + if (node.isTerminal()) node.mTerminalId = terminalCount++; + } + } + + MakedictLog.i("Computing addresses..."); + BinaryDictEncoderUtils.computeAddresses(dict, flatNodes, formatOptions); + if (MakedictLog.DBG) BinaryDictEncoderUtils.checkFlatPtNodeArrayList(flatNodes); + + writeTerminalData(flatNodes, terminalCount); + + final PtNodeArray lastNodeArray = flatNodes.get(flatNodes.size() - 1); + final int bufferSize = lastNodeArray.mCachedAddressAfterUpdate + lastNodeArray.mCachedSize; + mTrieBuf = new byte[bufferSize]; + + MakedictLog.i("Writing file..."); + for (PtNodeArray nodeArray : flatNodes) { + BinaryDictEncoderUtils.writePlacedPtNodeArray(dict, this, nodeArray, formatOptions); + } + if (MakedictLog.DBG) { + BinaryDictEncoderUtils.showStatistics(flatNodes); + MakedictLog.i("has " + terminalCount + " terminals."); + } + mTrieOutStream.write(mTrieBuf); + + MakedictLog.i("Done"); + close(); + } + + @Override + public void setPosition(int position) { + if (mTrieBuf == null || position < 0 || position >- mTrieBuf.length) return; + mTriePos = position; + } + + @Override + public int getPosition() { + return mTriePos; + } + + @Override + public void writePtNodeCount(int ptNodeCount) { + final int countSize = BinaryDictIOUtils.getPtNodeCountSize(ptNodeCount); + // ptNodeCount must fit on one byte or two bytes. + // Please see comments in FormatSpec + if (countSize != 1 && countSize != 2) { + throw new RuntimeException("Strange size from getPtNodeCountSize : " + countSize); + } + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, ptNodeCount, + countSize); + } + + private void writePtNodeFlags(final PtNode ptNode, final int parentAddress, + final FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + BinaryDictEncoderUtils.makePtNodeFlags(ptNode, mTriePos, childrenPos, + formatOptions), + FormatSpec.PTNODE_FLAGS_SIZE); + } + + private void writeParentPosition(int parentPos, final PtNode ptNode, + final FormatOptions formatOptions) { + if (parentPos != FormatSpec.NO_PARENT_ADDRESS) { + parentPos -= ptNode.mCachedAddressAfterUpdate; + } + mTriePos = BinaryDictEncoderUtils.writeParentAddress(mTrieBuf, mTriePos, parentPos, + formatOptions); + } + + private void writeCharacters(final int[] characters, final boolean hasSeveralChars) { + mTriePos = CharEncoding.writeCharArray(characters, mTrieBuf, mTriePos); + if (hasSeveralChars) { + mTrieBuf[mTriePos++] = FormatSpec.PTNODE_CHARACTERS_TERMINATOR; + } + } + + private void writeTerminalId(final int terminalId) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, terminalId, + FormatSpec.PTNODE_TERMINAL_ID_SIZE); + } + + private void writeChildrenPosition(PtNode ptNode, FormatOptions formatOptions) { + final int childrenPos = BinaryDictEncoderUtils.getChildrenPosition(ptNode, formatOptions); + if (formatOptions.mSupportsDynamicUpdate) { + mTriePos += BinaryDictEncoderUtils.writeSignedChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } else { + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, childrenPos); + } + } + + private void writeShortcuts(ArrayList<WeightedString> shortcuts) { + if (null == shortcuts || shortcuts.isEmpty()) return; + + final int indexOfShortcutByteSize = mTriePos; + mTriePos += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; + final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); + while (shortcutIterator.hasNext()) { + final WeightedString target = shortcutIterator.next(); + final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( + shortcutIterator.hasNext(), + target.mFrequency); + mTrieBuf[mTriePos++] = (byte)shortcutFlags; + final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos, + target.mWord); + mTriePos += shortcutShift; + } + final int shortcutByteSize = mTriePos - indexOfShortcutByteSize; + if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { + throw new RuntimeException("Shortcut list too large : " + shortcutByteSize); + } + BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, indexOfShortcutByteSize, + shortcutByteSize, FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); + } + + private void writeBigrams(ArrayList<WeightedString> bigrams, FusionDictionary dict) { + if (bigrams == null) return; + + final Iterator<WeightedString> bigramIterator = bigrams.iterator(); + while (bigramIterator.hasNext()) { + final WeightedString bigram = bigramIterator.next(); + final PtNode target = + FusionDictionary.findWordInTree(dict.mRootNodeArray, bigram.mWord); + final int addressOfBigram = target.mCachedAddressAfterUpdate; + final int unigramFrequencyForThisWord = target.mFrequency; + final int offset = addressOfBigram + - (mTriePos + FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); + int bigramFlags = BinaryDictEncoderUtils.makeBigramFlags(bigramIterator.hasNext(), + offset, bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); + mTrieBuf[mTriePos++] = (byte) bigramFlags; + mTriePos += BinaryDictEncoderUtils.writeChildrenPosition(mTrieBuf, + mTriePos, Math.abs(offset)); + } + } + + @Override + public void writeForwardLinkAddress(int forwardLinkAddress) { + mTriePos = BinaryDictEncoderUtils.writeUIntToBuffer(mTrieBuf, mTriePos, + forwardLinkAddress, FormatSpec.FORWARD_LINK_ADDRESS_SIZE); + } + + @Override + public void writePtNode(final PtNode ptNode, final int parentPosition, + final FormatOptions formatOptions, final FusionDictionary dict) { + writePtNodeFlags(ptNode, parentPosition, formatOptions); + writeParentPosition(parentPosition, ptNode, formatOptions); + writeCharacters(ptNode.mChars, ptNode.hasSeveralChars()); + if (ptNode.isTerminal()) { + writeTerminalId(ptNode.mTerminalId); + } + writeChildrenPosition(ptNode, formatOptions); + writeShortcuts(ptNode.mShortcutTargets); + writeBigrams(ptNode.mBigrams, dict); + } + + private void writeTerminalData(final ArrayList<PtNodeArray> flatNodes, + final int terminalCount) throws IOException { + final byte[] freqBuf = new byte[terminalCount * FormatSpec.FREQUENCY_AND_FLAGS_SIZE]; + final byte[] terminalAddressTableBuf = + new byte[terminalCount * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE]; + for (final PtNodeArray nodeArray : flatNodes) { + for (final PtNode ptNode : nodeArray.mData) { + if (ptNode.isTerminal()) { + BinaryDictEncoderUtils.writeUIntToBuffer(freqBuf, + ptNode.mTerminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE, + ptNode.mFrequency, FormatSpec.FREQUENCY_AND_FLAGS_SIZE); + BinaryDictEncoderUtils.writeUIntToBuffer(terminalAddressTableBuf, + ptNode.mTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE, + ptNode.mCachedAddressAfterUpdate + mHeaderSize, + FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + } + } + } + mFreqOutStream.write(freqBuf); + mTerminalAddressTableOutStream.write(terminalAddressTableBuf); + } +} |