diff options
11 files changed, 648 insertions, 403 deletions
diff --git a/java/res/xml/method.xml b/java/res/xml/method.xml index 0014dc260..0730b0100 100644 --- a/java/res/xml/method.xml +++ b/java/res/xml/method.xml @@ -24,7 +24,7 @@ keyboard_locale: script_name/keyboard_layout_set af: Afrikaans/qwerty ar: Arabic/arabic - (az: Azerbaijani/qwerty) # disabled temporarily. waiting for strnig resources. + (az: Azerbaijani/qwerty) # disabled temporarily. waiting for string resources. be: Belarusian/east_slavic bg: Bulgarian/bulgarian bg: Bulgarian/bulgarian_bds @@ -55,7 +55,7 @@ it: Italian/qwerty iw: Hebrew/hebrew # "he" is official language code of Hebrew. ka: Georgian/georgian - (kk: Kazakh/east_slavic) # disabled temporarily. waiting for strnig resources. + (kk: Kazakh/east_slavic) # disabled temporarily. waiting for string resources. km: Khmer/khmer ky: Kyrgyz/east_slavic lo: Lao/lao diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java index 216492b4d..8109321b6 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java @@ -225,20 +225,26 @@ public final class BinaryDictDecoderUtils { * * @param buffer the OutputStream to write to. * @param word the string to write. + * @return the size written, in bytes. */ - static void writeString(final OutputStream buffer, final String word) throws IOException { + static int writeString(final OutputStream buffer, final String word) throws IOException { final int length = word.length(); + int written = 0; for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { final int codePoint = word.codePointAt(i); - if (1 == getCharSize(codePoint)) { + final int charSize = getCharSize(codePoint); + if (1 == charSize) { buffer.write((byte) codePoint); } else { buffer.write((byte) (0xFF & (codePoint >> 16))); buffer.write((byte) (0xFF & (codePoint >> 8))); buffer.write((byte) (0xFF & codePoint)); } + written += charSize; } buffer.write(FormatSpec.PTNODE_CHARACTERS_TERMINATOR); + written += FormatSpec.PTNODE_TERMINATOR_SIZE; + return written; } /** diff --git a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java index 0f7d2f6c9..9a28629b1 100644 --- a/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java +++ b/java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java @@ -301,35 +301,6 @@ public final class BinaryDictIOUtils { } /** - * Write a string to a stream. - * - * @param destination the stream to write. - * @param word the string to be written. - * @return the size written, in bytes. - * @throws IOException - */ - private static int writeString(final OutputStream destination, final String word) - throws IOException { - int size = 0; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - if (CharEncoding.getCharSize(codePoint) == 1) { - destination.write((byte)codePoint); - size++; - } else { - destination.write((byte)(0xFF & (codePoint >> 16))); - destination.write((byte)(0xFF & (codePoint >> 8))); - destination.write((byte)(0xFF & codePoint)); - size += 3; - } - } - destination.write((byte)FormatSpec.PTNODE_CHARACTERS_TERMINATOR); - size += FormatSpec.PTNODE_TERMINATOR_SIZE; - return size; - } - - /** * Write a PtNode to an output stream from a PtNodeInfo. * A PtNode is an in-memory representation of a node in the patricia trie. * A PtNode info is a container for low-level information about how the @@ -387,7 +358,7 @@ public final class BinaryDictIOUtils { destination.write((byte)BinaryDictEncoderUtils.makeShortcutFlags( shortcutIterator.hasNext(), target.mFrequency)); size++; - size += writeString(destination, target.mWord); + size += CharEncoding.writeString(destination, target.mWord); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java index 3dbeee099..b8636eecd 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java @@ -43,7 +43,7 @@ public interface DictDecoder { public FileHeader readHeader() throws IOException, UnsupportedFormatException; /** - * Reads PtNode from nodeAddress. + * Reads PtNode from ptNodePos. * @param ptNodePos the position of PtNode. * @param formatOptions the format options. * @return PtNodeInfo. diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java new file mode 100644 index 000000000..00f401ea7 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentReader.java @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; +import com.android.inputmethod.latin.makedict.DictDecoder.DictionaryBufferFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +/** + * An auxiliary class for reading SparseTable and data written by SparseTableContentWriter. + */ +public class SparseTableContentReader { + + /** + * An interface of a function which is passed to SparseTableContentReader.read. + */ + public interface SparseTableContentReaderInterface { + /** + * Reads data. + * + * @param buffer the DictBuffer. The position of the buffer is set to the head of data. + */ + public void read(final DictBuffer buffer); + } + + private final int mContentCount; + private final int mBlockSize; + protected final File mBaseDir; + private final File mLookupTableFile; + private final File[] mAddressTableFiles; + private final File[] mContentFiles; + private DictBuffer mLookupTableBuffer; + private final DictBuffer[] mAddressTableBuffers; + private final DictBuffer[] mContentBuffers; + private final DictionaryBufferFactory mFactory; + + /** + * Sole constructor of SparseTableContentReader. + * + * @param name the name of SparseTable. + * @param blockSize the block size of the content table. + * @param baseDir the directory which contains the files of the content table. + * @param contentFilenames the file names of content files. + * @param contentIds the ids of contents. These ids are used for a suffix of a name of + * address files and content files. + * @param factory the DictionaryBufferFactory which is used for opening the files. + */ + public SparseTableContentReader(final String name, final int blockSize, final File baseDir, + final String[] contentFilenames, final String[] contentIds, + final DictionaryBufferFactory factory) { + if (contentFilenames.length != contentIds.length) { + throw new RuntimeException("The length of contentFilenames and the length of" + + " contentIds are different " + contentFilenames.length + ", " + + contentIds.length); + } + mBlockSize = blockSize; + mBaseDir = baseDir; + mFactory = factory; + mContentCount = contentFilenames.length; + mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + mAddressTableFiles = new File[mContentCount]; + mContentFiles = new File[mContentCount]; + for (int i = 0; i < mContentCount; ++i) { + mAddressTableFiles[i] = new File(mBaseDir, + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + } + mAddressTableBuffers = new DictBuffer[mContentCount]; + mContentBuffers = new DictBuffer[mContentCount]; + } + + public void openBuffers() throws FileNotFoundException, IOException { + mLookupTableBuffer = mFactory.getDictionaryBuffer(mLookupTableFile); + for (int i = 0; i < mContentCount; ++i) { + mAddressTableBuffers[i] = mFactory.getDictionaryBuffer(mAddressTableFiles[i]); + mContentBuffers[i] = mFactory.getDictionaryBuffer(mContentFiles[i]); + } + } + + protected void read(final int contentIndex, final int index, + final SparseTableContentReaderInterface reader) { + if (index < 0 || (index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES + >= mLookupTableBuffer.limit()) { + return; + } + + mLookupTableBuffer.position((index / mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); + final int posInAddressTable = mLookupTableBuffer.readInt(); + if (posInAddressTable == SparseTable.NOT_EXIST) { + return; + } + + mAddressTableBuffers[contentIndex].position( + (posInAddressTable + index % mBlockSize) * SparseTable.SIZE_OF_INT_IN_BYTES); + final int address = mAddressTableBuffers[contentIndex].readInt(); + if (address == SparseTable.NOT_EXIST) { + return; + } + + mContentBuffers[contentIndex].position(address); + reader.read(mContentBuffers[contentIndex]); + } +}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java new file mode 100644 index 000000000..49f0fd624 --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTableContentWriter.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * An auxiliary class for writing data associated with SparseTable to files. + */ +public class SparseTableContentWriter { + public interface SparseTableContentWriterInterface { + public void write(final OutputStream outStream) throws IOException; + } + + private final int mContentCount; + private final SparseTable mSparseTable; + private final File mLookupTableFile; + protected final File mBaseDir; + private final File[] mAddressTableFiles; + private final File[] mContentFiles; + protected final OutputStream[] mContentOutStreams; + + /** + * Sole constructor of SparseTableContentWriter. + * + * @param name the name of SparseTable. + * @param initialCapacity the initial capacity of SparseTable. + * @param blockSize the block size of the content table. + * @param baseDir the directory which contains the files of the content table. + * @param contentFilenames the file names of content files. + * @param contentIds the ids of contents. These ids are used for a suffix of a name of address + * files and content files. + */ + public SparseTableContentWriter(final String name, final int initialCapacity, + final int blockSize, final File baseDir, final String[] contentFilenames, + final String[] contentIds) { + if (contentFilenames.length != contentIds.length) { + throw new RuntimeException("The length of contentFilenames and the length of" + + " contentIds are different " + contentFilenames.length + ", " + + contentIds.length); + } + mContentCount = contentFilenames.length; + mSparseTable = new SparseTable(initialCapacity, blockSize, mContentCount); + mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); + mAddressTableFiles = new File[mContentCount]; + mContentFiles = new File[mContentCount]; + mBaseDir = baseDir; + for (int i = 0; i < mContentCount; ++i) { + mAddressTableFiles[i] = new File(mBaseDir, + name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); + mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); + } + mContentOutStreams = new OutputStream[mContentCount]; + } + + public void openStreams() throws FileNotFoundException { + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); + } + } + + protected void write(final int contentIndex, final int index, + final SparseTableContentWriterInterface writer) throws IOException { + mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); + writer.write(mContentOutStreams[contentIndex]); + mContentOutStreams[contentIndex].flush(); + } + + public void closeStreams() throws IOException { + mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); + for (int i = 0; i < mContentCount; ++i) { + mContentOutStreams[i].close(); + } + } +}
\ No newline at end of file diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 734223ec2..add03c323 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -51,10 +51,30 @@ public class Ver4DictDecoder extends AbstractDictDecoder { protected DictBuffer mDictBuffer; private DictBuffer mFrequencyBuffer; private DictBuffer mTerminalAddressTableBuffer; - private DictBuffer mBigramBuffer; - private DictBuffer mShortcutBuffer; - private SparseTable mBigramAddressTable; - private SparseTable mShortcutAddressTable; + private BigramContentReader mBigramReader; + private ShortcutContentReader mShortcutReader; + + /** + * Raw PtNode info straight out of a trie file in version 4 dictionary. + */ + protected static final class Ver4PtNodeInfo { + public final int mFlags; + public final int[] mCharacters; + public final int mTerminalId; + public final int mChildrenPos; + public final int mParentPos; + public final int mNodeSize; + + public Ver4PtNodeInfo(final int flags, final int[] characters, final int terminalId, + final int childrenPos, final int parentPos, final int nodeSize) { + mFlags = flags; + mCharacters = characters; + mTerminalId = terminalId; + mChildrenPos = childrenPos; + mParentPos = parentPos; + mNodeSize = nodeSize; + } + } @UsedForTesting /* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) { @@ -108,10 +128,12 @@ public class Ver4DictDecoder extends AbstractDictDecoder { mFrequencyBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_FREQUENCY)); mTerminalAddressTableBuffer = mBufferFactory.getDictionaryBuffer( getFile(FILETYPE_TERMINAL_ADDRESS_TABLE)); - mBigramBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_BIGRAM_FREQ)); - loadBigramAddressSparseTable(); - mShortcutBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_SHORTCUT)); - loadShortcutAddressSparseTable(); + mBigramReader = new BigramContentReader(mDictDirectory.getName(), + mDictDirectory, mBufferFactory, false); + mBigramReader.openBuffers(); + mShortcutReader = new ShortcutContentReader(mDictDirectory.getName(), mDictDirectory, + mBufferFactory); + mShortcutReader.openBuffers(); } @Override @@ -136,29 +158,112 @@ public class Ver4DictDecoder extends AbstractDictDecoder { return header; } - private void loadBigramAddressSparseTable() throws IOException { - final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - final File freqsFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.BIGRAM_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX - + FormatSpec.BIGRAM_FREQ_CONTENT_ID); - mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { freqsFile }, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); + /** + * An auxiliary class for reading bigrams. + */ + protected static class BigramContentReader extends SparseTableContentReader { + private final boolean mHasTimestamp; + + public BigramContentReader(final String name, final File baseDir, + final DictionaryBufferFactory factory, final boolean hasTimestamp) { + super(name + FormatSpec.BIGRAM_FILE_EXTENSION, + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + getContentFilenames(name, hasTimestamp), getContentIds(hasTimestamp), factory); + mHasTimestamp = hasTimestamp; + } + + // TODO: Consolidate this method and BigramContentWriter.getContentFilenames. + private static String[] getContentFilenames(final String name, final boolean hasTimestamp) { + final String[] contentFilenames; + if (hasTimestamp) { + contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION, + name + FormatSpec.BIGRAM_FILE_EXTENSION }; + } else { + contentFilenames = new String[] { name + FormatSpec.BIGRAM_FILE_EXTENSION }; + } + return contentFilenames; + } + + // TODO: Consolidate this method and BigramContentWriter.getContentIds. + private static String[] getContentIds(final boolean hasTimestamp) { + final String[] contentIds; + if (hasTimestamp) { + contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID, + FormatSpec.BIGRAM_TIMESTAMP_CONTENT_ID }; + } else { + contentIds = new String[] { FormatSpec.BIGRAM_FREQ_CONTENT_ID }; + } + return contentIds; + } + + public ArrayList<PendingAttribute> readTargetsAndFrequencies(final int terminalId, + final DictBuffer terminalAddressTableBuffer) { + final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList(); + read(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId, + new SparseTableContentReaderInterface() { + @Override + public void read(final DictBuffer buffer) { + while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, + // remaining bigram entries are ignored. + final int bigramFlags = buffer.readUnsignedByte(); + final int targetTerminalId = buffer.readUnsignedInt24(); + terminalAddressTableBuffer.position(targetTerminalId + * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); + final int targetAddress = + terminalAddressTableBuffer.readUnsignedInt24(); + bigrams.add(new PendingAttribute(bigramFlags + & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, + targetAddress)); + if (0 == (bigramFlags + & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { + break; + } + } + if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { + throw new RuntimeException("Too many bigrams in a PtNode (" + + bigrams.size() + " but max is " + + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); + } + } + }); + if (bigrams.isEmpty()) return null; + return bigrams; + } } - // TODO: Let's have something like SparseTableContentsReader in this class. - private void loadShortcutAddressSparseTable() throws IOException { - final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - final File contentFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX - + FormatSpec.SHORTCUT_CONTENT_ID); - final File timestampsFile = new File(mDictDirectory, mDictDirectory.getName() - + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX - + FormatSpec.SHORTCUT_CONTENT_ID); - mShortcutAddressTable = SparseTable.readFromFiles(lookupIndexFile, - new File[] { contentFile, timestampsFile }, - FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE); + /** + * An auxiliary class for reading shortcuts. + */ + protected static class ShortcutContentReader extends SparseTableContentReader { + public ShortcutContentReader(final String name, final File baseDir, + final DictionaryBufferFactory factory) { + super(name + FormatSpec.SHORTCUT_FILE_EXTENSION, + FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir, + new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION }, + new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, factory); + } + + public ArrayList<WeightedString> readShortcuts(final int terminalId) { + final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList(); + read(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId, + new SparseTableContentReaderInterface() { + @Override + public void read(final DictBuffer buffer) { + while (true) { + final int flags = buffer.readUnsignedByte(); + final String word = CharEncoding.readString(buffer); + shortcuts.add(new WeightedString(word, + flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); + if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) { + break; + } + } + } + }); + if (shortcuts.isEmpty()) return null; + return shortcuts; + } } protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader { @@ -172,102 +277,82 @@ public class Ver4DictDecoder extends AbstractDictDecoder { } } - private ArrayList<WeightedString> readShortcuts(final int terminalId) { - if (mShortcutAddressTable.get(0, terminalId) == SparseTable.NOT_EXIST) return null; - - final ArrayList<WeightedString> ret = CollectionUtils.newArrayList(); - final int posOfShortcuts = mShortcutAddressTable.get(FormatSpec.SHORTCUT_CONTENT_INDEX, - terminalId); - mShortcutBuffer.position(posOfShortcuts); - while (true) { - final int flags = mShortcutBuffer.readUnsignedByte(); - final String word = CharEncoding.readString(mShortcutBuffer); - ret.add(new WeightedString(word, - flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY)); - if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - return ret; - } + private final int[] mCharacterBufferForReadingVer4PtNodeInfo + = new int[FormatSpec.MAX_WORD_LENGTH]; + /** + * Reads PtNode from ptNodePos in the trie file and returns Ver4PtNodeInfo. + * + * @param ptNodePos the position of PtNode. + * @param options the format options. + * @return Ver4PtNodeInfo. + */ // TODO: Make this buffer thread safe. // TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH. - private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; - @Override - public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { - int addressPointer = ptNodePos; + protected Ver4PtNodeInfo readVer4PtNodeInfo(final int ptNodePos, final FormatOptions options) { + int readingPos = ptNodePos; final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); - addressPointer += FormatSpec.PTNODE_FLAGS_SIZE; + readingPos += FormatSpec.PTNODE_FLAGS_SIZE; - final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options); + final int parentPos = PtNodeReader.readParentAddress(mDictBuffer, options); if (BinaryDictIOUtils.supportsDynamicUpdate(options)) { - addressPointer += FormatSpec.PARENT_ADDRESS_SIZE; + readingPos += FormatSpec.PARENT_ADDRESS_SIZE; } final int characters[]; if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { int index = 0; int character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character); + readingPos += CharEncoding.getCharSize(character); while (FormatSpec.INVALID_CHARACTER != character && index < FormatSpec.MAX_WORD_LENGTH) { - mCharacterBuffer[index++] = character; + mCharacterBufferForReadingVer4PtNodeInfo[index++] = character; character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character); + readingPos += CharEncoding.getCharSize(character); } - characters = Arrays.copyOfRange(mCharacterBuffer, 0, index); + characters = Arrays.copyOfRange(mCharacterBufferForReadingVer4PtNodeInfo, 0, index); } else { final int character = CharEncoding.readChar(mDictBuffer); - addressPointer += CharEncoding.getCharSize(character); + readingPos += CharEncoding.getCharSize(character); characters = new int[] { character }; } final int terminalId; if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { terminalId = PtNodeReader.readTerminalId(mDictBuffer); - addressPointer += FormatSpec.PTNODE_TERMINAL_ID_SIZE; + readingPos += FormatSpec.PTNODE_TERMINAL_ID_SIZE; } else { terminalId = PtNode.NOT_A_TERMINAL; } + int childrenPos = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); + if (childrenPos != FormatSpec.NO_CHILDREN_ADDRESS) { + childrenPos += readingPos; + } + readingPos += BinaryDictIOUtils.getChildrenAddressSize(flags, options); + + return new Ver4PtNodeInfo(flags, characters, terminalId, childrenPos, parentPos, + readingPos - ptNodePos); + } + + @Override + public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) { + final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(ptNodePos, options); + final int frequency; - if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { - frequency = PtNodeReader.readFrequency(mFrequencyBuffer, terminalId); + if (0 != (FormatSpec.FLAG_IS_TERMINAL & nodeInfo.mFlags)) { + frequency = PtNodeReader.readFrequency(mFrequencyBuffer, nodeInfo.mTerminalId); } else { frequency = PtNode.NOT_A_TERMINAL; } - int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options); - if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { - childrenAddress += addressPointer; - } - addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options); - final ArrayList<WeightedString> shortcutTargets = readShortcuts(terminalId); - - final ArrayList<PendingAttribute> bigrams; - if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { - bigrams = new ArrayList<PendingAttribute>(); - final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId); - mBigramBuffer.position(posOfBigrams); - while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, - // remaining bigram entries are ignored. - final int bigramFlags = mBigramBuffer.readUnsignedByte(); - final int targetTerminalId = mBigramBuffer.readUnsignedInt24(); - mTerminalAddressTableBuffer.position( - targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE); - final int targetAddress = mTerminalAddressTableBuffer.readUnsignedInt24(); - bigrams.add(new PendingAttribute( - bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY, - targetAddress)); - if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break; - } - if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { - throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size() - + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")"); - } - } else { - bigrams = null; - } - return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency, - parentAddress, childrenAddress, shortcutTargets, bigrams); + + final ArrayList<WeightedString> shortcutTargets = mShortcutReader.readShortcuts( + nodeInfo.mTerminalId); + final ArrayList<PendingAttribute> bigrams = mBigramReader.readTargetsAndFrequencies( + nodeInfo.mTerminalId, mTerminalAddressTableBuffer); + + return new PtNodeInfo(ptNodePos, ptNodePos + nodeInfo.mNodeSize, nodeInfo.mFlags, + nodeInfo.mCharacters, frequency, nodeInfo.mParentPos, nodeInfo.mChildrenPos, + shortcutTargets, bigrams); } private void deleteDictFiles() { @@ -318,10 +403,14 @@ public class Ver4DictDecoder extends AbstractDictDecoder { @Override public boolean readAndFollowForwardLink() { - final int nextAddress = mDictBuffer.readUnsignedInt24(); - if (nextAddress >= 0 && nextAddress < mDictBuffer.limit()) { - mDictBuffer.position(nextAddress); - return true; + final int forwardLinkPos = mDictBuffer.position(); + int nextRelativePos = BinaryDictDecoderUtils.readSInt24(mDictBuffer); + if (nextRelativePos != FormatSpec.NO_FORWARD_LINK_ADDRESS) { + final int nextPos = forwardLinkPos + nextRelativePos; + if (nextPos >= 0 && nextPos < mDictBuffer.limit()) { + mDictBuffer.position(nextPos); + return true; + } } return false; } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 5d5ab0462..842427434 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -57,62 +57,6 @@ public class Ver4DictEncoder implements DictEncoder { mDictPlacedDir = dictPlacedDir; } - private interface SparseTableContentWriterInterface { - public void write(final OutputStream outStream) throws IOException; - } - - private static class SparseTableContentWriter { - private final int mContentCount; - private final SparseTable mSparseTable; - private final File mLookupTableFile; - protected final File mBaseDir; - private final File[] mAddressTableFiles; - private final File[] mContentFiles; - protected final OutputStream[] mContentOutStreams; - - public SparseTableContentWriter(final String name, final int initialCapacity, - final int blockSize, final File baseDir, final String[] contentFilenames, - final String[] contentIds) { - if (contentFilenames.length != contentIds.length) { - throw new RuntimeException("The length of contentFilenames and the length of" - + " contentIds are different " + contentFilenames.length + ", " - + contentIds.length); - } - mContentCount = contentFilenames.length; - mSparseTable = new SparseTable(initialCapacity, blockSize, mContentCount); - mLookupTableFile = new File(baseDir, name + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX); - mAddressTableFiles = new File[mContentCount]; - mContentFiles = new File[mContentCount]; - mBaseDir = baseDir; - for (int i = 0; i < mContentCount; ++i) { - mAddressTableFiles[i] = new File(mBaseDir, - name + FormatSpec.CONTENT_TABLE_FILE_SUFFIX + contentIds[i]); - mContentFiles[i] = new File(mBaseDir, contentFilenames[i] + contentIds[i]); - } - mContentOutStreams = new OutputStream[mContentCount]; - } - - public void openStreams() throws FileNotFoundException { - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i] = new FileOutputStream(mContentFiles[i]); - } - } - - protected void write(final int contentIndex, final int index, - final SparseTableContentWriterInterface writer) throws IOException { - mSparseTable.set(contentIndex, index, (int) mContentFiles[contentIndex].length()); - writer.write(mContentOutStreams[contentIndex]); - mContentOutStreams[contentIndex].flush(); - } - - public void closeStreams() throws IOException { - mSparseTable.writeToFiles(mLookupTableFile, mAddressTableFiles); - for (int i = 0; i < mContentCount; ++i) { - mContentOutStreams[i].close(); - } - } - } - private static class BigramContentWriter extends SparseTableContentWriter { private final boolean mWriteTimestamp; diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 32c07e106..d670aad43 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -26,7 +26,6 @@ import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncodin import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; @@ -60,9 +59,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50; private static final int NUM_OF_SHORTCUTS = 5; - private static final int USE_BYTE_ARRAY = 1; - private static final int USE_BYTE_BUFFER = 2; - private static final ArrayList<String> sWords = CollectionUtils.newArrayList(); private static final SparseArray<List<Integer>> sEmptyBigrams = CollectionUtils.newSparseArray(); @@ -71,21 +67,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { CollectionUtils.newSparseArray(); private static final HashMap<String, List<String>> sShortcuts = CollectionUtils.newHashMap(); - private static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); - private static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = - new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); - private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = - new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */, - true /* hasTimestamp */); - - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - public BinaryDictDecoderEncoderTests() { this(System.currentTimeMillis(), DEFAULT_MAX_UNIGRAMS); } @@ -124,17 +105,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } - private DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - return new Ver4DictEncoder(getContext().getCacheDir()); - } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { - return new Ver3DictEncoder(file); - } else { - throw new RuntimeException("The format option has a wrong version : " - + formatOptions.mVersion); - } - } - private void generateWords(final int number, final Random random, final int[] codePointSet) { final Set<String> wordSet = CollectionUtils.newHashSet(); while (wordSet.size() < number) { @@ -186,7 +156,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictEncoder dictEncoder = getDictEncoder(file, formatOptions); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); now = System.currentTimeMillis(); // If you need to dump the dict to a textual file, uncomment the line below and the @@ -241,54 +212,21 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private String outputOptions(final int bufferType, final FormatSpec.FormatOptions formatOptions) { String result = " : buffer type = " - + ((bufferType == USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); + + ((bufferType == BinaryDictUtils.USE_BYTE_BUFFER) ? "byte buffer" : "byte array"); result += " : version = " + formatOptions.mVersion; return result + ", supportsDynamicUpdate = " + formatOptions.mSupportsDynamicUpdate; } - private DictionaryOptions getDictionaryOptions(final String id, final String version) { - final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), - false, false); - options.mAttributes.put("version", version); - options.mAttributes.put("dictionary", id); - return options; - } - - private File setUpDictionaryFile(final String name, final String version) { - File file = null; - try { - file = new File(getContext().getCacheDir(), name + "." + version - + TEST_DICT_FILE_EXTENSION); - file.createNewFile(); - } catch (IOException e) { - // do nothing - } - assertTrue("Failed to create the dictionary file.", file.exists()); - return file; - } - - private DictDecoder getDictDecoder(final File file, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { - if (formatOptions.mVersion == FormatSpec.VERSION4) { - final FileHeader header = new FileHeader(0, dictOptions, formatOptions); - return FormatSpec.getDictDecoder(new File(getContext().getCacheDir(), - header.getId() + "." + header.getVersion()), bufferType); - } else { - return FormatSpec.getDictDecoder(file, bufferType); - } - } // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List<String> words, final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + final HashMap<String, List<String>> shortcutMap, final int bufferType) { long now, diff = -1; FusionDictionary dict = null; try { - final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, - dictOptions); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; @@ -310,17 +248,17 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final String dictName = "runReadAndWrite"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, shortcuts); addBigrams(dict, words, bigrams); checkDictionary(dict, words, bigrams, shortcuts); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType, - formatOptions, dict.mOptions); + final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -349,8 +287,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final byte[] buffer = new byte[50 * 3]; final DictBuffer dictBuffer = new ByteArrayDictBuffer(buffer); for (final String word : sWords) { - Log.d("testReadAndWriteString", "write : " + word); - Arrays.fill(buffer, (byte)0); + Arrays.fill(buffer, (byte) 0); CharEncoding.writeString(buffer, 0, word); dictBuffer.position(0); final String str = CharEncoding.readString(dictBuffer); @@ -361,12 +298,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadAndWriteWithByteBuffer() { final List<String> results = CollectionUtils.newArrayList(); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -376,12 +319,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadAndWriteWithByteArray() { final List<String> results = CollectionUtils.newArrayList(); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -437,8 +386,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words, - final SparseArray<List<Integer>> bigrams, final int bufferType, - final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + final SparseArray<List<Integer>> bigrams, final int bufferType) { FileInputStream inStream = null; final TreeMap<Integer, String> resultWords = CollectionUtils.newTreeMap(); @@ -448,8 +396,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, - dictOptions); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -476,20 +423,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatSpec.FormatOptions formatOptions, final String message) { final String dictName = "runReadUnigrams"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.getDictionaryOptions(dictName, dictVersion)); addUnigrams(words.size(), dict, words, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, - formatOptions, dict.mOptions); + long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType); long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, - bufferType, formatOptions, dict.mOptions); + bufferType); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap + " : " + message + " : " + outputOptions(bufferType, formatOptions); @@ -508,13 +455,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadUnigramsAndBigramsBinaryWithByteBuffer() { final ArrayList<String> results = CollectionUtils.newArrayList(); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, - VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION2); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -524,13 +476,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testReadUnigramsAndBigramsBinaryWithByteArray() { final ArrayList<String> results = CollectionUtils.newArrayList(); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, - VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION2); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -578,16 +535,16 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FormatOptions formatOptions, final String message) { final String dictName = "testGetTerminalPosition"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - getDictionaryOptions(dictName, dictVersion)); + BinaryDictUtils.getDictionaryOptions(dictName, dictVersion)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); - final DictDecoder dictDecoder = getDictDecoder(file, DictDecoder.USE_BYTEARRAY, - formatOptions, dict.mOptions); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { @@ -638,19 +595,29 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testGetTerminalPosition() { final ArrayList<String> results = CollectionUtils.newArrayList(); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION2); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); - - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION2); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, BinaryDictUtils.VERSION2); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_ARRAY, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); + + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, BinaryDictUtils.VERSION2); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(BinaryDictUtils.USE_BYTE_BUFFER, + BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP); for (final String result : results) { Log.d(TAG, result); @@ -660,7 +627,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private void runTestDeleteWord(final FormatOptions formatOptions) { final String dictName = "testDeleteWord"; final String dictVersion = Long.toString(System.currentTimeMillis()); - final File file = setUpDictionaryFile(dictName, dictVersion); + final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, + getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new FusionDictionary.DictionaryOptions( @@ -668,15 +636,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); timeWritingDictToFile(file, dict, formatOptions); - final DictUpdater dictUpdater; - if (formatOptions.mVersion == 3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else if (formatOptions.mVersion == 4) { - dictUpdater = new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatOptions.mVersion - + " doesn't exist."); - } + final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, @@ -696,7 +656,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } public void testDeleteWord() { - runTestDeleteWord(VERSION3_WITH_DYNAMIC_UPDATE); - runTestDeleteWord(VERSION4_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(BinaryDictUtils.VERSION4_WITH_DYNAMIC_UPDATE); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index afe5adb73..592f86e94 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -23,6 +23,7 @@ import android.util.Log; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; @@ -30,24 +31,16 @@ import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.Random; @LargeTest public class BinaryDictIOUtilsTests extends AndroidTestCase { private static final String TAG = BinaryDictIOUtilsTests.class.getSimpleName(); - private static final FormatSpec.FormatOptions FORMAT_OPTIONS = - new FormatSpec.FormatOptions(3, true); private static final ArrayList<String> sWords = CollectionUtils.newArrayList(); public static final int DEFAULT_MAX_UNIGRAMS = 1500; private final int mMaxUnigrams; - private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; - - private static final int VERSION3 = 3; - private static final int VERSION4 = 4; - private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", @@ -141,7 +134,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int position = FormatSpec.NOT_VALID_WORD; try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_READONLY_BYTEBUFFER); position = dictDecoder.getTerminalPosition(word); } catch (IOException e) { @@ -159,7 +152,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { * @throws IOException * @throws UnsupportedFormatException */ - private static PtNodeInfo findWordByBinaryDictReader(final DictDecoder dictDecoder, + private static PtNodeInfo findWordByDictDecoder(final DictDecoder dictDecoder, final String word) throws IOException, UnsupportedFormatException { int position = dictDecoder.getTerminalPosition(word); if (position != FormatSpec.NOT_VALID_WORD) { @@ -176,7 +169,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { PtNodeInfo info = null; try { dictDecoder.openDictBuffer(); - info = findWordByBinaryDictReader(dictDecoder, word); + info = findWordByDictDecoder(dictDecoder, word); } catch (IOException e) { } catch (UnsupportedFormatException e) { } @@ -186,16 +179,10 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { // return amount of time to insert a word private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList<WeightedString> bigrams, - final ArrayList<WeightedString> shortcuts, final int formatVersion) { + final ArrayList<WeightedString> shortcuts, final FormatOptions formatOptions) { long amountOfTime = -1; try { - final DictUpdater dictUpdater; - if (formatVersion == VERSION3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" - + " exist."); - } + final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); if (!exist) { assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); @@ -212,18 +199,14 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { return amountOfTime; } - private void deleteWord(final File file, final String word, final int formatVersion) { + private void deleteWord(final File file, final String word, final FormatOptions formatOptions) { try { - final DictUpdater dictUpdater; - if (formatVersion == VERSION3) { - dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); - } else { - throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" - + " exist."); - } + final DictUpdater dictUpdater = BinaryDictUtils.getDictUpdater(file, formatOptions); dictUpdater.deleteWord(word); } catch (IOException e) { + Log.e(TAG, "Raised an IOException while deleting a word", e); } catch (UnsupportedFormatException e) { + Log.e(TAG, "Raised an UnsupportedFormatException while deleting a word", e); } } @@ -242,23 +225,21 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } } - private void runTestInsertWord(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - fail("IOException while creating temporary file: " + e); - } + private void runTestInsertWord(final FormatOptions formatOptions) { + final String testName = "testInsertWord"; + final String version = Long.toString(System.currentTimeMillis()); + final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, + getContext().getCacheDir()); // set an initial dictionary. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + BinaryDictUtils.getDictionaryOptions(testName, version)); dict.add("abcd", 10, null, false); try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); + dictEncoder.writeDictionary(dict, formatOptions); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } catch (UnsupportedFormatException e) { @@ -266,54 +247,52 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - insertAndCheckWord(file, "abcde", 10, false, null, null, formatVersion); + insertAndCheckWord(file, "abcde", 10, false, null, null, formatOptions); - insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatVersion); + insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatOptions); checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatVersion); + insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatOptions); checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); // update the existing word. - insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatVersion); + insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatOptions); // split 1 - insertAndCheckWord(file, "ab", 20, false, null, null, formatVersion); + insertAndCheckWord(file, "ab", 20, false, null, null, formatOptions); // split 2 - insertAndCheckWord(file, "ami", 30, false, null, null, formatVersion); + insertAndCheckWord(file, "ami", 30, false, null, null, formatOptions); - deleteWord(file, "ami", formatVersion); + deleteWord(file, "ami", formatOptions); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami")); - insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatVersion); + insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatOptions); - deleteWord(file, "abcd", formatVersion); + deleteWord(file, "abcd", formatOptions); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); } public void testInsertWord() { - runTestInsertWord(VERSION3); + runTestInsertWord(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); } - private void runTestInsertWordWithBigrams(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - fail("IOException while creating temporary file: " + e); - } + private void runTestInsertWordWithBigrams(final FormatOptions formatOptions) { + final String testName = "testInsertWordWithBigrams"; + final String version = Long.toString(System.currentTimeMillis()); + File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, + getContext().getCacheDir()); // set an initial dictionary. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String,String>(), false, false)); + BinaryDictUtils.getDictionaryOptions(testName, version)); dict.add("abcd", 10, null, false); dict.add("efgh", 15, null, false); try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); + dictEncoder.writeDictionary(dict, formatOptions); } catch (IOException e) { fail("IOException while writing an initial dictionary : " + e); } catch (UnsupportedFormatException e) { @@ -323,8 +302,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { final ArrayList<WeightedString> banana = new ArrayList<WeightedString>(); banana.add(new WeightedString("banana", 10)); - insertAndCheckWord(file, "banana", 0, false, null, null, formatVersion); - insertAndCheckWord(file, "recursive", 60, true, banana, null, formatVersion); + insertAndCheckWord(file, "banana", 0, false, null, null, formatOptions); + insertAndCheckWord(file, "recursive", 60, true, banana, null, formatOptions); final PtNodeInfo info = findWordFromFile(file, "recursive"); int bananaPos = getWordPosition(file, "banana"); @@ -334,27 +313,24 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } public void testInsertWordWithBigrams() { - runTestInsertWordWithBigrams(VERSION3); + runTestInsertWordWithBigrams(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); } - private void runTestRandomWords(final int formatVersion) { - File file = null; - try { - file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, - getContext().getCacheDir()); - } catch (IOException e) { - } - assertNotNull(file); + private void runTestRandomWords(final FormatOptions formatOptions) { + final String testName = "testRandomWord"; + final String version = Long.toString(System.currentTimeMillis()); + final File file = BinaryDictUtils.getDictFile(testName, version, formatOptions, + getContext().getCacheDir()); // set an initial dictionary. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), - new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false, - false)); + BinaryDictUtils.getDictionaryOptions(testName, version)); dict.add("initial", 10, null, false); try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); - dictEncoder.writeDictionary(dict, FORMAT_OPTIONS); + final DictEncoder dictEncoder = BinaryDictUtils.getDictEncoder(file, formatOptions, + getContext().getCacheDir()); + dictEncoder.writeDictionary(dict, formatOptions); } catch (IOException e) { assertTrue(false); } catch (UnsupportedFormatException e) { @@ -366,7 +342,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int cnt = 0; for (final String word : sWords) { final long diff = insertAndCheckWord(file, word, - cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatVersion); + cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatOptions); maxTimeToInsert = Math.max(maxTimeToInsert, diff); minTimeToInsert = Math.min(minTimeToInsert, diff); sum += diff; @@ -377,13 +353,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } - Log.d(TAG, "Test version " + formatVersion); + Log.d(TAG, "Test version " + formatOptions); Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms."); Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms."); Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms."); } public void testRandomWords() { - runTestRandomWords(VERSION3); + runTestRandomWords(BinaryDictUtils.VERSION3_WITH_DYNAMIC_UPDATE); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java new file mode 100644 index 000000000..f476738f3 --- /dev/null +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictUtils.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; +import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; + +import java.io.File; +import java.util.HashMap; + +public class BinaryDictUtils { + public static final int USE_BYTE_ARRAY = 1; + public static final int USE_BYTE_BUFFER = 2; + + public static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + + public static final FormatSpec.FormatOptions VERSION2 = new FormatSpec.FormatOptions(2); + public static final FormatSpec.FormatOptions VERSION3_WITHOUT_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); + public static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE_AND_TIMESTAMP = + new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */, + true /* hasTimestamp */); + + public static DictionaryOptions getDictionaryOptions(final String id, final String version) { + final DictionaryOptions options = new DictionaryOptions(new HashMap<String, String>(), + false /* germanUmlautProcessing */, false /* frenchLigatureProcessing */); + options.mAttributes.put("dictionary", id); + options.mAttributes.put("version", version); + return options; + } + + public static File getDictFile(final String name, final String version, + final FormatOptions formatOptions, final File directory) { + if (formatOptions.mVersion == 2 || formatOptions.mVersion == 3) { + return new File(directory, name + "." + version + TEST_DICT_FILE_EXTENSION); + } else if (formatOptions.mVersion == 4) { + return new File(directory, name + "." + version); + } else { + throw new RuntimeException("the format option has a wrong version : " + + formatOptions.mVersion); + } + } + + public static DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions, + final File cacheDir) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new Ver4DictEncoder(cacheDir); + } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { + return new Ver3DictEncoder(file); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } + } + + public static DictUpdater getDictUpdater(final File file, final FormatOptions formatOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else if (formatOptions.mVersion == 3) { + return new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } + } +} |