diff options
Diffstat (limited to 'java/src/com/android/inputmethod')
3 files changed, 87 insertions, 57 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java index 96d057a44..9c327faed 100644 --- a/java/src/com/android/inputmethod/latin/makedict/SparseTable.java +++ b/java/src/com/android/inputmethod/latin/makedict/SparseTable.java @@ -17,6 +17,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.utils.CollectionUtils; import java.io.File; import java.io.FileInputStream; @@ -37,35 +38,39 @@ public class SparseTable { /** * mLookupTable is indexed by terminal ID, containing exactly one entry for every mBlockSize * terminals. - * It contains at index i = j / mBlockSize the index in mContentsTable where the values for - * terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized integer array. + * It contains at index i = j / mBlockSize the index in each ArrayList in mContentsTables where + * the values for terminals with IDs j to j + mBlockSize - 1 are stored as an mBlockSize-sized + * integer array. */ private final ArrayList<Integer> mLookupTable; - private final ArrayList<Integer> mContentTable; + private final ArrayList<ArrayList<Integer>> mContentTables; private final int mBlockSize; + private final int mContentTableCount; public static final int NOT_EXIST = -1; + public static final int SIZE_OF_INT_IN_BYTES = 4; @UsedForTesting - public SparseTable(final int initialCapacity, final int blockSize) { + public SparseTable(final int initialCapacity, final int blockSize, + final int contentTableCount) { mBlockSize = blockSize; final int lookupTableSize = initialCapacity / mBlockSize + (initialCapacity % mBlockSize > 0 ? 1 : 0); mLookupTable = new ArrayList<Integer>(Collections.nCopies(lookupTableSize, NOT_EXIST)); - mContentTable = new ArrayList<Integer>(); + mContentTableCount = contentTableCount; + mContentTables = CollectionUtils.newArrayList(); + for (int i = 0; i < mContentTableCount; ++i) { + mContentTables.add(new ArrayList<Integer>()); + } } @UsedForTesting - public SparseTable(final int[] lookupTable, final int[] contentTable, final int blockSize) { + public SparseTable(final ArrayList<Integer> lookupTable, + final ArrayList<ArrayList<Integer>> contentTables, final int blockSize) { mBlockSize = blockSize; - mLookupTable = new ArrayList<Integer>(lookupTable.length); - for (int i = 0; i < lookupTable.length; ++i) { - mLookupTable.add(lookupTable[i]); - } - mContentTable = new ArrayList<Integer>(contentTable.length); - for (int i = 0; i < contentTable.length; ++i) { - mContentTable.add(contentTable[i]); - } + mContentTableCount = contentTables.size(); + mLookupTable = lookupTable; + mContentTables = contentTables; } /** @@ -75,8 +80,8 @@ public class SparseTable { * Otherwise, IndexOutOfBoundsException will be raised. */ @UsedForTesting - private static void convertByteArrayToIntegerArray(final byte[] byteArray, - final ArrayList<Integer> integerArray) { + private static ArrayList<Integer> convertByteArrayToIntegerArray(final byte[] byteArray) { + final ArrayList<Integer> integerArray = new ArrayList<Integer>(byteArray.length / 4); for (int i = 0; i < byteArray.length; i += 4) { int value = 0; for (int j = i; j < i + 4; ++j) { @@ -85,39 +90,43 @@ public class SparseTable { } integerArray.add(value); } + return integerArray; } @UsedForTesting - public SparseTable(final byte[] lookupTable, final byte[] contentTable, final int blockSize) { - mBlockSize = blockSize; - mLookupTable = new ArrayList<Integer>(lookupTable.length / 4); - mContentTable = new ArrayList<Integer>(contentTable.length / 4); - convertByteArrayToIntegerArray(lookupTable, mLookupTable); - convertByteArrayToIntegerArray(contentTable, mContentTable); + public int get(final int contentTableIndex, final int index) { + if (!contains(index)) { + return NOT_EXIST; + } + return mContentTables.get(contentTableIndex).get( + mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); } @UsedForTesting - public int get(final int index) { - if (index < 0 || index / mBlockSize >= mLookupTable.size() - || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { - return NOT_EXIST; + public ArrayList<Integer> getAll(final int index) { + final ArrayList<Integer> ret = CollectionUtils.newArrayList(); + for (int i = 0; i < mContentTableCount; ++i) { + ret.add(get(i, index)); } - return mContentTable.get(mLookupTable.get(index / mBlockSize) + (index % mBlockSize)); + return ret; } @UsedForTesting - public void set(final int index, final int value) { + public void set(final int contentTableIndex, final int index, final int value) { if (mLookupTable.get(index / mBlockSize) == NOT_EXIST) { - mLookupTable.set(index / mBlockSize, mContentTable.size()); - for (int i = 0; i < mBlockSize; ++i) { - mContentTable.add(NOT_EXIST); + mLookupTable.set(index / mBlockSize, mContentTables.get(contentTableIndex).size()); + for (int i = 0; i < mContentTableCount; ++i) { + for (int j = 0; j < mBlockSize; ++j) { + mContentTables.get(i).add(NOT_EXIST); + } } } - mContentTable.set(mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); + mContentTables.get(contentTableIndex).set( + mLookupTable.get(index / mBlockSize) + (index % mBlockSize), value); } - public void remove(final int index) { - set(index, NOT_EXIST); + public void remove(final int indexOfContent, final int index) { + set(indexOfContent, index, NOT_EXIST); } @UsedForTesting @@ -127,7 +136,8 @@ public class SparseTable { @UsedForTesting /* package */ int getContentTableSize() { - return mContentTable.size(); + // This class always has at least one content table. + return mContentTables.get(0).size(); } @UsedForTesting @@ -136,36 +146,51 @@ public class SparseTable { } public boolean contains(final int index) { - return get(index) != NOT_EXIST; + if (index < 0 || index / mBlockSize >= mLookupTable.size() + || mLookupTable.get(index / mBlockSize) == NOT_EXIST) { + return false; + } + return true; } @UsedForTesting - public void write(final OutputStream lookupOutStream, final OutputStream contentOutStream) + public void write(final OutputStream lookupOutStream, final OutputStream[] contentOutStreams) throws IOException { + if (contentOutStreams.length != mContentTableCount) { + throw new RuntimeException(contentOutStreams.length + " streams are given, but the" + + " table has " + mContentTableCount + " content tables."); + } for (final int index : mLookupTable) { - BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, 4); + BinaryDictEncoderUtils.writeUIntToStream(lookupOutStream, index, SIZE_OF_INT_IN_BYTES); } - for (final int index : mContentTable) { - BinaryDictEncoderUtils.writeUIntToStream(contentOutStream, index, 4); + for (int i = 0; i < contentOutStreams.length; ++i) { + for (final int data : mContentTables.get(i)) { + BinaryDictEncoderUtils.writeUIntToStream(contentOutStreams[i], data, + SIZE_OF_INT_IN_BYTES); + } } } @UsedForTesting - public void writeToFiles(final File lookupTableFile, final File contentFile) + public void writeToFiles(final File lookupTableFile, final File[] contentFiles) throws IOException { - FileOutputStream lookupTableOutStream = null; - FileOutputStream contentOutStream = null; + FileOutputStream lookupTableOutStream = null; + final FileOutputStream[] contentTableOutStreams = new FileOutputStream[mContentTableCount]; try { lookupTableOutStream = new FileOutputStream(lookupTableFile); - contentOutStream = new FileOutputStream(contentFile); - write(lookupTableOutStream, contentOutStream); + for (int i = 0; i < contentFiles.length; ++i) { + contentTableOutStreams[i] = new FileOutputStream(contentFiles[i]); + } + write(lookupTableOutStream, contentTableOutStreams); } finally { if (lookupTableOutStream != null) { lookupTableOutStream.close(); } - if (contentOutStream != null) { - contentOutStream.close(); + for (int i = 0; i < contentTableOutStreams.length; ++i) { + if (contentTableOutStreams[i] != null) { + contentTableOutStreams[i].close(); + } } } } @@ -185,10 +210,15 @@ public class SparseTable { } @UsedForTesting - public static SparseTable readFromFiles(final File lookupTableFile, final File contentFile, + public static SparseTable readFromFiles(final File lookupTableFile, final File[] contentFiles, final int blockSize) throws IOException { final byte[] lookupTable = readFileToByteArray(lookupTableFile); - final byte[] content = readFileToByteArray(contentFile); - return new SparseTable(lookupTable, content, blockSize); + final ArrayList<ArrayList<Integer>> contentTables = + new ArrayList<ArrayList<Integer>>(contentFiles.length); + for (int i = 0; i < contentFiles.length; ++i) { + contentTables.add(convertByteArrayToIntegerArray(readFileToByteArray(contentFiles[i]))); + } + return new SparseTable(convertByteArrayToIntegerArray(readFileToByteArray(lookupTableFile)), + contentTables, blockSize); } } diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 89370f0dc..27cb1c244 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -131,7 +131,7 @@ public class Ver4DictDecoder extends DictDecoder { mDictDirectory.getName() + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION); final File contentFile = new File(mDictDirectory, mDictDirectory.getName() + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION); - mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, contentFile, + mBigramAddressTable = SparseTable.readFromFiles(lookupIndexFile, new File[] { contentFile }, FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); } @@ -208,7 +208,7 @@ public class Ver4DictDecoder extends DictDecoder { final ArrayList<PendingAttribute> bigrams; if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { bigrams = new ArrayList<PendingAttribute>(); - final int posOfBigrams = mBigramAddressTable.get(terminalId); + final int posOfBigrams = mBigramAddressTable.get(0 /* contentTableIndex */, terminalId); mBigramBuffer.position(posOfBigrams); while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) { // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 4c25faf88..d8a2dc821 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -136,7 +136,7 @@ public class Ver4DictEncoder implements DictEncoder { writeTerminalData(flatNodes, terminalCount); mBigramAddressTable = new SparseTable(terminalCount, - FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE); + FormatSpec.BIGRAM_ADDRESS_TABLE_BLOCK_SIZE, 1 /* contentTableCount */); writeBigrams(flatNodes, dict); writeBigramAddressSparseTable(); @@ -231,8 +231,7 @@ public class Ver4DictEncoder implements DictEncoder { while (shortcutIterator.hasNext()) { final WeightedString target = shortcutIterator.next(); final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), - target.mFrequency); + shortcutIterator.hasNext(), target.mFrequency); mTrieBuf[mTriePos++] = (byte)shortcutFlags; final int shortcutShift = CharEncoding.writeString(mTrieBuf, mTriePos, target.mWord); @@ -254,7 +253,8 @@ public class Ver4DictEncoder implements DictEncoder { for (final PtNode ptNode : nodeArray.mData) { if (ptNode.mBigrams != null) { final int startPos = bigramBuffer.size(); - mBigramAddressTable.set(ptNode.mTerminalId, startPos); + mBigramAddressTable.set(0 /* contentTableIndex */, ptNode.mTerminalId, + startPos); final Iterator<WeightedString> bigramIterator = ptNode.mBigrams.iterator(); while (bigramIterator.hasNext()) { final WeightedString bigram = bigramIterator.next(); @@ -280,7 +280,7 @@ public class Ver4DictEncoder implements DictEncoder { new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_LOOKUP_TABLE_FILE_EXTENSION); final File contentFile = new File(mDictDir, mBaseFilename + FormatSpec.BIGRAM_ADDRESS_TABLE_FILE_EXTENSION); - mBigramAddressTable.writeToFiles(lookupIndexFile, contentFile); + mBigramAddressTable.writeToFiles(lookupIndexFile, new File[] { contentFile }); } @Override |