aboutsummaryrefslogtreecommitdiffstats
path: root/java/src/com/android/inputmethod/latin/makedict
diff options
context:
space:
mode:
Diffstat (limited to 'java/src/com/android/inputmethod/latin/makedict')
-rw-r--r--java/src/com/android/inputmethod/latin/makedict/DictDecoder.java2
-rw-r--r--java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java214
2 files changed, 131 insertions, 85 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
index 3dbeee099..b8636eecd 100644
--- a/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/DictDecoder.java
@@ -43,7 +43,7 @@ public interface DictDecoder {
public FileHeader readHeader() throws IOException, UnsupportedFormatException;
/**
- * Reads PtNode from nodeAddress.
+ * Reads PtNode from ptNodePos.
* @param ptNodePos the position of PtNode.
* @param formatOptions the format options.
* @return PtNodeInfo.
diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
index 64538c102..add03c323 100644
--- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
+++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
@@ -51,9 +51,30 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
protected DictBuffer mDictBuffer;
private DictBuffer mFrequencyBuffer;
private DictBuffer mTerminalAddressTableBuffer;
- private DictBuffer mShortcutBuffer;
private BigramContentReader mBigramReader;
- private SparseTable mShortcutAddressTable;
+ private ShortcutContentReader mShortcutReader;
+
+ /**
+ * Raw PtNode info straight out of a trie file in version 4 dictionary.
+ */
+ protected static final class Ver4PtNodeInfo {
+ public final int mFlags;
+ public final int[] mCharacters;
+ public final int mTerminalId;
+ public final int mChildrenPos;
+ public final int mParentPos;
+ public final int mNodeSize;
+
+ public Ver4PtNodeInfo(final int flags, final int[] characters, final int terminalId,
+ final int childrenPos, final int parentPos, final int nodeSize) {
+ mFlags = flags;
+ mCharacters = characters;
+ mTerminalId = terminalId;
+ mChildrenPos = childrenPos;
+ mParentPos = parentPos;
+ mNodeSize = nodeSize;
+ }
+ }
@UsedForTesting
/* package */ Ver4DictDecoder(final File dictDirectory, final int factoryFlag) {
@@ -110,8 +131,9 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
mBigramReader = new BigramContentReader(mDictDirectory.getName(),
mDictDirectory, mBufferFactory, false);
mBigramReader.openBuffers();
- mShortcutBuffer = mBufferFactory.getDictionaryBuffer(getFile(FILETYPE_SHORTCUT));
- loadShortcutAddressSparseTable();
+ mShortcutReader = new ShortcutContentReader(mDictDirectory.getName(), mDictDirectory,
+ mBufferFactory);
+ mShortcutReader.openBuffers();
}
@Override
@@ -136,21 +158,6 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
return header;
}
- // TODO: Let's have something like SparseTableContentsReader in this class.
- private void loadShortcutAddressSparseTable() throws IOException {
- final File lookupIndexFile = new File(mDictDirectory, mDictDirectory.getName()
- + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.LOOKUP_TABLE_FILE_SUFFIX);
- final File contentFile = new File(mDictDirectory, mDictDirectory.getName()
- + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX
- + FormatSpec.SHORTCUT_CONTENT_ID);
- final File timestampsFile = new File(mDictDirectory, mDictDirectory.getName()
- + FormatSpec.SHORTCUT_FILE_EXTENSION + FormatSpec.CONTENT_TABLE_FILE_SUFFIX
- + FormatSpec.SHORTCUT_CONTENT_ID);
- mShortcutAddressTable = SparseTable.readFromFiles(lookupIndexFile,
- new File[] { contentFile, timestampsFile },
- FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE);
- }
-
/**
* An auxiliary class for reading bigrams.
*/
@@ -194,34 +201,71 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
final ArrayList<PendingAttribute> bigrams = CollectionUtils.newArrayList();
read(FormatSpec.BIGRAM_FREQ_CONTENT_INDEX, terminalId,
new SparseTableContentReaderInterface() {
- @Override
- public void read(final DictBuffer buffer) {
- while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE,
- // remaining bigram entries are ignored.
- final int bigramFlags = buffer.readUnsignedByte();
- final int targetTerminalId = buffer.readUnsignedInt24();
- terminalAddressTableBuffer.position(
- targetTerminalId * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
- final int targetAddress = terminalAddressTableBuffer.readUnsignedInt24();
- bigrams.add(new PendingAttribute(
- bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY,
- targetAddress));
- if (0 == (bigramFlags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) {
- break;
+ @Override
+ public void read(final DictBuffer buffer) {
+ while (bigrams.size() < FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ // If bigrams.size() reaches FormatSpec.MAX_BIGRAMS_IN_A_PTNODE,
+ // remaining bigram entries are ignored.
+ final int bigramFlags = buffer.readUnsignedByte();
+ final int targetTerminalId = buffer.readUnsignedInt24();
+ terminalAddressTableBuffer.position(targetTerminalId
+ * FormatSpec.TERMINAL_ADDRESS_TABLE_ADDRESS_SIZE);
+ final int targetAddress =
+ terminalAddressTableBuffer.readUnsignedInt24();
+ bigrams.add(new PendingAttribute(bigramFlags
+ & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY,
+ targetAddress));
+ if (0 == (bigramFlags
+ & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) {
+ break;
+ }
+ }
+ if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
+ throw new RuntimeException("Too many bigrams in a PtNode ("
+ + bigrams.size() + " but max is "
+ + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")");
+ }
}
- }
- if (bigrams.size() >= FormatSpec.MAX_BIGRAMS_IN_A_PTNODE) {
- throw new RuntimeException("Too many bigrams in a PtNode (" + bigrams.size()
- + " but max is " + FormatSpec.MAX_BIGRAMS_IN_A_PTNODE + ")");
- }
- }
- });
+ });
if (bigrams.isEmpty()) return null;
return bigrams;
}
}
+ /**
+ * An auxiliary class for reading shortcuts.
+ */
+ protected static class ShortcutContentReader extends SparseTableContentReader {
+ public ShortcutContentReader(final String name, final File baseDir,
+ final DictionaryBufferFactory factory) {
+ super(name + FormatSpec.SHORTCUT_FILE_EXTENSION,
+ FormatSpec.SHORTCUT_ADDRESS_TABLE_BLOCK_SIZE, baseDir,
+ new String[] { name + FormatSpec.SHORTCUT_FILE_EXTENSION },
+ new String[] { FormatSpec.SHORTCUT_CONTENT_ID }, factory);
+ }
+
+ public ArrayList<WeightedString> readShortcuts(final int terminalId) {
+ final ArrayList<WeightedString> shortcuts = CollectionUtils.newArrayList();
+ read(FormatSpec.SHORTCUT_CONTENT_INDEX, terminalId,
+ new SparseTableContentReaderInterface() {
+ @Override
+ public void read(final DictBuffer buffer) {
+ while (true) {
+ final int flags = buffer.readUnsignedByte();
+ final String word = CharEncoding.readString(buffer);
+ shortcuts.add(new WeightedString(word,
+ flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY));
+ if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) {
+ break;
+ }
+ }
+ }
+ });
+ if (shortcuts.isEmpty()) return null;
+ return shortcuts;
+ }
+ }
+
protected static class PtNodeReader extends AbstractDictDecoder.PtNodeReader {
protected static int readFrequency(final DictBuffer frequencyBuffer, final int terminalId) {
frequencyBuffer.position(terminalId * FormatSpec.FREQUENCY_AND_FLAGS_SIZE + 1);
@@ -233,80 +277,82 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
}
}
- private ArrayList<WeightedString> readShortcuts(final int terminalId) {
- if (mShortcutAddressTable.get(0, terminalId) == SparseTable.NOT_EXIST) return null;
-
- final ArrayList<WeightedString> ret = CollectionUtils.newArrayList();
- final int posOfShortcuts = mShortcutAddressTable.get(FormatSpec.SHORTCUT_CONTENT_INDEX,
- terminalId);
- mShortcutBuffer.position(posOfShortcuts);
- while (true) {
- final int flags = mShortcutBuffer.readUnsignedByte();
- final String word = CharEncoding.readString(mShortcutBuffer);
- ret.add(new WeightedString(word,
- flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY));
- if (0 == (flags & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT)) break;
- }
- return ret;
- }
+ private final int[] mCharacterBufferForReadingVer4PtNodeInfo
+ = new int[FormatSpec.MAX_WORD_LENGTH];
+ /**
+ * Reads PtNode from ptNodePos in the trie file and returns Ver4PtNodeInfo.
+ *
+ * @param ptNodePos the position of PtNode.
+ * @param options the format options.
+ * @return Ver4PtNodeInfo.
+ */
// TODO: Make this buffer thread safe.
// TODO: Support words longer than FormatSpec.MAX_WORD_LENGTH.
- private final int[] mCharacterBuffer = new int[FormatSpec.MAX_WORD_LENGTH];
- @Override
- public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) {
- int addressPointer = ptNodePos;
+ protected Ver4PtNodeInfo readVer4PtNodeInfo(final int ptNodePos, final FormatOptions options) {
+ int readingPos = ptNodePos;
final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer);
- addressPointer += FormatSpec.PTNODE_FLAGS_SIZE;
+ readingPos += FormatSpec.PTNODE_FLAGS_SIZE;
- final int parentAddress = PtNodeReader.readParentAddress(mDictBuffer, options);
+ final int parentPos = PtNodeReader.readParentAddress(mDictBuffer, options);
if (BinaryDictIOUtils.supportsDynamicUpdate(options)) {
- addressPointer += FormatSpec.PARENT_ADDRESS_SIZE;
+ readingPos += FormatSpec.PARENT_ADDRESS_SIZE;
}
final int characters[];
if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) {
int index = 0;
int character = CharEncoding.readChar(mDictBuffer);
- addressPointer += CharEncoding.getCharSize(character);
+ readingPos += CharEncoding.getCharSize(character);
while (FormatSpec.INVALID_CHARACTER != character
&& index < FormatSpec.MAX_WORD_LENGTH) {
- mCharacterBuffer[index++] = character;
+ mCharacterBufferForReadingVer4PtNodeInfo[index++] = character;
character = CharEncoding.readChar(mDictBuffer);
- addressPointer += CharEncoding.getCharSize(character);
+ readingPos += CharEncoding.getCharSize(character);
}
- characters = Arrays.copyOfRange(mCharacterBuffer, 0, index);
+ characters = Arrays.copyOfRange(mCharacterBufferForReadingVer4PtNodeInfo, 0, index);
} else {
final int character = CharEncoding.readChar(mDictBuffer);
- addressPointer += CharEncoding.getCharSize(character);
+ readingPos += CharEncoding.getCharSize(character);
characters = new int[] { character };
}
final int terminalId;
if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
terminalId = PtNodeReader.readTerminalId(mDictBuffer);
- addressPointer += FormatSpec.PTNODE_TERMINAL_ID_SIZE;
+ readingPos += FormatSpec.PTNODE_TERMINAL_ID_SIZE;
} else {
terminalId = PtNode.NOT_A_TERMINAL;
}
+ int childrenPos = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options);
+ if (childrenPos != FormatSpec.NO_CHILDREN_ADDRESS) {
+ childrenPos += readingPos;
+ }
+ readingPos += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
+
+ return new Ver4PtNodeInfo(flags, characters, terminalId, childrenPos, parentPos,
+ readingPos - ptNodePos);
+ }
+
+ @Override
+ public PtNodeInfo readPtNode(int ptNodePos, FormatOptions options) {
+ final Ver4PtNodeInfo nodeInfo = readVer4PtNodeInfo(ptNodePos, options);
+
final int frequency;
- if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) {
- frequency = PtNodeReader.readFrequency(mFrequencyBuffer, terminalId);
+ if (0 != (FormatSpec.FLAG_IS_TERMINAL & nodeInfo.mFlags)) {
+ frequency = PtNodeReader.readFrequency(mFrequencyBuffer, nodeInfo.mTerminalId);
} else {
frequency = PtNode.NOT_A_TERMINAL;
}
- int childrenAddress = PtNodeReader.readChildrenAddress(mDictBuffer, flags, options);
- if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) {
- childrenAddress += addressPointer;
- }
- addressPointer += BinaryDictIOUtils.getChildrenAddressSize(flags, options);
- final ArrayList<WeightedString> shortcutTargets = readShortcuts(terminalId);
- final ArrayList<PendingAttribute> bigrams =
- mBigramReader.readTargetsAndFrequencies(terminalId,
- mTerminalAddressTableBuffer);
-
- return new PtNodeInfo(ptNodePos, addressPointer, flags, characters, frequency,
- parentAddress, childrenAddress, shortcutTargets, bigrams);
+
+ final ArrayList<WeightedString> shortcutTargets = mShortcutReader.readShortcuts(
+ nodeInfo.mTerminalId);
+ final ArrayList<PendingAttribute> bigrams = mBigramReader.readTargetsAndFrequencies(
+ nodeInfo.mTerminalId, mTerminalAddressTableBuffer);
+
+ return new PtNodeInfo(ptNodePos, ptNodePos + nodeInfo.mNodeSize, nodeInfo.mFlags,
+ nodeInfo.mCharacters, frequency, nodeInfo.mParentPos, nodeInfo.mChildrenPos,
+ shortcutTargets, bigrams);
}
private void deleteDictFiles() {
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-06-04 07:10:45 +0000