diff options
Diffstat (limited to 'tools/dicttool')
7 files changed, 51 insertions, 45 deletions
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index d8fcbeeaf..6c4cbcf9d 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -17,9 +17,10 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; import org.xml.sax.SAXException; @@ -184,15 +185,14 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodedSpec.mFile); - dictDecoder.openDictBuffer( - new Ver3DictDecoder.DictionaryBufferFromByteArrayFactory()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, + DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); } - return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } } } catch (IOException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index a0fcaabd0..5c7e8b4f2 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -17,18 +17,19 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.MakedictLog; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; +import com.android.inputmethod.latin.makedict.Ver4DictEncoder; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; @@ -44,9 +45,9 @@ import org.xml.sax.SAXException; public class DictionaryMaker { static class Arguments { - private static final String OPTION_VERSION_1 = "-1"; private static final String OPTION_VERSION_2 = "-2"; private static final String OPTION_VERSION_3 = "-3"; + private static final String OPTION_VERSION_4 = "-4"; private static final String OPTION_INPUT_SOURCE = "-s"; private static final String OPTION_INPUT_BIGRAM_XML = "-b"; private static final String OPTION_INPUT_SHORTCUT_XML = "-c"; @@ -127,12 +128,12 @@ public class DictionaryMaker { + "| [-s <combined format input]" + "| [-s <binary input>] [-d <binary output>] [-x <xml output>] " + " [-o <combined output>]" - + "[-1] [-2] [-3]\n" + + "[-2] [-3] [-4]\n" + "\n" + " Converts a source dictionary file to one or several outputs.\n" + " Source can be an XML file, with an optional XML bigrams file, or a\n" + " binary dictionary file.\n" - + " Binary version 1 (Ice Cream Sandwich), 2 (Jelly Bean), 3, XML and\n" + + " Binary version 2 (Jelly Bean), 3, 4, XML and\n" + " combined format outputs are supported."; } @@ -159,8 +160,8 @@ public class DictionaryMaker { // Do nothing, this is the default } else if (OPTION_VERSION_3.equals(arg)) { outputBinaryFormatVersion = 3; - } else if (OPTION_VERSION_1.equals(arg)) { - outputBinaryFormatVersion = 1; + } else if (OPTION_VERSION_4.equals(arg)) { + outputBinaryFormatVersion = 4; } else if (OPTION_HELP.equals(arg)) { displayHelp(); } else { @@ -266,10 +267,8 @@ public class DictionaryMaker { private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { final File file = new File(binaryFilename); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file); - dictDecoder.openDictBuffer( - new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); - return BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, null); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } /** @@ -358,8 +357,13 @@ public class DictionaryMaker { throws FileNotFoundException, IOException, UnsupportedFormatException { final File outputFile = new File(outputFilename); final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version); - BinaryDictEncoder.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, - formatOptions); + final DictEncoder dictEncoder; + if (version == 4) { + dictEncoder = new Ver4DictEncoder(outputFile); + } else { + dictEncoder = new Ver3DictEncoder(outputFile); + } + dictEncoder.writeDictionary(dict, formatOptions); } /** diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java index d790d0652..66fd084cd 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java @@ -17,7 +17,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -121,7 +121,7 @@ public class Diff extends Dicttool.Command { private static void diffWords(final FusionDictionary dict0, final FusionDictionary dict1) { boolean hasDifferences = false; for (final Word word0 : dict0) { - final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray, + final PtNode word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray, word0.mWord); if (null == word1) { // This word is not in dict1 @@ -151,7 +151,7 @@ public class Diff extends Dicttool.Command { } } for (final Word word1 : dict1) { - final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray, + final PtNode word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray, word1.mWord); if (null == word0) { // This word is not in dict0 diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java index fa8c5f776..350f42772 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -65,20 +65,20 @@ public class Info extends Dicttool.Command { private static void showWordInfo(final FusionDictionary dict, final String word, final boolean plumbing) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); - if (null == group) { + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); + if (null == ptNode) { System.out.println(word + " is not in the dictionary"); return; } System.out.println("Word: " + word); - System.out.println(" Freq: " + group.getFrequency()); - if (group.getIsNotAWord()) { + System.out.println(" Freq: " + ptNode.getFrequency()); + if (ptNode.getIsNotAWord()) { System.out.println(" Is not a word"); } - if (group.getIsBlacklistEntry()) { + if (ptNode.getIsBlacklistEntry()) { System.out.println(" Is a blacklist entry"); } - final ArrayList<WeightedString> shortcutTargets = group.getShortcutTargets(); + final ArrayList<WeightedString> shortcutTargets = ptNode.getShortcutTargets(); if (null == shortcutTargets || shortcutTargets.isEmpty()) { System.out.println(" No shortcuts"); } else { @@ -88,7 +88,7 @@ public class Info extends Dicttool.Command { ? "whitelist" : shortcutTarget.mFrequency) + ")"); } } - final ArrayList<WeightedString> bigrams = group.getBigrams(); + final ArrayList<WeightedString> bigrams = ptNode.getBigrams(); if (null == bigrams || bigrams.isEmpty()) { System.out.println(" No bigrams"); } else { diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java index 451b145e9..1eff497c1 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtilsTests.java @@ -16,14 +16,15 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; -import com.android.inputmethod.latin.makedict.BinaryDictEncoder; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.DictEncoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; -import com.android.inputmethod.latin.makedict.Ver3DictDecoder; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; import junit.framework.TestCase; @@ -53,12 +54,13 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { final File dst = File.createTempFile("testGetRawDict", ".tmp"); dst.deleteOnExit(); + final OutputStream out = Compress.getCompressedStream( Compress.getCompressedStream( Compress.getCompressedStream( new BufferedOutputStream(new FileOutputStream(dst))))); - - BinaryDictEncoder.writeDictionaryBinary(out, dict, new FormatOptions(2, false)); + final DictEncoder dictEncoder = new Ver3DictEncoder(out); + dictEncoder.writeDictionary(dict, new FormatOptions(2, false)); // Test for an actually compressed dictionary and its contents final BinaryDictOffdeviceUtils.DecoderChainSpec decodeSpec = @@ -67,11 +69,10 @@ public class BinaryDictOffdeviceUtilsTests extends TestCase { assertEquals("Wrong decode spec", BinaryDictOffdeviceUtils.COMPRESSION, step); } assertEquals("Wrong decode spec", 3, decodeSpec.mDecoderSpec.size()); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(decodeSpec.mFile); - dictDecoder.openDictBuffer( - new Ver3DictDecoder.DictionaryBufferFromReadOnlyByteBufferFactory()); - final FusionDictionary resultDict = BinaryDictDecoderUtils.readDictionaryBinary(dictDecoder, - null /* dict : an optional dictionary to add words to, or null */); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodeSpec.mFile); + final FusionDictionary resultDict = dictDecoder.readDictionaryBinary( + null /* dict : an optional dictionary to add words to, or null */, + false /* deleteDictIfBroken */); assertEquals("Dictionary can't be read back correctly", FusionDictionary.findWordInTree(resultDict.mRootNodeArray, "foo").getFrequency(), TEST_FREQ); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java index fe6738303..55058238c 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/BinaryDictEncoderFlattenTreeTests.java @@ -25,7 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; /** - * Unit tests for BinaryDictEncoder.flattenTree(). + * Unit tests for BinaryDictEncoderUtils.flattenTree(). */ public class BinaryDictEncoderFlattenTreeTests extends TestCase { // Test the flattened array contains the expected number of nodes, and @@ -39,7 +39,8 @@ public class BinaryDictEncoderFlattenTreeTests extends TestCase { dict.add("ftb", 1, null, false /* isNotAWord */); dict.add("bar", 1, null, false /* isNotAWord */); dict.add("fool", 1, null, false /* isNotAWord */); - final ArrayList<PtNodeArray> result = BinaryDictEncoder.flattenTree(dict.mRootNodeArray); + final ArrayList<PtNodeArray> result = + BinaryDictEncoderUtils.flattenTree(dict.mRootNodeArray); assertEquals(4, result.size()); while (!result.isEmpty()) { final PtNodeArray n = result.remove(0); diff --git a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java index 22c0ceb4c..659650a05 100644 --- a/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java +++ b/tools/dicttool/tests/com/android/inputmethod/latin/makedict/FusionDictionaryTest.java @@ -17,7 +17,7 @@ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.Word; @@ -72,8 +72,8 @@ public class FusionDictionaryTest extends TestCase { assertNotNull(dict); for (final String word : words) { if (--limit < 0) return; - final CharGroup cg = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); - assertNotNull(cg); + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); + assertNotNull(ptNode); } } |