diff options
Diffstat (limited to 'tools/dicttool/src')
7 files changed, 52 insertions, 61 deletions
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index c2c77d61a..6c4cbcf9d 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -16,7 +16,9 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; @@ -30,8 +32,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.ArrayList; import javax.xml.parsers.ParserConfigurationException; @@ -98,7 +98,7 @@ public final class BinaryDictOffdeviceUtils { // over and over, ending in a stack overflow. Hence we limit the depth at which we try // decoding the file. if (depth > MAX_DECODE_DEPTH) return null; - if (BinaryDictInputOutput.isBinaryDictionary(src)) { + if (BinaryDictDecoderUtils.isBinaryDictionary(src)) { spec.mFile = src; return spec; } @@ -185,16 +185,14 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final FileInputStream inStream = new FileInputStream(decodedSpec.mFile); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, decodedSpec.mFile.length()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, + DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); } - return BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), null); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } } } catch (IOException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java index 092ee767f..4b6716936 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java @@ -19,7 +19,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -117,7 +117,7 @@ public class CombinedInputOutput { final boolean processLigatures = FRENCH_LIGATURE_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); attributes.remove(OPTIONS_TAG); - final FusionDictionary dict = new FusionDictionary(new Node(), new DictionaryOptions( + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions( attributes, processUmlauts, processLigatures)); String line; diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index cc890f60c..5c7e8b4f2 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -16,21 +16,22 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.MakedictLog; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; +import com.android.inputmethod.latin.makedict.Ver4DictEncoder; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.Arrays; import java.util.LinkedList; @@ -44,9 +45,9 @@ import org.xml.sax.SAXException; public class DictionaryMaker { static class Arguments { - private static final String OPTION_VERSION_1 = "-1"; private static final String OPTION_VERSION_2 = "-2"; private static final String OPTION_VERSION_3 = "-3"; + private static final String OPTION_VERSION_4 = "-4"; private static final String OPTION_INPUT_SOURCE = "-s"; private static final String OPTION_INPUT_BIGRAM_XML = "-b"; private static final String OPTION_INPUT_SHORTCUT_XML = "-c"; @@ -127,12 +128,12 @@ public class DictionaryMaker { + "| [-s <combined format input]" + "| [-s <binary input>] [-d <binary output>] [-x <xml output>] " + " [-o <combined output>]" - + "[-1] [-2] [-3]\n" + + "[-2] [-3] [-4]\n" + "\n" + " Converts a source dictionary file to one or several outputs.\n" + " Source can be an XML file, with an optional XML bigrams file, or a\n" + " binary dictionary file.\n" - + " Binary version 1 (Ice Cream Sandwich), 2 (Jelly Bean), 3, XML and\n" + + " Binary version 2 (Jelly Bean), 3, 4, XML and\n" + " combined format outputs are supported."; } @@ -159,8 +160,8 @@ public class DictionaryMaker { // Do nothing, this is the default } else if (OPTION_VERSION_3.equals(arg)) { outputBinaryFormatVersion = 3; - } else if (OPTION_VERSION_1.equals(arg)) { - outputBinaryFormatVersion = 1; + } else if (OPTION_VERSION_4.equals(arg)) { + outputBinaryFormatVersion = 4; } else if (OPTION_HELP.equals(arg)) { displayHelp(); } else { @@ -176,7 +177,7 @@ public class DictionaryMaker { inputUnigramXml = filename; } else if (CombinedInputOutput.isCombinedDictionary(filename)) { inputCombined = filename; - } else if (BinaryDictInputOutput.isBinaryDictionary(filename)) { + } else if (BinaryDictDecoderUtils.isBinaryDictionary(filename)) { inputBinary = filename; } else { throw new IllegalArgumentException( @@ -198,7 +199,7 @@ public class DictionaryMaker { } } else { if (null == inputBinary && null == inputUnigramXml) { - if (BinaryDictInputOutput.isBinaryDictionary(arg)) { + if (BinaryDictDecoderUtils.isBinaryDictionary(arg)) { inputBinary = arg; } else if (CombinedInputOutput.isCombinedDictionary(arg)) { inputCombined = arg; @@ -265,24 +266,9 @@ public class DictionaryMaker { */ private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { - FileInputStream inStream = null; - - try { - final File file = new File(binaryFilename); - inStream = new FileInputStream(file); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - return BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), null); - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } - } + final File file = new File(binaryFilename); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } /** @@ -371,8 +357,13 @@ public class DictionaryMaker { throws FileNotFoundException, IOException, UnsupportedFormatException { final File outputFile = new File(outputFilename); final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version); - BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, - formatOptions); + final DictEncoder dictEncoder; + if (version == 4) { + dictEncoder = new Ver4DictEncoder(outputFile); + } else { + dictEncoder = new Ver3DictEncoder(outputFile); + } + dictEncoder.writeDictionary(dict, formatOptions); } /** diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java index 5c3e87e10..66fd084cd 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java @@ -17,7 +17,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -121,7 +121,8 @@ public class Diff extends Dicttool.Command { private static void diffWords(final FusionDictionary dict0, final FusionDictionary dict1) { boolean hasDifferences = false; for (final Word word0 : dict0) { - final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRoot, word0.mWord); + final PtNode word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray, + word0.mWord); if (null == word1) { // This word is not in dict1 System.out.println("Deleted: " + word0.mWord + " " + word0.mFrequency); @@ -150,7 +151,8 @@ public class Diff extends Dicttool.Command { } } for (final Word word1 : dict1) { - final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRoot, word1.mWord); + final PtNode word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray, + word1.mWord); if (null == word0) { // This word is not in dict0 System.out.println("Added: " + word1.mWord + " " + word1.mFrequency); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java index f2894544f..350f42772 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -65,20 +65,20 @@ public class Info extends Dicttool.Command { private static void showWordInfo(final FusionDictionary dict, final String word, final boolean plumbing) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word); - if (null == group) { + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); + if (null == ptNode) { System.out.println(word + " is not in the dictionary"); return; } System.out.println("Word: " + word); - System.out.println(" Freq: " + group.getFrequency()); - if (group.getIsNotAWord()) { + System.out.println(" Freq: " + ptNode.getFrequency()); + if (ptNode.getIsNotAWord()) { System.out.println(" Is not a word"); } - if (group.getIsBlacklistEntry()) { + if (ptNode.getIsBlacklistEntry()) { System.out.println(" Is a blacklist entry"); } - final ArrayList<WeightedString> shortcutTargets = group.getShortcutTargets(); + final ArrayList<WeightedString> shortcutTargets = ptNode.getShortcutTargets(); if (null == shortcutTargets || shortcutTargets.isEmpty()) { System.out.println(" No shortcuts"); } else { @@ -88,7 +88,7 @@ public class Info extends Dicttool.Command { ? "whitelist" : shortcutTarget.mFrequency) + ")"); } } - final ArrayList<WeightedString> bigrams = group.getBigrams(); + final ArrayList<WeightedString> bigrams = ptNode.getBigrams(); if (null == bigrams || bigrams.isEmpty()) { System.out.println(" No bigrams"); } else { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java index 972b6e7e6..9174238da 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java @@ -16,9 +16,9 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictIOTests; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests; import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutputTest; import com.android.inputmethod.latin.makedict.FusionDictionaryTest; import java.lang.reflect.Constructor; @@ -37,9 +37,9 @@ public class Test extends Dicttool.Command { private static final Class<?>[] sClassesToTest = { BinaryDictOffdeviceUtilsTests.class, FusionDictionaryTest.class, - BinaryDictInputOutputTest.class, - BinaryDictIOUtilsTests.class, - BinaryDictIOTests.class + BinaryDictDecoderEncoderTests.class, + BinaryDictEncoderFlattenTreeTests.class, + BinaryDictIOUtilsTests.class }; private ArrayList<Method> mAllTestMethods = new ArrayList<Method>(); private ArrayList<String> mUsedTestMethods = new ArrayList<String>(); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java index 1fd2cba7a..4e99bf979 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -124,8 +124,8 @@ public class XmlDictInputOutput { GERMAN_UMLAUT_PROCESSING_OPTION.equals(optionsString); final boolean processLigatures = FRENCH_LIGATURE_PROCESSING_OPTION.equals(optionsString); - mDictionary = new FusionDictionary(new Node(), new DictionaryOptions(attributes, - processUmlauts, processLigatures)); + mDictionary = new FusionDictionary(new PtNodeArray(), + new DictionaryOptions(attributes, processUmlauts, processLigatures)); } else { mState = UNKNOWN; } |