diff options
Diffstat (limited to 'tools/dicttool/src')
11 files changed, 174 insertions, 61 deletions
diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java index c2c77d61a..6c4cbcf9d 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/BinaryDictOffdeviceUtils.java @@ -16,7 +16,9 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; @@ -30,8 +32,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.ArrayList; import javax.xml.parsers.ParserConfigurationException; @@ -98,7 +98,7 @@ public final class BinaryDictOffdeviceUtils { // over and over, ending in a stack overflow. Hence we limit the depth at which we try // decoding the file. if (depth > MAX_DECODE_DEPTH) return null; - if (BinaryDictInputOutput.isBinaryDictionary(src)) { + if (BinaryDictDecoderUtils.isBinaryDictionary(src)) { spec.mFile = src; return spec; } @@ -185,16 +185,14 @@ public final class BinaryDictOffdeviceUtils { crash(filename, new RuntimeException( filename + " does not seem to be a dictionary file")); } else { - final FileInputStream inStream = new FileInputStream(decodedSpec.mFile); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, decodedSpec.mFile.length()); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(decodedSpec.mFile, + DictDecoder.USE_BYTEARRAY); if (report) { System.out.println("Format : Binary dictionary format"); System.out.println("Packaging : " + decodedSpec.describeChain()); System.out.println("Uncompressed size : " + decodedSpec.mFile.length()); } - return BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), null); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } } } catch (IOException e) { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java index 092ee767f..4b6716936 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CombinedInputOutput.java @@ -19,7 +19,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -117,7 +117,7 @@ public class CombinedInputOutput { final boolean processLigatures = FRENCH_LIGATURE_PROCESSING_OPTION.equals(attributes.get(OPTIONS_TAG)); attributes.remove(OPTIONS_TAG); - final FusionDictionary dict = new FusionDictionary(new Node(), new DictionaryOptions( + final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions( attributes, processUmlauts, processLigatures)); String line; diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CommandList.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CommandList.java index 0e0095bd6..0d93c7fa9 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CommandList.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/CommandList.java @@ -27,5 +27,6 @@ public class CommandList { Dicttool.addCommand("package", Package.Packager.class); Dicttool.addCommand("unpackage", Package.Unpackager.class); Dicttool.addCommand("makedict", Makedict.class); + Dicttool.addCommand("test", Test.class); } } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Crypt.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Crypt.java index f8990231e..4612ae4ab 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Crypt.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Crypt.java @@ -40,10 +40,12 @@ public class Crypt { public Encrypter() { } + @Override public String getHelp() { return COMMAND + " <src_filename> <dst_filename>: Encrypts a file"; } + @Override public void run() { throw new UnsupportedOperationException(); } @@ -55,10 +57,12 @@ public class Crypt { public Decrypter() { } + @Override public String getHelp() { return COMMAND + " <src_filename> <dst_filename>: Decrypts a file"; } + @Override public void run() { throw new UnsupportedOperationException(); } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java index cc890f60c..5c7e8b4f2 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/DictionaryMaker.java @@ -16,21 +16,22 @@ package com.android.inputmethod.latin.dicttool; -import com.android.inputmethod.latin.makedict.BinaryDictInputOutput; +import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils; +import com.android.inputmethod.latin.makedict.DictDecoder; +import com.android.inputmethod.latin.makedict.DictEncoder; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.MakedictLog; import com.android.inputmethod.latin.makedict.UnsupportedFormatException; +import com.android.inputmethod.latin.makedict.Ver3DictEncoder; +import com.android.inputmethod.latin.makedict.Ver4DictEncoder; import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; import java.util.Arrays; import java.util.LinkedList; @@ -44,9 +45,9 @@ import org.xml.sax.SAXException; public class DictionaryMaker { static class Arguments { - private static final String OPTION_VERSION_1 = "-1"; private static final String OPTION_VERSION_2 = "-2"; private static final String OPTION_VERSION_3 = "-3"; + private static final String OPTION_VERSION_4 = "-4"; private static final String OPTION_INPUT_SOURCE = "-s"; private static final String OPTION_INPUT_BIGRAM_XML = "-b"; private static final String OPTION_INPUT_SHORTCUT_XML = "-c"; @@ -127,12 +128,12 @@ public class DictionaryMaker { + "| [-s <combined format input]" + "| [-s <binary input>] [-d <binary output>] [-x <xml output>] " + " [-o <combined output>]" - + "[-1] [-2] [-3]\n" + + "[-2] [-3] [-4]\n" + "\n" + " Converts a source dictionary file to one or several outputs.\n" + " Source can be an XML file, with an optional XML bigrams file, or a\n" + " binary dictionary file.\n" - + " Binary version 1 (Ice Cream Sandwich), 2 (Jelly Bean), 3, XML and\n" + + " Binary version 2 (Jelly Bean), 3, 4, XML and\n" + " combined format outputs are supported."; } @@ -159,8 +160,8 @@ public class DictionaryMaker { // Do nothing, this is the default } else if (OPTION_VERSION_3.equals(arg)) { outputBinaryFormatVersion = 3; - } else if (OPTION_VERSION_1.equals(arg)) { - outputBinaryFormatVersion = 1; + } else if (OPTION_VERSION_4.equals(arg)) { + outputBinaryFormatVersion = 4; } else if (OPTION_HELP.equals(arg)) { displayHelp(); } else { @@ -176,7 +177,7 @@ public class DictionaryMaker { inputUnigramXml = filename; } else if (CombinedInputOutput.isCombinedDictionary(filename)) { inputCombined = filename; - } else if (BinaryDictInputOutput.isBinaryDictionary(filename)) { + } else if (BinaryDictDecoderUtils.isBinaryDictionary(filename)) { inputBinary = filename; } else { throw new IllegalArgumentException( @@ -198,7 +199,7 @@ public class DictionaryMaker { } } else { if (null == inputBinary && null == inputUnigramXml) { - if (BinaryDictInputOutput.isBinaryDictionary(arg)) { + if (BinaryDictDecoderUtils.isBinaryDictionary(arg)) { inputBinary = arg; } else if (CombinedInputOutput.isCombinedDictionary(arg)) { inputCombined = arg; @@ -265,24 +266,9 @@ public class DictionaryMaker { */ private static FusionDictionary readBinaryFile(final String binaryFilename) throws FileNotFoundException, IOException, UnsupportedFormatException { - FileInputStream inStream = null; - - try { - final File file = new File(binaryFilename); - inStream = new FileInputStream(file); - final ByteBuffer buffer = inStream.getChannel().map( - FileChannel.MapMode.READ_ONLY, 0, file.length()); - return BinaryDictInputOutput.readDictionaryBinary( - new BinaryDictInputOutput.ByteBufferWrapper(buffer), null); - } finally { - if (inStream != null) { - try { - inStream.close(); - } catch (IOException e) { - // do nothing - } - } - } + final File file = new File(binaryFilename); + final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file); + return dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); } /** @@ -371,8 +357,13 @@ public class DictionaryMaker { throws FileNotFoundException, IOException, UnsupportedFormatException { final File outputFile = new File(outputFilename); final FormatSpec.FormatOptions formatOptions = new FormatSpec.FormatOptions(version); - BinaryDictInputOutput.writeDictionaryBinary(new FileOutputStream(outputFilename), dict, - formatOptions); + final DictEncoder dictEncoder; + if (version == 4) { + dictEncoder = new Ver4DictEncoder(outputFile); + } else { + dictEncoder = new Ver3DictEncoder(outputFile); + } + dictEncoder.writeDictionary(dict, formatOptions); } /** diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java index 7b311c3ec..cacee5268 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Dicttool.java @@ -72,15 +72,21 @@ public class Dicttool { return command; } - private void execute(final String[] arguments) { + /** + * Executes the specified command with the specified arguments. + * @param arguments the arguments passed to dicttool. + * @return 0 for success, an error code otherwise (always 1 at the moment) + */ + private int execute(final String[] arguments) { final Command command = getCommand(arguments); try { command.run(); + return 0; } catch (Exception e) { System.out.println("Exception while processing command " + command.getClass().getSimpleName() + " : " + e); e.printStackTrace(); - return; + return 1; } } @@ -89,6 +95,7 @@ public class Dicttool { help(); return; } - new Dicttool().execute(arguments); + // Exit with the success/error code from #execute() as status. + System.exit(new Dicttool().execute(arguments)); } } diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java index 5c3e87e10..66fd084cd 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Diff.java @@ -17,7 +17,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -121,7 +121,8 @@ public class Diff extends Dicttool.Command { private static void diffWords(final FusionDictionary dict0, final FusionDictionary dict1) { boolean hasDifferences = false; for (final Word word0 : dict0) { - final CharGroup word1 = FusionDictionary.findWordInTree(dict1.mRoot, word0.mWord); + final PtNode word1 = FusionDictionary.findWordInTree(dict1.mRootNodeArray, + word0.mWord); if (null == word1) { // This word is not in dict1 System.out.println("Deleted: " + word0.mWord + " " + word0.mFrequency); @@ -150,7 +151,8 @@ public class Diff extends Dicttool.Command { } } for (final Word word1 : dict1) { - final CharGroup word0 = FusionDictionary.findWordInTree(dict0.mRoot, word1.mWord); + final PtNode word0 = FusionDictionary.findWordInTree(dict0.mRootNodeArray, + word1.mWord); if (null == word0) { // This word is not in dict0 System.out.println("Added: " + word1.mWord + " " + word1.mFrequency); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java index f2894544f..350f42772 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Info.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.FusionDictionary; -import com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -65,20 +65,20 @@ public class Info extends Dicttool.Command { private static void showWordInfo(final FusionDictionary dict, final String word, final boolean plumbing) { - final CharGroup group = FusionDictionary.findWordInTree(dict.mRoot, word); - if (null == group) { + final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, word); + if (null == ptNode) { System.out.println(word + " is not in the dictionary"); return; } System.out.println("Word: " + word); - System.out.println(" Freq: " + group.getFrequency()); - if (group.getIsNotAWord()) { + System.out.println(" Freq: " + ptNode.getFrequency()); + if (ptNode.getIsNotAWord()) { System.out.println(" Is not a word"); } - if (group.getIsBlacklistEntry()) { + if (ptNode.getIsBlacklistEntry()) { System.out.println(" Is a blacklist entry"); } - final ArrayList<WeightedString> shortcutTargets = group.getShortcutTargets(); + final ArrayList<WeightedString> shortcutTargets = ptNode.getShortcutTargets(); if (null == shortcutTargets || shortcutTargets.isEmpty()) { System.out.println(" No shortcuts"); } else { @@ -88,7 +88,7 @@ public class Info extends Dicttool.Command { ? "whitelist" : shortcutTarget.mFrequency) + ")"); } } - final ArrayList<WeightedString> bigrams = group.getBigrams(); + final ArrayList<WeightedString> bigrams = ptNode.getBigrams(); if (null == bigrams || bigrams.isEmpty()) { System.out.println(" No bigrams"); } else { diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java index b29480764..9274dcd2e 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Package.java @@ -22,9 +22,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.InputStream; import java.io.IOException; -import java.io.OutputStream; public class Package { private Package() { @@ -39,10 +37,12 @@ public class Package { public Packager() { } + @Override public String getHelp() { return COMMAND + " <src_filename> <dst_filename>: Package a file for distribution"; } + @Override public void run() throws IOException { if (mArgs.length != 2) { throw new RuntimeException("Too many/too few arguments for command " + COMMAND); @@ -67,11 +67,13 @@ public class Package { public Unpackager() { } + @Override public String getHelp() { return COMMAND + " <src_filename> <dst_filename>: Detects how a file is packaged and\n" + "decrypts/uncompresses as necessary to produce a raw binary file."; } + @Override public void run() throws FileNotFoundException, IOException { if (mArgs.length != 2) { throw new RuntimeException("Too many/too few arguments for command " + COMMAND); diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java new file mode 100644 index 000000000..9174238da --- /dev/null +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/Test.java @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.android.inputmethod.latin.dicttool; + +import com.android.inputmethod.latin.makedict.BinaryDictDecoderEncoderTests; +import com.android.inputmethod.latin.makedict.BinaryDictEncoderFlattenTreeTests; +import com.android.inputmethod.latin.makedict.BinaryDictIOUtilsTests; +import com.android.inputmethod.latin.makedict.FusionDictionaryTest; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; + +/** + * Dicttool command implementing self-tests. + */ +public class Test extends Dicttool.Command { + public static final String COMMAND = "test"; + private long mSeed = System.currentTimeMillis(); + private int mMaxUnigrams = BinaryDictIOUtilsTests.DEFAULT_MAX_UNIGRAMS; + + private static final Class<?>[] sClassesToTest = { + BinaryDictOffdeviceUtilsTests.class, + FusionDictionaryTest.class, + BinaryDictDecoderEncoderTests.class, + BinaryDictEncoderFlattenTreeTests.class, + BinaryDictIOUtilsTests.class + }; + private ArrayList<Method> mAllTestMethods = new ArrayList<Method>(); + private ArrayList<String> mUsedTestMethods = new ArrayList<String>(); + + public Test() { + for (final Class<?> c : sClassesToTest) { + for (final Method m : c.getDeclaredMethods()) { + if (m.getName().startsWith("test") && Void.TYPE == m.getReturnType() + && 0 == m.getParameterTypes().length) { + mAllTestMethods.add(m); + } + } + } + } + + @Override + public String getHelp() { + final StringBuilder s = new StringBuilder("test [-s seed] [-m maxUnigrams] [testName...]\n" + + "If seed is not specified, the current time is used.\nTest list is:\n"); + for (final Method m : mAllTestMethods) { + s.append(" "); + s.append(m.getName()); + s.append("\n"); + } + return s.toString(); + } + + @Override + public void run() throws IllegalAccessException, InstantiationException, + InvocationTargetException { + int i = 0; + while (i < mArgs.length) { + final String arg = mArgs[i++]; + if ("-s".equals(arg)) { + mSeed = Long.parseLong(mArgs[i++]); + } else if ("-m".equals(arg)) { + mMaxUnigrams = Integer.parseInt(mArgs[i++]); + } else { + mUsedTestMethods.add(arg); + } + } + runChosenTests(); + } + + private void runChosenTests() throws IllegalAccessException, InstantiationException, + InvocationTargetException { + for (final Method m : mAllTestMethods) { + final Class<?> declaringClass = m.getDeclaringClass(); + if (!mUsedTestMethods.isEmpty() && !mUsedTestMethods.contains(m.getName())) continue; + // Some of the test classes expose a two-argument constructor, taking a long as a + // seed for Random, and an int for a vocabulary size to test the dictionary with. They + // correspond respectively to the -s and -m numerical arguments to this command, which + // are stored in mSeed and mMaxUnigrams. If the two-arguments constructor is present, + // then invoke it; otherwise, invoke the default constructor. + Constructor<?> twoArgsConstructor = null; + try { + twoArgsConstructor = declaringClass.getDeclaredConstructor(Long.TYPE, Integer.TYPE); + } catch (NoSuchMethodException e) { + // No constructor with two args + } + final Object instance = null == twoArgsConstructor ? declaringClass.newInstance() + : twoArgsConstructor.newInstance(mSeed, mMaxUnigrams); + m.invoke(instance); + } + } +} diff --git a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java index 1fd2cba7a..4e99bf979 100644 --- a/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java +++ b/tools/dicttool/src/com/android/inputmethod/latin/dicttool/XmlDictInputOutput.java @@ -18,7 +18,7 @@ package com.android.inputmethod.latin.dicttool; import com.android.inputmethod.latin.makedict.FusionDictionary; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; -import com.android.inputmethod.latin.makedict.FusionDictionary.Node; +import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.makedict.Word; @@ -124,8 +124,8 @@ public class XmlDictInputOutput { GERMAN_UMLAUT_PROCESSING_OPTION.equals(optionsString); final boolean processLigatures = FRENCH_LIGATURE_PROCESSING_OPTION.equals(optionsString); - mDictionary = new FusionDictionary(new Node(), new DictionaryOptions(attributes, - processUmlauts, processLigatures)); + mDictionary = new FusionDictionary(new PtNodeArray(), + new DictionaryOptions(attributes, processUmlauts, processLigatures)); } else { mState = UNKNOWN; } |