diff options
6 files changed, 128 insertions, 27 deletions
diff --git a/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java index 709ea3310..c4f7ec91f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/DictUpdater.java @@ -16,6 +16,7 @@ package com.android.inputmethod.latin.makedict; +import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import java.io.IOException; @@ -24,6 +25,7 @@ import java.util.ArrayList; /** * An interface of a binary dictionary updater. */ +@UsedForTesting public interface DictUpdater extends DictDecoder { /** @@ -31,6 +33,7 @@ public interface DictUpdater extends DictDecoder { * * @param word the word to be deleted. */ + @UsedForTesting public void deleteWord(final String word) throws IOException, UnsupportedFormatException; /** @@ -43,6 +46,7 @@ public interface DictUpdater extends DictDecoder { * @param isBlackListEntry whether this should be a blacklist entry. */ // TODO: Support batch insertion. + @UsedForTesting public void insertWord(final String word, final int frequency, final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, final boolean isNotAWord, diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java index fa7ae310a..07adda625 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver3DictUpdater.java @@ -57,7 +57,7 @@ public class Ver3DictUpdater extends Ver3DictDecoder implements DictUpdater { public void deleteWord(final String word) throws IOException, UnsupportedFormatException { if (mOutStream == null) openStreamAndBuffer(); mDictBuffer.position(0); - super.readHeader(); + readHeader(); final int wordPos = getTerminalPosition(word); if (wordPos != FormatSpec.NOT_VALID_WORD) { mDictBuffer.position(wordPos); diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index bab24e301..53729075f 100644 --- a/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -48,7 +48,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder { private final File mDictDirectory; private final DictionaryBufferFactory mBufferFactory; - private DictBuffer mDictBuffer; + protected DictBuffer mDictBuffer; private DictBuffer mFrequencyBuffer; private DictBuffer mTerminalAddressTableBuffer; private DictBuffer mBigramBuffer; diff --git a/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java new file mode 100644 index 000000000..3d8f186ba --- /dev/null +++ b/java/src/com/android/inputmethod/latin/makedict/Ver4DictUpdater.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.inputmethod.latin.makedict; + +import com.android.inputmethod.annotations.UsedForTesting; +import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +/** + * An implementation of DictUpdater for version 4 binary dictionary. + */ +@UsedForTesting +public class Ver4DictUpdater extends Ver4DictDecoder implements DictUpdater { + + @UsedForTesting + public Ver4DictUpdater(final File dictDirectory, final int factoryType) { + // DictUpdater must have an updatable DictBuffer. + super(dictDirectory, ((factoryType & MASK_DICTBUFFER) == USE_BYTEARRAY) + ? USE_BYTEARRAY : USE_WRITABLE_BYTEBUFFER); + } + + @Override + public void deleteWord(final String word) throws IOException, UnsupportedFormatException { + if (mDictBuffer == null) openDictBuffer(); + readHeader(); + final int wordPos = getTerminalPosition(word); + if (wordPos != FormatSpec.NOT_VALID_WORD) { + mDictBuffer.position(wordPos); + final int flags = PtNodeReader.readPtNodeOptionFlags(mDictBuffer); + mDictBuffer.position(wordPos); + mDictBuffer.put((byte) DynamicBinaryDictIOUtils.markAsDeleted(flags)); + } + } + + @Override + public void insertWord(final String word, final int frequency, + final ArrayList<WeightedString> bigramStrings, final ArrayList<WeightedString> shortcuts, + final boolean isNotAWord, final boolean isBlackListEntry) + throws IOException, UnsupportedFormatException { + // TODO: Implement this method. + } +} diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index aa1658301..0189b3334 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -646,7 +646,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } - public void testDeleteWord() { + private void runTestDeleteWord(final FormatOptions formatOptions) { final String dictName = "testDeleteWord"; final String dictVersion = Long.toString(System.currentTimeMillis()); final File file = setUpDictionaryFile(dictName, dictVersion); @@ -655,10 +655,17 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { new FusionDictionary.DictionaryOptions( new HashMap<String, String>(), false, false)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); - timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); + timeWritingDictToFile(file, dict, formatOptions); - final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file, - DictDecoder.USE_WRITABLE_BYTEBUFFER); + final DictUpdater dictUpdater; + if (formatOptions.mVersion == 3) { + dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else if (formatOptions.mVersion == 4) { + dictUpdater = new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("DictUpdater for version " + formatOptions.mVersion + + " doesn't exist."); + } try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, @@ -676,4 +683,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } catch (UnsupportedFormatException e) { } } + + public void testDeleteWord() { + runTestDeleteWord(VERSION3_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(VERSION4_WITH_DYNAMIC_UPDATE); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index acd65856c..afe5adb73 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -45,6 +45,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final int VERSION3 = 3; + private static final int VERSION4 = 4; + private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", @@ -183,11 +186,16 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { // return amount of time to insert a word private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList<WeightedString> bigrams, - final ArrayList<WeightedString> shortcuts) { + final ArrayList<WeightedString> shortcuts, final int formatVersion) { long amountOfTime = -1; try { - final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file, - DictDecoder.USE_WRITABLE_BYTEBUFFER); + final DictUpdater dictUpdater; + if (formatVersion == VERSION3) { + dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" + + " exist."); + } if (!exist) { assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); @@ -204,10 +212,15 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { return amountOfTime; } - private void deleteWord(final File file, final String word) { + private void deleteWord(final File file, final String word, final int formatVersion) { try { - final Ver3DictUpdater dictUpdater = new Ver3DictUpdater(file, - DictDecoder.USE_WRITABLE_BYTEBUFFER); + final DictUpdater dictUpdater; + if (formatVersion == VERSION3) { + dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" + + " exist."); + } dictUpdater.deleteWord(word); } catch (IOException e) { } catch (UnsupportedFormatException e) { @@ -229,7 +242,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } } - public void testInsertWord() { + private void runTestInsertWord(final int formatVersion) { File file = null; try { file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, @@ -253,33 +266,37 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - insertAndCheckWord(file, "abcde", 10, false, null, null); + insertAndCheckWord(file, "abcde", 10, false, null, null, formatVersion); - insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null); + insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatVersion); checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - insertAndCheckWord(file, "abcdabcd", 10, false, null, null); + insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatVersion); checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); // update the existing word. - insertAndCheckWord(file, "abcdabcd", 15, true, null, null); + insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatVersion); // split 1 - insertAndCheckWord(file, "ab", 20, false, null, null); + insertAndCheckWord(file, "ab", 20, false, null, null, formatVersion); // split 2 - insertAndCheckWord(file, "ami", 30, false, null, null); + insertAndCheckWord(file, "ami", 30, false, null, null, formatVersion); - deleteWord(file, "ami"); + deleteWord(file, "ami", formatVersion); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami")); - insertAndCheckWord(file, "abcdabfg", 30, false, null, null); + insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatVersion); - deleteWord(file, "abcd"); + deleteWord(file, "abcd", formatVersion); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); } - public void testInsertWordWithBigrams() { + public void testInsertWord() { + runTestInsertWord(VERSION3); + } + + private void runTestInsertWordWithBigrams(final int formatVersion) { File file = null; try { file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, @@ -306,8 +323,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { final ArrayList<WeightedString> banana = new ArrayList<WeightedString>(); banana.add(new WeightedString("banana", 10)); - insertAndCheckWord(file, "banana", 0, false, null, null); - insertAndCheckWord(file, "recursive", 60, true, banana, null); + insertAndCheckWord(file, "banana", 0, false, null, null, formatVersion); + insertAndCheckWord(file, "recursive", 60, true, banana, null, formatVersion); final PtNodeInfo info = findWordFromFile(file, "recursive"); int bananaPos = getWordPosition(file, "banana"); @@ -316,7 +333,11 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { assertEquals(info.mBigrams.get(0).mAddress, bananaPos); } - public void testRandomWords() { + public void testInsertWordWithBigrams() { + runTestInsertWordWithBigrams(VERSION3); + } + + private void runTestRandomWords(final int formatVersion) { File file = null; try { file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, @@ -345,7 +366,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int cnt = 0; for (final String word : sWords) { final long diff = insertAndCheckWord(file, word, - cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null); + cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatVersion); maxTimeToInsert = Math.max(maxTimeToInsert, diff); minTimeToInsert = Math.min(minTimeToInsert, diff); sum += diff; @@ -356,8 +377,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } + Log.d(TAG, "Test version " + formatVersion); Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms."); Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms."); Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms."); } + + public void testRandomWords() { + runTestRandomWords(VERSION3); + } } |