diff options
Diffstat (limited to 'tests/src')
7 files changed, 288 insertions, 215 deletions
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index cf85d97a0..ded8eaa97 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -32,6 +32,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; + // Note that these are corresponding definitions in native code in + // latinime::DynamicPatriciaTriePolicy. + private static final String SET_NEEDS_TO_DECAY_FOR_TESTING_KEY = + "SET_NEEDS_TO_DECAY_FOR_TESTING"; + private static final int DUMMY_PROBABILITY = 0; @Override @@ -45,14 +50,20 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { - binaryDictionary.flushWithGC(); + // Entries having low probability would be suppressed once in 3 GCs. + final int count = 3; + for (int i = 0; i < count; i++) { + binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY); + binaryDictionary.flushWithGC(); + } } private void forcePassingLongTime(final BinaryDictionary binaryDictionary) { // Currently, probabilities are decayed when GC is run. All entries that have never been - // typed in 32 GCs are removed. - final int count = 32; + // typed in 128 GCs would be removed. + final int count = 128; for (int i = 0; i < count; i++) { + binaryDictionary.getPropertyForTests(SET_NEEDS_TO_DECAY_FOR_TESTING_KEY); binaryDictionary.flushWithGC(); } } diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 6a21522f9..5b8f0e977 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; +import android.text.TextUtils; import android.util.Pair; import com.android.inputmethod.latin.makedict.CodePointUtils; @@ -126,7 +127,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { public void testRandomlyAddUnigramWord() { final int wordCount = 1000; final int codePointSetSize = 50; - final int seed = 123456789; + final long seed = System.currentTimeMillis(); File dictFile = null; try { @@ -223,7 +224,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 50; - final int seed = 11111; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); File dictFile = null; try { @@ -234,43 +236,42 @@ public class BinaryDictionaryTests extends AndroidTestCase { BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); - // Test a word that isn't contained within the dictionary. - final Random random = new Random(seed); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final int[] unigramProbabilities = new int[wordCount]; + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = + new HashMap<Pair<String, String>, Integer>(); + for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); - unigramProbabilities[i] = unigramProbability; + unigramProbabilities.put(word, unigramProbability); binaryDictionary.addUnigramWord(word, unigramProbability); } - final int[][] probabilities = new int[wordCount][wordCount]; - - for (int i = 0; i < wordCount; ++i) { - for (int j = 0; j < wordCount; ++j) { - probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; - } - } - for (int i = 0; i < bigramCount; i++) { - final int word0Index = random.nextInt(wordCount); - final int word1Index = random.nextInt(wordCount); - final String word0 = words.get(word0Index); - final String word1 = words.get(word1Index); + final String word0 = words.get(random.nextInt(wordCount)); + final String word1 = words.get(random.nextInt(wordCount)); + if (TextUtils.equals(word0, word1)) { + continue; + } + final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + bigramWords.add(bigram); final int bigramProbability = random.nextInt(0xF); - probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( - unigramProbabilities[word1Index], bigramProbability); + bigramProbabilities.put(bigram, bigramProbability); binaryDictionary.addBigramWords(word0, word1, bigramProbability); } - for (int i = 0; i < words.size(); i++) { - for (int j = 0; j < words.size(); j++) { - assertEquals(probabilities[i][j], - binaryDictionary.getBigramProbability(words.get(i), words.get(j))); - } + for (final Pair<String, String> bigram : bigramWords) { + final int unigramProbability = unigramProbabilities.get(bigram.second); + final int bigramProbability = bigramProbabilities.get(bigram); + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + assertEquals(probability, + binaryDictionary.getBigramProbability(bigram.first, bigram.second)); } dictFile.delete(); @@ -419,8 +420,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 30; - // TODO: Use various seeds such as a current timestamp to make this test more random. - final int seed = 314159265; + final long seed = System.currentTimeMillis(); + final Random random = new Random(seed); File dictFile = null; try { @@ -432,35 +433,32 @@ public class BinaryDictionaryTests extends AndroidTestCase { BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); - // Test a word that isn't contained within the dictionary. - final Random random = new Random(seed); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final int[] unigramProbabilities = new int[wordCount]; + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = + new HashMap<Pair<String, String>, Integer>(); + for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); final int unigramProbability = random.nextInt(0xFF); - unigramProbabilities[i] = unigramProbability; + unigramProbabilities.put(word, unigramProbability); binaryDictionary.addUnigramWord(word, unigramProbability); } - final int[][] probabilities = new int[wordCount][wordCount]; - - for (int i = 0; i < wordCount; ++i) { - for (int j = 0; j < wordCount; ++j) { - probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; - } - } - for (int i = 0; i < bigramCount; i++) { - final int word0Index = random.nextInt(wordCount); - final int word1Index = random.nextInt(wordCount); - final String word0 = words.get(word0Index); - final String word1 = words.get(word1Index); + final String word0 = words.get(random.nextInt(wordCount)); + final String word1 = words.get(random.nextInt(wordCount)); + if (TextUtils.equals(word0, word1)) { + continue; + } + final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + bigramWords.add(bigram); final int bigramProbability = random.nextInt(0xF); - probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( - unigramProbabilities[word1Index], bigramProbability); + bigramProbabilities.put(bigram, bigramProbability); binaryDictionary.addBigramWords(word0, word1, bigramProbability); } @@ -470,12 +468,15 @@ public class BinaryDictionaryTests extends AndroidTestCase { 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - for (int i = 0; i < words.size(); i++) { - for (int j = 0; j < words.size(); j++) { - assertEquals(probabilities[i][j], - binaryDictionary.getBigramProbability(words.get(i), words.get(j))); - } + for (final Pair<String, String> bigram : bigramWords) { + final int unigramProbability = unigramProbabilities.get(bigram.second); + final int bigramProbability = bigramProbabilities.get(bigram); + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + assertEquals(probability, + binaryDictionary.getBigramProbability(bigram.first, bigram.second)); } + dictFile.delete(); } @@ -487,8 +488,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { final float addBigramProb = 0.8f; final float removeBigramProb = 0.2f; final int codePointSetSize = 30; - final int seed = 141421356; + final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; @@ -539,6 +540,9 @@ public class BinaryDictionaryTests extends AndroidTestCase { } final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); + if (TextUtils.equals(word0, word1)) { + continue; + } final int bigramProbability = random.nextInt(0xF); final Pair<String, String> bigram = new Pair<String, String>(word0, word1); bigramWords.add(bigram); @@ -586,8 +590,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { public void testAddManyUnigramsAndFlushWithGC() { final int flashWithGCIterationCount = 3; final int codePointSetSize = 50; - final int seed = 22360679; + final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; @@ -632,8 +636,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int codePointSetSize = 50; final int unigramCountPerIteration = 1000; final int bigramCountPerIteration = 2000; - final int seed = 1123581321; - + final long seed = System.currentTimeMillis(); final Random random = new Random(seed); File dictFile = null; @@ -661,6 +664,9 @@ public class BinaryDictionaryTests extends AndroidTestCase { for (int j = 0; j < bigramCountPerIteration; j++) { final String word0 = words.get(random.nextInt(words.size())); final String word1 = words.get(random.nextInt(words.size())); + if (TextUtils.equals(word0, word1)) { + continue; + } bigrams.add(new Pair<String, String>(word0, word1)); final int bigramProbability = random.nextInt(0xF); binaryDictionary.addBigramWords(word0, word1, bigramProbability); diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java index cc2569f5e..6bc8b9dd5 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java @@ -179,10 +179,17 @@ public class InputLogicTests extends InputTestsBase { } public void testDoubleSpace() { - final String STRING_TO_TYPE = "this "; - final String EXPECTED_RESULT = "this. "; - type(STRING_TO_TYPE); - assertEquals("double space make a period", EXPECTED_RESULT, mEditText.getText().toString()); + // U+1F607 is an emoji + final String[] STRINGS_TO_TYPE = + new String[] { "this ", "a+ ", "\u1F607 ", ".. ", ") ", "( ", "% " }; + final String[] EXPECTED_RESULTS = + new String[] { "this. ", "a+. ", "\u1F607. ", ".. ", "). ", "( ", "% " }; + for (int i = 0; i < STRINGS_TO_TYPE.length; ++i) { + mEditText.setText(""); + type(STRINGS_TO_TYPE[i]); + assertEquals("double space processing", EXPECTED_RESULTS[i], + mEditText.getText().toString()); + } } public void testCancelDoubleSpace() { diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index a4d94262f..0189b3334 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -104,7 +104,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } sStarBigrams.put(0, new ArrayList<Integer>()); - for (int i = 1; i < sWords.size(); ++i) { + // MAX - 1 because we added one above already + final int maxBigrams = Math.min(sWords.size(), FormatSpec.MAX_BIGRAMS_IN_A_PTNODE - 1); + for (int i = 1; i < maxBigrams; ++i) { sStarBigrams.get(0).add(i); } @@ -544,8 +546,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word, - int index, boolean contained) { - final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; + final boolean contained) { long diff = -1; int position = -1; try { @@ -603,7 +604,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Test a word that is contained within the dictionary. long sum = 0; for (int i = 0; i < sWords.size(); ++i) { - final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), i, true); + final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), true); sum += time == -1 ? 0 : time; } Log.d(TAG, "per search : " + (((double)sum) / sWords.size() / 1000000) + " : " + message @@ -616,11 +617,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { for (int i = 0; i < 1000; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); if (sWords.indexOf(word) != -1) continue; - checkGetTerminalPosition(dictDecoder, word, i, false); + checkGetTerminalPosition(dictDecoder, word, false); } } - private void runGetTerminalPositionTests(final ArrayList<String> results, final int bufferType, + private void runGetTerminalPositionTests(final int bufferType, final FormatOptions formatOptions) { runGetTerminalPosition(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram"); } @@ -628,24 +629,24 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testGetTerminalPosition() { final ArrayList<String> results = CollectionUtils.newArrayList(); - runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION2); - runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION2); + runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION2); - runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); - runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION2); + runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); } } - public void testDeleteWord() { + private void runTestDeleteWord(final FormatOptions formatOptions) { final String dictName = "testDeleteWord"; final String dictVersion = Long.toString(System.currentTimeMillis()); final File file = setUpDictionaryFile(dictName, dictVersion); @@ -654,31 +655,37 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { new FusionDictionary.DictionaryOptions( new HashMap<String, String>(), false, false)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); - timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); + timeWritingDictToFile(file, dict, formatOptions); - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, DictDecoder.USE_BYTEARRAY); - try { - dictDecoder.openDictBuffer(); - } catch (IOException e) { - // ignore - Log.e(TAG, "IOException while opening the buffer", e); + final DictUpdater dictUpdater; + if (formatOptions.mVersion == 3) { + dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else if (formatOptions.mVersion == 4) { + dictUpdater = new Ver4DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("DictUpdater for version " + formatOptions.mVersion + + " doesn't exist."); } - assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - dictDecoder.getTerminalPosition(sWords.get(0))); - DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(0)); + dictUpdater.getTerminalPosition(sWords.get(0))); + dictUpdater.deleteWord(sWords.get(0)); assertEquals(FormatSpec.NOT_VALID_WORD, - dictDecoder.getTerminalPosition(sWords.get(0))); + dictUpdater.getTerminalPosition(sWords.get(0))); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, - dictDecoder.getTerminalPosition(sWords.get(5))); - DynamicBinaryDictIOUtils.deleteWord(dictDecoder, sWords.get(5)); + dictUpdater.getTerminalPosition(sWords.get(5))); + dictUpdater.deleteWord(sWords.get(5)); assertEquals(FormatSpec.NOT_VALID_WORD, - dictDecoder.getTerminalPosition(sWords.get(5))); + dictUpdater.getTerminalPosition(sWords.get(5))); } catch (IOException e) { } catch (UnsupportedFormatException e) { } } + + public void testDeleteWord() { + runTestDeleteWord(VERSION3_WITH_DYNAMIC_UPDATE); + runTestDeleteWord(VERSION4_WITH_DYNAMIC_UPDATE); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java index a83749499..afe5adb73 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtilsTests.java @@ -27,9 +27,7 @@ import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; import com.android.inputmethod.latin.utils.CollectionUtils; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -47,6 +45,9 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; + private static final int VERSION3 = 3; + private static final int VERSION4 = 4; + private static final String[] CHARACTERS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", @@ -185,47 +186,42 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { // return amount of time to insert a word private long insertAndCheckWord(final File file, final String word, final int frequency, final boolean exist, final ArrayList<WeightedString> bigrams, - final ArrayList<WeightedString> shortcuts) { - BufferedOutputStream outStream = null; + final ArrayList<WeightedString> shortcuts, final int formatVersion) { long amountOfTime = -1; try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, - DictDecoder.USE_WRITABLE_BYTEBUFFER); - dictDecoder.openDictBuffer(); - outStream = new BufferedOutputStream(new FileOutputStream(file, true)); + final DictUpdater dictUpdater; + if (formatVersion == VERSION3) { + dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" + + " exist."); + } if (!exist) { assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } final long now = System.nanoTime(); - DynamicBinaryDictIOUtils.insertWord(dictDecoder, outStream, word, frequency, bigrams, - shortcuts, false, false); + dictUpdater.insertWord(word, frequency, bigrams, shortcuts, false, false); amountOfTime = System.nanoTime() - now; - outStream.flush(); MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); - outStream.close(); } catch (IOException e) { Log.e(TAG, "Raised an IOException while inserting a word", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "Raised an UnsupportedFormatException error while inserting a word", e); - } finally { - if (outStream != null) { - try { - outStream.close(); - } catch (IOException e) { - Log.e(TAG, "Failed to close the output stream", e); - } - } } return amountOfTime; } - private void deleteWord(final File file, final String word) { + private void deleteWord(final File file, final String word, final int formatVersion) { try { - final Ver3DictDecoder dictDecoder = new Ver3DictDecoder(file, - DictDecoder.USE_WRITABLE_BYTEBUFFER); - dictDecoder.openDictBuffer(); - DynamicBinaryDictIOUtils.deleteWord(dictDecoder, word); + final DictUpdater dictUpdater; + if (formatVersion == VERSION3) { + dictUpdater = new Ver3DictUpdater(file, DictDecoder.USE_WRITABLE_BYTEBUFFER); + } else { + throw new RuntimeException("DictUpdater for version " + formatVersion + " doesn't" + + " exist."); + } + dictUpdater.deleteWord(word); } catch (IOException e) { } catch (UnsupportedFormatException e) { } @@ -246,7 +242,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } } - public void testInsertWord() { + private void runTestInsertWord(final int formatVersion) { File file = null; try { file = File.createTempFile("testInsertWord", TEST_DICT_FILE_EXTENSION, @@ -270,33 +266,37 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { } MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); - insertAndCheckWord(file, "abcde", 10, false, null, null); + insertAndCheckWord(file, "abcde", 10, false, null, null, formatVersion); - insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null); + insertAndCheckWord(file, "abcdefghijklmn", 10, false, null, null, formatVersion); checkReverseLookup(file, "abcdefghijklmn", getWordPosition(file, "abcdefghijklmn")); - insertAndCheckWord(file, "abcdabcd", 10, false, null, null); + insertAndCheckWord(file, "abcdabcd", 10, false, null, null, formatVersion); checkReverseLookup(file, "abcdabcd", getWordPosition(file, "abcdabcd")); // update the existing word. - insertAndCheckWord(file, "abcdabcd", 15, true, null, null); + insertAndCheckWord(file, "abcdabcd", 15, true, null, null, formatVersion); // split 1 - insertAndCheckWord(file, "ab", 20, false, null, null); + insertAndCheckWord(file, "ab", 20, false, null, null, formatVersion); // split 2 - insertAndCheckWord(file, "ami", 30, false, null, null); + insertAndCheckWord(file, "ami", 30, false, null, null, formatVersion); - deleteWord(file, "ami"); + deleteWord(file, "ami", formatVersion); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "ami")); - insertAndCheckWord(file, "abcdabfg", 30, false, null, null); + insertAndCheckWord(file, "abcdabfg", 30, false, null, null, formatVersion); - deleteWord(file, "abcd"); + deleteWord(file, "abcd", formatVersion); assertEquals(FormatSpec.NOT_VALID_WORD, getWordPosition(file, "abcd")); } - public void testInsertWordWithBigrams() { + public void testInsertWord() { + runTestInsertWord(VERSION3); + } + + private void runTestInsertWordWithBigrams(final int formatVersion) { File file = null; try { file = File.createTempFile("testInsertWordWithBigrams", TEST_DICT_FILE_EXTENSION, @@ -323,8 +323,8 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { final ArrayList<WeightedString> banana = new ArrayList<WeightedString>(); banana.add(new WeightedString("banana", 10)); - insertAndCheckWord(file, "banana", 0, false, null, null); - insertAndCheckWord(file, "recursive", 60, true, banana, null); + insertAndCheckWord(file, "banana", 0, false, null, null, formatVersion); + insertAndCheckWord(file, "recursive", 60, true, banana, null, formatVersion); final PtNodeInfo info = findWordFromFile(file, "recursive"); int bananaPos = getWordPosition(file, "banana"); @@ -333,7 +333,11 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { assertEquals(info.mBigrams.get(0).mAddress, bananaPos); } - public void testRandomWords() { + public void testInsertWordWithBigrams() { + runTestInsertWordWithBigrams(VERSION3); + } + + private void runTestRandomWords(final int formatVersion) { File file = null; try { file = File.createTempFile("testRandomWord", TEST_DICT_FILE_EXTENSION, @@ -362,7 +366,7 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { int cnt = 0; for (final String word : sWords) { final long diff = insertAndCheckWord(file, word, - cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null); + cnt % FormatSpec.MAX_TERMINAL_FREQUENCY, false, null, null, formatVersion); maxTimeToInsert = Math.max(maxTimeToInsert, diff); minTimeToInsert = Math.min(minTimeToInsert, diff); sum += diff; @@ -373,8 +377,13 @@ public class BinaryDictIOUtilsTests extends AndroidTestCase { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, getWordPosition(file, word)); } + Log.d(TAG, "Test version " + formatVersion); Log.d(TAG, "max = " + ((double)maxTimeToInsert/1000000) + " ms."); Log.d(TAG, "min = " + ((double)minTimeToInsert/1000000) + " ms."); Log.d(TAG, "avg = " + ((double)sum/mMaxUnigrams/1000000) + " ms."); } + + public void testRandomWords() { + runTestRandomWords(VERSION3); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java index 132483d5e..aeb8552bd 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/SparseTableTests.java @@ -21,10 +21,8 @@ import android.test.suitebuilder.annotation.LargeTest; import android.util.Log; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Random; @@ -36,9 +34,6 @@ import java.util.Random; public class SparseTableTests extends AndroidTestCase { private static final String TAG = SparseTableTests.class.getSimpleName(); - private static final int[] SMALL_INDEX = { SparseTable.NOT_EXIST, 0 }; - private static final int[] BIG_INDEX = { SparseTable.NOT_EXIST, 1, 2, 3, 4, 5, 6, 7}; - private final Random mRandom; private final ArrayList<Integer> mRandomIndex; @@ -59,32 +54,21 @@ public class SparseTableTests extends AndroidTestCase { } } - public void testInitializeWithArray() { - final SparseTable table = new SparseTable(SMALL_INDEX, BIG_INDEX, BLOCK_SIZE); - for (int i = 0; i < 8; ++i) { - assertEquals(SparseTable.NOT_EXIST, table.get(i)); - } - assertEquals(SparseTable.NOT_EXIST, table.get(8)); - for (int i = 9; i < 16; ++i) { - assertEquals(i - 8, table.get(i)); - } - } - public void testSet() { - final SparseTable table = new SparseTable(16, BLOCK_SIZE); - table.set(3, 6); - table.set(8, 16); + final SparseTable table = new SparseTable(16, BLOCK_SIZE, 1); + table.set(0, 3, 6); + table.set(0, 8, 16); for (int i = 0; i < 16; ++i) { if (i == 3 || i == 8) { - assertEquals(i * 2, table.get(i)); + assertEquals(i * 2, table.get(0, i)); } else { - assertEquals(SparseTable.NOT_EXIST, table.get(i)); + assertEquals(SparseTable.NOT_EXIST, table.get(0, i)); } } } private void generateRandomIndex(final int size, final int prop) { - for (int i = 0; i < DEFAULT_SIZE; ++i) { + for (int i = 0; i < size; ++i) { if (mRandom.nextInt(100) < prop) { mRandomIndex.set(i, mRandom.nextInt()); } else { @@ -94,11 +78,11 @@ public class SparseTableTests extends AndroidTestCase { } private void runTestRandomSet() { - final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE); + final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, 1); int elementCount = 0; for (int i = 0; i < DEFAULT_SIZE; ++i) { if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { - table.set(i, mRandomIndex.get(i)); + table.set(0, i, mRandomIndex.get(i)); elementCount++; } } @@ -107,29 +91,24 @@ public class SparseTableTests extends AndroidTestCase { + table.getContentTableSize()); Log.d(TAG, "the table has " + elementCount + " elements"); for (int i = 0; i < DEFAULT_SIZE; ++i) { - assertEquals(table.get(i), (int)mRandomIndex.get(i)); + assertEquals(table.get(0, i), (int)mRandomIndex.get(i)); } // flush and reload OutputStream lookupOutStream = null; OutputStream contentOutStream = null; - InputStream lookupInStream = null; - InputStream contentInStream = null; try { final File lookupIndexFile = File.createTempFile("testRandomSet", ".small"); final File contentFile = File.createTempFile("testRandomSet", ".big"); lookupOutStream = new FileOutputStream(lookupIndexFile); contentOutStream = new FileOutputStream(contentFile); - table.write(lookupOutStream, contentOutStream); - lookupInStream = new FileInputStream(lookupIndexFile); - contentInStream = new FileInputStream(contentFile); - final byte[] lookupArray = new byte[(int) lookupIndexFile.length()]; - final byte[] contentArray = new byte[(int) contentFile.length()]; - lookupInStream.read(lookupArray); - contentInStream.read(contentArray); - final SparseTable newTable = new SparseTable(lookupArray, contentArray, BLOCK_SIZE); + table.write(lookupOutStream, new OutputStream[] { contentOutStream }); + lookupOutStream.flush(); + contentOutStream.flush(); + final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, + new File[] { contentFile }, BLOCK_SIZE); for (int i = 0; i < DEFAULT_SIZE; ++i) { - assertEquals(table.get(i), newTable.get(i)); + assertEquals(table.get(0, i), newTable.get(0, i)); } } catch (IOException e) { Log.d(TAG, "IOException while flushing and realoding", e); @@ -157,4 +136,60 @@ public class SparseTableTests extends AndroidTestCase { runTestRandomSet(); } } + + public void testMultipleContents() { + final int numOfContents = 5; + generateRandomIndex(DEFAULT_SIZE, 20); + final SparseTable table = new SparseTable(DEFAULT_SIZE, BLOCK_SIZE, numOfContents); + for (int i = 0; i < mRandomIndex.size(); ++i) { + if (mRandomIndex.get(i) != SparseTable.NOT_EXIST) { + for (int j = 0; j < numOfContents; ++j) { + table.set(j, i, mRandomIndex.get(i)); + } + } + } + + OutputStream lookupOutStream = null; + OutputStream[] contentsOutStream = new OutputStream[numOfContents]; + try { + final File lookupIndexFile = File.createTempFile("testMultipleContents", "small"); + lookupOutStream = new FileOutputStream(lookupIndexFile); + final File[] contentFiles = new File[numOfContents]; + for (int i = 0; i < numOfContents; ++i) { + contentFiles[i] = File.createTempFile("testMultipleContents", "big" + i); + contentsOutStream[i] = new FileOutputStream(contentFiles[i]); + } + table.write(lookupOutStream, contentsOutStream); + lookupOutStream.flush(); + for (int i = 0; i < numOfContents; ++i) { + contentsOutStream[i].flush(); + } + final SparseTable newTable = SparseTable.readFromFiles(lookupIndexFile, contentFiles, + BLOCK_SIZE); + for (int i = 0; i < numOfContents; ++i) { + for (int j = 0; j < DEFAULT_SIZE; ++j) { + assertEquals(table.get(i, j), newTable.get(i, j)); + } + } + } catch (IOException e) { + Log.d(TAG, "IOException while flushing and reloading", e); + } finally { + if (lookupOutStream != null) { + try { + lookupOutStream.close(); + } catch (IOException e) { + Log.d(TAG, "IOException while closing the stream", e); + } + } + for (int i = 0; i < numOfContents; ++i) { + if (contentsOutStream[i] != null) { + try { + contentsOutStream[i].close(); + } catch (IOException e) { + Log.d(TAG, "IOException while closing the stream.", e); + } + } + } + } + } } diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index ddc9546c5..7c1decb71 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -109,35 +109,54 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { dict.close(); } + /** + * Clear all entries in the user history dictionary. + * @param testFilenameSuffix file name suffix used for testing. + */ + private void clearHistory(final String testFilenameSuffix) { + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), + testFilenameSuffix /* locale */, mPrefs); + dict.clearAndFlushDictionary(); + dict.close(); + } + + /** + * Shut down executer and wait until all operations of user history are done. + * @param testFilenameSuffix file name suffix used for testing. + */ + private void waitForWriting(final String testFilenameSuffix) { + try { + final UserHistoryDictionary dict = + PersonalizationHelper.getUserHistoryDictionary(getContext(), + testFilenameSuffix, mPrefs); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } catch (InterruptedException e) { + Log.d(TAG, "InterruptedException: ", e); + } + } + public void testRandomWords() { - File dictFile = null; Log.d(TAG, "This test can be used for profiling."); Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true."); final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + final int numberOfWords = 1000; final Random random = new Random(123456); try { + clearHistory(testFilenameSuffix); addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, true /* checksContents */); } finally { - try { - final UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix, mPrefs); - Log.d(TAG, "waiting for writing ..."); - dict.shutdownExecutorForTests(); - while (!dict.isTerminatedForTests()) { - Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); - } - } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: " + e); - } - - final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix - + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; - dictFile = new File(getContext().getFilesDir(), fileName); - + Log.d(TAG, "waiting for writing ..."); + waitForWriting(testFilenameSuffix); + final File dictFile = new File(getContext().getFilesDir(), fileName); if (dictFile != null) { assertTrue(dictFile.exists()); assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); @@ -162,6 +181,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffixes[i] + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; dictFiles[i] = new File(getContext().getFilesDir(), fileName); + clearHistory(testFilenameSuffixes[i]); } final long start = System.currentTimeMillis(); @@ -178,19 +198,9 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Log.d(TAG, "testStressTestForSwitchingLanguageAndAddingWords took " + (end - start) + " ms"); } finally { - try { - Log.d(TAG, "waiting for writing ..."); - for (int i = 0; i < numberOfLanguages; i++) { - final UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffixes[i], mPrefs); - dict.shutdownExecutorForTests(); - while (!dict.isTerminatedForTests()) { - Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); - } - } - } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: " + e); + Log.d(TAG, "waiting for writing ..."); + for (int i = 0; i < numberOfLanguages; i++) { + waitForWriting(testFilenameSuffixes[i]); } for (final File file : dictFiles) { if (file != null) { @@ -203,33 +213,21 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } public void testAddManyWords() { - File dictFile = null; final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); final int numberOfWords = ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? 10000 : 1000; final Random random = new Random(123456); - - UserHistoryDictionary dict = - PersonalizationHelper.getUserHistoryDictionary(getContext(), - testFilenameSuffix, mPrefs); + clearHistory(testFilenameSuffix); try { addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, true /* checksContents */); - dict.close(); } finally { - try { - Log.d(TAG, "waiting for writing ..."); - dict.shutdownExecutorForTests(); - while (!dict.isTerminatedForTests()) { - Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); - } - } catch (InterruptedException e) { - Log.d(TAG, "InterruptedException: ", e); - } + Log.d(TAG, "waiting for writing ..."); + waitForWriting(testFilenameSuffix); final String fileName = UserHistoryDictionary.NAME + "." + testFilenameSuffix + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; - dictFile = new File(getContext().getFilesDir(), fileName); + final File dictFile = new File(getContext().getFilesDir(), fileName); if (dictFile != null) { assertTrue(dictFile.exists()); assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); |