diff options
Diffstat (limited to 'tests/src')
6 files changed, 581 insertions, 32 deletions
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 4d231cde7..96a2217a3 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -18,6 +18,7 @@ package com.android.inputmethod.latin; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; +import android.util.Pair; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.DictEncoder; @@ -151,7 +152,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); - probabilityMap.put(word, random.nextInt() & 0xFF); + probabilityMap.put(word, random.nextInt(0xFF)); } for (String word : probabilityMap.keySet()) { binaryDictionary.addUnigramWord(word, probabilityMap.get(word)); @@ -163,8 +164,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { } public void testAddBigramWords() { - // TODO: Add a test to check the frequency of the bigram score which uses current value - // calculated in the native code File dictFile = null; try { dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); @@ -179,6 +178,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int unigramProbability = 100; final int bigramProbability = 10; + final int updatedBigramProbability = 15; binaryDictionary.addUnigramWord("aaa", unigramProbability); binaryDictionary.addUnigramWord("abb", unigramProbability); binaryDictionary.addUnigramWord("bcc", unigramProbability); @@ -187,21 +187,49 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb")); assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc")); assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa")); assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); + + binaryDictionary.addBigramWords("aaa", "abb", updatedBigramProbability); + final int updatedProbability = binaryDictionary.calculateProbability(unigramProbability, + updatedBigramProbability); + assertEquals(updatedProbability, binaryDictionary.getBigramProbability("aaa", "abb")); assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("bcc", "aaa")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("bcc", "bbc")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("aaa", "aaa")); + + // Testing bigram link. + binaryDictionary.addUnigramWord("abcde", unigramProbability); + binaryDictionary.addUnigramWord("fghij", unigramProbability); + binaryDictionary.addBigramWords("abcde", "fghij", bigramProbability); + binaryDictionary.addUnigramWord("fgh", unigramProbability); + binaryDictionary.addUnigramWord("abc", unigramProbability); + binaryDictionary.addUnigramWord("f", unigramProbability); + assertEquals(probability, binaryDictionary.getBigramProbability("abcde", "fghij")); + assertEquals(Dictionary.NOT_A_PROBABILITY, + binaryDictionary.getBigramProbability("abcde", "fgh")); + binaryDictionary.addBigramWords("abcde", "fghij", updatedBigramProbability); + assertEquals(updatedProbability, binaryDictionary.getBigramProbability("abcde", "fghij")); dictFile.delete(); } public void testRandomlyAddBigramWords() { - // TODO: Add a test to check the frequency of the bigram score which uses current value - // calculated in the native code final int wordCount = 100; final int bigramCount = 1000; final int codePointSetSize = 50; @@ -222,29 +250,38 @@ public class BinaryDictionaryTests extends AndroidTestCase { // Test a word that isn't contained within the dictionary. final Random random = new Random(seed); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final int unigramProbability = 100; - final int bigramProbability = 10; + final int[] unigramProbabilities = new int[wordCount]; for (int i = 0; i < wordCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities[i] = unigramProbability; binaryDictionary.addUnigramWord(word, unigramProbability); } - final boolean[][] bigramRelations = new boolean[wordCount][wordCount]; + final int[][] probabilities = new int[wordCount][wordCount]; + + for (int i = 0; i < wordCount; ++i) { + for (int j = 0; j < wordCount; ++j) { + probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; + } + } + for (int i = 0; i < bigramCount; i++) { final int word0Index = random.nextInt(wordCount); final int word1Index = random.nextInt(wordCount); final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); - - bigramRelations[word0Index][word1Index] = true; + final int bigramProbability = random.nextInt(0xF); + probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( + unigramProbabilities[word1Index], bigramProbability); binaryDictionary.addBigramWords(word0, word1, bigramProbability); } for (int i = 0; i < words.size(); i++) { for (int j = 0; j < words.size(); j++) { - assertEquals(bigramRelations[i][j], - binaryDictionary.isValidBigram(words.get(i), words.get(j))); + assertEquals(probabilities[i][j], + binaryDictionary.getBigramProbability(words.get(i), words.get(j))); } } @@ -263,7 +300,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - final int unigramProbability = 100; final int bigramProbability = 10; binaryDictionary.addUnigramWord("aaa", unigramProbability); @@ -299,4 +335,319 @@ public class BinaryDictionaryTests extends AndroidTestCase { dictFile.delete(); } + + public void testFlushDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int probability = 100; + binaryDictionary.addUnigramWord("aaa", probability); + binaryDictionary.addUnigramWord("abcd", probability); + // Close without flushing. + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("aaa")); + assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency("abcd")); + + binaryDictionary.addUnigramWord("aaa", probability); + binaryDictionary.addUnigramWord("abcd", probability); + binaryDictionary.flush(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + assertEquals(probability, binaryDictionary.getFrequency("aaa")); + assertEquals(probability, binaryDictionary.getFrequency("abcd")); + binaryDictionary.addUnigramWord("bcde", probability); + binaryDictionary.flush(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + assertEquals(probability, binaryDictionary.getFrequency("bcde")); + binaryDictionary.close(); + + dictFile.delete(); + } + + public void testFlushWithGCDictionary() { + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + final int unigramProbability = 100; + final int bigramProbability = 10; + binaryDictionary.addUnigramWord("aaa", unigramProbability); + binaryDictionary.addUnigramWord("abb", unigramProbability); + binaryDictionary.addUnigramWord("bcc", unigramProbability); + binaryDictionary.addBigramWords("aaa", "abb", bigramProbability); + binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability); + binaryDictionary.addBigramWords("abb", "aaa", bigramProbability); + binaryDictionary.addBigramWords("abb", "bcc", bigramProbability); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final int probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); + assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "abb")); + assertEquals(probability, binaryDictionary.getBigramProbability("aaa", "bcc")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "aaa")); + assertEquals(probability, binaryDictionary.getBigramProbability("abb", "bcc")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "aaa")); + assertEquals(false, binaryDictionary.isValidBigram("bcc", "bbc")); + assertEquals(false, binaryDictionary.isValidBigram("aaa", "aaa")); + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + dictFile.delete(); + } + + // TODO: Evaluate performance of GC + public void testAddBigramWordsAndFlashWithGC() { + final int wordCount = 100; + final int bigramCount = 1000; + final int codePointSetSize = 30; + // TODO: Use various seeds such as a current timestamp to make this test more random. + final int seed = 314159265; + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + // Test a word that isn't contained within the dictionary. + final Random random = new Random(seed); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final int[] unigramProbabilities = new int[wordCount]; + for (int i = 0; i < wordCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities[i] = unigramProbability; + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + final int[][] probabilities = new int[wordCount][wordCount]; + + for (int i = 0; i < wordCount; ++i) { + for (int j = 0; j < wordCount; ++j) { + probabilities[i][j] = Dictionary.NOT_A_PROBABILITY; + } + } + + for (int i = 0; i < bigramCount; i++) { + final int word0Index = random.nextInt(wordCount); + final int word1Index = random.nextInt(wordCount); + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + probabilities[word0Index][word1Index] = binaryDictionary.calculateProbability( + unigramProbabilities[word1Index], bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + for (int i = 0; i < words.size(); i++) { + for (int j = 0; j < words.size(); j++) { + assertEquals(probabilities[i][j], + binaryDictionary.getBigramProbability(words.get(i), words.get(j))); + } + } + dictFile.delete(); + } + + public void testRandomOperetionsAndFlashWithGC() { + final int flashWithGCIterationCount = 50; + final int operationCountInEachIteration = 200; + final int initialUnigramCount = 100; + final float addUnigramProb = 0.5f; + final float addBigramProb = 0.8f; + final float removeBigramProb = 0.2f; + final int codePointSetSize = 30; + final int seed = 141421356; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + + BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<Pair<String, String>> bigramWords = new ArrayList<Pair<String,String>>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final HashMap<Pair<String, String>, Integer> bigramProbabilities = + new HashMap<Pair<String, String>, Integer>(); + for (int i = 0; i < initialUnigramCount; ++i) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + + for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) { + // Add unigram. + if (random.nextFloat() < addUnigramProb) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + // Add bigram. + if (random.nextFloat() < addBigramProb && words.size() > 2) { + final int word0Index = random.nextInt(words.size()); + int word1Index = random.nextInt(words.size() - 1); + if (word0Index <= word1Index) { + word1Index++; + } + final String word0 = words.get(word0Index); + final String word1 = words.get(word1Index); + final int bigramProbability = random.nextInt(0xF); + final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + bigramWords.add(bigram); + bigramProbabilities.put(bigram, bigramProbability); + binaryDictionary.addBigramWords(word0, word1, bigramProbability); + } + // Remove bigram. + if (random.nextFloat() < removeBigramProb && !bigramWords.isEmpty()) { + final int bigramIndex = random.nextInt(bigramWords.size()); + final Pair<String, String> bigram = bigramWords.get(bigramIndex); + bigramWords.remove(bigramIndex); + bigramProbabilities.remove(bigram); + binaryDictionary.removeBigramWords(bigram.first, bigram.second); + } + } + + // Test whether the all unigram operations are collectlly handled. + for (int i = 0; i < words.size(); i++) { + final String word = words.get(i); + final int unigramProbability = unigramProbabilities.get(word); + assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); + } + // Test whether the all bigram operations are collectlly handled. + for (int i = 0; i < bigramWords.size(); i++) { + final Pair<String, String> bigram = bigramWords.get(i); + final int unigramProbability = unigramProbabilities.get(bigram.second); + final int probability; + if (bigramProbabilities.containsKey(bigram)) { + final int bigramProbability = bigramProbabilities.get(bigram); + probability = binaryDictionary.calculateProbability(unigramProbability, + bigramProbability); + } else { + probability = Dictionary.NOT_A_PROBABILITY; + } + assertEquals(probability, + binaryDictionary.getBigramProbability(bigram.first, bigram.second)); + } + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + } + + dictFile.delete(); + } + + public void testAddManyUnigramsAndFlushWithGC() { + final int flashWithGCIterationCount = 3; + final int codePointSetSize = 50; + final int seed = 22360679; + + final Random random = new Random(seed); + + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary"); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } catch (UnsupportedFormatException e) { + fail("UnsupportedFormatException while writing an initial dictionary : " + e); + } + + final ArrayList<String> words = new ArrayList<String>(); + final HashMap<String, Integer> unigramProbabilities = new HashMap<String, Integer>(); + final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); + + BinaryDictionary binaryDictionary; + for (int i = 0; i < flashWithGCIterationCount; i++) { + binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + while(!binaryDictionary.needsToRunGC()) { + final String word = CodePointUtils.generateWord(random, codePointSet); + words.add(word); + final int unigramProbability = random.nextInt(0xFF); + unigramProbabilities.put(word, unigramProbability); + binaryDictionary.addUnigramWord(word, unigramProbability); + } + + for (int j = 0; j < words.size(); j++) { + final String word = words.get(j); + final int unigramProbability = unigramProbabilities.get(word); + assertEquals(word, unigramProbability, binaryDictionary.getFrequency(word)); + } + + binaryDictionary.flushWithGC(); + binaryDictionary.close(); + } + + dictFile.delete(); + } } diff --git a/tests/src/com/android/inputmethod/latin/InputLogicTests.java b/tests/src/com/android/inputmethod/latin/InputLogicTests.java index fe92be618..cc2569f5e 100644 --- a/tests/src/com/android/inputmethod/latin/InputLogicTests.java +++ b/tests/src/com/android/inputmethod/latin/InputLogicTests.java @@ -134,6 +134,13 @@ public class InputLogicTests extends InputTestsBase { assertEquals("simple auto-correct", EXPECTED_RESULT, mEditText.getText().toString()); } + public void testAutoCorrectWithQuote() { + final String STRING_TO_TYPE = "didn' "; + final String EXPECTED_RESULT = "didn't "; + type(STRING_TO_TYPE); + assertEquals("auto-correct with quote", EXPECTED_RESULT, mEditText.getText().toString()); + } + public void testAutoCorrectWithPeriod() { final String STRING_TO_TYPE = "tgis."; final String EXPECTED_RESULT = "this."; diff --git a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java index 4cf83339a..a594baf0b 100644 --- a/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java +++ b/tests/src/com/android/inputmethod/latin/SuggestedWordsTests.java @@ -64,4 +64,37 @@ public class SuggestedWordsTests extends AndroidTestCase { assertEquals("0", wordsWithoutTyped.getWord(0)); assertEquals(SuggestedWordInfo.KIND_CORRECTION, wordsWithoutTyped.getInfo(0).mKind); } + + // Helper for testGetTransformedWordInfo + private SuggestedWordInfo createWordInfo(final String s) { + // Use 100 as the frequency because the numerical value does not matter as + // long as it's > 1 and < INT_MAX. + return new SuggestedWordInfo(s, 100, + SuggestedWordInfo.KIND_TYPED, null /* sourceDict */, + SuggestedWordInfo.NOT_AN_INDEX /* indexOfTouchPointOfSecondWord */, + SuggestedWordInfo.NOT_A_CONFIDENCE /* autoCommitFirstWordConfidence */); + } + + // Helper for testGetTransformedWordInfo + private SuggestedWordInfo transformWordInfo(final String info, + final int trailingSingleQuotesCount) { + return Suggest.getTransformedSuggestedWordInfo(createWordInfo(info), + Locale.ENGLISH, false /* isAllUpperCase */, false /* isFirstCharCapitalized */, + trailingSingleQuotesCount); + } + + public void testGetTransformedSuggestedWordInfo() { + SuggestedWordInfo result = transformWordInfo("word", 0); + assertEquals(result.mWord, "word"); + result = transformWordInfo("word", 1); + assertEquals(result.mWord, "word'"); + result = transformWordInfo("word", 3); + assertEquals(result.mWord, "word'''"); + result = transformWordInfo("didn't", 0); + assertEquals(result.mWord, "didn't"); + result = transformWordInfo("didn't", 1); + assertEquals(result.mWord, "didn't"); + result = transformWordInfo("didn't", 3); + assertEquals(result.mWord, "didn't''"); + } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index 8bc0095a5..a4d94262f 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -25,6 +25,7 @@ import android.util.SparseArray; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; +import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -75,6 +76,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { new FormatSpec.FormatOptions(3, false /* supportsDynamicUpdate */); private static final FormatSpec.FormatOptions VERSION3_WITH_DYNAMIC_UPDATE = new FormatSpec.FormatOptions(3, true /* supportsDynamicUpdate */); + private static final FormatSpec.FormatOptions VERSION4_WITHOUT_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, false /* supportsDynamicUpdate */); + private static final FormatSpec.FormatOptions VERSION4_WITH_DYNAMIC_UPDATE = + new FormatSpec.FormatOptions(4, true /* supportsDynamicUpdate */); private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; @@ -114,6 +119,17 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } } + private DictEncoder getDictEncoder(final File file, final FormatOptions formatOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + return new Ver4DictEncoder(getContext().getCacheDir()); + } else if (formatOptions.mVersion == 3 || formatOptions.mVersion == 2) { + return new Ver3DictEncoder(file); + } else { + throw new RuntimeException("The format option has a wrong version : " + + formatOptions.mVersion); + } + } + private void generateWords(final int number, final Random random, final int[] codePointSet) { final Set<String> wordSet = CollectionUtils.newHashSet(); while (wordSet.size() < number) { @@ -165,7 +181,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictEncoder dictEncoder = new Ver3DictEncoder(file); + final DictEncoder dictEncoder = getDictEncoder(file, formatOptions); now = System.currentTimeMillis(); // If you need to dump the dict to a textual file, uncomment the line below and the @@ -246,16 +262,28 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return file; } + private DictDecoder getDictDecoder(final File file, final int bufferType, + final FormatOptions formatOptions, final DictionaryOptions dictOptions) { + if (formatOptions.mVersion == FormatSpec.VERSION4) { + final FileHeader header = new FileHeader(0, dictOptions, formatOptions); + return FormatSpec.getDictDecoder(new File(getContext().getCacheDir(), + header.getId() + "." + header.getVersion()), bufferType); + } else { + return FormatSpec.getDictDecoder(file, bufferType); + } + } // Tests for readDictionaryBinary and writeDictionaryBinary private long timeReadingAndCheckDict(final File file, final List<String> words, final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap, final int bufferType) { + final HashMap<String, List<String>> shortcutMap, final int bufferType, + final FormatOptions formatOptions, final DictionaryOptions dictOptions) { long now, diff = -1; FusionDictionary dict = null; try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); + final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, + dictOptions); now = System.currentTimeMillis(); dict = dictDecoder.readDictionaryBinary(null, false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; @@ -286,7 +314,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { checkDictionary(dict, words, bigrams, shortcuts); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); + final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType, + formatOptions, dict.mOptions); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -330,6 +359,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION2); runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -342,6 +373,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION2); runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadAndWriteTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -397,7 +430,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private long timeAndCheckReadUnigramsAndBigramsBinary(final File file, final List<String> words, - final SparseArray<List<Integer>> bigrams, final int bufferType) { + final SparseArray<List<Integer>> bigrams, final int bufferType, + final FormatOptions formatOptions, final DictionaryOptions dictOptions) { FileInputStream inStream = null; final TreeMap<Integer, String> resultWords = CollectionUtils.newTreeMap(); @@ -407,7 +441,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { long now = -1, diff = -1; try { - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, bufferType); + final DictDecoder dictDecoder = getDictDecoder(file, bufferType, formatOptions, + dictOptions); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; @@ -444,9 +479,10 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { timeWritingDictToFile(file, dict, formatOptions); - long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType); + long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, + formatOptions, dict.mOptions); long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, - bufferType); + bufferType, formatOptions, dict.mOptions); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap + " : " + message + " : " + outputOptions(bufferType, formatOptions); @@ -458,7 +494,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { formatOptions, "unigram")); results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, formatOptions, "chain")); - results.add(runReadUnigramsAndBigramsBinary(sWords, sChainBigrams, bufferType, + results.add(runReadUnigramsAndBigramsBinary(sWords, sStarBigrams, bufferType, formatOptions, "star")); } @@ -468,6 +504,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION2); runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -480,6 +518,8 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION2); runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runReadUnigramsAndBigramsTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); for (final String result : results) { Log.d(TAG, result); @@ -503,7 +543,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { address, fileHeader.mFormatOptions).mWord; } - private long runGetTerminalPosition(final DictDecoder dictDecoder, final String word, + private long checkGetTerminalPosition(final DictDecoder dictDecoder, final String word, int index, boolean contained) { final int expectedFrequency = (UNIGRAM_FREQ + index) % 255; long diff = -1; @@ -523,7 +563,9 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { return diff; } - public void testGetTerminalPosition() { + private void runGetTerminalPosition(final ArrayList<String> words, + final SparseArray<List<Integer>> bigrams, final int bufferType, + final FormatOptions formatOptions, final String message) { final String dictName = "testGetTerminalPosition"; final String dictVersion = Long.toString(System.currentTimeMillis()); final File file = setUpDictionaryFile(dictName, dictVersion); @@ -531,16 +573,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), getDictionaryOptions(dictName, dictVersion)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); - timeWritingDictToFile(file, dict, VERSION3_WITH_DYNAMIC_UPDATE); + addBigrams(dict, words, bigrams); + timeWritingDictToFile(file, dict, formatOptions); - final DictDecoder dictDecoder = FormatSpec.getDictDecoder(file, DictDecoder.USE_BYTEARRAY); + final DictDecoder dictDecoder = getDictDecoder(file, DictDecoder.USE_BYTEARRAY, + formatOptions, dict.mOptions); try { dictDecoder.openDictBuffer(); } catch (IOException e) { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer()); + assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); try { // too long word @@ -559,10 +603,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Test a word that is contained within the dictionary. long sum = 0; for (int i = 0; i < sWords.size(); ++i) { - final long time = runGetTerminalPosition(dictDecoder, sWords.get(i), i, true); + final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), i, true); sum += time == -1 ? 0 : time; } - Log.d(TAG, "per a search : " + (((double)sum) / sWords.size() / 1000000)); + Log.d(TAG, "per search : " + (((double)sum) / sWords.size() / 1000000) + " : " + message + + " : " + outputOptions(bufferType, formatOptions)); // Test a word that isn't contained within the dictionary. final Random random = new Random((int)System.currentTimeMillis()); @@ -571,7 +616,32 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { for (int i = 0; i < 1000; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); if (sWords.indexOf(word) != -1) continue; - runGetTerminalPosition(dictDecoder, word, i, false); + checkGetTerminalPosition(dictDecoder, word, i, false); + } + } + + private void runGetTerminalPositionTests(final ArrayList<String> results, final int bufferType, + final FormatOptions formatOptions) { + runGetTerminalPosition(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram"); + } + + public void testGetTerminalPosition() { + final ArrayList<String> results = CollectionUtils.newArrayList(); + + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION2); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_ARRAY, VERSION4_WITH_DYNAMIC_UPDATE); + + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION2); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION3_WITH_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITHOUT_DYNAMIC_UPDATE); + runGetTerminalPositionTests(results, USE_BYTE_BUFFER, VERSION4_WITH_DYNAMIC_UPDATE); + + for (final String result : results) { + Log.d(TAG, result); } } @@ -593,7 +663,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // ignore Log.e(TAG, "IOException while opening the buffer", e); } - assertTrue("Can't get the buffer", dictDecoder.isOpenedDictBuffer()); + assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); try { MoreAsserts.assertNotEqual(FormatSpec.NOT_VALID_WORD, diff --git a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java index bf44a1424..d605cdb84 100644 --- a/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java @@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS)); } catch (InterruptedException e) { } - for (int i = 0; i < 10 && i < numberOfWords; ++i) { + // Limit word count to check when using a Java on memory dictionary. + final int wordCountToCheck = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + numberOfWords : 10; + for (int i = 0; i < wordCountToCheck; ++i) { final String word = words.get(i); // This may fail as long as we use tryLock on inserting the bigram words assertTrue(dict.isInDictionaryForTests(word)); @@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { } } } + + public void testAddManyWords() { + File dictFile = null; + final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis(); + final int numberOfWords = + ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ? + 10000 : 1000; + final Random random = new Random(123456); + + UserHistoryPredictionDictionary dict = + PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(), + testFilenameSuffix, mPrefs); + try { + addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random, + true /* checksContents */); + dict.close(); + } finally { + try { + Log.d(TAG, "waiting for writing ..."); + dict.shutdownExecutorForTests(); + while (!dict.isTerminatedForTests()) { + Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS); + } + } catch (InterruptedException e) { + Log.d(TAG, "InterruptedException: ", e); + } + final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix + + ExpandableBinaryDictionary.DICT_FILE_EXTENSION; + dictFile = new File(getContext().getFilesDir(), fileName); + if (dictFile != null) { + assertTrue(dictFile.exists()); + assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE); + dictFile.delete(); + } + } + } + } diff --git a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java index c6fa943fe..eb9fb984e 100644 --- a/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java +++ b/tests/src/com/android/inputmethod/latin/utils/StringUtilsTests.java @@ -20,7 +20,14 @@ import com.android.inputmethod.latin.settings.SettingsValues; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; +import android.text.style.SuggestionSpan; +import android.text.style.URLSpan; +import android.text.SpannableStringBuilder; +import android.text.Spannable; +import android.text.Spanned; +import java.util.Arrays; +import java.util.List; import java.util.Locale; @SmallTest @@ -268,4 +275,44 @@ public class StringUtilsTests extends AndroidTestCase { final String bytesStr2 = StringUtils.byteArrayToHexString(bytes2); assertTrue(bytesStr.equals(bytesStr2)); } + + public void testJsonStringUtils() { + final Object[] objs = new Object[] { 1, "aaa", "bbb", 3 }; + final List<Object> objArray = Arrays.asList(objs); + final String str = StringUtils.listToJsonStr(objArray); + final List<Object> newObjArray = StringUtils.jsonStrToList(str); + for (int i = 0; i < objs.length; ++i) { + assertEquals(objs[i], newObjArray.get(i)); + } + } + + public void testConcatWithSuggestionSpansOnly() { + SpannableStringBuilder s = new SpannableStringBuilder("test string\ntest string\n" + + "test string\ntest string\ntest string\ntest string\ntest string\ntest string\n" + + "test string\ntest string\n"); + final int N = 10; + for (int i = 0; i < N; ++i) { + // Put a PARAGRAPH-flagged span that should not be found in the result. + s.setSpan(new SuggestionSpan(getContext(), + new String[] {"" + i}, Spannable.SPAN_PARAGRAPH), + i * 12, i * 12 + 12, Spannable.SPAN_PARAGRAPH); + // Put a normal suggestion span that should be found in the result. + s.setSpan(new SuggestionSpan(getContext(), new String[] {"" + i}, 0), i, i * 2, 0); + // Put a URL span than should not be found in the result. + s.setSpan(new URLSpan("http://a"), i, i * 2, 0); + } + + final CharSequence a = s.subSequence(0, 15); + final CharSequence b = s.subSequence(15, s.length()); + final Spanned result = + (Spanned)StringUtils.concatWithNonParagraphSuggestionSpansOnly(a, b); + + Object[] spans = result.getSpans(0, result.length(), SuggestionSpan.class); + for (int i = 0; i < spans.length; i++) { + final int flags = result.getSpanFlags(spans[i]); + assertEquals("Should not find a span with PARAGRAPH flag", + flags & Spannable.SPAN_PARAGRAPH, 0); + assertTrue("Should be a SuggestionSpan", spans[i] instanceof SuggestionSpan); + } + } } |