diff options
Diffstat (limited to 'tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java')
| -rw-r--r-- | tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java | 206 |
1 files changed, 166 insertions, 40 deletions
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index ae2205b36..28cce834c 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -46,6 +46,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; private static final int DUMMY_PROBABILITY = 0; + private static final int[] DICT_FORMAT_VERSIONS = + new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; private int mCurrentTime = 0; @@ -61,20 +63,29 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { super.tearDown(); } + private static boolean supportsBeginningOfSentence(final int formatVersion) { + return formatVersion > FormatSpec.VERSION401; + } + private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { - binaryDictionary.addUnigramWord(word, probability, "" /* shortcutTarget */, + binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, - false /* isNotAWord */, false /* isBlacklisted */, - mCurrentTime /* timestamp */); + false /* isBeginningOfSentence */, false /* isNotAWord */, + false /* isBlacklisted */, mCurrentTime /* timestamp */); } private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, final String word1, final int probability) { - binaryDictionary.addBigramWords(word0, word1, probability, + binaryDictionary.addNgramEntry(new PrevWordsInfo(word0), word1, probability, mCurrentTime /* timestamp */); } + private static boolean isValidBigram(final BinaryDictionary binaryDictionary, + final String word0, final String word1) { + return binaryDictionary.isValidNgram(new PrevWordsInfo(word0), word1); + } + private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { // 30 days. final int timeToElapse = (int)TimeUnit.SECONDS.convert(30, TimeUnit.DAYS); @@ -93,19 +104,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private File createEmptyDictionaryAndGetFile(final String dictId, final int formatVersion) throws IOException { - if (formatVersion == FormatSpec.VERSION4) { - return createEmptyVer4DictionaryAndGetFile(dictId); + if (formatVersion == FormatSpec.VERSION4 + || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING + || formatVersion == FormatSpec.VERSION4_DEV) { + return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion); } else { throw new IOException("Dictionary format version " + formatVersion + " is not supported."); } } - private File createEmptyVer4DictionaryAndGetFile(final String dictId) throws IOException { + private File createEmptyVer4DictionaryAndGetFile(final String dictId, final int formatVersion) + throws IOException { final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); FileUtils.deleteRecursively(file); - Map<String, String> attributeMap = new HashMap<String, String>(); + Map<String, String> attributeMap = new HashMap<>(); attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId); attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); @@ -113,12 +127,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { DictionaryHeader.ATTRIBUTE_VALUE_TRUE); attributeMap.put(DictionaryHeader.HAS_HISTORICAL_INFO_KEY, DictionaryHeader.ATTRIBUTE_VALUE_TRUE); - if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), FormatSpec.VERSION4, + if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion, LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) { return file; } else { throw new IOException("Empty dictionary " + file.getAbsolutePath() - + " cannot be created."); + + " cannot be created. Foramt version: " + formatVersion); } } @@ -131,7 +145,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testReadDictInJavaSide() { - testReadDictInJavaSide(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testReadDictInJavaSide(formatVersion); + } } private void testReadDictInJavaSide(final int formatVersion) { @@ -176,10 +192,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testControlCurrentTime() { - testControlCurrentTime(FormatSpec.VERSION4); - } - - private void testControlCurrentTime(final int formatVersion) { final int TEST_COUNT = 1000; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); @@ -195,7 +207,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testAddValidAndInvalidWords() { - testAddValidAndInvalidWords(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddValidAndInvalidWords(formatVersion); + } } private void testAddValidAndInvalidWords(final int formatVersion) { @@ -219,26 +233,28 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.isValidWord("b")); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("a", "b")); + assertFalse(isValidBigram(binaryDictionary, "a", "b")); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "c")); + assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("x", "y")); + assertFalse(isValidBigram(binaryDictionary, "x", "y")); addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); - assertFalse(binaryDictionary.isValidBigram("x", "y")); + assertFalse(isValidBigram(binaryDictionary, "x", "y")); binaryDictionary.close(); dictFile.delete(); } public void testDecayingProbability() { - testDecayingProbability(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testDecayingProbability(formatVersion); + } } private void testDecayingProbability(final int formatVersion) { @@ -269,9 +285,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); - assertFalse(binaryDictionary.isValidBigram("a", "b")); + assertFalse(isValidBigram(binaryDictionary, "a", "b")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); @@ -282,18 +298,20 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); - assertTrue(binaryDictionary.isValidBigram("a", "b")); + assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingLongTime(binaryDictionary); - assertFalse(binaryDictionary.isValidBigram("a", "b")); + assertFalse(isValidBigram(binaryDictionary, "a", "b")); binaryDictionary.close(); dictFile.delete(); } public void testAddManyUnigramsToDecayingDict() { - testAddManyUnigramsToDecayingDict(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddManyUnigramsToDecayingDict(formatVersion); + } } private void testAddManyUnigramsToDecayingDict(final int formatVersion) { @@ -315,7 +333,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<String> words = new ArrayList<>(); for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); @@ -352,7 +370,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testOverflowUnigrams() { - testOverflowUnigrams(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testOverflowUnigrams(formatVersion); + } } private void testOverflowUnigrams(final int formatVersion) { @@ -411,7 +431,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testAddManyBigramsToDecayingDict() { - testAddManyBigramsToDecayingDict(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testAddManyBigramsToDecayingDict(formatVersion); + } } private void testAddManyBigramsToDecayingDict(final int formatVersion) { @@ -434,8 +456,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final ArrayList<String> words = new ArrayList<String>(); - final ArrayList<Pair<String, String>> bigrams = new ArrayList<Pair<String, String>>(); + final ArrayList<String> words = new ArrayList<>(); + final ArrayList<Pair<String, String>> bigrams = new ArrayList<>(); for (int i = 0; i < unigramCount; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); @@ -449,7 +471,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final String word0 = words.get(word0Index); final String word1 = words.get(word1Index); - final Pair<String, String> bigram = new Pair<String, String>(word0, word1); + final Pair<String, String> bigram = new Pair<>(word0, word1); bigrams.add(bigram); } @@ -485,7 +507,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } public void testOverflowBigrams() { - testOverflowBigrams(FormatSpec.VERSION4); + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testOverflowBigrams(formatVersion); + } } private void testOverflowBigrams(final int formatVersion) { @@ -511,7 +535,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final ArrayList<String> words = new ArrayList<String>(); + final ArrayList<String> words = new ArrayList<>(); for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); @@ -534,8 +558,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { for (int j = 0; j < weakBigramTypedCount; j++) { addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); } - assertTrue(binaryDictionary.isValidBigram(strong, target)); - assertTrue(binaryDictionary.isValidBigram(weak, target)); + assertTrue(isValidBigram(binaryDictionary, strong, target)); + assertTrue(isValidBigram(binaryDictionary, weak, target)); for (int i = 0; i < bigramCount; i++) { final int word0Index = random.nextInt(words.size()); @@ -556,10 +580,112 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { Integer.parseInt(binaryDictionary.getPropertyForTest( BinaryDictionary.BIGRAM_COUNT_QUERY)); assertTrue(bigramCountBeforeGC > bigramCountAfterGC); - assertTrue(binaryDictionary.isValidBigram(strong, target)); - assertFalse(binaryDictionary.isValidBigram(weak, target)); + assertTrue(isValidBigram(binaryDictionary, strong, target)); + assertFalse(isValidBigram(binaryDictionary, weak, target)); break; } } } + + public void testDictMigration() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, formatVersion); + } + } + + private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) { + setCurrentTimeForTestMode(mCurrentTime); + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("aaa")); + addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + assertFalse(binaryDictionary.isValidWord("bbb")); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); + addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); + addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); + addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); + + assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); + assertTrue(binaryDictionary.migrateTo(toFormatVersion)); + assertTrue(binaryDictionary.isValidDictionary()); + assertEquals(toFormatVersion, binaryDictionary.getFormatVersion()); + assertTrue(binaryDictionary.isValidWord("aaa")); + assertFalse(binaryDictionary.isValidWord("bbb")); + assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); + addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + assertTrue(binaryDictionary.isValidWord("bbb")); + assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); + assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); + addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); + binaryDictionary.close(); + dictFile.delete(); + } + + public void testBeginningOfSentence() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + if (supportsBeginningOfSentence(formatVersion)) { + testBeginningOfSentence(formatVersion); + } + } + } + + private void testBeginningOfSentence(final int formatVersion) { + setCurrentTimeForTestMode(mCurrentTime); + File dictFile = null; + try { + dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); + } catch (IOException e) { + fail("IOException while writing an initial dictionary : " + e); + } + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + + binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */, + BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, + mCurrentTime); + final PrevWordsInfo prevWordsInfoStartOfSentence = PrevWordsInfo.BEGINNING_OF_SENTENCE; + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, + mCurrentTime); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, + mCurrentTime); + addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY, + mCurrentTime); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + + forcePassingLongTime(binaryDictionary); + assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + + addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, + mCurrentTime); + addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); + binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY, + mCurrentTime); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); + assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + binaryDictionary.close(); + dictFile.delete(); + } } |