diff options
Diffstat (limited to 'tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java')
| -rw-r--r-- | tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java | 437 |
1 files changed, 232 insertions, 205 deletions
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index ae184268c..fa70f9988 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -20,7 +20,7 @@ import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.LargeTest; import android.util.Pair; -import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; +import com.android.inputmethod.latin.NgramContext.WordInfo; import com.android.inputmethod.latin.makedict.BinaryDictIOUtils; import com.android.inputmethod.latin.makedict.CodePointUtils; import com.android.inputmethod.latin.makedict.DictDecoder; @@ -37,6 +37,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.Random; @@ -49,6 +50,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private static final int DUMMY_PROBABILITY = 0; private static final int[] DICT_FORMAT_VERSIONS = new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; + private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary"; private int mCurrentTime = 0; @@ -56,10 +58,15 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { protected void setUp() throws Exception { super.setUp(); mCurrentTime = 0; + mDictFilesToBeDeleted.clear(); } @Override protected void tearDown() throws Exception { + for (final File dictFile : mDictFilesToBeDeleted) { + dictFile.delete(); + } + mDictFilesToBeDeleted.clear(); stopTestModeInNativeCode(); super.tearDown(); } @@ -68,23 +75,46 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { return formatVersion > FormatSpec.VERSION401; } - private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, - final int probability) { - binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, - BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, - false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, mCurrentTime /* timestamp */); + private static boolean supportsNgram(final int formatVersion) { + return formatVersion >= FormatSpec.VERSION4_DEV; + } + + private void onInputWord(final BinaryDictionary binaryDictionary, final String word, + final boolean isValidWord) { + binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO, + word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } - private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, - final String word1, final int probability) { - binaryDictionary.addNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1, probability, + private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word, + final boolean isValidWord, final String prevWord) { + binaryDictionary.updateEntriesForWordWithNgramContext( + new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } + private void onInputWordWithPrevWords(final BinaryDictionary binaryDictionary, + final String word, final boolean isValidWord, final String prevWord, + final String prevPrevWord) { + binaryDictionary.updateEntriesForWordWithNgramContext( + new NgramContext(new WordInfo(prevWord), new WordInfo(prevPrevWord)), word, + isValidWord, 1 /* count */, mCurrentTime /* timestamp */); + } + + private void onInputWordWithBeginningOfSentenceContext( + final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) { + binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE, + word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); + } + private static boolean isValidBigram(final BinaryDictionary binaryDictionary, final String word0, final String word1) { - return binaryDictionary.isValidNgram(new PrevWordsInfo(new WordInfo(word0)), word1); + return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); + } + + private static boolean isValidTrigram(final BinaryDictionary binaryDictionary, + final String word0, final String word1, final String word2) { + return binaryDictionary.isValidNgram( + new NgramContext(new WordInfo(word1), new WordInfo(word0)), word2); } private void forcePassingShortTime(final BinaryDictionary binaryDictionary) { @@ -103,25 +133,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.flushWithGC(); } - private File createEmptyDictionaryAndGetFile(final String dictId, - final int formatVersion) throws IOException { + private HashSet<File> mDictFilesToBeDeleted = new HashSet<>(); + + private File createEmptyDictionaryAndGetFile(final int formatVersion) { if (formatVersion == FormatSpec.VERSION4 || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING || formatVersion == FormatSpec.VERSION4_DEV) { - return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion); + try { + final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion); + mDictFilesToBeDeleted.add(dictFile); + return dictFile; + } catch (final IOException e) { + fail(e.toString()); + } } else { - throw new IOException("Dictionary format version " + formatVersion - + " is not supported."); + fail("Dictionary format version " + formatVersion + " is not supported."); } + return null; } - private File createEmptyVer4DictionaryAndGetFile(final String dictId, final int formatVersion) + private File createEmptyVer4DictionaryAndGetFile(final int formatVersion) throws IOException { - final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, + final File file = File.createTempFile(DICTIONARY_ID, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); FileUtils.deleteRecursively(file); Map<String, String> attributeMap = new HashMap<>(); - attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId); + attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, DICTIONARY_ID); attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY, String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()))); attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY, @@ -137,6 +174,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } } + private BinaryDictionary getBinaryDictionary(final File dictFile) { + return new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + } + private static int setCurrentTimeForTestMode(final int currentTime) { return BinaryDictionaryUtils.setCurrentTimeForTest(currentTime); } @@ -153,19 +196,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private void testReadDictInJavaSide(final int formatVersion) { setCurrentTimeForTestMode(mCurrentTime); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a"); binaryDictionary.flushWithGC(); binaryDictionary.close(); @@ -189,7 +224,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } catch (UnsupportedFormatException e) { fail("Unsupported format: " + e); } - dictFile.delete(); } public void testControlCurrentTime() { @@ -214,42 +248,49 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } private void testAddValidAndInvalidWords(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "a", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); - addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "a", false /* isValidWord */); + onInputWord(binaryDictionary, "a", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "b", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("b")); - addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertFalse(isValidBigram(binaryDictionary, "a", "b")); - addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); - addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. - addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); - addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); - binaryDictionary.close(); - dictFile.delete(); + if (!supportsNgram(formatVersion)) { + return; + } + + onInputWordWithPrevWords(binaryDictionary, "c", false /* isValidWord */, "b", "a"); + assertFalse(isValidTrigram(binaryDictionary, "a", "b", "c")); + assertFalse(isValidBigram(binaryDictionary, "b", "c")); + onInputWordWithPrevWords(binaryDictionary, "c", false /* isValidWord */, "b", "a"); + assertTrue(isValidTrigram(binaryDictionary, "a", "b", "c")); + assertTrue(isValidBigram(binaryDictionary, "b", "c")); + + onInputWordWithPrevWords(binaryDictionary, "d", true /* isValidWord */, "b", "a"); + assertTrue(isValidTrigram(binaryDictionary, "a", "b", "d")); + assertTrue(isValidBigram(binaryDictionary, "b", "d")); + + onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "b", "a"); + assertTrue(isValidTrigram(binaryDictionary, "a", "b", "cd")); } public void testDecayingProbability() { @@ -259,54 +300,67 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } private void testDecayingProbability(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); assertFalse(isValidBigram(binaryDictionary, "a", "b")); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); + onInputWord(binaryDictionary, "a", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingLongTime(binaryDictionary); assertFalse(isValidBigram(binaryDictionary, "a", "b")); + if (!supportsNgram(formatVersion)) { + return; + } + + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab"); + onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab"); + assertTrue(isValidTrigram(binaryDictionary, "ab", "bc", "cd")); + forcePassingShortTime(binaryDictionary); + assertFalse(isValidTrigram(binaryDictionary, "ab", "bc", "cd")); + + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab"); + onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab"); + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab"); + onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab"); + onInputWord(binaryDictionary, "ab", true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab"); + onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab"); + forcePassingShortTime(binaryDictionary); + assertTrue(isValidTrigram(binaryDictionary, "ab", "bc", "cd")); + forcePassingLongTime(binaryDictionary); + assertFalse(isValidTrigram(binaryDictionary, "ab", "bc", "cd")); + binaryDictionary.close(); - dictFile.delete(); } public void testAddManyUnigramsToDecayingDict() { @@ -321,16 +375,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); @@ -342,31 +388,31 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final int maxUnigramCount = Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); + binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY)); while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { forcePassingShortTime(binaryDictionary); } final int unigramCountAfterGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY)); assertTrue(unigramCountBeforeGC > unigramCountAfterGC); } } - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY)) > 0); - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY)) <= maxUnigramCount); forcePassingLongTime(binaryDictionary); - assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest( + assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY))); } @@ -384,26 +430,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); setCurrentTimeForTestMode(mCurrentTime); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final String strong = "strong"; final String weak = "weak"; for (int j = 0; j < strongUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, strong, true /* isValidWord */); } for (int j = 0; j < weakUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, weak, true /* isValidWord */); } assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(weak)); @@ -411,17 +449,17 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); for (int j = 0; j < eachUnigramTypedCount; j++) { - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY)); assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(weak)); binaryDictionary.flushWithGC(); final int unigramCountAfterGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.UNIGRAM_COUNT_QUERY)); assertTrue(unigramCountBeforeGC > unigramCountAfterGC); assertFalse(binaryDictionary.isValidWord(weak)); @@ -445,16 +483,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); setCurrentTimeForTestMode(mCurrentTime); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final ArrayList<String> words = new ArrayList<>(); @@ -477,33 +508,34 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } final int maxBigramCount = Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); - addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, bigram.first, true /* isValidWord */); + onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */, + bigram.first); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY)); while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { forcePassingShortTime(binaryDictionary); } final int bigramCountAfterGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY)); assertTrue(bigramCountBeforeGC > bigramCountAfterGC); } } - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY)) > 0); - assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest( + assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY)) <= maxBigramCount); forcePassingLongTime(binaryDictionary); - assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest( + assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY))); } @@ -523,17 +555,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); setCurrentTimeForTestMode(mCurrentTime); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); final ArrayList<String> words = new ArrayList<>(); @@ -541,23 +565,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); for (int j = 0; j < unigramTypedCount; j++) { - addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, word, true /* isValidWord */); } } final String strong = "strong"; final String weak = "weak"; final String target = "target"; for (int j = 0; j < unigramTypedCount; j++) { - addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY); + onInputWord(binaryDictionary, weak, true /* isValidWord */); + onInputWord(binaryDictionary, strong, true /* isValidWord */); } binaryDictionary.flushWithGC(); for (int j = 0; j < strongBigramTypedCount; j++) { - addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong); } for (int j = 0; j < weakBigramTypedCount; j++) { - addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak); } assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); @@ -570,15 +593,15 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word1 = words.get(word1Index); for (int j = 0; j < eachBigramTypedCount; j++) { - addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0); } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY)); binaryDictionary.flushWithGC(); final int bigramCountAfterGC = - Integer.parseInt(binaryDictionary.getPropertyForTest( + Integer.parseInt(binaryDictionary.getPropertyForGettingStats( BinaryDictionary.BIGRAM_COUNT_QUERY)); assertTrue(bigramCountBeforeGC > bigramCountAfterGC); assertTrue(isValidBigram(binaryDictionary, strong, target)); @@ -596,28 +619,20 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) { setCurrentTimeForTestMode(mCurrentTime); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); + final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); + onInputWord(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("aaa")); - addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); - assertFalse(binaryDictionary.isValidWord("bbb")); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); - addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); - addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + onInputWord(binaryDictionary, "ccc", true /* isValidWord */); + + onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); - addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); + assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); @@ -627,14 +642,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); - addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWord(binaryDictionary, "bbb", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); - addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); + onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); binaryDictionary.close(); - dictFile.delete(); } public void testBeginningOfSentence() { @@ -647,46 +661,59 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { private void testBeginningOfSentence(final int formatVersion) { setCurrentTimeForTestMode(mCurrentTime); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, mCurrentTime); - final PrevWordsInfo prevWordsInfoStartOfSentence = PrevWordsInfo.BEGINNING_OF_SENTENCE; - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); - binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY, - mCurrentTime); - assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); - assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); - + final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); forcePassingLongTime(binaryDictionary); - assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); - assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); + onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); + assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); + assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); + binaryDictionary.close(); + } + + public void testRemoveUnigrams() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + testRemoveUnigrams(formatVersion); + } + } + + private void testRemoveUnigrams(final int formatVersion) { + final int unigramInputCount = 20; + setCurrentTimeForTestMode(mCurrentTime); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); + + onInputWord(binaryDictionary, "aaa", false /* isValidWord */); + assertFalse(binaryDictionary.isValidWord("aaa")); + for (int i = 0; i < unigramInputCount; i++) { + onInputWord(binaryDictionary, "aaa", false /* isValidWord */); + } + assertTrue(binaryDictionary.isValidWord("aaa")); + assertTrue(binaryDictionary.removeUnigramEntry("aaa")); + assertFalse(binaryDictionary.isValidWord("aaa")); - addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY, - mCurrentTime); - addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); - binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY, - mCurrentTime); - assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa")); - assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb")); binaryDictionary.close(); - dictFile.delete(); } } |