aboutsummaryrefslogtreecommitdiffstats
path: root/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
diff options
context:
space:
mode:
Diffstat (limited to 'tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java')
-rw-r--r--tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java594
1 files changed, 361 insertions, 233 deletions
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
index ae184268c..039330c87 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -20,9 +20,9 @@ import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.LargeTest;
import android.util.Pair;
-import com.android.inputmethod.latin.PrevWordsInfo.WordInfo;
+import com.android.inputmethod.latin.NgramContext.WordInfo;
+import com.android.inputmethod.latin.common.CodePointUtils;
import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
-import com.android.inputmethod.latin.makedict.CodePointUtils;
import com.android.inputmethod.latin.makedict.DictDecoder;
import com.android.inputmethod.latin.makedict.DictionaryHeader;
import com.android.inputmethod.latin.makedict.FormatSpec;
@@ -32,13 +32,14 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
import com.android.inputmethod.latin.utils.FileUtils;
import com.android.inputmethod.latin.utils.LocaleUtils;
+import com.android.inputmethod.latin.utils.WordInputEventForPersonalization;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Locale;
-import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
@@ -48,7 +49,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private static final String TEST_LOCALE = "test";
private static final int DUMMY_PROBABILITY = 0;
private static final int[] DICT_FORMAT_VERSIONS =
- new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV };
+ new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV };
+ private static final String DICTIONARY_ID = "TestDecayingBinaryDictionary";
private int mCurrentTime = 0;
@@ -56,35 +58,64 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
protected void setUp() throws Exception {
super.setUp();
mCurrentTime = 0;
+ mDictFilesToBeDeleted.clear();
+ setCurrentTimeForTestMode(mCurrentTime);
}
@Override
protected void tearDown() throws Exception {
+ for (final File dictFile : mDictFilesToBeDeleted) {
+ dictFile.delete();
+ }
+ mDictFilesToBeDeleted.clear();
stopTestModeInNativeCode();
super.tearDown();
}
- private static boolean supportsBeginningOfSentence(final int formatVersion) {
- return formatVersion > FormatSpec.VERSION401;
+ private static boolean supportsCountBasedNgram(final int formatVersion) {
+ return formatVersion >= FormatSpec.VERSION403;
}
- private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
- final int probability) {
- binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
- BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
- false /* isBeginningOfSentence */, false /* isNotAWord */,
- false /* isBlacklisted */, mCurrentTime /* timestamp */);
+ private static boolean supportsNgram(final int formatVersion) {
+ return formatVersion >= FormatSpec.VERSION403;
}
- private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0,
- final String word1, final int probability) {
- binaryDictionary.addNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1, probability,
+ private void onInputWord(final BinaryDictionary binaryDictionary, final String word,
+ final boolean isValidWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO,
+ word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
+ }
+
+ private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word,
+ final boolean isValidWord, final String prevWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(
+ new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */,
mCurrentTime /* timestamp */);
}
+ private void onInputWordWithPrevWords(final BinaryDictionary binaryDictionary,
+ final String word, final boolean isValidWord, final String prevWord,
+ final String prevPrevWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(
+ new NgramContext(new WordInfo(prevWord), new WordInfo(prevPrevWord)), word,
+ isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
+ }
+
+ private void onInputWordWithBeginningOfSentenceContext(
+ final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) {
+ binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE,
+ word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */);
+ }
+
private static boolean isValidBigram(final BinaryDictionary binaryDictionary,
final String word0, final String word1) {
- return binaryDictionary.isValidNgram(new PrevWordsInfo(new WordInfo(word0)), word1);
+ return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1);
+ }
+
+ private static boolean isValidTrigram(final BinaryDictionary binaryDictionary,
+ final String word0, final String word1, final String word2) {
+ return binaryDictionary.isValidNgram(
+ new NgramContext(new WordInfo(word1), new WordInfo(word0)), word2);
}
private void forcePassingShortTime(final BinaryDictionary binaryDictionary) {
@@ -103,25 +134,33 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
binaryDictionary.flushWithGC();
}
- private File createEmptyDictionaryAndGetFile(final String dictId,
- final int formatVersion) throws IOException {
- if (formatVersion == FormatSpec.VERSION4
- || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING
- || formatVersion == FormatSpec.VERSION4_DEV) {
- return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion);
- } else {
- throw new IOException("Dictionary format version " + formatVersion
- + " is not supported.");
- }
+ private HashSet<File> mDictFilesToBeDeleted = new HashSet<>();
+
+ private File createEmptyDictionaryAndGetFile(final int formatVersion) {
+ return createEmptyDictionaryWithAttributeMapAndGetFile(formatVersion,
+ new HashMap<String, String>());
}
- private File createEmptyVer4DictionaryAndGetFile(final String dictId, final int formatVersion)
+ private File createEmptyDictionaryWithAttributeMapAndGetFile(final int formatVersion,
+ final HashMap<String, String> attributeMap) {
+ try {
+ final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion,
+ attributeMap);
+ mDictFilesToBeDeleted.add(dictFile);
+ return dictFile;
+ } catch (final IOException e) {
+ fail(e.toString());
+ }
+ return null;
+ }
+
+ private File createEmptyVer4DictionaryAndGetFile(final int formatVersion,
+ final HashMap<String, String> attributeMap)
throws IOException {
- final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION,
+ final File file = File.createTempFile(DICTIONARY_ID, TEST_DICT_FILE_EXTENSION,
getContext().getCacheDir());
FileUtils.deleteRecursively(file);
- Map<String, String> attributeMap = new HashMap<>();
- attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, dictId);
+ attributeMap.put(DictionaryHeader.DICTIONARY_ID_KEY, DICTIONARY_ID);
attributeMap.put(DictionaryHeader.DICTIONARY_VERSION_KEY,
String.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())));
attributeMap.put(DictionaryHeader.USES_FORGETTING_CURVE_KEY,
@@ -131,10 +170,15 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion,
LocaleUtils.constructLocaleFromString(TEST_LOCALE), attributeMap)) {
return file;
- } else {
- throw new IOException("Empty dictionary " + file.getAbsolutePath()
- + " cannot be created. Foramt version: " + formatVersion);
}
+ throw new IOException("Empty dictionary " + file.getAbsolutePath()
+ + " cannot be created. Foramt version: " + formatVersion);
+ }
+
+ private static BinaryDictionary getBinaryDictionary(final File dictFile) {
+ return new BinaryDictionary(dictFile.getAbsolutePath(),
+ 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
+ Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
}
private static int setCurrentTimeForTestMode(final int currentTime) {
@@ -153,19 +197,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private void testReadDictInJavaSide(final int formatVersion) {
setCurrentTimeForTestMode(mCurrentTime);
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ab", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a");
binaryDictionary.flushWithGC();
binaryDictionary.close();
@@ -189,7 +225,6 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
} catch (UnsupportedFormatException e) {
fail("Unsupported format: " + e);
}
- dictFile.delete();
}
public void testControlCurrentTime() {
@@ -214,42 +249,43 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
private void testAddValidAndInvalidWords(final int formatVersion) {
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "a", false /* isValidWord */);
assertFalse(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY);
+ onInputWord(binaryDictionary, "a", false /* isValidWord */);
+ onInputWord(binaryDictionary, "a", false /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- assertTrue(binaryDictionary.isValidWord("b"));
-
- addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a");
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
- addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a");
+ assertTrue(binaryDictionary.isValidWord("b"));
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
- addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "c"));
// Add bigrams of not valid unigrams.
- addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x");
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
- addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x");
assertFalse(isValidBigram(binaryDictionary, "x", "y"));
- binaryDictionary.close();
- dictFile.delete();
+ if (!supportsNgram(formatVersion)) {
+ return;
+ }
+
+ onInputWordWithPrevWords(binaryDictionary, "c", true /* isValidWord */, "b", "a");
+ assertTrue(isValidTrigram(binaryDictionary, "a", "b", "c"));
+ assertTrue(isValidBigram(binaryDictionary, "b", "c"));
+ onInputWordWithPrevWords(binaryDictionary, "d", false /* isValidWord */, "c", "b");
+ assertFalse(isValidTrigram(binaryDictionary, "b", "c", "d"));
+ assertFalse(isValidBigram(binaryDictionary, "c", "d"));
+
+ onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "b", "a");
+ assertTrue(isValidTrigram(binaryDictionary, "a", "b", "cd"));
}
public void testDecayingProbability() {
@@ -259,54 +295,80 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
private void testDecayingProbability(final int formatVersion) {
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingShortTime(binaryDictionary);
+ if (supportsCountBasedNgram(formatVersion)) {
+ // Count based ngram language model doesn't support decaying based on the elapsed time.
+ assertTrue(binaryDictionary.isValidWord("a"));
+ } else {
+ assertFalse(binaryDictionary.isValidWord("a"));
+ }
+ forcePassingLongTime(binaryDictionary);
assertFalse(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingShortTime(binaryDictionary);
assertTrue(binaryDictionary.isValidWord("a"));
forcePassingLongTime(binaryDictionary);
assertFalse(binaryDictionary.isValidWord("a"));
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingShortTime(binaryDictionary);
+ if (supportsCountBasedNgram(formatVersion)) {
+ assertTrue(isValidBigram(binaryDictionary, "a", "b"));
+ } else {
+ assertFalse(isValidBigram(binaryDictionary, "a", "b"));
+ }
+ forcePassingLongTime(binaryDictionary);
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
+ onInputWord(binaryDictionary, "a", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a");
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingShortTime(binaryDictionary);
assertTrue(isValidBigram(binaryDictionary, "a", "b"));
forcePassingLongTime(binaryDictionary);
assertFalse(isValidBigram(binaryDictionary, "a", "b"));
+ if (!supportsNgram(formatVersion)) {
+ return;
+ }
+
+ onInputWord(binaryDictionary, "ab", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
+ onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
+ assertTrue(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
+ forcePassingLongTime(binaryDictionary);
+ assertFalse(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
+
+ onInputWord(binaryDictionary, "ab", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
+ onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
+ onInputWord(binaryDictionary, "ab", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
+ onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
+ onInputWord(binaryDictionary, "ab", true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, "bc", true /* isValidWord */, "ab");
+ onInputWordWithPrevWords(binaryDictionary, "cd", true /* isValidWord */, "bc", "ab");
+ forcePassingShortTime(binaryDictionary);
+ assertTrue(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
+ forcePassingLongTime(binaryDictionary);
+ assertFalse(isValidTrigram(binaryDictionary, "ab", "bc", "cd"));
+
binaryDictionary.close();
- dictFile.delete();
}
public void testAddManyUnigramsToDecayingDict() {
@@ -321,16 +383,8 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final int codePointSetSize = 50;
final long seed = System.currentTimeMillis();
final Random random = new Random(seed);
-
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
setCurrentTimeForTestMode(mCurrentTime);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
@@ -342,31 +396,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
final int maxUnigramCount = Integer.parseInt(
- binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
+ binaryDictionary.getPropertyForGettingStats(
+ BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY));
for (int i = 0; i < unigramTypedCount; i++) {
final String word = words.get(random.nextInt(words.size()));
- addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, word, true /* isValidWord */);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
forcePassingShortTime(binaryDictionary);
}
final int unigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
}
}
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
+ assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY)) > 0);
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
+ assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY)) <= maxUnigramCount);
forcePassingLongTime(binaryDictionary);
- assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest(
+ assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY)));
}
@@ -384,26 +439,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final int codePointSetSize = 50;
final long seed = System.currentTimeMillis();
final Random random = new Random(seed);
-
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
setCurrentTimeForTestMode(mCurrentTime);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final String strong = "strong";
final String weak = "weak";
for (int j = 0; j < strongUnigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, strong, true /* isValidWord */);
}
for (int j = 0; j < weakUnigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, weak, true /* isValidWord */);
}
assertTrue(binaryDictionary.isValidWord(strong));
assertTrue(binaryDictionary.isValidWord(weak));
@@ -411,17 +458,17 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
for (int i = 0; i < unigramCount; i++) {
final String word = CodePointUtils.generateWord(random, codePointSet);
for (int j = 0; j < eachUnigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, word, true /* isValidWord */);
}
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int unigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
assertTrue(binaryDictionary.isValidWord(strong));
assertTrue(binaryDictionary.isValidWord(weak));
binaryDictionary.flushWithGC();
final int unigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.UNIGRAM_COUNT_QUERY));
assertTrue(unigramCountBeforeGC > unigramCountAfterGC);
assertFalse(binaryDictionary.isValidWord(weak));
@@ -438,6 +485,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
private void testAddManyBigramsToDecayingDict(final int formatVersion) {
+ final int maxUnigramCount = 5000;
+ final int maxBigramCount = 10000;
+ final HashMap<String, String> attributeMap = new HashMap<>();
+ attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, String.valueOf(maxUnigramCount));
+ attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, String.valueOf(maxBigramCount));
+
final int unigramCount = 5000;
final int bigramCount = 30000;
final int bigramTypedCount = 100000;
@@ -445,16 +498,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final long seed = System.currentTimeMillis();
final Random random = new Random(seed);
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
setCurrentTimeForTestMode(mCurrentTime);
+ final File dictFile = createEmptyDictionaryWithAttributeMapAndGetFile(formatVersion,
+ attributeMap);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final ArrayList<String> words = new ArrayList<>();
@@ -476,34 +523,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
bigrams.add(bigram);
}
- final int maxBigramCount = Integer.parseInt(
- binaryDictionary.getPropertyForTest(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY));
for (int i = 0; i < bigramTypedCount; ++i) {
final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size()));
- addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, bigram.first, true /* isValidWord */);
+ onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */,
+ bigram.first);
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY));
while (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
forcePassingShortTime(binaryDictionary);
}
final int bigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY));
assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
}
}
-
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
+ forcePassingShortTime(binaryDictionary);
+ assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY)) > 0);
- assertTrue(Integer.parseInt(binaryDictionary.getPropertyForTest(
+ assertTrue(Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY)) <= maxBigramCount);
forcePassingLongTime(binaryDictionary);
- assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForTest(
+ assertEquals(0, Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY)));
}
@@ -514,6 +559,12 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
}
private void testOverflowBigrams(final int formatVersion) {
+ final int maxUnigramCount = 5000;
+ final int maxBigramCount = 10000;
+ final HashMap<String, String> attributeMap = new HashMap<>();
+ attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, String.valueOf(maxUnigramCount));
+ attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, String.valueOf(maxBigramCount));
+
final int bigramCount = 20000;
final int unigramCount = 1000;
final int unigramTypedCount = 20;
@@ -523,17 +574,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final int codePointSetSize = 50;
final long seed = System.currentTimeMillis();
final Random random = new Random(seed);
-
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
setCurrentTimeForTestMode(mCurrentTime);
+ final File dictFile = createEmptyDictionaryWithAttributeMapAndGetFile(formatVersion,
+ attributeMap);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
final ArrayList<String> words = new ArrayList<>();
@@ -541,23 +585,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final String word = CodePointUtils.generateWord(random, codePointSet);
words.add(word);
for (int j = 0; j < unigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, word, true /* isValidWord */);
}
}
final String strong = "strong";
final String weak = "weak";
final String target = "target";
for (int j = 0; j < unigramTypedCount; j++) {
- addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, weak, true /* isValidWord */);
+ onInputWord(binaryDictionary, strong, true /* isValidWord */);
}
binaryDictionary.flushWithGC();
for (int j = 0; j < strongBigramTypedCount; j++) {
- addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong);
}
for (int j = 0; j < weakBigramTypedCount; j++) {
- addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak);
}
assertTrue(isValidBigram(binaryDictionary, strong, target));
assertTrue(isValidBigram(binaryDictionary, weak, target));
@@ -570,15 +613,15 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final String word1 = words.get(word1Index);
for (int j = 0; j < eachBigramTypedCount; j++) {
- addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0);
}
if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
final int bigramCountBeforeGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY));
binaryDictionary.flushWithGC();
final int bigramCountAfterGC =
- Integer.parseInt(binaryDictionary.getPropertyForTest(
+ Integer.parseInt(binaryDictionary.getPropertyForGettingStats(
BinaryDictionary.BIGRAM_COUNT_QUERY));
assertTrue(bigramCountBeforeGC > bigramCountAfterGC);
assertTrue(isValidBigram(binaryDictionary, strong, target));
@@ -596,97 +639,182 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) {
setCurrentTimeForTestMode(mCurrentTime);
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
+ final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
+ onInputWord(binaryDictionary, "aaa", true /* isValidWord */);
assertTrue(binaryDictionary.isValidWord("aaa"));
- addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
- assertFalse(binaryDictionary.isValidWord("bbb"));
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY);
- addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY);
- addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+ onInputWord(binaryDictionary, "ccc", true /* isValidWord */);
+
+ onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa");
assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
- addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
+ onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa");
+ assertFalse(binaryDictionary.isValidWord("bbb"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
+ if (supportsNgram(toFormatVersion)) {
+ onInputWordWithPrevWords(binaryDictionary, "xyz", true, "abc", "aaa");
+ assertTrue(isValidTrigram(binaryDictionary, "aaa", "abc", "xyz"));
+ onInputWordWithPrevWords(binaryDictionary, "def", false, "abc", "aaa");
+ assertFalse(isValidTrigram(binaryDictionary, "aaa", "abc", "def"));
+ }
+
assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.migrateTo(toFormatVersion));
assertTrue(binaryDictionary.isValidDictionary());
assertEquals(toFormatVersion, binaryDictionary.getFormatVersion());
assertTrue(binaryDictionary.isValidWord("aaa"));
assertFalse(binaryDictionary.isValidWord("bbb"));
- assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc"));
- addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY);
- assertTrue(binaryDictionary.isValidWord("bbb"));
+ if (supportsCountBasedNgram(toFormatVersion)) {
+ assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc"));
+ onInputWord(binaryDictionary, "bbb", false /* isValidWord */);
+ assertTrue(binaryDictionary.isValidWord("bbb"));
+ }
assertTrue(isValidBigram(binaryDictionary, "aaa", "abc"));
assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb"));
- addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY);
- assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
+ if (supportsCountBasedNgram(toFormatVersion)) {
+ onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa");
+ assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
+ }
+ if (supportsNgram(toFormatVersion)) {
+ assertTrue(isValidTrigram(binaryDictionary, "aaa", "abc", "xyz"));
+ assertFalse(isValidTrigram(binaryDictionary, "aaa", "abc", "def"));
+ onInputWordWithPrevWords(binaryDictionary, "def", false, "abc", "aaa");
+ assertTrue(isValidTrigram(binaryDictionary, "aaa", "abc", "def"));
+ }
+
binaryDictionary.close();
- dictFile.delete();
}
public void testBeginningOfSentence() {
for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- if (supportsBeginningOfSentence(formatVersion)) {
- testBeginningOfSentence(formatVersion);
- }
+ testBeginningOfSentence(formatVersion);
}
}
private void testBeginningOfSentence(final int formatVersion) {
setCurrentTimeForTestMode(mCurrentTime);
- File dictFile = null;
- try {
- dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion);
- } catch (IOException e) {
- fail("IOException while writing an initial dictionary : " + e);
- }
- final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
- 0 /* offset */, dictFile.length(), true /* useFullEditDistance */,
- Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */,
- BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */,
- true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */,
- mCurrentTime);
- final PrevWordsInfo prevWordsInfoStartOfSentence = PrevWordsInfo.BEGINNING_OF_SENTENCE;
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY,
- mCurrentTime);
- assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa"));
- binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY,
- mCurrentTime);
- addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY,
- mCurrentTime);
- assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa"));
- assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb"));
-
+ Dictionary.NOT_A_PROBABILITY /* shortcutProbability */,
+ true /* isBeginningOfSentence */, true /* isNotAWord */,
+ false /* isPossiblyOffensive */, mCurrentTime);
+ final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE;
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
+ assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
+ assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
+ assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
forcePassingLongTime(binaryDictionary);
- assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa"));
- assertFalse(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb"));
-
- addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "aaa", DUMMY_PROBABILITY,
- mCurrentTime);
- addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY);
- binaryDictionary.addNgramEntry(prevWordsInfoStartOfSentence, "bbb", DUMMY_PROBABILITY,
- mCurrentTime);
- assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "aaa"));
- assertTrue(binaryDictionary.isValidNgram(prevWordsInfoStartOfSentence, "bbb"));
+ assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
+ assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
+ onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */);
+ assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa"));
+ assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb"));
binaryDictionary.close();
- dictFile.delete();
+ }
+
+ public void testRemoveUnigrams() {
+ for (final int formatVersion : DICT_FORMAT_VERSIONS) {
+ testRemoveUnigrams(formatVersion);
+ }
+ }
+
+ private void testRemoveUnigrams(final int formatVersion) {
+ final int unigramInputCount = 20;
+ setCurrentTimeForTestMode(mCurrentTime);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
+
+ onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
+ assertFalse(binaryDictionary.isValidWord("aaa"));
+ for (int i = 0; i < unigramInputCount; i++) {
+ onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
+ }
+ assertTrue(binaryDictionary.isValidWord("aaa"));
+ assertTrue(binaryDictionary.removeUnigramEntry("aaa"));
+ assertFalse(binaryDictionary.isValidWord("aaa"));
+ onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
+ assertFalse(binaryDictionary.isValidWord("aaa"));
+ onInputWord(binaryDictionary, "aaa", false /* isValidWord */);
+ assertTrue(binaryDictionary.isValidWord("aaa"));
+ assertTrue(binaryDictionary.removeUnigramEntry("aaa"));
+ assertFalse(binaryDictionary.isValidWord("aaa"));
+ binaryDictionary.close();
+ }
+
+ public void testUpdateEntriesForInputEvents() {
+ for (final int formatVersion : DICT_FORMAT_VERSIONS) {
+ testUpdateEntriesForInputEvents(formatVersion);
+ }
+ }
+
+ private void testUpdateEntriesForInputEvents(final int formatVersion) {
+ setCurrentTimeForTestMode(mCurrentTime);
+ final int codePointSetSize = 20;
+ final int EVENT_COUNT = 1000;
+ final double CONTINUE_RATE = 0.9;
+ final long seed = System.currentTimeMillis();
+ final Random random = new Random(seed);
+ final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
+
+ final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
+ final ArrayList<String> unigrams = new ArrayList<>();
+ final ArrayList<Pair<String, String>> bigrams = new ArrayList<>();
+ final ArrayList<Pair<Pair<String, String>, String>> trigrams = new ArrayList<>();
+
+ final WordInputEventForPersonalization[] inputEvents =
+ new WordInputEventForPersonalization[EVENT_COUNT];
+ NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
+ int prevWordCount = 0;
+ for (int i = 0; i < inputEvents.length; i++) {
+ final String word = CodePointUtils.generateWord(random, codePointSet);
+ inputEvents[i] = new WordInputEventForPersonalization(word, ngramContext,
+ true /* isValid */, mCurrentTime);
+ unigrams.add(word);
+ if (prevWordCount >= 2) {
+ final Pair<String, String> prevWordsPair = bigrams.get(bigrams.size() - 1);
+ trigrams.add(new Pair<>(prevWordsPair, word));
+ }
+ if (prevWordCount >= 1) {
+ bigrams.add(new Pair<>(ngramContext.getNthPrevWord(1 /* n */).toString(), word));
+ }
+ if (random.nextDouble() > CONTINUE_RATE) {
+ ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
+ prevWordCount = 0;
+ } else {
+ ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
+ prevWordCount++;
+ }
+ }
+ final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
+ binaryDictionary.updateEntriesForInputEvents(inputEvents);
+
+ for (final String word : unigrams) {
+ assertTrue(binaryDictionary.isInDictionary(word));
+ }
+ for (final Pair<String, String> bigram : bigrams) {
+ assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second));
+ }
+ if (!supportsNgram(formatVersion)) {
+ return;
+ }
+ for (final Pair<Pair<String, String>, String> trigram : trigrams) {
+ assertTrue(isValidTrigram(binaryDictionary, trigram.first.first, trigram.first.second,
+ trigram.second));
+ }
}
}
generated by cgit v1.2.3-83-g751a (git 2.47.1) at 2025-05-07 10:15:39 +0000