diff options
Diffstat (limited to 'tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java')
| -rw-r--r-- | tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java | 691 |
1 files changed, 254 insertions, 437 deletions
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index 6ba18d665..fcaa8cdca 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -21,14 +21,15 @@ import android.test.suitebuilder.annotation.LargeTest; import android.text.TextUtils; import android.util.Pair; -import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; -import com.android.inputmethod.latin.makedict.CodePointUtils; +import com.android.inputmethod.latin.NgramContext.WordInfo; +import com.android.inputmethod.latin.common.CodePointUtils; +import com.android.inputmethod.latin.common.Constants; +import com.android.inputmethod.latin.makedict.DictionaryHeader; import com.android.inputmethod.latin.makedict.FormatSpec; import com.android.inputmethod.latin.makedict.WeightedString; import com.android.inputmethod.latin.makedict.WordProperty; import com.android.inputmethod.latin.utils.BinaryDictionaryUtils; import com.android.inputmethod.latin.utils.FileUtils; -import com.android.inputmethod.latin.utils.LanguageModelParam; import java.io.File; import java.io.IOException; @@ -36,7 +37,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; -import java.util.Map; import java.util.Random; // TODO Use the seed passed as an argument for makedict test. @@ -45,42 +45,73 @@ public class BinaryDictionaryTests extends AndroidTestCase { private static final String TEST_DICT_FILE_EXTENSION = ".testDict"; private static final String TEST_LOCALE = "test"; private static final int[] DICT_FORMAT_VERSIONS = - new int[] { FormatSpec.VERSION4, FormatSpec.VERSION4_DEV }; + new int[] { FormatSpec.VERSION402, FormatSpec.VERSION403, FormatSpec.VERSION4_DEV }; + private static final String DICTIONARY_ID = "TestBinaryDictionary"; - private static boolean canCheckBigramProbability(final int formatVersion) { - return formatVersion > FormatSpec.VERSION401; + private static boolean supportsNgram(final int formatVersion) { + return formatVersion >= FormatSpec.VERSION403; } - private static boolean supportsBeginningOfSentence(final int formatVersion) { - return formatVersion > FormatSpec.VERSION401; + private HashSet<File> mDictFilesToBeDeleted = new HashSet<>(); + + @Override + protected void setUp() throws Exception { + super.setUp(); + mDictFilesToBeDeleted.clear(); } - private File createEmptyDictionaryAndGetFile(final String dictId, - final int formatVersion) throws IOException { - if (formatVersion == FormatSpec.VERSION4 - || formatVersion == FormatSpec.VERSION4_ONLY_FOR_TESTING - || formatVersion == FormatSpec.VERSION4_DEV) { - return createEmptyVer4DictionaryAndGetFile(dictId, formatVersion); - } else { - throw new IOException("Dictionary format version " + formatVersion - + " is not supported."); + @Override + protected void tearDown() throws Exception { + for (final File dictFile : mDictFilesToBeDeleted) { + dictFile.delete(); } + mDictFilesToBeDeleted.clear(); + super.tearDown(); + } + + private File createEmptyDictionaryAndGetFile(final int formatVersion) { + return createEmptyDictionaryWithAttributesAndGetFile(formatVersion, + new HashMap<String, String>()); + } + + private File createEmptyDictionaryWithAttributesAndGetFile(final int formatVersion, + final HashMap<String, String> attributeMap) { + try { + final File dictFile = createEmptyVer4DictionaryAndGetFile(formatVersion, + attributeMap); + mDictFilesToBeDeleted.add(dictFile); + return dictFile; + } catch (final IOException e) { + fail(e.toString()); + } + return null; } - private File createEmptyVer4DictionaryAndGetFile(final String dictId, - final int formatVersion) throws IOException { - final File file = File.createTempFile(dictId, TEST_DICT_FILE_EXTENSION, + private File createEmptyVer4DictionaryAndGetFile(final int formatVersion, + final HashMap<String, String> attributeMap) throws IOException { + final File file = File.createTempFile(DICTIONARY_ID, TEST_DICT_FILE_EXTENSION, getContext().getCacheDir()); file.delete(); file.mkdir(); - Map<String, String> attributeMap = new HashMap<>(); if (BinaryDictionaryUtils.createEmptyDictFile(file.getAbsolutePath(), formatVersion, Locale.ENGLISH, attributeMap)) { return file; - } else { - throw new IOException("Empty dictionary " + file.getAbsolutePath() - + " cannot be created. Format version: " + formatVersion); } + throw new IOException("Empty dictionary " + file.getAbsolutePath() + + " cannot be created. Format version: " + formatVersion); + } + + private static BinaryDictionary getBinaryDictionary(final File dictFile) { + return new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + } + + private BinaryDictionary getEmptyBinaryDictionary(final int formatVersion) { + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + return new BinaryDictionary(dictFile.getAbsolutePath(), + 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, + Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); } public void testIsValidDictionary() { @@ -90,24 +121,15 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testIsValidDictionary(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); assertTrue("binaryDictionary must be valid for existing valid dictionary file.", binaryDictionary.isValidDictionary()); binaryDictionary.close(); assertFalse("binaryDictionary must be invalid after closing.", binaryDictionary.isValidDictionary()); FileUtils.deleteRecursively(dictFile); - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), 0 /* offset */, - dictFile.length(), true /* useFullEditDistance */, Locale.getDefault(), - TEST_LOCALE, true /* isUpdatable */); + binaryDictionary = getBinaryDictionary(dictFile); assertFalse("binaryDictionary must be invalid for not existing dictionary file.", binaryDictionary.isValidDictionary()); binaryDictionary.close(); @@ -120,15 +142,10 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testConstructingDictionaryOnMemory(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); FileUtils.deleteRecursively(dictFile); assertFalse(dictFile.exists()); - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), + final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), true /* useFullEditDistance */, Locale.getDefault(), TEST_LOCALE, formatVersion, new HashMap<String, String>()); assertTrue(binaryDictionary.isValidDictionary()); @@ -143,7 +160,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(formatVersion, binaryDictionary.getFormatVersion()); assertEquals(probability, binaryDictionary.getFrequency("word")); binaryDictionary.close(); - dictFile.delete(); } public void testAddTooLongWord() { @@ -153,16 +169,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testAddTooLongWord(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final StringBuffer stringBuilder = new StringBuffer(); for (int i = 0; i < Constants.DICTIONARY_MAX_WORD_LENGTH; i++) { stringBuilder.append('a'); @@ -177,7 +184,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { // Too long short cut. binaryDictionary.addUnigramEntry("a", probability, invalidLongWord, 10 /* shortcutProbability */, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isBlacklisted */, + false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); addUnigramWord(binaryDictionary, "abc", probability); final int updatedProbability = 200; @@ -188,39 +195,60 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(probability, binaryDictionary.getFrequency("aaa")); assertEquals(updatedProbability, binaryDictionary.getFrequency(validLongWord)); - assertEquals(BinaryDictionary.NOT_A_PROBABILITY, - binaryDictionary.getFrequency(invalidLongWord)); + assertEquals(Dictionary.NOT_A_PROBABILITY, binaryDictionary.getFrequency(invalidLongWord)); assertEquals(updatedProbability, binaryDictionary.getFrequency("abc")); - dictFile.delete(); } private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, - BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, + Dictionary.NOT_A_PROBABILITY /* shortcutProbability */, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + false /* isPossiblyOffensive */, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); } private static void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, final String word1, final int probability) { - binaryDictionary.addNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1, probability, + binaryDictionary.addNgramEntry(new NgramContext(new WordInfo(word0)), word1, probability, + BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); + } + + private static void addTrigramEntry(final BinaryDictionary binaryDictionary, final String word0, + final String word1, final String word2, final int probability) { + final NgramContext ngramContext = + new NgramContext(new WordInfo[] { new WordInfo(word1), new WordInfo(word0) } ); + binaryDictionary.addNgramEntry(ngramContext, word2, probability, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); } private static boolean isValidBigram(final BinaryDictionary binaryDictionary, final String word0, final String word1) { - return binaryDictionary.isValidNgram(new PrevWordsInfo(new WordInfo(word0)), word1); + return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); } private static void removeBigramEntry(final BinaryDictionary binaryDictionary, final String word0, final String word1) { - binaryDictionary.removeNgramEntry(new PrevWordsInfo(new WordInfo(word0)), word1); + binaryDictionary.removeNgramEntry(new NgramContext(new WordInfo(word0)), word1); + } + + private static void removeTrigramEntry(final BinaryDictionary binaryDictionary, + final String word0, final String word1, final String word2) { + final NgramContext ngramContext = + new NgramContext(new WordInfo[] { new WordInfo(word1), new WordInfo(word0) } ); + binaryDictionary.removeNgramEntry(ngramContext, word2); } private static int getBigramProbability(final BinaryDictionary binaryDictionary, final String word0, final String word1) { - return binaryDictionary.getNgramProbability(new PrevWordsInfo(new WordInfo(word0)), word1); + return binaryDictionary.getNgramProbability(new NgramContext(new WordInfo(word0)), word1); + } + + private static int getTrigramProbability(final BinaryDictionary binaryDictionary, + final String word0, final String word1, final String word2) { + final NgramContext ngramContext = + new NgramContext(new WordInfo[] { new WordInfo(word1), new WordInfo(word0) } ); + return binaryDictionary.getNgramProbability(ngramContext, word2); } public void testAddUnigramWord() { @@ -230,16 +258,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testAddUnigramWord(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final int probability = 100; addUnigramWord(binaryDictionary, "aaa", probability); // Reallocate and create. @@ -263,8 +282,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(probability, binaryDictionary.getFrequency("aaaa")); assertEquals(probability, binaryDictionary.getFrequency("a")); assertEquals(updatedProbability, binaryDictionary.getFrequency("aaa")); - - dictFile.delete(); } public void testRandomlyAddUnigramWord() { @@ -277,16 +294,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int wordCount = 1000; final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final HashMap<String, Integer> probabilityMap = new HashMap<>(); // Test a word that isn't contained within the dictionary. @@ -302,7 +310,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { for (String word : probabilityMap.keySet()) { assertEquals(word, (int)probabilityMap.get(word), binaryDictionary.getFrequency(word)); } - dictFile.delete(); } public void testAddBigramWords() { @@ -312,15 +319,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testAddBigramWords(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final int unigramProbability = 100; final int bigramProbability = 150; @@ -337,18 +336,14 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertTrue(isValidBigram(binaryDictionary, "aaa", "bcc")); assertTrue(isValidBigram(binaryDictionary, "abb", "aaa")); assertTrue(isValidBigram(binaryDictionary, "abb", "bcc")); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc")); - } + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc")); addBigramWords(binaryDictionary, "aaa", "abb", updatedBigramProbability); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(updatedBigramProbability, - getBigramProbability(binaryDictionary, "aaa", "abb")); - } + assertEquals(updatedBigramProbability, + getBigramProbability(binaryDictionary, "aaa", "abb")); assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa")); assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc")); @@ -368,19 +363,12 @@ public class BinaryDictionaryTests extends AndroidTestCase { addUnigramWord(binaryDictionary, "abc", unigramProbability); addUnigramWord(binaryDictionary, "f", unigramProbability); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(bigramProbability, - getBigramProbability(binaryDictionary, "abcde", "fghij")); - } + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abcde", "fghij")); assertEquals(Dictionary.NOT_A_PROBABILITY, getBigramProbability(binaryDictionary, "abcde", "fgh")); addBigramWords(binaryDictionary, "abcde", "fghij", updatedBigramProbability); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(updatedBigramProbability, - getBigramProbability(binaryDictionary, "abcde", "fghij")); - } - - dictFile.delete(); + assertEquals(updatedBigramProbability, + getBigramProbability(binaryDictionary, "abcde", "fghij")); } public void testRandomlyAddBigramWords() { @@ -395,16 +383,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final ArrayList<String> words = new ArrayList<>(); final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>(); @@ -439,13 +418,9 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int bigramProbability = bigramProbabilities.get(bigram); assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, isValidBigram(binaryDictionary, bigram.first, bigram.second)); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(bigramProbability, - getBigramProbability(binaryDictionary, bigram.first, bigram.second)); - } + assertEquals(bigramProbability, + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); } - - dictFile.delete(); } public void testRemoveBigramWords() { @@ -455,15 +430,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testRemoveBigramWords(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final int unigramProbability = 100; final int bigramProbability = 150; addUnigramWord(binaryDictionary, "aaa", unigramProbability); @@ -496,8 +463,45 @@ public class BinaryDictionaryTests extends AndroidTestCase { // Test remove non-existing bigram operation. removeBigramEntry(binaryDictionary, "aaa", "abb"); removeBigramEntry(binaryDictionary, "bcc", "aaa"); + } + + public void testAddTrigramWords() { + for (final int formatVersion : DICT_FORMAT_VERSIONS) { + if (supportsNgram(formatVersion)) { + testAddTrigramWords(formatVersion); + } + } + } + + private void testAddTrigramWords(final int formatVersion) { + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); + final int unigramProbability = 100; + final int trigramProbability = 150; + final int updatedTrigramProbability = 200; + addUnigramWord(binaryDictionary, "aaa", unigramProbability); + addUnigramWord(binaryDictionary, "abb", unigramProbability); + addUnigramWord(binaryDictionary, "bcc", unigramProbability); + + addBigramWords(binaryDictionary, "abb", "bcc", 10); + addBigramWords(binaryDictionary, "abb", "aaa", 10); + + addTrigramEntry(binaryDictionary, "aaa", "abb", "bcc", trigramProbability); + addTrigramEntry(binaryDictionary, "bcc", "abb", "aaa", trigramProbability); + + assertEquals(trigramProbability, + getTrigramProbability(binaryDictionary, "aaa", "abb", "bcc")); + assertEquals(trigramProbability, + getTrigramProbability(binaryDictionary, "bcc", "abb", "aaa")); + assertFalse(isValidBigram(binaryDictionary, "aaa", "abb")); + + addTrigramEntry(binaryDictionary, "bcc", "abb", "aaa", updatedTrigramProbability); + assertEquals(updatedTrigramProbability, + getTrigramProbability(binaryDictionary, "bcc", "abb", "aaa")); - dictFile.delete(); + removeTrigramEntry(binaryDictionary, "aaa", "abb", "bcc"); + assertEquals(Dictionary.NOT_A_PROBABILITY, + getTrigramProbability(binaryDictionary, "aaa", "abb", "bcc")); + assertTrue(isValidBigram(binaryDictionary, "abb", "bcc")); } public void testFlushDictionary() { @@ -507,15 +511,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testFlushDictionary(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final int probability = 100; addUnigramWord(binaryDictionary, "aaa", probability); @@ -535,23 +532,16 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.flush(); binaryDictionary.close(); - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - + binaryDictionary = getBinaryDictionary(dictFile); assertEquals(probability, binaryDictionary.getFrequency("aaa")); assertEquals(probability, binaryDictionary.getFrequency("abcd")); addUnigramWord(binaryDictionary, "bcde", probability); binaryDictionary.flush(); binaryDictionary.close(); - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + binaryDictionary = getBinaryDictionary(dictFile); assertEquals(probability, binaryDictionary.getFrequency("bcde")); binaryDictionary.close(); - - dictFile.delete(); } public void testFlushWithGCDictionary() { @@ -561,16 +551,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testFlushWithGCDictionary(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final int unigramProbability = 100; final int bigramProbability = 150; addUnigramWord(binaryDictionary, "aaa", unigramProbability); @@ -583,25 +565,19 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.flushWithGC(); binaryDictionary.close(); - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + binaryDictionary = getBinaryDictionary(dictFile); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("abb")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bcc")); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc")); - } + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "abb")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bcc")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "aaa")); + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "abb", "bcc")); assertFalse(isValidBigram(binaryDictionary, "bcc", "aaa")); assertFalse(isValidBigram(binaryDictionary, "bcc", "bbc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "aaa")); binaryDictionary.flushWithGC(); binaryDictionary.close(); - - dictFile.delete(); } public void testAddBigramWordsAndFlashWithGC() { @@ -618,16 +594,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final ArrayList<String> words = new ArrayList<>(); final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>(); @@ -660,22 +628,15 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.flushWithGC(); binaryDictionary.close(); - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - + binaryDictionary = getBinaryDictionary(dictFile); for (final Pair<String, String> bigram : bigramWords) { final int bigramProbability = bigramProbabilities.get(bigram); assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, isValidBigram(binaryDictionary, bigram.first, bigram.second)); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(bigramProbability, - getBigramProbability(binaryDictionary, bigram.first, bigram.second)); - } + assertEquals(bigramProbability, + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); } - - dictFile.delete(); } public void testRandomOperationsAndFlashWithGC() { @@ -685,6 +646,12 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testRandomOperationsAndFlashWithGC(final int formatVersion) { + final int maxUnigramCount = 5000; + final int maxBigramCount = 10000; + final HashMap<String, String> attributeMap = new HashMap<>(); + attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, String.valueOf(maxUnigramCount)); + attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, String.valueOf(maxBigramCount)); + final int flashWithGCIterationCount = 50; final int operationCountInEachIteration = 200; final int initialUnigramCount = 100; @@ -695,17 +662,10 @@ public class BinaryDictionaryTests extends AndroidTestCase { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); + final File dictFile = createEmptyDictionaryWithAttributesAndGetFile(formatVersion, + attributeMap); + BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - - BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); final ArrayList<String> words = new ArrayList<>(); final ArrayList<Pair<String, String>> bigramWords = new ArrayList<>(); final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); @@ -722,9 +682,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.close(); for (int gcCount = 0; gcCount < flashWithGCIterationCount; gcCount++) { - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + binaryDictionary = getBinaryDictionary(dictFile); for (int opCount = 0; opCount < operationCountInEachIteration; opCount++) { // Add unigram. if (random.nextFloat() < addUnigramProb) { @@ -781,18 +739,14 @@ public class BinaryDictionaryTests extends AndroidTestCase { probability = Dictionary.NOT_A_PROBABILITY; } - if (canCheckBigramProbability(formatVersion)) { - assertEquals(probability, - getBigramProbability(binaryDictionary, bigram.first, bigram.second)); - } + assertEquals(probability, + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); assertEquals(probability != Dictionary.NOT_A_PROBABILITY, isValidBigram(binaryDictionary, bigram.first, bigram.second)); } binaryDictionary.flushWithGC(); binaryDictionary.close(); } - - dictFile.delete(); } public void testAddManyUnigramsAndFlushWithGC() { @@ -808,12 +762,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final ArrayList<String> words = new ArrayList<>(); final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); @@ -821,9 +770,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { BinaryDictionary binaryDictionary; for (int i = 0; i < flashWithGCIterationCount; i++) { - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + binaryDictionary = getBinaryDictionary(dictFile); while(!binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); @@ -841,8 +788,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { binaryDictionary.flushWithGC(); binaryDictionary.close(); } - - dictFile.delete(); } public void testUnigramAndBigramCount() { @@ -852,19 +797,20 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testUnigramAndBigramCount(final int formatVersion) { + final int maxUnigramCount = 5000; + final int maxBigramCount = 10000; + final HashMap<String, String> attributeMap = new HashMap<>(); + attributeMap.put(DictionaryHeader.MAX_UNIGRAM_COUNT_KEY, String.valueOf(maxUnigramCount)); + attributeMap.put(DictionaryHeader.MAX_BIGRAM_COUNT_KEY, String.valueOf(maxBigramCount)); + final int flashWithGCIterationCount = 10; final int codePointSetSize = 50; final int unigramCountPerIteration = 1000; final int bigramCountPerIteration = 2000; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } + final File dictFile = createEmptyDictionaryWithAttributesAndGetFile(formatVersion, + attributeMap); final ArrayList<String> words = new ArrayList<>(); final HashSet<Pair<String, String>> bigrams = new HashSet<>(); @@ -872,9 +818,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { BinaryDictionary binaryDictionary; for (int i = 0; i < flashWithGCIterationCount; i++) { - binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + binaryDictionary = getBinaryDictionary(dictFile); for (int j = 0; j < unigramCountPerIteration; j++) { final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); @@ -892,83 +836,20 @@ public class BinaryDictionaryTests extends AndroidTestCase { addBigramWords(binaryDictionary, word0, word1, bigramProbability); } assertEquals(new HashSet<>(words).size(), Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.UNIGRAM_COUNT_QUERY))); assertEquals(new HashSet<>(bigrams).size(), Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.flushWithGC(); assertEquals(new HashSet<>(words).size(), Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.UNIGRAM_COUNT_QUERY))); assertEquals(new HashSet<>(bigrams).size(), Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); + binaryDictionary.getPropertyForGettingStats( + BinaryDictionary.BIGRAM_COUNT_QUERY))); binaryDictionary.close(); } - - dictFile.delete(); - } - - public void testAddMultipleDictionaryEntries() { - for (final int formatVersion : DICT_FORMAT_VERSIONS) { - testAddMultipleDictionaryEntries(formatVersion); - } - } - - private void testAddMultipleDictionaryEntries(final int formatVersion) { - final int codePointSetSize = 20; - final int lmParamCount = 1000; - final double bigramContinueRate = 0.9; - final long seed = System.currentTimeMillis(); - final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - - final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); - final HashMap<Pair<String, String>, Integer> bigramProbabilities = new HashMap<>(); - - final LanguageModelParam[] languageModelParams = new LanguageModelParam[lmParamCount]; - String prevWord = null; - for (int i = 0; i < languageModelParams.length; i++) { - final String word = CodePointUtils.generateWord(random, codePointSet); - final int probability = random.nextInt(0xFF); - final int bigramProbability = probability + random.nextInt(0xFF - probability); - unigramProbabilities.put(word, probability); - if (prevWord == null) { - languageModelParams[i] = new LanguageModelParam(word, probability, - BinaryDictionary.NOT_A_VALID_TIMESTAMP); - } else { - languageModelParams[i] = new LanguageModelParam(prevWord, word, probability, - bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP); - bigramProbabilities.put(new Pair<>(prevWord, word), - bigramProbability); - } - prevWord = (random.nextDouble() < bigramContinueRate) ? word : null; - } - - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); - binaryDictionary.addMultipleDictionaryEntries(languageModelParams); - - for (Map.Entry<String, Integer> entry : unigramProbabilities.entrySet()) { - assertEquals((int)entry.getValue(), binaryDictionary.getFrequency(entry.getKey())); - } - - for (Map.Entry<Pair<String, String>, Integer> entry : bigramProbabilities.entrySet()) { - final String word0 = entry.getKey().first; - final String word1 = entry.getKey().second; - final int bigramProbability = entry.getValue(); - assertEquals(bigramProbability != Dictionary.NOT_A_PROBABILITY, - isValidBigram(binaryDictionary, word0, word1)); - if (canCheckBigramProbability(formatVersion)) { - assertEquals(bigramProbability, - getBigramProbability(binaryDictionary, word0, word1)); - } - } } public void testGetWordProperties() { @@ -984,16 +865,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int BIGRAM_COUNT = 1000; final int codePointSetSize = 20; final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); + final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord", false /* isBeginningOfSentence */); @@ -1008,11 +881,11 @@ public class BinaryDictionaryTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); final int unigramProbability = random.nextInt(0xFF); final boolean isNotAWord = random.nextBoolean(); - final boolean isBlacklisted = random.nextBoolean(); + final boolean isPossiblyOffensive = random.nextBoolean(); // TODO: Add tests for historical info. binaryDictionary.addUnigramEntry(word, unigramProbability, - null /* shortcutTarget */, BinaryDictionary.NOT_A_PROBABILITY, - false /* isBeginningOfSentence */, isNotAWord, isBlacklisted, + null /* shortcutTarget */, Dictionary.NOT_A_PROBABILITY, + false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { binaryDictionary.flushWithGC(); @@ -1024,8 +897,8 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(word, wordProperty.mWord); assertTrue(wordProperty.isValid()); assertEquals(isNotAWord, wordProperty.mIsNotAWord); - assertEquals(isBlacklisted, wordProperty.mIsBlacklistEntry); - assertEquals(false, wordProperty.mHasBigrams); + assertEquals(isPossiblyOffensive, wordProperty.mIsPossiblyOffensive); + assertEquals(false, wordProperty.mHasNgrams); assertEquals(false, wordProperty.mHasShortcuts); assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability); assertTrue(wordProperty.mShortcutTargets.isEmpty()); @@ -1062,14 +935,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { final HashSet<String> bigramWord1s = bigrams.get(word0); final WordProperty wordProperty = binaryDictionary.getWordProperty(word0, false /* isBeginningOfSentence */); - assertEquals(bigramWord1s.size(), wordProperty.mBigrams.size()); - for (int j = 0; j < wordProperty.mBigrams.size(); j++) { - final String word1 = wordProperty.mBigrams.get(j).mWord; + assertEquals(bigramWord1s.size(), wordProperty.mNgrams.size()); + // TODO: Support ngram. + for (final WeightedString bigramTarget : wordProperty.getBigrams()) { + final String word1 = bigramTarget.mWord; assertTrue(bigramWord1s.contains(word1)); - if (canCheckBigramProbability(formatVersion)) { - final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1)); - assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability()); - } + final int bigramProbability = bigramProbabilities.get(new Pair<>(word0, word1)); + assertEquals(bigramProbability, bigramTarget.getProbability()); } } } @@ -1087,16 +959,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int BIGRAM_COUNT = 1000; final int codePointSetSize = 20; final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final WordProperty invalidWordProperty = binaryDictionary.getWordProperty("dummyWord", false /* isBeginningOfSentence */); @@ -1155,15 +1018,16 @@ public class BinaryDictionaryTests extends AndroidTestCase { wordProperty.mProbabilityInfo.mProbability); wordSet.remove(word0); final HashSet<String> bigramWord1s = bigrams.get(word0); - for (int j = 0; j < wordProperty.mBigrams.size(); j++) { - final String word1 = wordProperty.mBigrams.get(j).mWord; - assertTrue(bigramWord1s.contains(word1)); - final Pair<String, String> bigram = new Pair<>(word0, word1); - if (canCheckBigramProbability(formatVersion)) { + // TODO: Support ngram. + if (wordProperty.mHasNgrams) { + for (final WeightedString bigramTarget : wordProperty.getBigrams()) { + final String word1 = bigramTarget.mWord; + assertTrue(bigramWord1s.contains(word1)); + final Pair<String, String> bigram = new Pair<>(word0, word1); final int bigramProbability = bigramProbabilitiesToCheckLater.get(bigram); - assertEquals(bigramProbability, wordProperty.mBigrams.get(j).getProbability()); + assertEquals(bigramProbability, bigramTarget.getProbability()); + bigramSet.remove(bigram); } - bigramSet.remove(bigram); } token = result.mNextToken; } while (token != 0); @@ -1178,21 +1042,13 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testAddShortcuts(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final int unigramProbability = 100; final int shortcutProbability = 10; binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", shortcutProbability, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); + false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); WordProperty wordProperty = binaryDictionary.getWordProperty("aaa", false /* isBeginningOfSentence */); assertEquals(1, wordProperty.mShortcutTargets.size()); @@ -1201,7 +1057,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int updatedShortcutProbability = 2; binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", updatedShortcutProbability, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isBlacklisted */, 0 /* timestamp */); + false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); wordProperty = binaryDictionary.getWordProperty("aaa", false /* isBeginningOfSentence */); assertEquals(1, wordProperty.mShortcutTargets.size()); @@ -1210,7 +1066,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { wordProperty.mShortcutTargets.get(0).getProbability()); binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy", shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, 0 /* timestamp */); + false /* isPossiblyOffensive */, 0 /* timestamp */); final HashMap<String, Integer> shortcutTargets = new HashMap<>(); shortcutTargets.put("zzz", updatedShortcutProbability); shortcutTargets.put("yyy", shortcutProbability); @@ -1254,16 +1110,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final ArrayList<String> words = new ArrayList<>(); final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>(); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); for (int i = 0; i < UNIGRAM_COUNT; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); @@ -1282,7 +1129,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int unigramProbability = unigramProbabilities.get(word); binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget, shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, 0 /* timestamp */); + false /* isPossiblyOffensive */, 0 /* timestamp */); if (shortcutTargets.containsKey(word)) { final HashMap<String, Integer> shortcutTargetsOfWord = shortcutTargets.get(word); shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); @@ -1314,6 +1161,15 @@ public class BinaryDictionaryTests extends AndroidTestCase { } } + public void testPossiblyOffensiveAttributeMaintained() { + final BinaryDictionary binaryDictionary = + getEmptyBinaryDictionary(FormatSpec.VERSION403); + binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY, + false, true, true, 0); + WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false); + assertEquals(true, wordProperty.mIsPossiblyOffensive); + } + public void testDictMigration() { for (final int formatVersion : DICT_FORMAT_VERSIONS) { testDictMigration(FormatSpec.VERSION4_ONLY_FOR_TESTING, formatVersion); @@ -1321,15 +1177,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testDictMigration(final int fromFormatVersion, final int toFormatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(fromFormatVersion); final int unigramProbability = 100; addUnigramWord(binaryDictionary, "aaa", unigramProbability); addUnigramWord(binaryDictionary, "bbb", unigramProbability); @@ -1338,11 +1186,11 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int shortcutProbability = 10; binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isBlacklisted */, 0 /* timestamp */); + false /* isPossiblyOffensive */, 0 /* timestamp */); binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */, Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */, - true /* isNotAWord */, true /* isBlacklisted */, 0 /* timestamp */); - binaryDictionary.addNgramEntry(PrevWordsInfo.BEGINNING_OF_SENTENCE, + true /* isNotAWord */, true /* isPossiblyOffensive */, 0 /* timestamp */); + binaryDictionary.addNgramEntry(NgramContext.BEGINNING_OF_SENTENCE, "aaa", bigramProbability, 0 /* timestamp */); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); @@ -1353,11 +1201,9 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(toFormatVersion, binaryDictionary.getFormatVersion()); assertEquals(unigramProbability, binaryDictionary.getFrequency("aaa")); assertEquals(unigramProbability, binaryDictionary.getFrequency("bbb")); - if (canCheckBigramProbability(toFormatVersion)) { - assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb")); - assertEquals(bigramProbability, binaryDictionary.getNgramProbability( - PrevWordsInfo.BEGINNING_OF_SENTENCE, "aaa")); - } + assertEquals(bigramProbability, getBigramProbability(binaryDictionary, "aaa", "bbb")); + assertEquals(bigramProbability, binaryDictionary.getNgramProbability( + NgramContext.BEGINNING_OF_SENTENCE, "aaa")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); WordProperty wordProperty = binaryDictionary.getWordProperty("ccc", false /* isBeginningOfSentence */); @@ -1365,7 +1211,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord); wordProperty = binaryDictionary.getWordProperty("ddd", false /* isBeginningOfSentence */); - assertTrue(wordProperty.mIsBlacklistEntry); + assertTrue(wordProperty.mIsPossiblyOffensive); assertTrue(wordProperty.mIsNotAWord); } @@ -1381,16 +1227,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int codePointSetSize = 50; final long seed = System.currentTimeMillis(); final Random random = new Random(seed); - - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", fromFormatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(fromFormatVersion); final ArrayList<String> words = new ArrayList<>(); final ArrayList<Pair<String, String>> bigrams = new ArrayList<>(); @@ -1434,55 +1271,43 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals((int)unigramProbabilities.get(word), binaryDictionary.getFrequency(word)); } assertEquals(unigramProbabilities.size(), Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.UNIGRAM_COUNT_QUERY))); + binaryDictionary.getPropertyForGettingStats(BinaryDictionary.UNIGRAM_COUNT_QUERY))); for (final Pair<String, String> bigram : bigrams) { - if (canCheckBigramProbability(toFormatVersion)) { - assertEquals((int)bigramProbabilities.get(bigram), - getBigramProbability(binaryDictionary, bigram.first, bigram.second)); - } + assertEquals((int)bigramProbabilities.get(bigram), + getBigramProbability(binaryDictionary, bigram.first, bigram.second)); assertTrue(isValidBigram(binaryDictionary, bigram.first, bigram.second)); } assertEquals(bigramProbabilities.size(), Integer.parseInt( - binaryDictionary.getPropertyForTest(BinaryDictionary.BIGRAM_COUNT_QUERY))); + binaryDictionary.getPropertyForGettingStats(BinaryDictionary.BIGRAM_COUNT_QUERY))); } public void testBeginningOfSentence() { for (final int formatVersion : DICT_FORMAT_VERSIONS) { - if (supportsBeginningOfSentence(formatVersion)) { - testBeginningOfSentence(formatVersion); - } + testBeginningOfSentence(formatVersion); } } private void testBeginningOfSentence(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); final int dummyProbability = 0; - final PrevWordsInfo prevWordsInfoBeginningOfSentence = PrevWordsInfo.BEGINNING_OF_SENTENCE; + final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; final int bigramProbability = 200; addUnigramWord(binaryDictionary, "aaa", dummyProbability); - binaryDictionary.addNgramEntry(prevWordsInfoBeginningOfSentence, "aaa", bigramProbability, + binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); assertEquals(bigramProbability, - binaryDictionary.getNgramProbability(prevWordsInfoBeginningOfSentence, "aaa")); - binaryDictionary.addNgramEntry(prevWordsInfoBeginningOfSentence, "aaa", bigramProbability, + binaryDictionary.getNgramProbability(beginningOfSentenceContext, "aaa")); + binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); addUnigramWord(binaryDictionary, "bbb", dummyProbability); - binaryDictionary.addNgramEntry(prevWordsInfoBeginningOfSentence, "bbb", bigramProbability, + binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", bigramProbability, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); binaryDictionary.flushWithGC(); assertEquals(bigramProbability, - binaryDictionary.getNgramProbability(prevWordsInfoBeginningOfSentence, "aaa")); + binaryDictionary.getNgramProbability(beginningOfSentenceContext, "aaa")); assertEquals(bigramProbability, - binaryDictionary.getNgramProbability(prevWordsInfoBeginningOfSentence, "bbb")); + binaryDictionary.getNgramProbability(beginningOfSentenceContext, "bbb")); } public void testGetMaxFrequencyOfExactMatches() { @@ -1492,15 +1317,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { } private void testGetMaxFrequencyOfExactMatches(final int formatVersion) { - File dictFile = null; - try { - dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary", formatVersion); - } catch (IOException e) { - fail("IOException while writing an initial dictionary : " + e); - } - final BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(), - 0 /* offset */, dictFile.length(), true /* useFullEditDistance */, - Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */); + final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); addUnigramWord(binaryDictionary, "abc", 10); addUnigramWord(binaryDictionary, "aBc", 15); assertEquals(15, binaryDictionary.getMaxFrequencyOfExactMatches("abc")); |