diff options
Diffstat (limited to 'tests/src/com')
13 files changed, 74 insertions, 455 deletions
diff --git a/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java b/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java index 67e76464b..d75e61ad1 100644 --- a/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java +++ b/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java @@ -17,19 +17,20 @@ package com.android.inputmethod.compat; import android.graphics.Typeface; -import android.os.Build; import android.test.AndroidTestCase; import android.test.suitebuilder.annotation.SmallTest; import android.text.SpannableString; import android.text.Spanned; import android.text.style.StyleSpan; +import com.android.inputmethod.latin.common.Constants; + import java.util.Locale; @SmallTest public class LocaleSpanCompatUtilsTests extends AndroidTestCase { public void testInstantiatable() { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR1) { + if (!Constants.JELLY_BEAN_MR1_OR_HIGHER) { // LocaleSpan isn't yet available. return; } diff --git a/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java b/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java index bcf06f085..557cc214d 100644 --- a/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java +++ b/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java @@ -16,15 +16,14 @@ package com.android.inputmethod.keyboard.layout; -import android.os.Build; +import com.android.inputmethod.latin.common.Constants; /** * This class offers label strings of Devanagari letters that need the dotted circle to draw * its glyph. */ class DevanagariLetterConstants { - private static final boolean NEEDS_DOTTED_CIRCLE = - Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN; + private static final boolean NEEDS_DOTTED_CIRCLE = !Constants.JELLY_BEAN_OR_HIGHER; // U+25CC: "◌" DOTTED CIRCLE private static final String DOTTED_CIRCLE = NEEDS_DOTTED_CIRCLE ? "\u25CC" : ""; diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java index baac0883e..a51558e43 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java @@ -700,8 +700,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); - binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */, - Dictionary.NOT_A_PROBABILITY /* shortcutProbability */, + binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isPossiblyOffensive */, mCurrentTime); final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java index a8f0d81af..161879841 100644 --- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java @@ -182,8 +182,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { addUnigramWord(binaryDictionary, validLongWord, probability); addUnigramWord(binaryDictionary, invalidLongWord, probability); // Too long short cut. - binaryDictionary.addUnigramEntry("a", probability, invalidLongWord, - 10 /* shortcutProbability */, false /* isBeginningOfSentence */, + binaryDictionary.addUnigramEntry("a", probability, false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP); addUnigramWord(binaryDictionary, "abc", probability); @@ -201,8 +200,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { - binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, - Dictionary.NOT_A_PROBABILITY /* shortcutProbability */, + binaryDictionary.addUnigramEntry(word, probability, false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isPossiblyOffensive */, BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */); @@ -884,7 +882,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { final boolean isPossiblyOffensive = random.nextBoolean(); // TODO: Add tests for historical info. binaryDictionary.addUnigramEntry(word, unigramProbability, - null /* shortcutTarget */, Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive, BinaryDictionary.NOT_A_VALID_TIMESTAMP); if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) { @@ -899,9 +896,7 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertEquals(isNotAWord, wordProperty.mIsNotAWord); assertEquals(isPossiblyOffensive, wordProperty.mIsPossiblyOffensive); assertEquals(false, wordProperty.mHasNgrams); - assertEquals(false, wordProperty.mHasShortcuts); assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability); - assertTrue(wordProperty.mShortcutTargets.isEmpty()); } for (int i = 0; i < BIGRAM_COUNT; i++) { @@ -1035,137 +1030,10 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertTrue(bigramSet.isEmpty()); } - public void testAddShortcuts() { - for (final int formatVersion : DICT_FORMAT_VERSIONS) { - testAddShortcuts(formatVersion); - } - } - - private void testAddShortcuts(final int formatVersion) { - final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); - - final int unigramProbability = 100; - final int shortcutProbability = 10; - binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", - shortcutProbability, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); - WordProperty wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(1, wordProperty.mShortcutTargets.size()); - assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); - assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability()); - final int updatedShortcutProbability = 2; - binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz", - updatedShortcutProbability, false /* isBeginningOfSentence */, - false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); - wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(1, wordProperty.mShortcutTargets.size()); - assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord); - assertEquals(updatedShortcutProbability, - wordProperty.mShortcutTargets.get(0).getProbability()); - binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy", - shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isPossiblyOffensive */, 0 /* timestamp */); - final HashMap<String, Integer> shortcutTargets = new HashMap<>(); - shortcutTargets.put("zzz", updatedShortcutProbability); - shortcutTargets.put("yyy", shortcutProbability); - wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(2, wordProperty.mShortcutTargets.size()); - for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); - assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), - shortcutTarget.getProbability()); - shortcutTargets.remove(shortcutTarget.mWord); - } - shortcutTargets.put("zzz", updatedShortcutProbability); - shortcutTargets.put("yyy", shortcutProbability); - binaryDictionary.flushWithGC(); - wordProperty = binaryDictionary.getWordProperty("aaa", - false /* isBeginningOfSentence */); - assertEquals(2, wordProperty.mShortcutTargets.size()); - for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord)); - assertEquals((int)shortcutTargets.get(shortcutTarget.mWord), - shortcutTarget.getProbability()); - shortcutTargets.remove(shortcutTarget.mWord); - } - } - - public void testAddManyShortcuts() { - for (final int formatVersion : DICT_FORMAT_VERSIONS) { - testAddManyShortcuts(formatVersion); - } - } - - private void testAddManyShortcuts(final int formatVersion) { - final long seed = System.currentTimeMillis(); - final Random random = new Random(seed); - final int UNIGRAM_COUNT = 1000; - final int SHORTCUT_COUNT = 10000; - final int codePointSetSize = 20; - final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random); - - final ArrayList<String> words = new ArrayList<>(); - final HashMap<String, Integer> unigramProbabilities = new HashMap<>(); - final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>(); - final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion); - - for (int i = 0; i < UNIGRAM_COUNT; i++) { - final String word = CodePointUtils.generateWord(random, codePointSet); - final int unigramProbability = random.nextInt(0xFF); - addUnigramWord(binaryDictionary, word, unigramProbability); - words.add(word); - unigramProbabilities.put(word, unigramProbability); - if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDictionary.flushWithGC(); - } - } - for (int i = 0; i < SHORTCUT_COUNT; i++) { - final String shortcutTarget = CodePointUtils.generateWord(random, codePointSet); - final int shortcutProbability = random.nextInt(0xF); - final String word = words.get(random.nextInt(words.size())); - final int unigramProbability = unigramProbabilities.get(word); - binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget, - shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, - false /* isPossiblyOffensive */, 0 /* timestamp */); - if (shortcutTargets.containsKey(word)) { - final HashMap<String, Integer> shortcutTargetsOfWord = shortcutTargets.get(word); - shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); - } else { - final HashMap<String, Integer> shortcutTargetsOfWord = new HashMap<>(); - shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability); - shortcutTargets.put(word, shortcutTargetsOfWord); - } - if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { - binaryDictionary.flushWithGC(); - } - } - - for (final String word : words) { - final WordProperty wordProperty = binaryDictionary.getWordProperty(word, - false /* isBeginningOfSentence */); - assertEquals((int)unigramProbabilities.get(word), - wordProperty.mProbabilityInfo.mProbability); - if (!shortcutTargets.containsKey(word)) { - // The word does not have shortcut targets. - continue; - } - assertEquals(shortcutTargets.get(word).size(), wordProperty.mShortcutTargets.size()); - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - final String targetCodePonts = shortcutTarget.mWord; - assertEquals((int)shortcutTargets.get(word).get(targetCodePonts), - shortcutTarget.getProbability()); - } - } - } - public void testPossiblyOffensiveAttributeMaintained() { final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(FormatSpec.VERSION403); - binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY, - false, true, true, 0); + binaryDictionary.addUnigramEntry("ddd", 100, false, true, true, 0); WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false); assertEquals(true, wordProperty.mIsPossiblyOffensive); } @@ -1184,11 +1052,11 @@ public class BinaryDictionaryTests extends AndroidTestCase { final int bigramProbability = 150; addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability); final int shortcutProbability = 10; - binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability, + binaryDictionary.addUnigramEntry("ccc", unigramProbability, false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */); - binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */, - Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */, + binaryDictionary.addUnigramEntry("ddd", unigramProbability, + false /* isBeginningOfSentence */, true /* isNotAWord */, true /* isPossiblyOffensive */, 0 /* timestamp */); binaryDictionary.addNgramEntry(NgramContext.BEGINNING_OF_SENTENCE, "aaa", bigramProbability, 0 /* timestamp */); @@ -1207,8 +1075,6 @@ public class BinaryDictionaryTests extends AndroidTestCase { assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); WordProperty wordProperty = binaryDictionary.getWordProperty("ccc", false /* isBeginningOfSentence */); - assertEquals(1, wordProperty.mShortcutTargets.size()); - assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord); wordProperty = binaryDictionary.getWordProperty("ddd", false /* isBeginningOfSentence */); assertTrue(wordProperty.mIsPossiblyOffensive); diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java index 07d7c3225..6c6f62872 100644 --- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java +++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java @@ -35,19 +35,19 @@ public class FusionDictionaryTests extends AndroidTestCase { FusionDictionary dict = new FusionDictionary(new PtNodeArray(), new DictionaryOptions(new HashMap<String,String>())); - dict.add("abc", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("abc", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc")); - dict.add("aa", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("aa", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa")); - dict.add("babcd", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("babcd", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); - dict.add("bacde", new ProbabilityInfo(10), null, false /* isNotAWord */, + dict.add("bacde", new ProbabilityInfo(10), false /* isNotAWord */, false /* isPossiblyOffensive */); assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba")); assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd")); diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java index d833b9736..39da9fcd6 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java @@ -57,15 +57,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private static final int UNIGRAM_FREQ = 10; private static final int BIGRAM_FREQ = 50; private static final int TOLERANCE_OF_BIGRAM_FREQ = 5; - private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50; - private static final int NUM_OF_SHORTCUTS = 5; private static final ArrayList<String> sWords = new ArrayList<>(); private static final ArrayList<String> sWordsWithVariousCodePoints = new ArrayList<>(); private static final SparseArray<List<Integer>> sEmptyBigrams = new SparseArray<>(); private static final SparseArray<List<Integer>> sStarBigrams = new SparseArray<>(); private static final SparseArray<List<Integer>> sChainBigrams = new SparseArray<>(); - private static final HashMap<String, List<String>> sShortcuts = new HashMap<>(); final Random mRandom; @@ -95,16 +92,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { for (int i = 1; i < maxBigrams; ++i) { sStarBigrams.get(0).add(i); } - - sShortcuts.clear(); - for (int i = 0; i < NUM_OF_NODES_HAVING_SHORTCUTS; ++i) { - final int from = Math.abs(mRandom.nextInt()) % sWords.size(); - sShortcuts.put(sWords.get(from), new ArrayList<String>()); - for (int j = 0; j < NUM_OF_SHORTCUTS; ++j) { - final int to = Math.abs(mRandom.nextInt()) % sWords.size(); - sShortcuts.get(sWords.get(from)).add(sWords.get(to)); - } - } } @Override @@ -142,17 +129,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { * Adds unigrams to the dictionary. */ private static void addUnigrams(final int number, final FusionDictionary dict, - final List<String> words, final HashMap<String, List<String>> shortcutMap) { + final List<String> words) { for (int i = 0; i < number; ++i) { final String word = words.get(i); final ArrayList<WeightedString> shortcuts = new ArrayList<>(); - if (shortcutMap != null && shortcutMap.containsKey(word)) { - for (final String shortcut : shortcutMap.get(word)) { - shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ)); - } - } - dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), - (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */, + dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), false /* isNotAWord */, false /* isPossiblyOffensive */); } } @@ -200,8 +181,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { } private static void checkDictionary(final FusionDictionary dict, final List<String> words, - final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap) { + final SparseArray<List<Integer>> bigrams) { assertNotNull(dict); // check unigram @@ -219,19 +199,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { assertNotNull(words.get(w1) + "," + words.get(w2), ptNode.getBigram(words.get(w2))); } } - - // check shortcut - if (shortcutMap != null) { - for (final Entry<String, List<String>> entry : shortcutMap.entrySet()) { - assertTrue(words.contains(entry.getKey())); - final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray, - entry.getKey()); - for (final String word : entry.getValue()) { - assertNotNull("shortcut not found: " + entry.getKey() + ", " + word, - ptNode.getShortcut(word)); - } - } - } } private static String outputOptions(final int bufferType, @@ -244,8 +211,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // Tests for readDictionaryBinary and writeDictionaryBinary private static long timeReadingAndCheckDict(final File file, final List<String> words, - final SparseArray<List<Integer>> bigrams, - final HashMap<String, List<String>> shortcutMap, final int bufferType) { + final SparseArray<List<Integer>> bigrams, final int bufferType) { long now, diff = -1; FusionDictionary dict = null; @@ -261,13 +227,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { Log.e(TAG, "Unsupported format", e); } - checkDictionary(dict, words, bigrams, shortcutMap); + checkDictionary(dict, words, bigrams); return diff; } // Tests for readDictionaryBinary and writeDictionaryBinary private String runReadAndWrite(final List<String> words, - final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcuts, + final SparseArray<List<Integer>> bigrams, final int bufferType, final FormatSpec.FormatOptions formatOptions, final String message) { @@ -278,12 +244,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, shortcuts); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, bigrams); - checkDictionary(dict, words, bigrams, shortcuts); + checkDictionary(dict, words, bigrams); final long write = timeWritingDictToFile(file, dict, formatOptions); - final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType); + final long read = timeReadingAndCheckDict(file, words, bigrams, bufferType); return "PROF: read=" + read + "ms, write=" + write + "ms :" + message + " : " + outputOptions(bufferType, formatOptions); @@ -291,20 +257,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { private void runReadAndWriteTests(final List<String> results, final int bufferType, final FormatSpec.FormatOptions formatOptions) { - results.add(runReadAndWrite(sWords, sEmptyBigrams, null /* shortcuts */, bufferType, + results.add(runReadAndWrite(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram")); - results.add(runReadAndWrite(sWords, sChainBigrams, null /* shortcuts */, bufferType, + results.add(runReadAndWrite(sWords, sChainBigrams, bufferType, formatOptions, "chain")); - results.add(runReadAndWrite(sWords, sStarBigrams, null /* shortcuts */, bufferType, + results.add(runReadAndWrite(sWords, sStarBigrams, bufferType, formatOptions, "star")); - results.add(runReadAndWrite(sWords, sEmptyBigrams, sShortcuts, bufferType, formatOptions, + results.add(runReadAndWrite(sWords, sEmptyBigrams, bufferType, formatOptions, "unigram with shortcuts")); - results.add(runReadAndWrite(sWords, sChainBigrams, sShortcuts, bufferType, formatOptions, + results.add(runReadAndWrite(sWords, sChainBigrams, bufferType, formatOptions, "chain with shortcuts")); - results.add(runReadAndWrite(sWords, sStarBigrams, sShortcuts, bufferType, formatOptions, + results.add(runReadAndWrite(sWords, sStarBigrams, bufferType, formatOptions, "star with shortcuts")); results.add(runReadAndWrite(sWordsWithVariousCodePoints, sEmptyBigrams, - null /* shortcuts */, bufferType, formatOptions, + bufferType, formatOptions, "unigram with various code points")); } @@ -326,7 +292,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { FormatSpec.MINIMUM_SUPPORTED_STATIC_VERSION); final FusionDictionary sourcedict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), sourcedict, words, null /* shortcutMap */); + addUnigrams(words.size(), sourcedict, words); dictEncoder.writeDictionary(sourcedict, formatOptions); // Read the dictionary @@ -472,7 +438,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // making the dictionary from lists of words. final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, null /* shortcutMap */); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); @@ -482,7 +448,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { // TODO: Abandon the Java code, and implement the v4 dictionary reading code in native. long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType, !formatOptions.mHasTimestamp /* checkProbability */); - long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */, + long fullReading = timeReadingAndCheckDict(file, words, bigrams, bufferType); return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap @@ -567,7 +533,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); + addUnigrams(sWords.size(), dict, sWords); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); @@ -636,12 +602,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { public void testVer2DictGetWordProperty() { final FormatOptions formatOptions = BinaryDictUtils.STATIC_OPTIONS; final ArrayList<String> words = sWords; - final HashMap<String, List<String>> shortcuts = sShortcuts; final String dictName = "testGetWordProperty"; final String dictVersion = Long.toString(System.currentTimeMillis()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, shortcuts); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, sEmptyBigrams); final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, getContext().getCacheDir()); @@ -655,30 +620,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { false /* isBeginningOfSentence */); assertEquals(word, wordProperty.mWord); assertEquals(UNIGRAM_FREQ, wordProperty.getProbability()); - if (shortcuts.containsKey(word)) { - assertEquals(shortcuts.get(word).size(), wordProperty.mShortcutTargets.size()); - final List<String> shortcutList = shortcuts.get(word); - assertTrue(wordProperty.mHasShortcuts); - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutList.contains(shortcutTarget.mWord)); - assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability()); - shortcutList.remove(shortcutTarget.mWord); - } - assertTrue(shortcutList.isEmpty()); - } } } public void testVer2DictIteration() { final FormatOptions formatOptions = BinaryDictUtils.STATIC_OPTIONS; final ArrayList<String> words = sWords; - final HashMap<String, List<String>> shortcuts = sShortcuts; final SparseArray<List<Integer>> bigrams = sEmptyBigrams; final String dictName = "testGetWordProperty"; final String dictVersion = Long.toString(System.currentTimeMillis()); final FusionDictionary dict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(words.size(), dict, words, shortcuts); + addUnigrams(words.size(), dict, words); addBigrams(dict, words, bigrams); final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions, getContext().getCacheDir()); @@ -708,17 +661,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase { final String word0 = wordProperty.mWord; assertEquals(UNIGRAM_FREQ, wordProperty.mProbabilityInfo.mProbability); wordSet.remove(word0); - if (shortcuts.containsKey(word0)) { - assertEquals(shortcuts.get(word0).size(), wordProperty.mShortcutTargets.size()); - final List<String> shortcutList = shortcuts.get(word0); - assertNotNull(wordProperty.mShortcutTargets); - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - assertTrue(shortcutList.contains(shortcutTarget.mWord)); - assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability()); - shortcutList.remove(shortcutTarget.mWord); - } - assertTrue(shortcutList.isEmpty()); - } if (wordProperty.mHasNgrams) { for (final WeightedString bigramTarget : wordProperty.getBigrams()) { final String word1 = bigramTarget.mWord; diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java index ce905c499..bd5136583 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java +++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java @@ -16,9 +16,7 @@ package com.android.inputmethod.latin.makedict; -import com.android.inputmethod.annotations.UsedForTesting; import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding; -import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer; import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode; import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray; @@ -92,38 +90,6 @@ public class BinaryDictEncoderUtils { } /** - * Compute the size of a shortcut in bytes. - */ - private static int getShortcutSize(final WeightedString shortcut, - final HashMap<Integer, Integer> codePointToOneByteCodeMap) { - int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE; - final String word = shortcut.mWord; - final int length = word.length(); - for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { - final int codePoint = word.codePointAt(i); - size += CharEncoding.getCharSize(codePoint, codePointToOneByteCodeMap); - } - size += FormatSpec.PTNODE_TERMINATOR_SIZE; - return size; - } - - /** - * Compute the size of a shortcut list in bytes. - * - * This is known in advance and does not change according to position in the file - * like address lists do. - */ - static int getShortcutListSize(final ArrayList<WeightedString> shortcutList, - final HashMap<Integer, Integer> codePointToOneByteCodeMap) { - if (null == shortcutList || shortcutList.isEmpty()) return 0; - int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - for (final WeightedString shortcut : shortcutList) { - size += getShortcutSize(shortcut, codePointToOneByteCodeMap); - } - return size; - } - - /** * Compute the maximum size of a PtNode, assuming 3-byte addresses for everything. * * @param ptNode the PtNode to compute the size of. @@ -137,8 +103,6 @@ public class BinaryDictEncoderUtils { size += FormatSpec.PTNODE_FREQUENCY_SIZE; } size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address - // TODO: Use codePointToOneByteCodeMap for shortcuts. - size += getShortcutListSize(ptNode.mShortcutTargets, null /* codePointToOneByteCodeMap */); if (null != ptNode.mBigrams) { size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE + FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE) @@ -241,27 +205,6 @@ public class BinaryDictEncoderUtils { } } - @UsedForTesting - static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value, - final int size) { - switch(size) { - case 4: - dictBuffer.put((byte) ((value >> 24) & 0xFF)); - /* fall through */ - case 3: - dictBuffer.put((byte) ((value >> 16) & 0xFF)); - /* fall through */ - case 2: - dictBuffer.put((byte) ((value >> 8) & 0xFF)); - /* fall through */ - case 1: - dictBuffer.put((byte) (value & 0xFF)); - break; - default: - /* nop */ - } - } - // End utility methods // This method is responsible for finding a nice ordering of the nodes that favors run-time @@ -391,9 +334,6 @@ public class BinaryDictEncoderUtils { nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray, nodeSize + size, ptNode.mChildren)); } - // TODO: Use codePointToOneByteCodeMap for shortcuts. - nodeSize += getShortcutListSize(ptNode.mShortcutTargets, - null /* codePointToOneByteCodeMap */); if (null != ptNode.mBigrams) { for (WeightedString bigram : ptNode.mBigrams) { final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray, @@ -568,14 +508,13 @@ public class BinaryDictEncoderUtils { * @param hasMultipleChars whether the PtNode has multiple chars. * @param isTerminal whether the PtNode is terminal. * @param childrenAddressSize the size of a children address. - * @param hasShortcuts whether the PtNode has shortcuts. * @param hasBigrams whether the PtNode has bigrams. * @param isNotAWord whether the PtNode is not a word. * @param isPossiblyOffensive whether the PtNode is a possibly offensive entry. * @return the flags */ static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal, - final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams, + final int childrenAddressSize, final boolean hasBigrams, final boolean isNotAWord, final boolean isPossiblyOffensive) { byte flags = 0; if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; @@ -596,7 +535,6 @@ public class BinaryDictEncoderUtils { default: throw new RuntimeException("Node with a strange address"); } - if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS; if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD; if (isPossiblyOffensive) flags |= FormatSpec.FLAG_IS_POSSIBLY_OFFENSIVE; @@ -606,7 +544,6 @@ public class BinaryDictEncoderUtils { /* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset) { return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(), getByteSize(childrenOffset), - node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(), node.mBigrams != null && !node.mBigrams.isEmpty(), node.mIsNotAWord, node.mIsPossiblyOffensive); } @@ -621,7 +558,7 @@ public class BinaryDictEncoderUtils { * @param word the second bigram, for debugging purposes * @return the flags */ - /* package */ static final int makeBigramFlags(final boolean more, final int offset, + /* package */ static int makeBigramFlags(final boolean more, final int offset, final int bigramFrequency, final int unigramFrequency, final String word) { int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) + (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0); @@ -690,19 +627,7 @@ public class BinaryDictEncoderUtils { return discretizedFrequency > 0 ? discretizedFrequency : 0; } - /** - * Makes the flag value for a shortcut. - * - * @param more whether there are more attributes after this one. - * @param frequency the frequency of the attribute, 0..15 - * @return the flags - */ - static final int makeShortcutFlags(final boolean more, final int frequency) { - return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0) - + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY); - } - - /* package */ static final int getChildrenPosition(final PtNode ptNode, + /* package */ static int getChildrenPosition(final PtNode ptNode, final HashMap<Integer, Integer> codePointToOneByteCodeMap) { int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate + getNodeHeaderSize(ptNode, codePointToOneByteCodeMap); diff --git a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java index a42f0a93d..52060bed2 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java +++ b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java @@ -82,7 +82,6 @@ public final class FusionDictionary implements Iterable<WordProperty> { public static final class PtNode { private static final int NOT_A_TERMINAL = -1; final int mChars[]; - ArrayList<WeightedString> mShortcutTargets; ArrayList<WeightedString> mBigrams; // null == mProbabilityInfo indicates this is not a terminal. ProbabilityInfo mProbabilityInfo; @@ -100,26 +99,23 @@ public final class FusionDictionary implements Iterable<WordProperty> { int mCachedAddressBeforeUpdate; // The address of this PtNode (before update) int mCachedAddressAfterUpdate; // The address of this PtNode (after update) - public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, - final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo, - final boolean isNotAWord, final boolean isPossiblyOffensive) { + public PtNode(final int[] chars, final ArrayList<WeightedString> bigrams, + final ProbabilityInfo probabilityInfo, final boolean isNotAWord, + final boolean isPossiblyOffensive) { mChars = chars; mProbabilityInfo = probabilityInfo; mTerminalId = probabilityInfo == null ? NOT_A_TERMINAL : probabilityInfo.mProbability; - mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = null; mIsNotAWord = isNotAWord; mIsPossiblyOffensive = isPossiblyOffensive; } - public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets, - final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo, - final boolean isNotAWord, final boolean isPossiblyOffensive, - final PtNodeArray children) { + public PtNode(final int[] chars, final ArrayList<WeightedString> bigrams, + final ProbabilityInfo probabilityInfo, final boolean isNotAWord, + final boolean isPossiblyOffensive, final PtNodeArray children) { mChars = chars; mProbabilityInfo = probabilityInfo; - mShortcutTargets = shortcutTargets; mBigrams = bigrams; mChildren = children; mIsNotAWord = isNotAWord; @@ -153,14 +149,6 @@ public final class FusionDictionary implements Iterable<WordProperty> { return mIsPossiblyOffensive; } - public ArrayList<WeightedString> getShortcutTargets() { - // We don't want write permission to escape outside the package, so we return a copy - if (null == mShortcutTargets) return null; - final ArrayList<WeightedString> copyOfShortcutTargets = - new ArrayList<>(mShortcutTargets); - return copyOfShortcutTargets; - } - public ArrayList<WeightedString> getBigrams() { // We don't want write permission to escape outside the package, so we return a copy if (null == mBigrams) return null; @@ -191,24 +179,6 @@ public final class FusionDictionary implements Iterable<WordProperty> { } /** - * Gets the shortcut target for the given word. Returns null if the word is not in the - * shortcut list. - */ - public WeightedString getShortcut(final String word) { - // TODO: Don't do a linear search - if (mShortcutTargets != null) { - final int size = mShortcutTargets.size(); - for (int i = 0; i < size; ++i) { - WeightedString shortcut = mShortcutTargets.get(i); - if (shortcut.mWord.equals(word)) { - return shortcut; - } - } - } - return null; - } - - /** * Gets the bigram for the given word. * Returns null if the word is not in the bigrams list. */ @@ -232,27 +202,9 @@ public final class FusionDictionary implements Iterable<WordProperty> { * updated if they are higher than the existing ones. */ void update(final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, final ArrayList<WeightedString> bigrams, final boolean isNotAWord, final boolean isPossiblyOffensive) { mProbabilityInfo = ProbabilityInfo.max(mProbabilityInfo, probabilityInfo); - if (shortcutTargets != null) { - if (mShortcutTargets == null) { - mShortcutTargets = shortcutTargets; - } else { - final int size = shortcutTargets.size(); - for (int i = 0; i < size; ++i) { - final WeightedString shortcut = shortcutTargets.get(i); - final WeightedString existingShortcut = getShortcut(shortcut.mWord); - if (existingShortcut == null) { - mShortcutTargets.add(shortcut); - } else { - existingShortcut.mProbabilityInfo = ProbabilityInfo.max( - existingShortcut.mProbabilityInfo, shortcut.mProbabilityInfo); - } - } - } - } if (bigrams != null) { if (mBigrams == null) { mBigrams = bigrams; @@ -312,19 +264,16 @@ public final class FusionDictionary implements Iterable<WordProperty> { * Helper method to add a word as a string. * * This method adds a word to the dictionary with the given frequency. Optional - * lists of bigrams and shortcuts can be passed here. For each word inside, + * lists of bigrams can be passed here. For each word inside, * they will be added to the dictionary as necessary. - * - * @param word the word to add. + * @param word the word to add. * @param probabilityInfo probability information of the word. - * @param shortcutTargets a list of shortcut targets for this word, or null. * @param isNotAWord true if this should not be considered a word (e.g. shortcut only) * @param isPossiblyOffensive true if this word is possibly offensive */ public void add(final String word, final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord, - final boolean isPossiblyOffensive) { - add(getCodePoints(word), probabilityInfo, shortcutTargets, isNotAWord, isPossiblyOffensive); + final boolean isNotAWord, final boolean isPossiblyOffensive) { + add(getCodePoints(word), probabilityInfo, isNotAWord, isPossiblyOffensive); } /** @@ -358,7 +307,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { if (ptNode0 != null) { final PtNode ptNode1 = findWordInTree(mRootNodeArray, word1); if (ptNode1 == null) { - add(getCodePoints(word1), new ProbabilityInfo(0), null, false /* isNotAWord */, + add(getCodePoints(word1), new ProbabilityInfo(0), false /* isNotAWord */, false /* isPossiblyOffensive */); // The PtNode for the first word may have moved by the above insertion, // if word1 and word2 share a common stem that happens not to have been @@ -376,15 +325,12 @@ public final class FusionDictionary implements Iterable<WordProperty> { * * The shortcuts, if any, have to be in the dictionary already. If they aren't, * an exception is thrown. - * - * @param word the word, as an int array. + * @param word the word, as an int array. * @param probabilityInfo the probability information of the word. - * @param shortcutTargets an optional list of shortcut targets for this word (null if none). * @param isNotAWord true if this is not a word for spellcheking purposes (shortcut only or so) * @param isPossiblyOffensive true if this word is possibly offensive */ private void add(final int[] word, final ProbabilityInfo probabilityInfo, - final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord, final boolean isPossiblyOffensive) { assert(probabilityInfo.mProbability <= FormatSpec.MAX_TERMINAL_FREQUENCY); if (word.length >= DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH) { @@ -414,7 +360,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { // No node at this point to accept the word. Create one. final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]); final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length), - shortcutTargets, null /* bigrams */, probabilityInfo, isNotAWord, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive); currentNodeArray.mData.add(insertionIndex, newPtNode); if (DBG) checkStack(currentNodeArray); @@ -425,14 +371,14 @@ public final class FusionDictionary implements Iterable<WordProperty> { // The new word is a prefix of an existing word, but the node on which it // should end already exists as is. Since the old PtNode was not a terminal, // make it one by filling in its frequency and other attributes - currentPtNode.update(probabilityInfo, shortcutTargets, null, isNotAWord, + currentPtNode.update(probabilityInfo, null, isNotAWord, isPossiblyOffensive); } else { // The new word matches the full old word and extends past it. // We only have to create a new node and add it to the end of this. final PtNode newNode = new PtNode( Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), - shortcutTargets, null /* bigrams */, probabilityInfo, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive); currentPtNode.mChildren = new PtNodeArray(); currentPtNode.mChildren.mData.add(newNode); @@ -441,7 +387,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { if (0 == differentCharIndex) { // Exact same word. Update the frequency if higher. This will also add the // new shortcuts to the existing shortcut list if it already exists. - currentPtNode.update(probabilityInfo, shortcutTargets, null, + currentPtNode.update(probabilityInfo, null, currentPtNode.mIsNotAWord && isNotAWord, currentPtNode.mIsPossiblyOffensive || isPossiblyOffensive); } else { @@ -450,7 +396,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { PtNodeArray newChildren = new PtNodeArray(); final PtNode newOldWord = new PtNode( Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex, - currentPtNode.mChars.length), currentPtNode.mShortcutTargets, + currentPtNode.mChars.length), currentPtNode.mBigrams, currentPtNode.mProbabilityInfo, currentPtNode.mIsNotAWord, currentPtNode.mIsPossiblyOffensive, currentPtNode.mChildren); @@ -460,17 +406,17 @@ public final class FusionDictionary implements Iterable<WordProperty> { if (charIndex + differentCharIndex >= word.length) { newParent = new PtNode( Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), - shortcutTargets, null /* bigrams */, probabilityInfo, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive, newChildren); } else { newParent = new PtNode( Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex), - null /* shortcutTargets */, null /* bigrams */, - null /* probabilityInfo */, false /* isNotAWord */, - false /* isPossiblyOffensive */, newChildren); + null /* bigrams */, null /* probabilityInfo */, + false /* isNotAWord */, false /* isPossiblyOffensive */, + newChildren); final PtNode newWord = new PtNode(Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length), - shortcutTargets, null /* bigrams */, probabilityInfo, + null /* bigrams */, probabilityInfo, isNotAWord, isPossiblyOffensive); final int addIndex = word[charIndex + differentCharIndex] > currentPtNode.mChars[differentCharIndex] ? 1 : 0; @@ -532,7 +478,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { private static int findInsertionIndex(final PtNodeArray nodeArray, int character) { final ArrayList<PtNode> data = nodeArray.mData; final PtNode reference = new PtNode(new int[] { character }, - null /* shortcutTargets */, null /* bigrams */, null /* probabilityInfo */, + null /* bigrams */, null /* probabilityInfo */, false /* isNotAWord */, false /* isPossiblyOffensive */); int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR); return result >= 0 ? result : -result - 1; @@ -669,8 +615,7 @@ public final class FusionDictionary implements Iterable<WordProperty> { } if (currentPtNode.isTerminal()) { return new WordProperty(mCurrentString.toString(), - currentPtNode.mProbabilityInfo, - currentPtNode.mShortcutTargets, currentPtNode.mBigrams, + currentPtNode.mProbabilityInfo, currentPtNode.mBigrams, currentPtNode.mIsNotAWord, currentPtNode.mIsPossiblyOffensive); } } else { diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java index 5c261a94d..7ee1df92b 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java @@ -285,7 +285,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder { // Insert unigrams into the fusion dictionary. for (final WordProperty wordProperty : wordProperties) { fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, - wordProperty.mShortcutTargets, wordProperty.mIsNotAWord, + wordProperty.mIsNotAWord, wordProperty.mIsPossiblyOffensive); } // Insert bigrams into the fusion dictionary. diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java index b52b8c485..c63b972eb 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java @@ -240,37 +240,6 @@ public class Ver2DictEncoder implements DictEncoder { } /** - * Write a shortcut attributes list to mBuffer. - * - * @param shortcuts the shortcut attributes list. - */ - private void writeShortcuts(final ArrayList<WeightedString> shortcuts, - final HashMap<Integer, Integer> codePointToOneByteCodeMap) { - if (null == shortcuts || shortcuts.isEmpty()) return; - - final int indexOfShortcutByteSize = mPosition; - mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE; - final Iterator<WeightedString> shortcutIterator = shortcuts.iterator(); - while (shortcutIterator.hasNext()) { - final WeightedString target = shortcutIterator.next(); - final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags( - shortcutIterator.hasNext(), - target.getProbability()); - mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags, - FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE); - final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord, - codePointToOneByteCodeMap); - mPosition += shortcutShift; - } - final int shortcutByteSize = mPosition - indexOfShortcutByteSize; - if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) { - throw new RuntimeException("Shortcut list too large"); - } - BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize, - FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE); - } - - /** * Write a bigram attributes list to mBuffer. * * @param bigrams the bigram attributes list. @@ -305,8 +274,6 @@ public class Ver2DictEncoder implements DictEncoder { writeCharacters(ptNode.mChars, ptNode.hasSeveralChars(), codePointToOneByteCodeMap); writeFrequency(ptNode.getProbability()); writeChildrenPosition(ptNode, codePointToOneByteCodeMap); - // TODO: Use codePointToOneByteCodeMap for shortcuts. - writeShortcuts(ptNode.mShortcutTargets, null /* codePointToOneByteCodeMap */); writeBigrams(ptNode.mBigrams, dict); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java index 7d858760e..dbf9b7ac2 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java @@ -16,9 +16,7 @@ package com.android.inputmethod.latin.makedict; -import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map.Entry; @@ -49,7 +47,7 @@ public class Ver2DictEncoderTests extends AndroidTestCase { new FormatSpec.FormatOptions(FormatSpec.VERSION2); final FusionDictionary sourcedict = new FusionDictionary(new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); - addUnigrams(sourcedict, words, null /* shortcutMap */); + addUnigrams(sourcedict, words); final CodePointTable codePointTable = Ver2DictEncoder.makeCodePointTable(sourcedict); // Check if mCodePointOccurrenceArray is correct @@ -73,17 +71,10 @@ public class Ver2DictEncoderTests extends AndroidTestCase { /** * Adds unigrams to the dictionary. */ - private static void addUnigrams(final FusionDictionary dict, final List<String> words, - final HashMap<String, List<String>> shortcutMap) { + private static void addUnigrams(final FusionDictionary dict, final List<String> words) { for (final String word : words) { - final ArrayList<WeightedString> shortcuts = new ArrayList<>(); - if (shortcutMap != null && shortcutMap.containsKey(word)) { - for (final String shortcut : shortcutMap.get(word)) { - shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ)); - } - } dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), - (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */, + false /* isNotAWord */, false /* isPossiblyOffensive */); } } diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java index 63ea89c1d..746431dfa 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java @@ -84,7 +84,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder { // Insert unigrams into the fusion dictionary. for (final WordProperty wordProperty : wordProperties) { fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo, - wordProperty.mShortcutTargets, wordProperty.mIsNotAWord, + wordProperty.mIsNotAWord, wordProperty.mIsPossiblyOffensive); } // Insert bigrams into the fusion dictionary. diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java index 1e4bd768c..6e7b37d54 100644 --- a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java +++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java @@ -74,26 +74,10 @@ public class Ver4DictEncoder implements DictEncoder { throw new IOException("Cannot create dictionary file"); } for (final WordProperty wordProperty : dict) { - // TODO: switch to addMultipleDictionaryEntries when they support shortcuts - if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) { - if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(), - null /* shortcutTarget */, 0 /* shortcutProbability */, - wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, - wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) { - MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord); - } - } else { - for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) { - if (!binaryDict.addUnigramEntry(wordProperty.mWord, - wordProperty.getProbability(), - shortcutTarget.mWord, shortcutTarget.getProbability(), - wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, - wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) { - MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord - + ", shortcutTarget: " + shortcutTarget.mWord); - return; - } - } + if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(), + wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord, + wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) { + MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord); } if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) { if (!binaryDict.flushWithGC()) { |