aboutsummaryrefslogtreecommitdiffstats
path: root/tests/src
diff options
context:
space:
mode:
authorDan Zivkovic <zivkovic@google.com>2015-02-10 14:54:38 -0800
committerDan Zivkovic <zivkovic@google.com>2015-02-11 13:02:41 -0800
commit12d80ebead6a1d7f704a5a3af3b6fe3313ceab05 (patch)
tree3ab839123291ba254aec3189f6c2f20dd30016c8 /tests/src
parent2979fad21384bb595ba2baca8f5bbbfc053be921 (diff)
downloadlatinime-12d80ebead6a1d7f704a5a3af3b6fe3313ceab05.tar.gz
latinime-12d80ebead6a1d7f704a5a3af3b6fe3313ceab05.tar.xz
latinime-12d80ebead6a1d7f704a5a3af3b6fe3313ceab05.zip
Remove shortcut support from LatinIME.
Note this change does not affect the native decoder interface. Change-Id: I73b7dc008a5acaf75a31a36a2d332b5afabd82d0
Diffstat (limited to 'tests/src')
-rw-r--r--tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java5
-rw-r--r--tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java5
-rw-r--r--tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java3
-rw-r--r--tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java146
-rw-r--r--tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java8
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java102
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java81
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java103
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java2
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java33
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java15
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java2
-rw-r--r--tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java24
13 files changed, 74 insertions, 455 deletions
diff --git a/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java b/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java
index 67e76464b..d75e61ad1 100644
--- a/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java
+++ b/tests/src/com/android/inputmethod/compat/LocaleSpanCompatUtilsTests.java
@@ -17,19 +17,20 @@
package com.android.inputmethod.compat;
import android.graphics.Typeface;
-import android.os.Build;
import android.test.AndroidTestCase;
import android.test.suitebuilder.annotation.SmallTest;
import android.text.SpannableString;
import android.text.Spanned;
import android.text.style.StyleSpan;
+import com.android.inputmethod.latin.common.Constants;
+
import java.util.Locale;
@SmallTest
public class LocaleSpanCompatUtilsTests extends AndroidTestCase {
public void testInstantiatable() {
- if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR1) {
+ if (!Constants.JELLY_BEAN_MR1_OR_HIGHER) {
// LocaleSpan isn't yet available.
return;
}
diff --git a/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java b/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java
index bcf06f085..557cc214d 100644
--- a/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java
+++ b/tests/src/com/android/inputmethod/keyboard/layout/DevanagariLetterConstants.java
@@ -16,15 +16,14 @@
package com.android.inputmethod.keyboard.layout;
-import android.os.Build;
+import com.android.inputmethod.latin.common.Constants;
/**
* This class offers label strings of Devanagari letters that need the dotted circle to draw
* its glyph.
*/
class DevanagariLetterConstants {
- private static final boolean NEEDS_DOTTED_CIRCLE =
- Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN;
+ private static final boolean NEEDS_DOTTED_CIRCLE = !Constants.JELLY_BEAN_OR_HIGHER;
// U+25CC: "◌" DOTTED CIRCLE
private static final String DOTTED_CIRCLE = NEEDS_DOTTED_CIRCLE ? "\u25CC" : "";
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
index baac0883e..a51558e43 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java
@@ -700,8 +700,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase {
final File dictFile = createEmptyDictionaryAndGetFile(formatVersion);
final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile);
- binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY, "" /* shortcutTarget */,
- Dictionary.NOT_A_PROBABILITY /* shortcutProbability */,
+ binaryDictionary.addUnigramEntry("", DUMMY_PROBABILITY,
true /* isBeginningOfSentence */, true /* isNotAWord */,
false /* isPossiblyOffensive */, mCurrentTime);
final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE;
diff --git a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
index a8f0d81af..161879841 100644
--- a/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java
@@ -182,8 +182,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
addUnigramWord(binaryDictionary, validLongWord, probability);
addUnigramWord(binaryDictionary, invalidLongWord, probability);
// Too long short cut.
- binaryDictionary.addUnigramEntry("a", probability, invalidLongWord,
- 10 /* shortcutProbability */, false /* isBeginningOfSentence */,
+ binaryDictionary.addUnigramEntry("a", probability, false /* isBeginningOfSentence */,
false /* isNotAWord */, false /* isPossiblyOffensive */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
addUnigramWord(binaryDictionary, "abc", probability);
@@ -201,8 +200,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
private static void addUnigramWord(final BinaryDictionary binaryDictionary, final String word,
final int probability) {
- binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */,
- Dictionary.NOT_A_PROBABILITY /* shortcutProbability */,
+ binaryDictionary.addUnigramEntry(word, probability,
false /* isBeginningOfSentence */, false /* isNotAWord */,
false /* isPossiblyOffensive */,
BinaryDictionary.NOT_A_VALID_TIMESTAMP /* timestamp */);
@@ -884,7 +882,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final boolean isPossiblyOffensive = random.nextBoolean();
// TODO: Add tests for historical info.
binaryDictionary.addUnigramEntry(word, unigramProbability,
- null /* shortcutTarget */, Dictionary.NOT_A_PROBABILITY,
false /* isBeginningOfSentence */, isNotAWord, isPossiblyOffensive,
BinaryDictionary.NOT_A_VALID_TIMESTAMP);
if (binaryDictionary.needsToRunGC(false /* mindsBlockByGC */)) {
@@ -899,9 +896,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertEquals(isNotAWord, wordProperty.mIsNotAWord);
assertEquals(isPossiblyOffensive, wordProperty.mIsPossiblyOffensive);
assertEquals(false, wordProperty.mHasNgrams);
- assertEquals(false, wordProperty.mHasShortcuts);
assertEquals(unigramProbability, wordProperty.mProbabilityInfo.mProbability);
- assertTrue(wordProperty.mShortcutTargets.isEmpty());
}
for (int i = 0; i < BIGRAM_COUNT; i++) {
@@ -1035,137 +1030,10 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertTrue(bigramSet.isEmpty());
}
- public void testAddShortcuts() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testAddShortcuts(formatVersion);
- }
- }
-
- private void testAddShortcuts(final int formatVersion) {
- final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
-
- final int unigramProbability = 100;
- final int shortcutProbability = 10;
- binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz",
- shortcutProbability, false /* isBeginningOfSentence */,
- false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */);
- WordProperty wordProperty = binaryDictionary.getWordProperty("aaa",
- false /* isBeginningOfSentence */);
- assertEquals(1, wordProperty.mShortcutTargets.size());
- assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
- assertEquals(shortcutProbability, wordProperty.mShortcutTargets.get(0).getProbability());
- final int updatedShortcutProbability = 2;
- binaryDictionary.addUnigramEntry("aaa", unigramProbability, "zzz",
- updatedShortcutProbability, false /* isBeginningOfSentence */,
- false /* isNotAWord */, false /* isPossiblyOffensive */, 0 /* timestamp */);
- wordProperty = binaryDictionary.getWordProperty("aaa",
- false /* isBeginningOfSentence */);
- assertEquals(1, wordProperty.mShortcutTargets.size());
- assertEquals("zzz", wordProperty.mShortcutTargets.get(0).mWord);
- assertEquals(updatedShortcutProbability,
- wordProperty.mShortcutTargets.get(0).getProbability());
- binaryDictionary.addUnigramEntry("aaa", unigramProbability, "yyy",
- shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */,
- false /* isPossiblyOffensive */, 0 /* timestamp */);
- final HashMap<String, Integer> shortcutTargets = new HashMap<>();
- shortcutTargets.put("zzz", updatedShortcutProbability);
- shortcutTargets.put("yyy", shortcutProbability);
- wordProperty = binaryDictionary.getWordProperty("aaa",
- false /* isBeginningOfSentence */);
- assertEquals(2, wordProperty.mShortcutTargets.size());
- for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
- assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
- assertEquals((int)shortcutTargets.get(shortcutTarget.mWord),
- shortcutTarget.getProbability());
- shortcutTargets.remove(shortcutTarget.mWord);
- }
- shortcutTargets.put("zzz", updatedShortcutProbability);
- shortcutTargets.put("yyy", shortcutProbability);
- binaryDictionary.flushWithGC();
- wordProperty = binaryDictionary.getWordProperty("aaa",
- false /* isBeginningOfSentence */);
- assertEquals(2, wordProperty.mShortcutTargets.size());
- for (WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
- assertTrue(shortcutTargets.containsKey(shortcutTarget.mWord));
- assertEquals((int)shortcutTargets.get(shortcutTarget.mWord),
- shortcutTarget.getProbability());
- shortcutTargets.remove(shortcutTarget.mWord);
- }
- }
-
- public void testAddManyShortcuts() {
- for (final int formatVersion : DICT_FORMAT_VERSIONS) {
- testAddManyShortcuts(formatVersion);
- }
- }
-
- private void testAddManyShortcuts(final int formatVersion) {
- final long seed = System.currentTimeMillis();
- final Random random = new Random(seed);
- final int UNIGRAM_COUNT = 1000;
- final int SHORTCUT_COUNT = 10000;
- final int codePointSetSize = 20;
- final int[] codePointSet = CodePointUtils.generateCodePointSet(codePointSetSize, random);
-
- final ArrayList<String> words = new ArrayList<>();
- final HashMap<String, Integer> unigramProbabilities = new HashMap<>();
- final HashMap<String, HashMap<String, Integer>> shortcutTargets = new HashMap<>();
- final BinaryDictionary binaryDictionary = getEmptyBinaryDictionary(formatVersion);
-
- for (int i = 0; i < UNIGRAM_COUNT; i++) {
- final String word = CodePointUtils.generateWord(random, codePointSet);
- final int unigramProbability = random.nextInt(0xFF);
- addUnigramWord(binaryDictionary, word, unigramProbability);
- words.add(word);
- unigramProbabilities.put(word, unigramProbability);
- if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- binaryDictionary.flushWithGC();
- }
- }
- for (int i = 0; i < SHORTCUT_COUNT; i++) {
- final String shortcutTarget = CodePointUtils.generateWord(random, codePointSet);
- final int shortcutProbability = random.nextInt(0xF);
- final String word = words.get(random.nextInt(words.size()));
- final int unigramProbability = unigramProbabilities.get(word);
- binaryDictionary.addUnigramEntry(word, unigramProbability, shortcutTarget,
- shortcutProbability, false /* isBeginningOfSentence */, false /* isNotAWord */,
- false /* isPossiblyOffensive */, 0 /* timestamp */);
- if (shortcutTargets.containsKey(word)) {
- final HashMap<String, Integer> shortcutTargetsOfWord = shortcutTargets.get(word);
- shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability);
- } else {
- final HashMap<String, Integer> shortcutTargetsOfWord = new HashMap<>();
- shortcutTargetsOfWord.put(shortcutTarget, shortcutProbability);
- shortcutTargets.put(word, shortcutTargetsOfWord);
- }
- if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) {
- binaryDictionary.flushWithGC();
- }
- }
-
- for (final String word : words) {
- final WordProperty wordProperty = binaryDictionary.getWordProperty(word,
- false /* isBeginningOfSentence */);
- assertEquals((int)unigramProbabilities.get(word),
- wordProperty.mProbabilityInfo.mProbability);
- if (!shortcutTargets.containsKey(word)) {
- // The word does not have shortcut targets.
- continue;
- }
- assertEquals(shortcutTargets.get(word).size(), wordProperty.mShortcutTargets.size());
- for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
- final String targetCodePonts = shortcutTarget.mWord;
- assertEquals((int)shortcutTargets.get(word).get(targetCodePonts),
- shortcutTarget.getProbability());
- }
- }
- }
-
public void testPossiblyOffensiveAttributeMaintained() {
final BinaryDictionary binaryDictionary =
getEmptyBinaryDictionary(FormatSpec.VERSION403);
- binaryDictionary.addUnigramEntry("ddd", 100, null, Dictionary.NOT_A_PROBABILITY,
- false, true, true, 0);
+ binaryDictionary.addUnigramEntry("ddd", 100, false, true, true, 0);
WordProperty wordProperty = binaryDictionary.getWordProperty("ddd", false);
assertEquals(true, wordProperty.mIsPossiblyOffensive);
}
@@ -1184,11 +1052,11 @@ public class BinaryDictionaryTests extends AndroidTestCase {
final int bigramProbability = 150;
addBigramWords(binaryDictionary, "aaa", "bbb", bigramProbability);
final int shortcutProbability = 10;
- binaryDictionary.addUnigramEntry("ccc", unigramProbability, "xxx", shortcutProbability,
+ binaryDictionary.addUnigramEntry("ccc", unigramProbability,
false /* isBeginningOfSentence */, false /* isNotAWord */,
false /* isPossiblyOffensive */, 0 /* timestamp */);
- binaryDictionary.addUnigramEntry("ddd", unigramProbability, null /* shortcutTarget */,
- Dictionary.NOT_A_PROBABILITY, false /* isBeginningOfSentence */,
+ binaryDictionary.addUnigramEntry("ddd", unigramProbability,
+ false /* isBeginningOfSentence */,
true /* isNotAWord */, true /* isPossiblyOffensive */, 0 /* timestamp */);
binaryDictionary.addNgramEntry(NgramContext.BEGINNING_OF_SENTENCE,
"aaa", bigramProbability, 0 /* timestamp */);
@@ -1207,8 +1075,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb"));
WordProperty wordProperty = binaryDictionary.getWordProperty("ccc",
false /* isBeginningOfSentence */);
- assertEquals(1, wordProperty.mShortcutTargets.size());
- assertEquals("xxx", wordProperty.mShortcutTargets.get(0).mWord);
wordProperty = binaryDictionary.getWordProperty("ddd",
false /* isBeginningOfSentence */);
assertTrue(wordProperty.mIsPossiblyOffensive);
diff --git a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java
index 07d7c3225..6c6f62872 100644
--- a/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java
+++ b/tests/src/com/android/inputmethod/latin/FusionDictionaryTests.java
@@ -35,19 +35,19 @@ public class FusionDictionaryTests extends AndroidTestCase {
FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
new DictionaryOptions(new HashMap<String,String>()));
- dict.add("abc", new ProbabilityInfo(10), null, false /* isNotAWord */,
+ dict.add("abc", new ProbabilityInfo(10), false /* isNotAWord */,
false /* isPossiblyOffensive */);
assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa"));
assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "abc"));
- dict.add("aa", new ProbabilityInfo(10), null, false /* isNotAWord */,
+ dict.add("aa", new ProbabilityInfo(10), false /* isNotAWord */,
false /* isPossiblyOffensive */);
assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aaa"));
assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "aa"));
- dict.add("babcd", new ProbabilityInfo(10), null, false /* isNotAWord */,
+ dict.add("babcd", new ProbabilityInfo(10), false /* isNotAWord */,
false /* isPossiblyOffensive */);
- dict.add("bacde", new ProbabilityInfo(10), null, false /* isNotAWord */,
+ dict.add("bacde", new ProbabilityInfo(10), false /* isNotAWord */,
false /* isPossiblyOffensive */);
assertNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "ba"));
assertNotNull(FusionDictionary.findWordInTree(dict.mRootNodeArray, "babcd"));
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
index d833b9736..39da9fcd6 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderEncoderTests.java
@@ -57,15 +57,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
private static final int UNIGRAM_FREQ = 10;
private static final int BIGRAM_FREQ = 50;
private static final int TOLERANCE_OF_BIGRAM_FREQ = 5;
- private static final int NUM_OF_NODES_HAVING_SHORTCUTS = 50;
- private static final int NUM_OF_SHORTCUTS = 5;
private static final ArrayList<String> sWords = new ArrayList<>();
private static final ArrayList<String> sWordsWithVariousCodePoints = new ArrayList<>();
private static final SparseArray<List<Integer>> sEmptyBigrams = new SparseArray<>();
private static final SparseArray<List<Integer>> sStarBigrams = new SparseArray<>();
private static final SparseArray<List<Integer>> sChainBigrams = new SparseArray<>();
- private static final HashMap<String, List<String>> sShortcuts = new HashMap<>();
final Random mRandom;
@@ -95,16 +92,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
for (int i = 1; i < maxBigrams; ++i) {
sStarBigrams.get(0).add(i);
}
-
- sShortcuts.clear();
- for (int i = 0; i < NUM_OF_NODES_HAVING_SHORTCUTS; ++i) {
- final int from = Math.abs(mRandom.nextInt()) % sWords.size();
- sShortcuts.put(sWords.get(from), new ArrayList<String>());
- for (int j = 0; j < NUM_OF_SHORTCUTS; ++j) {
- final int to = Math.abs(mRandom.nextInt()) % sWords.size();
- sShortcuts.get(sWords.get(from)).add(sWords.get(to));
- }
- }
}
@Override
@@ -142,17 +129,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
* Adds unigrams to the dictionary.
*/
private static void addUnigrams(final int number, final FusionDictionary dict,
- final List<String> words, final HashMap<String, List<String>> shortcutMap) {
+ final List<String> words) {
for (int i = 0; i < number; ++i) {
final String word = words.get(i);
final ArrayList<WeightedString> shortcuts = new ArrayList<>();
- if (shortcutMap != null && shortcutMap.containsKey(word)) {
- for (final String shortcut : shortcutMap.get(word)) {
- shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ));
- }
- }
- dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ),
- (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */,
+ dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ), false /* isNotAWord */,
false /* isPossiblyOffensive */);
}
}
@@ -200,8 +181,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
}
private static void checkDictionary(final FusionDictionary dict, final List<String> words,
- final SparseArray<List<Integer>> bigrams,
- final HashMap<String, List<String>> shortcutMap) {
+ final SparseArray<List<Integer>> bigrams) {
assertNotNull(dict);
// check unigram
@@ -219,19 +199,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
assertNotNull(words.get(w1) + "," + words.get(w2), ptNode.getBigram(words.get(w2)));
}
}
-
- // check shortcut
- if (shortcutMap != null) {
- for (final Entry<String, List<String>> entry : shortcutMap.entrySet()) {
- assertTrue(words.contains(entry.getKey()));
- final PtNode ptNode = FusionDictionary.findWordInTree(dict.mRootNodeArray,
- entry.getKey());
- for (final String word : entry.getValue()) {
- assertNotNull("shortcut not found: " + entry.getKey() + ", " + word,
- ptNode.getShortcut(word));
- }
- }
- }
}
private static String outputOptions(final int bufferType,
@@ -244,8 +211,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// Tests for readDictionaryBinary and writeDictionaryBinary
private static long timeReadingAndCheckDict(final File file, final List<String> words,
- final SparseArray<List<Integer>> bigrams,
- final HashMap<String, List<String>> shortcutMap, final int bufferType) {
+ final SparseArray<List<Integer>> bigrams, final int bufferType) {
long now, diff = -1;
FusionDictionary dict = null;
@@ -261,13 +227,13 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
Log.e(TAG, "Unsupported format", e);
}
- checkDictionary(dict, words, bigrams, shortcutMap);
+ checkDictionary(dict, words, bigrams);
return diff;
}
// Tests for readDictionaryBinary and writeDictionaryBinary
private String runReadAndWrite(final List<String> words,
- final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcuts,
+ final SparseArray<List<Integer>> bigrams,
final int bufferType, final FormatSpec.FormatOptions formatOptions,
final String message) {
@@ -278,12 +244,12 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(words.size(), dict, words, shortcuts);
+ addUnigrams(words.size(), dict, words);
addBigrams(dict, words, bigrams);
- checkDictionary(dict, words, bigrams, shortcuts);
+ checkDictionary(dict, words, bigrams);
final long write = timeWritingDictToFile(file, dict, formatOptions);
- final long read = timeReadingAndCheckDict(file, words, bigrams, shortcuts, bufferType);
+ final long read = timeReadingAndCheckDict(file, words, bigrams, bufferType);
return "PROF: read=" + read + "ms, write=" + write + "ms :" + message
+ " : " + outputOptions(bufferType, formatOptions);
@@ -291,20 +257,20 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
private void runReadAndWriteTests(final List<String> results, final int bufferType,
final FormatSpec.FormatOptions formatOptions) {
- results.add(runReadAndWrite(sWords, sEmptyBigrams, null /* shortcuts */, bufferType,
+ results.add(runReadAndWrite(sWords, sEmptyBigrams, bufferType,
formatOptions, "unigram"));
- results.add(runReadAndWrite(sWords, sChainBigrams, null /* shortcuts */, bufferType,
+ results.add(runReadAndWrite(sWords, sChainBigrams, bufferType,
formatOptions, "chain"));
- results.add(runReadAndWrite(sWords, sStarBigrams, null /* shortcuts */, bufferType,
+ results.add(runReadAndWrite(sWords, sStarBigrams, bufferType,
formatOptions, "star"));
- results.add(runReadAndWrite(sWords, sEmptyBigrams, sShortcuts, bufferType, formatOptions,
+ results.add(runReadAndWrite(sWords, sEmptyBigrams, bufferType, formatOptions,
"unigram with shortcuts"));
- results.add(runReadAndWrite(sWords, sChainBigrams, sShortcuts, bufferType, formatOptions,
+ results.add(runReadAndWrite(sWords, sChainBigrams, bufferType, formatOptions,
"chain with shortcuts"));
- results.add(runReadAndWrite(sWords, sStarBigrams, sShortcuts, bufferType, formatOptions,
+ results.add(runReadAndWrite(sWords, sStarBigrams, bufferType, formatOptions,
"star with shortcuts"));
results.add(runReadAndWrite(sWordsWithVariousCodePoints, sEmptyBigrams,
- null /* shortcuts */, bufferType, formatOptions,
+ bufferType, formatOptions,
"unigram with various code points"));
}
@@ -326,7 +292,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
FormatSpec.MINIMUM_SUPPORTED_STATIC_VERSION);
final FusionDictionary sourcedict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(words.size(), sourcedict, words, null /* shortcutMap */);
+ addUnigrams(words.size(), sourcedict, words);
dictEncoder.writeDictionary(sourcedict, formatOptions);
// Read the dictionary
@@ -472,7 +438,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// making the dictionary from lists of words.
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(words.size(), dict, words, null /* shortcutMap */);
+ addUnigrams(words.size(), dict, words);
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
@@ -482,7 +448,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
// TODO: Abandon the Java code, and implement the v4 dictionary reading code in native.
long wordMap = timeAndCheckReadUnigramsAndBigramsBinary(file, words, bigrams, bufferType,
!formatOptions.mHasTimestamp /* checkProbability */);
- long fullReading = timeReadingAndCheckDict(file, words, bigrams, null /* shortcutMap */,
+ long fullReading = timeReadingAndCheckDict(file, words, bigrams,
bufferType);
return "readDictionaryBinary=" + fullReading + ", readUnigramsAndBigramsBinary=" + wordMap
@@ -567,7 +533,7 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
+ addUnigrams(sWords.size(), dict, sWords);
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
@@ -636,12 +602,11 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
public void testVer2DictGetWordProperty() {
final FormatOptions formatOptions = BinaryDictUtils.STATIC_OPTIONS;
final ArrayList<String> words = sWords;
- final HashMap<String, List<String>> shortcuts = sShortcuts;
final String dictName = "testGetWordProperty";
final String dictVersion = Long.toString(System.currentTimeMillis());
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(words.size(), dict, words, shortcuts);
+ addUnigrams(words.size(), dict, words);
addBigrams(dict, words, sEmptyBigrams);
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
getContext().getCacheDir());
@@ -655,30 +620,18 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
false /* isBeginningOfSentence */);
assertEquals(word, wordProperty.mWord);
assertEquals(UNIGRAM_FREQ, wordProperty.getProbability());
- if (shortcuts.containsKey(word)) {
- assertEquals(shortcuts.get(word).size(), wordProperty.mShortcutTargets.size());
- final List<String> shortcutList = shortcuts.get(word);
- assertTrue(wordProperty.mHasShortcuts);
- for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
- assertTrue(shortcutList.contains(shortcutTarget.mWord));
- assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability());
- shortcutList.remove(shortcutTarget.mWord);
- }
- assertTrue(shortcutList.isEmpty());
- }
}
}
public void testVer2DictIteration() {
final FormatOptions formatOptions = BinaryDictUtils.STATIC_OPTIONS;
final ArrayList<String> words = sWords;
- final HashMap<String, List<String>> shortcuts = sShortcuts;
final SparseArray<List<Integer>> bigrams = sEmptyBigrams;
final String dictName = "testGetWordProperty";
final String dictVersion = Long.toString(System.currentTimeMillis());
final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(words.size(), dict, words, shortcuts);
+ addUnigrams(words.size(), dict, words);
addBigrams(dict, words, bigrams);
final File file = BinaryDictUtils.getDictFile(dictName, dictVersion, formatOptions,
getContext().getCacheDir());
@@ -708,17 +661,6 @@ public class BinaryDictDecoderEncoderTests extends AndroidTestCase {
final String word0 = wordProperty.mWord;
assertEquals(UNIGRAM_FREQ, wordProperty.mProbabilityInfo.mProbability);
wordSet.remove(word0);
- if (shortcuts.containsKey(word0)) {
- assertEquals(shortcuts.get(word0).size(), wordProperty.mShortcutTargets.size());
- final List<String> shortcutList = shortcuts.get(word0);
- assertNotNull(wordProperty.mShortcutTargets);
- for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
- assertTrue(shortcutList.contains(shortcutTarget.mWord));
- assertEquals(UNIGRAM_FREQ, shortcutTarget.getProbability());
- shortcutList.remove(shortcutTarget.mWord);
- }
- assertTrue(shortcutList.isEmpty());
- }
if (wordProperty.mHasNgrams) {
for (final WeightedString bigramTarget : wordProperty.getBigrams()) {
final String word1 = bigramTarget.mWord;
diff --git a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
index ce905c499..bd5136583 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java
@@ -16,9 +16,7 @@
package com.android.inputmethod.latin.makedict;
-import com.android.inputmethod.annotations.UsedForTesting;
import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.CharEncoding;
-import com.android.inputmethod.latin.makedict.BinaryDictDecoderUtils.DictBuffer;
import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNode;
import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
@@ -92,38 +90,6 @@ public class BinaryDictEncoderUtils {
}
/**
- * Compute the size of a shortcut in bytes.
- */
- private static int getShortcutSize(final WeightedString shortcut,
- final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
- int size = FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE;
- final String word = shortcut.mWord;
- final int length = word.length();
- for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) {
- final int codePoint = word.codePointAt(i);
- size += CharEncoding.getCharSize(codePoint, codePointToOneByteCodeMap);
- }
- size += FormatSpec.PTNODE_TERMINATOR_SIZE;
- return size;
- }
-
- /**
- * Compute the size of a shortcut list in bytes.
- *
- * This is known in advance and does not change according to position in the file
- * like address lists do.
- */
- static int getShortcutListSize(final ArrayList<WeightedString> shortcutList,
- final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
- if (null == shortcutList || shortcutList.isEmpty()) return 0;
- int size = FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE;
- for (final WeightedString shortcut : shortcutList) {
- size += getShortcutSize(shortcut, codePointToOneByteCodeMap);
- }
- return size;
- }
-
- /**
* Compute the maximum size of a PtNode, assuming 3-byte addresses for everything.
*
* @param ptNode the PtNode to compute the size of.
@@ -137,8 +103,6 @@ public class BinaryDictEncoderUtils {
size += FormatSpec.PTNODE_FREQUENCY_SIZE;
}
size += FormatSpec.PTNODE_MAX_ADDRESS_SIZE; // For children address
- // TODO: Use codePointToOneByteCodeMap for shortcuts.
- size += getShortcutListSize(ptNode.mShortcutTargets, null /* codePointToOneByteCodeMap */);
if (null != ptNode.mBigrams) {
size += (FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE
+ FormatSpec.PTNODE_ATTRIBUTE_MAX_ADDRESS_SIZE)
@@ -241,27 +205,6 @@ public class BinaryDictEncoderUtils {
}
}
- @UsedForTesting
- static void writeUIntToDictBuffer(final DictBuffer dictBuffer, final int value,
- final int size) {
- switch(size) {
- case 4:
- dictBuffer.put((byte) ((value >> 24) & 0xFF));
- /* fall through */
- case 3:
- dictBuffer.put((byte) ((value >> 16) & 0xFF));
- /* fall through */
- case 2:
- dictBuffer.put((byte) ((value >> 8) & 0xFF));
- /* fall through */
- case 1:
- dictBuffer.put((byte) (value & 0xFF));
- break;
- default:
- /* nop */
- }
- }
-
// End utility methods
// This method is responsible for finding a nice ordering of the nodes that favors run-time
@@ -391,9 +334,6 @@ public class BinaryDictEncoderUtils {
nodeSize += getByteSize(getOffsetToTargetNodeArrayDuringUpdate(ptNodeArray,
nodeSize + size, ptNode.mChildren));
}
- // TODO: Use codePointToOneByteCodeMap for shortcuts.
- nodeSize += getShortcutListSize(ptNode.mShortcutTargets,
- null /* codePointToOneByteCodeMap */);
if (null != ptNode.mBigrams) {
for (WeightedString bigram : ptNode.mBigrams) {
final int offset = getOffsetToTargetPtNodeDuringUpdate(ptNodeArray,
@@ -568,14 +508,13 @@ public class BinaryDictEncoderUtils {
* @param hasMultipleChars whether the PtNode has multiple chars.
* @param isTerminal whether the PtNode is terminal.
* @param childrenAddressSize the size of a children address.
- * @param hasShortcuts whether the PtNode has shortcuts.
* @param hasBigrams whether the PtNode has bigrams.
* @param isNotAWord whether the PtNode is not a word.
* @param isPossiblyOffensive whether the PtNode is a possibly offensive entry.
* @return the flags
*/
static int makePtNodeFlags(final boolean hasMultipleChars, final boolean isTerminal,
- final int childrenAddressSize, final boolean hasShortcuts, final boolean hasBigrams,
+ final int childrenAddressSize, final boolean hasBigrams,
final boolean isNotAWord, final boolean isPossiblyOffensive) {
byte flags = 0;
if (hasMultipleChars) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS;
@@ -596,7 +535,6 @@ public class BinaryDictEncoderUtils {
default:
throw new RuntimeException("Node with a strange address");
}
- if (hasShortcuts) flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS;
if (hasBigrams) flags |= FormatSpec.FLAG_HAS_BIGRAMS;
if (isNotAWord) flags |= FormatSpec.FLAG_IS_NOT_A_WORD;
if (isPossiblyOffensive) flags |= FormatSpec.FLAG_IS_POSSIBLY_OFFENSIVE;
@@ -606,7 +544,6 @@ public class BinaryDictEncoderUtils {
/* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset) {
return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(),
getByteSize(childrenOffset),
- node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(),
node.mBigrams != null && !node.mBigrams.isEmpty(),
node.mIsNotAWord, node.mIsPossiblyOffensive);
}
@@ -621,7 +558,7 @@ public class BinaryDictEncoderUtils {
* @param word the second bigram, for debugging purposes
* @return the flags
*/
- /* package */ static final int makeBigramFlags(final boolean more, final int offset,
+ /* package */ static int makeBigramFlags(final boolean more, final int offset,
final int bigramFrequency, final int unigramFrequency, final String word) {
int bigramFlags = (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0)
+ (offset < 0 ? FormatSpec.FLAG_BIGRAM_ATTR_OFFSET_NEGATIVE : 0);
@@ -690,19 +627,7 @@ public class BinaryDictEncoderUtils {
return discretizedFrequency > 0 ? discretizedFrequency : 0;
}
- /**
- * Makes the flag value for a shortcut.
- *
- * @param more whether there are more attributes after this one.
- * @param frequency the frequency of the attribute, 0..15
- * @return the flags
- */
- static final int makeShortcutFlags(final boolean more, final int frequency) {
- return (more ? FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_HAS_NEXT : 0)
- + (frequency & FormatSpec.FLAG_BIGRAM_SHORTCUT_ATTR_FREQUENCY);
- }
-
- /* package */ static final int getChildrenPosition(final PtNode ptNode,
+ /* package */ static int getChildrenPosition(final PtNode ptNode,
final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
int positionOfChildrenPosField = ptNode.mCachedAddressAfterUpdate
+ getNodeHeaderSize(ptNode, codePointToOneByteCodeMap);
diff --git a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
index a42f0a93d..52060bed2 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/FusionDictionary.java
@@ -82,7 +82,6 @@ public final class FusionDictionary implements Iterable<WordProperty> {
public static final class PtNode {
private static final int NOT_A_TERMINAL = -1;
final int mChars[];
- ArrayList<WeightedString> mShortcutTargets;
ArrayList<WeightedString> mBigrams;
// null == mProbabilityInfo indicates this is not a terminal.
ProbabilityInfo mProbabilityInfo;
@@ -100,26 +99,23 @@ public final class FusionDictionary implements Iterable<WordProperty> {
int mCachedAddressBeforeUpdate; // The address of this PtNode (before update)
int mCachedAddressAfterUpdate; // The address of this PtNode (after update)
- public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets,
- final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo,
- final boolean isNotAWord, final boolean isPossiblyOffensive) {
+ public PtNode(final int[] chars, final ArrayList<WeightedString> bigrams,
+ final ProbabilityInfo probabilityInfo, final boolean isNotAWord,
+ final boolean isPossiblyOffensive) {
mChars = chars;
mProbabilityInfo = probabilityInfo;
mTerminalId = probabilityInfo == null ? NOT_A_TERMINAL : probabilityInfo.mProbability;
- mShortcutTargets = shortcutTargets;
mBigrams = bigrams;
mChildren = null;
mIsNotAWord = isNotAWord;
mIsPossiblyOffensive = isPossiblyOffensive;
}
- public PtNode(final int[] chars, final ArrayList<WeightedString> shortcutTargets,
- final ArrayList<WeightedString> bigrams, final ProbabilityInfo probabilityInfo,
- final boolean isNotAWord, final boolean isPossiblyOffensive,
- final PtNodeArray children) {
+ public PtNode(final int[] chars, final ArrayList<WeightedString> bigrams,
+ final ProbabilityInfo probabilityInfo, final boolean isNotAWord,
+ final boolean isPossiblyOffensive, final PtNodeArray children) {
mChars = chars;
mProbabilityInfo = probabilityInfo;
- mShortcutTargets = shortcutTargets;
mBigrams = bigrams;
mChildren = children;
mIsNotAWord = isNotAWord;
@@ -153,14 +149,6 @@ public final class FusionDictionary implements Iterable<WordProperty> {
return mIsPossiblyOffensive;
}
- public ArrayList<WeightedString> getShortcutTargets() {
- // We don't want write permission to escape outside the package, so we return a copy
- if (null == mShortcutTargets) return null;
- final ArrayList<WeightedString> copyOfShortcutTargets =
- new ArrayList<>(mShortcutTargets);
- return copyOfShortcutTargets;
- }
-
public ArrayList<WeightedString> getBigrams() {
// We don't want write permission to escape outside the package, so we return a copy
if (null == mBigrams) return null;
@@ -191,24 +179,6 @@ public final class FusionDictionary implements Iterable<WordProperty> {
}
/**
- * Gets the shortcut target for the given word. Returns null if the word is not in the
- * shortcut list.
- */
- public WeightedString getShortcut(final String word) {
- // TODO: Don't do a linear search
- if (mShortcutTargets != null) {
- final int size = mShortcutTargets.size();
- for (int i = 0; i < size; ++i) {
- WeightedString shortcut = mShortcutTargets.get(i);
- if (shortcut.mWord.equals(word)) {
- return shortcut;
- }
- }
- }
- return null;
- }
-
- /**
* Gets the bigram for the given word.
* Returns null if the word is not in the bigrams list.
*/
@@ -232,27 +202,9 @@ public final class FusionDictionary implements Iterable<WordProperty> {
* updated if they are higher than the existing ones.
*/
void update(final ProbabilityInfo probabilityInfo,
- final ArrayList<WeightedString> shortcutTargets,
final ArrayList<WeightedString> bigrams,
final boolean isNotAWord, final boolean isPossiblyOffensive) {
mProbabilityInfo = ProbabilityInfo.max(mProbabilityInfo, probabilityInfo);
- if (shortcutTargets != null) {
- if (mShortcutTargets == null) {
- mShortcutTargets = shortcutTargets;
- } else {
- final int size = shortcutTargets.size();
- for (int i = 0; i < size; ++i) {
- final WeightedString shortcut = shortcutTargets.get(i);
- final WeightedString existingShortcut = getShortcut(shortcut.mWord);
- if (existingShortcut == null) {
- mShortcutTargets.add(shortcut);
- } else {
- existingShortcut.mProbabilityInfo = ProbabilityInfo.max(
- existingShortcut.mProbabilityInfo, shortcut.mProbabilityInfo);
- }
- }
- }
- }
if (bigrams != null) {
if (mBigrams == null) {
mBigrams = bigrams;
@@ -312,19 +264,16 @@ public final class FusionDictionary implements Iterable<WordProperty> {
* Helper method to add a word as a string.
*
* This method adds a word to the dictionary with the given frequency. Optional
- * lists of bigrams and shortcuts can be passed here. For each word inside,
+ * lists of bigrams can be passed here. For each word inside,
* they will be added to the dictionary as necessary.
- *
- * @param word the word to add.
+ * @param word the word to add.
* @param probabilityInfo probability information of the word.
- * @param shortcutTargets a list of shortcut targets for this word, or null.
* @param isNotAWord true if this should not be considered a word (e.g. shortcut only)
* @param isPossiblyOffensive true if this word is possibly offensive
*/
public void add(final String word, final ProbabilityInfo probabilityInfo,
- final ArrayList<WeightedString> shortcutTargets, final boolean isNotAWord,
- final boolean isPossiblyOffensive) {
- add(getCodePoints(word), probabilityInfo, shortcutTargets, isNotAWord, isPossiblyOffensive);
+ final boolean isNotAWord, final boolean isPossiblyOffensive) {
+ add(getCodePoints(word), probabilityInfo, isNotAWord, isPossiblyOffensive);
}
/**
@@ -358,7 +307,7 @@ public final class FusionDictionary implements Iterable<WordProperty> {
if (ptNode0 != null) {
final PtNode ptNode1 = findWordInTree(mRootNodeArray, word1);
if (ptNode1 == null) {
- add(getCodePoints(word1), new ProbabilityInfo(0), null, false /* isNotAWord */,
+ add(getCodePoints(word1), new ProbabilityInfo(0), false /* isNotAWord */,
false /* isPossiblyOffensive */);
// The PtNode for the first word may have moved by the above insertion,
// if word1 and word2 share a common stem that happens not to have been
@@ -376,15 +325,12 @@ public final class FusionDictionary implements Iterable<WordProperty> {
*
* The shortcuts, if any, have to be in the dictionary already. If they aren't,
* an exception is thrown.
- *
- * @param word the word, as an int array.
+ * @param word the word, as an int array.
* @param probabilityInfo the probability information of the word.
- * @param shortcutTargets an optional list of shortcut targets for this word (null if none).
* @param isNotAWord true if this is not a word for spellcheking purposes (shortcut only or so)
* @param isPossiblyOffensive true if this word is possibly offensive
*/
private void add(final int[] word, final ProbabilityInfo probabilityInfo,
- final ArrayList<WeightedString> shortcutTargets,
final boolean isNotAWord, final boolean isPossiblyOffensive) {
assert(probabilityInfo.mProbability <= FormatSpec.MAX_TERMINAL_FREQUENCY);
if (word.length >= DecoderSpecificConstants.DICTIONARY_MAX_WORD_LENGTH) {
@@ -414,7 +360,7 @@ public final class FusionDictionary implements Iterable<WordProperty> {
// No node at this point to accept the word. Create one.
final int insertionIndex = findInsertionIndex(currentNodeArray, word[charIndex]);
final PtNode newPtNode = new PtNode(Arrays.copyOfRange(word, charIndex, word.length),
- shortcutTargets, null /* bigrams */, probabilityInfo, isNotAWord,
+ null /* bigrams */, probabilityInfo, isNotAWord,
isPossiblyOffensive);
currentNodeArray.mData.add(insertionIndex, newPtNode);
if (DBG) checkStack(currentNodeArray);
@@ -425,14 +371,14 @@ public final class FusionDictionary implements Iterable<WordProperty> {
// The new word is a prefix of an existing word, but the node on which it
// should end already exists as is. Since the old PtNode was not a terminal,
// make it one by filling in its frequency and other attributes
- currentPtNode.update(probabilityInfo, shortcutTargets, null, isNotAWord,
+ currentPtNode.update(probabilityInfo, null, isNotAWord,
isPossiblyOffensive);
} else {
// The new word matches the full old word and extends past it.
// We only have to create a new node and add it to the end of this.
final PtNode newNode = new PtNode(
Arrays.copyOfRange(word, charIndex + differentCharIndex, word.length),
- shortcutTargets, null /* bigrams */, probabilityInfo,
+ null /* bigrams */, probabilityInfo,
isNotAWord, isPossiblyOffensive);
currentPtNode.mChildren = new PtNodeArray();
currentPtNode.mChildren.mData.add(newNode);
@@ -441,7 +387,7 @@ public final class FusionDictionary implements Iterable<WordProperty> {
if (0 == differentCharIndex) {
// Exact same word. Update the frequency if higher. This will also add the
// new shortcuts to the existing shortcut list if it already exists.
- currentPtNode.update(probabilityInfo, shortcutTargets, null,
+ currentPtNode.update(probabilityInfo, null,
currentPtNode.mIsNotAWord && isNotAWord,
currentPtNode.mIsPossiblyOffensive || isPossiblyOffensive);
} else {
@@ -450,7 +396,7 @@ public final class FusionDictionary implements Iterable<WordProperty> {
PtNodeArray newChildren = new PtNodeArray();
final PtNode newOldWord = new PtNode(
Arrays.copyOfRange(currentPtNode.mChars, differentCharIndex,
- currentPtNode.mChars.length), currentPtNode.mShortcutTargets,
+ currentPtNode.mChars.length),
currentPtNode.mBigrams, currentPtNode.mProbabilityInfo,
currentPtNode.mIsNotAWord, currentPtNode.mIsPossiblyOffensive,
currentPtNode.mChildren);
@@ -460,17 +406,17 @@ public final class FusionDictionary implements Iterable<WordProperty> {
if (charIndex + differentCharIndex >= word.length) {
newParent = new PtNode(
Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex),
- shortcutTargets, null /* bigrams */, probabilityInfo,
+ null /* bigrams */, probabilityInfo,
isNotAWord, isPossiblyOffensive, newChildren);
} else {
newParent = new PtNode(
Arrays.copyOfRange(currentPtNode.mChars, 0, differentCharIndex),
- null /* shortcutTargets */, null /* bigrams */,
- null /* probabilityInfo */, false /* isNotAWord */,
- false /* isPossiblyOffensive */, newChildren);
+ null /* bigrams */, null /* probabilityInfo */,
+ false /* isNotAWord */, false /* isPossiblyOffensive */,
+ newChildren);
final PtNode newWord = new PtNode(Arrays.copyOfRange(word,
charIndex + differentCharIndex, word.length),
- shortcutTargets, null /* bigrams */, probabilityInfo,
+ null /* bigrams */, probabilityInfo,
isNotAWord, isPossiblyOffensive);
final int addIndex = word[charIndex + differentCharIndex]
> currentPtNode.mChars[differentCharIndex] ? 1 : 0;
@@ -532,7 +478,7 @@ public final class FusionDictionary implements Iterable<WordProperty> {
private static int findInsertionIndex(final PtNodeArray nodeArray, int character) {
final ArrayList<PtNode> data = nodeArray.mData;
final PtNode reference = new PtNode(new int[] { character },
- null /* shortcutTargets */, null /* bigrams */, null /* probabilityInfo */,
+ null /* bigrams */, null /* probabilityInfo */,
false /* isNotAWord */, false /* isPossiblyOffensive */);
int result = Collections.binarySearch(data, reference, PTNODE_COMPARATOR);
return result >= 0 ? result : -result - 1;
@@ -669,8 +615,7 @@ public final class FusionDictionary implements Iterable<WordProperty> {
}
if (currentPtNode.isTerminal()) {
return new WordProperty(mCurrentString.toString(),
- currentPtNode.mProbabilityInfo,
- currentPtNode.mShortcutTargets, currentPtNode.mBigrams,
+ currentPtNode.mProbabilityInfo, currentPtNode.mBigrams,
currentPtNode.mIsNotAWord, currentPtNode.mIsPossiblyOffensive);
}
} else {
diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
index 5c261a94d..7ee1df92b 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictDecoder.java
@@ -285,7 +285,7 @@ public class Ver2DictDecoder extends AbstractDictDecoder {
// Insert unigrams into the fusion dictionary.
for (final WordProperty wordProperty : wordProperties) {
fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo,
- wordProperty.mShortcutTargets, wordProperty.mIsNotAWord,
+ wordProperty.mIsNotAWord,
wordProperty.mIsPossiblyOffensive);
}
// Insert bigrams into the fusion dictionary.
diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
index b52b8c485..c63b972eb 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoder.java
@@ -240,37 +240,6 @@ public class Ver2DictEncoder implements DictEncoder {
}
/**
- * Write a shortcut attributes list to mBuffer.
- *
- * @param shortcuts the shortcut attributes list.
- */
- private void writeShortcuts(final ArrayList<WeightedString> shortcuts,
- final HashMap<Integer, Integer> codePointToOneByteCodeMap) {
- if (null == shortcuts || shortcuts.isEmpty()) return;
-
- final int indexOfShortcutByteSize = mPosition;
- mPosition += FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE;
- final Iterator<WeightedString> shortcutIterator = shortcuts.iterator();
- while (shortcutIterator.hasNext()) {
- final WeightedString target = shortcutIterator.next();
- final int shortcutFlags = BinaryDictEncoderUtils.makeShortcutFlags(
- shortcutIterator.hasNext(),
- target.getProbability());
- mPosition = BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, mPosition, shortcutFlags,
- FormatSpec.PTNODE_ATTRIBUTE_FLAGS_SIZE);
- final int shortcutShift = CharEncoding.writeString(mBuffer, mPosition, target.mWord,
- codePointToOneByteCodeMap);
- mPosition += shortcutShift;
- }
- final int shortcutByteSize = mPosition - indexOfShortcutByteSize;
- if (shortcutByteSize > FormatSpec.MAX_SHORTCUT_LIST_SIZE_IN_A_PTNODE) {
- throw new RuntimeException("Shortcut list too large");
- }
- BinaryDictEncoderUtils.writeUIntToBuffer(mBuffer, indexOfShortcutByteSize, shortcutByteSize,
- FormatSpec.PTNODE_SHORTCUT_LIST_SIZE_SIZE);
- }
-
- /**
* Write a bigram attributes list to mBuffer.
*
* @param bigrams the bigram attributes list.
@@ -305,8 +274,6 @@ public class Ver2DictEncoder implements DictEncoder {
writeCharacters(ptNode.mChars, ptNode.hasSeveralChars(), codePointToOneByteCodeMap);
writeFrequency(ptNode.getProbability());
writeChildrenPosition(ptNode, codePointToOneByteCodeMap);
- // TODO: Use codePointToOneByteCodeMap for shortcuts.
- writeShortcuts(ptNode.mShortcutTargets, null /* codePointToOneByteCodeMap */);
writeBigrams(ptNode.mBigrams, dict);
}
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java
index 7d858760e..dbf9b7ac2 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/Ver2DictEncoderTests.java
@@ -16,9 +16,7 @@
package com.android.inputmethod.latin.makedict;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
@@ -49,7 +47,7 @@ public class Ver2DictEncoderTests extends AndroidTestCase {
new FormatSpec.FormatOptions(FormatSpec.VERSION2);
final FusionDictionary sourcedict = new FusionDictionary(new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
- addUnigrams(sourcedict, words, null /* shortcutMap */);
+ addUnigrams(sourcedict, words);
final CodePointTable codePointTable = Ver2DictEncoder.makeCodePointTable(sourcedict);
// Check if mCodePointOccurrenceArray is correct
@@ -73,17 +71,10 @@ public class Ver2DictEncoderTests extends AndroidTestCase {
/**
* Adds unigrams to the dictionary.
*/
- private static void addUnigrams(final FusionDictionary dict, final List<String> words,
- final HashMap<String, List<String>> shortcutMap) {
+ private static void addUnigrams(final FusionDictionary dict, final List<String> words) {
for (final String word : words) {
- final ArrayList<WeightedString> shortcuts = new ArrayList<>();
- if (shortcutMap != null && shortcutMap.containsKey(word)) {
- for (final String shortcut : shortcutMap.get(word)) {
- shortcuts.add(new WeightedString(shortcut, UNIGRAM_FREQ));
- }
- }
dict.add(word, new ProbabilityInfo(UNIGRAM_FREQ),
- (shortcutMap == null) ? null : shortcuts, false /* isNotAWord */,
+ false /* isNotAWord */,
false /* isPossiblyOffensive */);
}
}
diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
index 63ea89c1d..746431dfa 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictDecoder.java
@@ -84,7 +84,7 @@ public class Ver4DictDecoder extends AbstractDictDecoder {
// Insert unigrams into the fusion dictionary.
for (final WordProperty wordProperty : wordProperties) {
fusionDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo,
- wordProperty.mShortcutTargets, wordProperty.mIsNotAWord,
+ wordProperty.mIsNotAWord,
wordProperty.mIsPossiblyOffensive);
}
// Insert bigrams into the fusion dictionary.
diff --git a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
index 1e4bd768c..6e7b37d54 100644
--- a/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
+++ b/tests/src/com/android/inputmethod/latin/makedict/Ver4DictEncoder.java
@@ -74,26 +74,10 @@ public class Ver4DictEncoder implements DictEncoder {
throw new IOException("Cannot create dictionary file");
}
for (final WordProperty wordProperty : dict) {
- // TODO: switch to addMultipleDictionaryEntries when they support shortcuts
- if (null == wordProperty.mShortcutTargets || wordProperty.mShortcutTargets.isEmpty()) {
- if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
- null /* shortcutTarget */, 0 /* shortcutProbability */,
- wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
- wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) {
- MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord);
- }
- } else {
- for (final WeightedString shortcutTarget : wordProperty.mShortcutTargets) {
- if (!binaryDict.addUnigramEntry(wordProperty.mWord,
- wordProperty.getProbability(),
- shortcutTarget.mWord, shortcutTarget.getProbability(),
- wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
- wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) {
- MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord
- + ", shortcutTarget: " + shortcutTarget.mWord);
- return;
- }
- }
+ if (!binaryDict.addUnigramEntry(wordProperty.mWord, wordProperty.getProbability(),
+ wordProperty.mIsBeginningOfSentence, wordProperty.mIsNotAWord,
+ wordProperty.mIsPossiblyOffensive, 0 /* timestamp */)) {
+ MakedictLog.e("Cannot add unigram entry for " + wordProperty.mWord);
}
if (binaryDict.needsToRunGC(true /* mindsBlockByGC */)) {
if (!binaryDict.flushWithGC()) {